fixup! Prerelease collector contrib v0.51.0 (open-telemetry#9975)

.github/CODEOWNERS

@@ -22,6 +22,7 @@ exporter/awscloudwatchlogsexporter                   @open-telemetry/collector-c
 exporter/awsemfexporter/                             @open-telemetry/collector-contrib-approvers @Aneurysm9 @shaochengwang @mxiamxia
 exporter/awskinesisexporter/                         @open-telemetry/collector-contrib-approvers @Aneurysm9 @MovieStoreGuy
 exporter/awsprometheusremotewriteexporter/           @open-telemetry/collector-contrib-approvers @Aneurysm9 @alolita
+exporter/awss3exporter/                              @open-telemetry/collector-contrib-approvers @pmm-sumo
 exporter/awsxrayexporter/                            @open-telemetry/collector-contrib-approvers @willarmiros
 exporter/azuremonitorexporter/                       @open-telemetry/collector-contrib-approvers @pcwiese
 exporter/carbonexporter/                             @open-telemetry/collector-contrib-approvers @pjanotti

CHANGELOG.md

@@ -98,6 +98,7 @@
 - `schemaprocessor`: Starting the initial work to allow from translating from semantic convention to another (#8371)
 - `saphanareceiver`: Added implementation of SAP HANA Metric Receiver (#8827)
 - `logstransformprocessor`: Add implementation of Logs Transform Processor (#9335)
+- `awss3exporter`: Add aws s3 exporter (#2835)
 
 ### 💡 Enhancements 💡
 

exporter/awss3exporter/README.md

@@ -1,8 +1,13 @@
 # AWS S3 Exporter for OpenTelemetry Collector
-This exporter converts OpenTelemetry metrics, logs and traces to supported format and upload to S3
+
+| Status                   |                       |
+| ------------------------ |-----------------------|
+| Stability                | [in development]      |
+| Supported pipeline types | traces, logs          |
+| Distributions            | [contrib]             |
 
 ## Schema supported
-This exporter targets to support parquet/json format
+This exporter targets to support proto/json and proto/binary format
 
 ## Exporter Configuration
 
@@ -12,8 +17,30 @@ The following exporter configuration parameters are supported.
 | :--------------------- | :--------------------------------------------------------------------------------- | ------- |
 | `region`               | AWS region.                                                                        |         |
 | `s3_bucket`            | S3 bucket                                                                          |         |
-| `s3_prefix`            | prefix for the S3 key.                                                             |         |
+| `s3_prefix`            | prefix for the S3 key (root directory inside bucket).                              |         |
 | `s3_partition`         | time granularity of S3 key: hour or minute                                         |"minute" |
+| `file_prefix`          | file prefix defined by user                                                        |         |
+| `marshaler_name`       | marshaler used to produce output data otlp_json or otlp_proto                      |         |
+
+# Example Configuration
+
+Following example configuration defines to store output in 'eu-central' region and bucket named 'databucket'.
+
+```yaml
+exporters:
+  awss3:
+    s3uploader:
+        region: 'eu-central-1'
+        s3_bucket: 'databucket'
+        s3_prefix: 'metric'
+        s3_partition: 'minute'
+```
+
+Logs and traces will be stored inside 'databucket' in the following path format.
+
+```console
+metric/year=XXXX/month=XX/day=XX/hour=XX/minute=XX
+```
 
 ## AWS Credential Configuration
 

exporter/awss3exporter/config.go

@@ -12,7 +12,7 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-package awss3exporter
+package awss3exporter // import "github.com/open-telemetry/opentelemetry-collector-contrib/exporter/awss3exporter"
 
 import (
 	"go.opentelemetry.io/collector/config"
@@ -22,22 +22,19 @@ import (
 // S3UploaderConfig contains aws s3 uploader related config to controls things
 // like bucket, prefix, batching, connections, retries, etc.
 type S3UploaderConfig struct {
-	Region              string `mapstructure:"region"`
-	S3Bucket            string `mapstructure:"s3_bucket"`
-	S3Prefix            string `mapstructure:"s3_prefix"`
-	S3Partition         string `mapstructure:"s3_partition"`
-	FilePrefix          string `mapstructure:"file_prefix"`
+	Region      string `mapstructure:"region"`
+	S3Bucket    string `mapstructure:"s3_bucket"`
+	S3Prefix    string `mapstructure:"s3_prefix"`
+	S3Partition string `mapstructure:"s3_partition"`
+	FilePrefix  string `mapstructure:"file_prefix"`
 }
 
 // Config contains the main configuration options for the awskinesis exporter
 type Config struct {
 	config.ExporterSettings `mapstructure:",squash"`
 
-	FileFormat string           `mapstructure:"file_format"`
-	S3Uploader S3UploaderConfig `mapstructure:"s3uploader"`
-
-	// MetricDescriptors is the list of override metric descriptors
-	MetricDescriptors []MetricDescriptor `mapstructure:"metric_descriptors"`
+	S3Uploader    S3UploaderConfig `mapstructure:"s3uploader"`
+	MarshalerName string           `mapstructure:"marshaler_name"`
 
 	// ResourceToTelemetrySettings is the option for converting resource attrihutes to telemetry attributes.
 	// "Enabled" - A boolean field to enable/disable this option. Default is `false`.

exporter/awss3exporter/config_test.go

@@ -0,0 +1,81 @@
+// Copyright 2021 OpenTelemetry Authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      shttp://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package awss3exporter
+
+import (
+	"path/filepath"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+	"go.opentelemetry.io/collector/component/componenttest"
+	"go.opentelemetry.io/collector/config"
+	"go.opentelemetry.io/collector/service/servicetest"
+)
+
+func TestLoadConfig(t *testing.T) {
+	factories, err := componenttest.NopFactories()
+	assert.NoError(t, err)
+
+	factory := NewFactory()
+	factories.Exporters[typeStr] = factory
+	cfg, err := servicetest.LoadConfigAndValidate(filepath.Join("testdata", "default.yaml"), factories)
+
+	require.NoError(t, err)
+	require.NotNil(t, cfg)
+
+	e := cfg.Exporters[config.NewComponentID(typeStr)]
+
+	assert.Equal(t, e,
+		&Config{
+			ExporterSettings: config.NewExporterSettings(config.NewComponentID(typeStr)),
+
+			S3Uploader: S3UploaderConfig{
+				Region:      "us-east-1",
+				S3Partition: "minute",
+			},
+			MarshalerName: "otlp_json",
+		},
+	)
+}
+
+func TestConfig(t *testing.T) {
+	factories, err := componenttest.NopFactories()
+	assert.Nil(t, err)
+
+	factory := NewFactory()
+	factories.Exporters[factory.Type()] = factory
+	cfg, err := servicetest.LoadConfigAndValidate(
+		filepath.Join("testdata", "config.yaml"), factories)
+
+	require.NoError(t, err)
+	require.NotNil(t, cfg)
+
+	e := cfg.Exporters[config.NewComponentID(typeStr)]
+
+	assert.Equal(t, e,
+		&Config{
+			ExporterSettings: config.NewExporterSettings(config.NewComponentID(typeStr)),
+
+			S3Uploader: S3UploaderConfig{
+				Region:      "us-east-1",
+				S3Bucket:    "foo",
+				S3Prefix:    "bar",
+				S3Partition: "minute",
+			},
+			MarshalerName: "otlp_json",
+		},
+	)
+}

exporter/awss3exporter/data_writer.go

@@ -0,0 +1,21 @@
+// Copyright 2022 OpenTelemetry Authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package awss3exporter // import "github.com/open-telemetry/opentelemetry-collector-contrib/exporter/awss3exporter"
+
+import "context"
+
+type DataWriter interface {
+	WriteBuffer(ctx context.Context, buf []byte, config *Config, metadata string, format string) error
+}

exporter/awss3exporter/exporter.go

@@ -12,7 +12,7 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-package awss3exporter
+package awss3exporter // import "github.com/open-telemetry/opentelemetry-collector-contrib/exporter/awss3exporter"
 
 import (
 	"context"
@@ -27,13 +27,11 @@ import (
 	"go.uber.org/zap"
 )
 
-var logsMarshaler = plog.NewJSONMarshaler()
-var traceMarshaler = ptrace.NewJSONMarshaler()
-
 type S3Exporter struct {
-	config           config.Exporter
-	metricTranslator metricTranslator
-	logger           *zap.Logger
+	config     config.Exporter
+	dataWriter DataWriter
+	logger     *zap.Logger
+	marshaler  Marshaler
 }
 
 func NewS3Exporter(config config.Exporter,
@@ -47,12 +45,22 @@ func NewS3Exporter(config config.Exporter,
 	expConfig := config.(*Config)
 	expConfig.logger = logger
 
-	expConfig.Validate()
+	validateConfig := expConfig.Validate()
+
+	if validateConfig != nil {
+		return nil, validateConfig
+	}
+
+	marshaler, err := NewMarshaler(expConfig.MarshalerName, logger)
+	if err != nil {
+		return nil, errors.New("unknown marshaler")
+	}
 
 	s3Exporter := &S3Exporter{
-		config:           config,
-		metricTranslator: newMetricTranslator(*expConfig),
-		logger:           logger,
+		config:     config,
+		dataWriter: &S3Writer{},
+		logger:     logger,
+		marshaler:  marshaler,
 	}
 	return s3Exporter, nil
 }

exporter/awss3exporter/factory.go

@@ -12,7 +12,7 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-package awss3exporter
+package awss3exporter // import "github.com/open-telemetry/opentelemetry-collector-contrib/exporter/awss3exporter"
 
 import (
 	"context"
@@ -24,15 +24,14 @@ import (
 
 const (
 	// The value of "type" key in configuration.
-	typeStr       = "awss3"
+	typeStr = "awss3"
 )
 
 // NewFactory creates a factory for S3 exporter.
 func NewFactory() component.ExporterFactory {
 	return component.NewExporterFactory(
 		typeStr,
 		createDefaultConfig,
-		component.WithMetricsExporter(createMetricsExporter),
 		component.WithLogsExporter(createLogsExporter),
 		component.WithTracesExporter(createTracesExporter))
 }
@@ -42,30 +41,15 @@ func createDefaultConfig() config.Exporter {
 		ExporterSettings: config.NewExporterSettings(config.NewComponentID(typeStr)),
 
 		S3Uploader: S3UploaderConfig{
-			Region:              "us-east-1",
-			S3Partition:         "minute",
+			Region:      "us-east-1",
+			S3Partition: "minute",
 		},
 
-		MetricDescriptors: make([]MetricDescriptor, 0),
-		logger:            nil,
+		MarshalerName: "otlp_json",
+		logger:        nil,
 	}
 }
 
-func createMetricsExporter(ctx context.Context,
-	params component.ExporterCreateSettings,
-	config config.Exporter) (component.MetricsExporter, error) {
-
-	s3Exporter, err := NewS3Exporter(config, params)
-	if err != nil {
-		return nil, err
-	}
-
-	return exporterhelper.NewMetricsExporter(
-		config,
-		params,
-		s3Exporter.ConsumeMetrics)
-}
-
 func createLogsExporter(ctx context.Context,
 	params component.ExporterCreateSettings,
 	config config.Exporter) (component.LogsExporter, error) {

exporter/awss3exporter/go.mod

@@ -1,3 +1,38 @@
 module github.com/open-telemetry/opentelemetry-collector-contrib/exporter/awss3exporter
 
 go 1.17
+
+require (
+	github.com/stretchr/testify v1.7.4
+	go.opentelemetry.io/collector v0.54.0
+	go.opentelemetry.io/collector/pdata v0.54.0
+	go.uber.org/zap v1.21.0
+)
+
+require (
+	github.com/cenkalti/backoff/v4 v4.1.3 // indirect
+	github.com/davecgh/go-spew v1.1.1 // indirect
+	github.com/gogo/protobuf v1.3.2 // indirect
+	github.com/golang/protobuf v1.5.2 // indirect
+	github.com/json-iterator/go v1.1.12 // indirect
+	github.com/knadh/koanf v1.4.2 // indirect
+	github.com/mitchellh/copystructure v1.2.0 // indirect
+	github.com/mitchellh/mapstructure v1.5.0 // indirect
+	github.com/mitchellh/reflectwalk v1.0.2 // indirect
+	github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect
+	github.com/modern-go/reflect2 v1.0.2 // indirect
+	github.com/pmezard/go-difflib v1.0.0 // indirect
+	go.opencensus.io v0.23.0 // indirect
+	go.opentelemetry.io/otel v1.7.0 // indirect
+	go.opentelemetry.io/otel/metric v0.30.0 // indirect
+	go.opentelemetry.io/otel/trace v1.7.0 // indirect
+	go.uber.org/atomic v1.9.0 // indirect
+	go.uber.org/multierr v1.8.0 // indirect
+	golang.org/x/net v0.0.0-20220225172249-27dd8689420f // indirect
+	golang.org/x/sys v0.0.0-20220128215802-99c3d69c2c27 // indirect
+	golang.org/x/text v0.3.7 // indirect
+	google.golang.org/genproto v0.0.0-20211208223120-3a66f561d7aa // indirect
+	google.golang.org/grpc v1.47.0 // indirect
+	google.golang.org/protobuf v1.28.0 // indirect
+	gopkg.in/yaml.v3 v3.0.1 // indirect
+)