Add Loki as a log exporter (first PR: overall structure) (#1900)

* added initial README

* add initial structure

* updated name of setting to better reflect its usage

* updated named return value to reflect usage

* fixed name in comment

* renamed setting per review recommendation

* updated attributes_for_labels examples to use both semantic conventional and non-conventional attribute names

* updated setting descriptions per review

* addressed lint build issue

* updated verbiage in readme

* Trigger rebuild and test, due to non-related unit test coverage failure in K8s observer

exporter/lokiexporter/Makefile

@@ -0,0 +1 @@
+include ../../Makefile.Common

exporter/lokiexporter/README.md

@@ -0,0 +1,62 @@
+# Loki Exporter
+
+Exports data via HTTP to [Loki](https://grafana.com/docs/loki/latest/).
+
+Supported pipeline types: logs
+
+## Getting Started
+
+The following settings are required:
+
+- `endpoint` (no default): The target URL to send Loki log streams to (
+  e.g.: https://loki.example.com:3100/loki/api/v1/push).
+
+
+- `attributes_for_labels` (no default): List of allowed attributes to be added as labels to Loki log 
+  streams. This is a safety net to help prevent accidentally adding dynamic labels that may significantly increase 
+  cardinality, thus having a performance impact on your Loki instance. See the 
+  [Loki label best practices](https://grafana.com/docs/loki/latest/best-practices/current-best-practices/) page for 
+  additional details on the types of labels you may want to associate with log streams.
+
+The following settings can be optionally configured:
+
+- `insecure` (default = false): When set to true disables verifying the server's certificate chain and host name. The
+  connection is still encrypted but server identity is not verified.
+- `ca_file` (no default) Path to the CA cert to verify the server being connected to. Should only be used if `insecure` 
+  is set to false.
+- `cert_file` (no default) Path to the TLS cert to use for client connections when TLS client auth is required. 
+  Should only be used if `insecure` is set to false.
+- `key_file` (no default) Path to the TLS key to use for TLS required connections. Should only be used if `insecure` is
+  set to false.
+
+
+- `timeout` (default = 30s): HTTP request time limit. For details see https://golang.org/pkg/net/http/#Client
+- `read_buffer_size` (default = 0): ReadBufferSize for HTTP client.
+- `write_buffer_size` (default = 512 * 1024): WriteBufferSize for HTTP client.
+
+
+- `headers` (no default): Name/value pairs added to the HTTP request headers. Loki by default uses the "X-Scope-OrgID"
+  header to identify the tenant the log is associated to.
+
+Example:
+
+```yaml
+loki:
+  endpoint: https://loki.example.com:3100/loki/api/v1/push
+  attributes_for_labels:
+    - container.name
+    - k8s.cluster.name
+    - severity
+  headers:
+    "X-Scope-OrgID": "example"
+```
+
+The full list of settings exposed for this exporter are documented [here](./config.go) with detailed sample
+configurations [here](./testdata/config.yaml).
+
+## Advanced Configuration
+
+Several helper files are leveraged to provide additional capabilities automatically:
+
+- [HTTP settings](https://github.com/open-telemetry/opentelemetry-collector/blob/master/config/confighttp/README.md)
+- [Queuing and retry settings](https://github.com/open-telemetry/opentelemetry-collector/blob/master/exporter/exporterhelper/README.md)

exporter/lokiexporter/config.go

@@ -0,0 +1,32 @@
+// Copyright The OpenTelemetry Authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package lokiexporter
+
+import (
+	"go.opentelemetry.io/collector/config/confighttp"
+	"go.opentelemetry.io/collector/config/configmodels"
+	"go.opentelemetry.io/collector/exporter/exporterhelper"
+)
+
+// Config defines configuration for Loki exporter.
+type Config struct {
+	configmodels.ExporterSettings `mapstructure:",squash"` // squash ensures fields are correctly decoded in embedded struct.
+	confighttp.HTTPClientSettings `mapstructure:",squash"` // squash ensures fields are correctly decoded in embedded struct.
+	exporterhelper.QueueSettings  `mapstructure:"sending_queue"`
+	exporterhelper.RetrySettings  `mapstructure:"retry_on_failure"`
+
+	// The attributes that are allowed to be added as labels on the log stream sent to Loki.
+	AttributesForLabels []string `mapstructure:"attributes_for_labels"`
+}

exporter/lokiexporter/config_test.go

@@ -0,0 +1,80 @@
+// Copyright The OpenTelemetry Authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package lokiexporter
+
+import (
+	"path"
+	"testing"
+	"time"
+
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+	"go.opentelemetry.io/collector/component/componenttest"
+	"go.opentelemetry.io/collector/config/confighttp"
+	"go.opentelemetry.io/collector/config/configmodels"
+	"go.opentelemetry.io/collector/config/configtest"
+	"go.opentelemetry.io/collector/config/configtls"
+	"go.opentelemetry.io/collector/exporter/exporterhelper"
+	"go.opentelemetry.io/collector/translator/conventions"
+)
+
+func TestLoadConfig(t *testing.T) {
+	factories, err := componenttest.ExampleComponents()
+	assert.Nil(t, err)
+
+	factory := NewFactory()
+	factories.Exporters[configmodels.Type(typeStr)] = factory
+	cfg, err := configtest.LoadConfigFile(t, path.Join(".", "testdata", "config.yaml"), factories)
+
+	require.NoError(t, err)
+	require.NotNil(t, cfg)
+
+	assert.Equal(t, 2, len(cfg.Exporters))
+
+	actualCfg := cfg.Exporters["loki/allsettings"].(*Config)
+	expectedCfg := Config{
+		ExporterSettings: configmodels.ExporterSettings{TypeVal: typeStr, NameVal: "loki/allsettings"},
+		HTTPClientSettings: confighttp.HTTPClientSettings{
+			Headers: map[string]string{
+				"x-scope-orgid": "example",
+			},
+			Endpoint: "https://loki:3100/loki/api/v1/push",
+			TLSSetting: configtls.TLSClientSetting{
+				TLSSetting: configtls.TLSSetting{
+					CAFile:   "/var/lib/mycert.pem",
+					CertFile: "certfile",
+					KeyFile:  "keyfile",
+				},
+				Insecure: true,
+			},
+			ReadBufferSize:  123,
+			WriteBufferSize: 345,
+			Timeout:         time.Second * 10,
+		},
+		RetrySettings: exporterhelper.RetrySettings{
+			Enabled:         true,
+			InitialInterval: 10 * time.Second,
+			MaxInterval:     1 * time.Minute,
+			MaxElapsedTime:  10 * time.Minute,
+		},
+		QueueSettings: exporterhelper.QueueSettings{
+			Enabled:      true,
+			NumConsumers: 2,
+			QueueSize:    10,
+		},
+		AttributesForLabels: []string{conventions.AttributeContainerName, conventions.AttributeK8sCluster, "severity"},
+	}
+	assert.Equal(t, &expectedCfg, actualCfg)
+}

exporter/lokiexporter/doc.go

@@ -0,0 +1,16 @@
+// Copyright The OpenTelemetry Authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+// Package lokiexporter implements an exporter that sends log data to Loki.
+package lokiexporter

exporter/lokiexporter/factory.go

@@ -0,0 +1,76 @@
+// Copyright The OpenTelemetry Authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package lokiexporter
+
+import (
+	"context"
+	"time"
+
+	"go.opentelemetry.io/collector/component"
+	"go.opentelemetry.io/collector/config/confighttp"
+	"go.opentelemetry.io/collector/config/configmodels"
+	"go.opentelemetry.io/collector/exporter/exporterhelper"
+)
+
+const typeStr = "loki"
+
+// NewFactory creates a factory for Loki exporter.
+func NewFactory() component.ExporterFactory {
+	return exporterhelper.NewFactory(
+		typeStr,
+		createDefaultConfig,
+		exporterhelper.WithLogs(createLogsExporter),
+	)
+}
+
+func createDefaultConfig() configmodels.Exporter {
+	return &Config{
+		ExporterSettings: configmodels.ExporterSettings{
+			TypeVal: configmodels.Type(typeStr),
+			NameVal: typeStr,
+		},
+		HTTPClientSettings: confighttp.HTTPClientSettings{
+			Endpoint: "",
+			Timeout:  30 * time.Second,
+			Headers:  map[string]string{},
+			// We almost read 0 bytes, so no need to tune ReadBufferSize.
+			WriteBufferSize: 512 * 1024,
+		},
+		RetrySettings:       exporterhelper.DefaultRetrySettings(),
+		QueueSettings:       exporterhelper.DefaultQueueSettings(),
+		AttributesForLabels: []string{},
+	}
+}
+
+func createLogsExporter(_ context.Context, params component.ExporterCreateParams, config configmodels.Exporter) (component.LogsExporter, error) {
+	expCfg := config.(*Config)
+
+	exp, err := newExporter(expCfg, params.Logger)
+	if err != nil {
+		return nil, err
+	}
+
+	return exporterhelper.NewLogsExporter(
+		expCfg,
+		params.Logger,
+		exp.pushLogData,
+		// explicitly disable since we rely on http.Client timeout logic.
+		exporterhelper.WithTimeout(exporterhelper.TimeoutSettings{Timeout: 0}),
+		exporterhelper.WithRetry(expCfg.RetrySettings),
+		exporterhelper.WithQueue(expCfg.QueueSettings),
+		exporterhelper.WithStart(exp.start),
+		exporterhelper.WithShutdown(exp.stop),
+	)
+}

exporter/lokiexporter/factory_test.go

@@ -0,0 +1,56 @@
+// Copyright The OpenTelemetry Authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package lokiexporter
+
+import (
+	"context"
+	"testing"
+	"time"
+
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+	"go.opentelemetry.io/collector/component"
+	"go.opentelemetry.io/collector/config/configcheck"
+	"go.opentelemetry.io/collector/testutil"
+	"go.uber.org/zap"
+)
+
+func TestCreateDefaultConfig(t *testing.T) {
+	factory := NewFactory()
+	cfg := factory.CreateDefaultConfig()
+	assert.NotNil(t, cfg, "failed to create default config")
+	assert.NoError(t, configcheck.ValidateConfig(cfg))
+	ocfg, ok := factory.CreateDefaultConfig().(*Config)
+	assert.True(t, ok)
+	assert.Equal(t, "", ocfg.HTTPClientSettings.Endpoint)
+	assert.Equal(t, 30*time.Second, ocfg.HTTPClientSettings.Timeout, "default timeout is 30 seconds")
+	assert.Equal(t, true, ocfg.RetrySettings.Enabled, "default retry is enabled")
+	assert.Equal(t, 300*time.Second, ocfg.RetrySettings.MaxElapsedTime, "default retry MaxElapsedTime")
+	assert.Equal(t, 5*time.Second, ocfg.RetrySettings.InitialInterval, "default retry InitialInterval")
+	assert.Equal(t, 30*time.Second, ocfg.RetrySettings.MaxInterval, "default retry MaxInterval")
+	assert.Equal(t, true, ocfg.QueueSettings.Enabled, "default sending queue is enabled")
+}
+
+func TestCreateLogExporter(t *testing.T) {
+	factory := NewFactory()
+	cfg := factory.CreateDefaultConfig().(*Config)
+	cfg.HTTPClientSettings.Endpoint = "http://" + testutil.GetAvailableLocalAddress(t)
+	cfg.AttributesForLabels = []string{"app", "level"}
+
+	creationParams := component.ExporterCreateParams{Logger: zap.NewNop()}
+	exp, err := factory.CreateLogsExporter(context.Background(), creationParams, cfg)
+	require.Nil(t, err)
+	require.NotNil(t, exp)
+}

exporter/lokiexporter/go.mod

@@ -0,0 +1,9 @@
+module github.com/open-telemetry/opentelemetry-collector-contrib/exporter/lokiexporter
+
+go 1.14
+
+require (
+	github.com/stretchr/testify v1.6.1
+	go.opentelemetry.io/collector v0.17.0
+	go.uber.org/zap v1.16.0
+)