Read-write deployment mode integration test (#3423)

* Add a first pass of a integration test for read-write mode.

* Add note about how to deal with integration tests timing out.

* Remove unnecessary min-ready-duration from read and backend instances.

* Address PR feedback.

* use one bucket for all storage
* use CommonStorageBackendFlags() to configure storage connectivity
* move replication factor flag to New(Read|Write|Backend)Instance()

docs/internal/contributing/how-integration-tests-work.md

@@ -22,6 +22,12 @@ If you want to run a single test you can use a filter. For example, to only run
 go test -v -tags=requires_docker ./integration -run "^TestChunksStorageAllIndexBackends$"
 ```
 
+When running all integration tests, the test process may time out before the tests complete. If this happens, you can increase `go test`'s default 10 minute timeout to something longer with the `-timeout` flag, for example:
+
+```
+go test -v -tags=requires_docker -timeout=20m ./integration/...
+```
+
 ### Supported environment variables
 
 - **`MIMIR_IMAGE`**<br />

integration/e2emimir/services.go

@@ -212,6 +212,47 @@ func NewSingleBinary(name string, flags map[string]string, options ...Option) *M
 	)
 }
 
+func NewReadInstance(name string, flags map[string]string, options ...Option) *MimirService {
+	return newMimirServiceFromOptions(
+		name,
+		map[string]string{
+			"-target":                           "read",
+			"-log.level":                        "warn",
+			"-ingester.ring.replication-factor": "1",
+		},
+		flags,
+		options...,
+	)
+}
+
+func NewWriteInstance(name string, flags map[string]string, options ...Option) *MimirService {
+	return newMimirServiceFromOptions(
+		name,
+		map[string]string{
+			"-target":                           "write",
+			"-log.level":                        "warn",
+			"-ingester.ring.replication-factor": "1",
+			// Speed up startup.
+			"-ingester.ring.min-ready-duration": "0s",
+		},
+		flags,
+		options...,
+	)
+}
+
+func NewBackendInstance(name string, flags map[string]string, options ...Option) *MimirService {
+	return newMimirServiceFromOptions(
+		name,
+		map[string]string{
+			"-target":                           "backend",
+			"-log.level":                        "warn",
+			"-ingester.ring.replication-factor": "1",
+		},
+		flags,
+		options...,
+	)
+}
+
 func NewAlertmanager(name string, flags map[string]string, options ...Option) *MimirService {
 	return newMimirServiceFromOptions(
 		name,

integration/read_write_mode_test.go

@@ -0,0 +1,72 @@
+// SPDX-License-Identifier: AGPL-3.0-only
+//go:build requires_docker
+// +build requires_docker
+
+package integration
+
+import (
+	"math"
+	"testing"
+	"time"
+
+	"github.com/grafana/e2e"
+	e2edb "github.com/grafana/e2e/db"
+	"github.com/prometheus/common/model"
+	"github.com/prometheus/prometheus/prompb"
+	"github.com/stretchr/testify/require"
+
+	"github.com/grafana/mimir/integration/e2emimir"
+)
+
+func TestReadWriteMode(t *testing.T) {
+	s, err := e2e.NewScenario(networkName)
+	require.NoError(t, err)
+	defer s.Close()
+
+	minio := e2edb.NewMinio(9000, mimirBucketName)
+	require.NoError(t, s.StartAndWaitReady(minio))
+
+	flags := mergeFlags(
+		CommonStorageBackendFlags(),
+		map[string]string{
+			"-memberlist.join": "mimir-backend-1",
+		},
+	)
+
+	readInstance := e2emimir.NewReadInstance("mimir-read-1", flags)
+	writeInstance := e2emimir.NewWriteInstance("mimir-write-1", flags)
+	backendInstance := e2emimir.NewBackendInstance("mimir-backend-1", flags)
+	require.NoError(t, s.StartAndWaitReady(readInstance, writeInstance, backendInstance))
+
+	c, err := e2emimir.NewClient(writeInstance.HTTPEndpoint(), readInstance.HTTPEndpoint(), "", "", "user-1")
+	require.NoError(t, err)
+
+	// Push some data to the cluster.
+	now := time.Now()
+	series, expectedVector := generateSeries("test_series_1", now, prompb.Label{Name: "foo", Value: "bar"})
+
+	res, err := c.Push(series)
+	require.NoError(t, err)
+	require.Equal(t, 200, res.StatusCode)
+
+	// Verify we can read the data we just pushed, both with an instant query and a range query.
+	result, err := c.Query("test_series_1", now)
+	require.NoError(t, err)
+	require.Equal(t, model.ValVector, result.Type())
+	require.Equal(t, expectedVector, result.(model.Vector))
+
+	_, err = c.QueryRange("test_series_1", now.Add(-5*time.Minute), now, 15*time.Second)
+	require.NoError(t, err)
+
+	minTime := time.Unix(math.MinInt64/1000+62135596801, 0).UTC()
+	maxTime := time.Unix(math.MaxInt64/1000-62135596801, 999999999).UTC()
+
+	// Verify we can retrieve the labels we just pushed.
+	labelValues, err := c.LabelValues("foo", minTime, maxTime, nil)
+	require.NoError(t, err)
+	require.Equal(t, model.LabelValues{"bar"}, labelValues)
+
+	labelNames, err := c.LabelNames(minTime, maxTime)
+	require.NoError(t, err)
+	require.Equal(t, []string{"__name__", "foo"}, labelNames)
+}