Fix write fanouts with aggregated namespaces

scripts/development/m3_stack/m3coordinator.yml

@@ -14,9 +14,13 @@ metrics:
 
 clusters:
   - namespaces:
-      - namespace: prometheus_metrics
+      - namespace: agg
+        type: aggregated
+        retention: 10h
+        resolution: 15s
+      - namespace: unagg
         type: unaggregated
-        retention: 48h
+        retention: 10m
     client:
       config:
         service:

scripts/development/m3_stack/start_m3.sh

@@ -6,9 +6,43 @@ echo "Bringing up nodes in the backgorund with docker compose, remember to run .
 docker-compose -f docker-compose.yml up -d --renew-anon-volumes
 echo "Nodes online"
 
+sleep 10
+
+echo "Initializing namespace"
+curl -vvvsSf -X POST 0.0.0.0:7201/api/v1/namespace -d '{
+  "name": "agg",
+  "options": {
+    "bootstrapEnabled": true,
+    "flushEnabled": true,
+    "writesToCommitLog": true,
+    "cleanupEnabled": true,
+    "snapshotEnabled": true,
+    "repairEnabled": false,
+    "retentionOptions": {
+      "retentionPeriodNanos": 172800000000000,
+      "blockSizeNanos": 7200000000000,
+      "bufferFutureNanos": 600000000000,
+      "bufferPastNanos": 600000000000,
+      "blockDataExpiry": true,
+      "blockDataExpiryAfterNotAccessPeriodNanos": 300000000000
+    },
+    "indexOptions": {
+      "enabled": true,
+      "blockSizeNanos": 7200000000000
+    }
+  }
+}'
+echo "Done initializing namespace"
+
+echo "Validating namespace"
+[ "$(curl -sSf localhost:7201/api/v1/namespace | jq .registry.namespaces.agg.indexOptions.enabled)" == true ]
+echo "Done validating namespace"
+
+sleep 10
+
 echo "Initializing namespace"
-curl -vvvsSf -X POST localhost:7201/api/v1/namespace -d '{
-  "name": "prometheus_metrics",
+curl -vvvsSf -X POST 0.0.0.0:7201/api/v1/namespace -d '{
+  "name": "unagg",
   "options": {
     "bootstrapEnabled": true,
     "flushEnabled": true,
@@ -33,9 +67,11 @@ curl -vvvsSf -X POST localhost:7201/api/v1/namespace -d '{
 echo "Done initializing namespace"
 
 echo "Validating namespace"
-[ "$(curl -sSf localhost:7201/api/v1/namespace | jq .registry.namespaces.prometheus_metrics.indexOptions.enabled)" == true ]
+[ "$(curl -sSf localhost:7201/api/v1/namespace | jq .registry.namespaces.unagg.indexOptions.enabled)" == true ]
 echo "Done validating namespace"
 
+sleep 10
+
 echo "Initializing topology"
 curl -vvvsSf -X POST localhost:7201/api/v1/placement/init -d '{
     "num_shards": 64,

scripts/docker-integration-tests/prometheus/test.sh

@@ -11,6 +11,13 @@ echo "Run m3dbnode and m3coordinator containers"
 docker-compose -f ${COMPOSE_FILE} up -d dbnode01
 docker-compose -f ${COMPOSE_FILE} up -d coordinator01
 
+# think of this as a defer func() in golang
+function defer {
+  docker-compose -f ${COMPOSE_FILE} down || echo "unable to shutdown containers" # CI fails to stop all containers sometimes
+}
+trap defer EXIT
+
+
 echo "Sleeping for a bit to ensure db up"
 sleep 15 # TODO Replace sleeps with logic to determine when to proceed
 
@@ -104,4 +111,14 @@ echo "Wait until the remote write endpoint generates and allows for data to be q
 ATTEMPTS=6 TIMEOUT=2 retry_with_backoff  \
   '[[ $(curl -sSf 0.0.0.0:9090/api/v1/query?query=prometheus_remote_storage_succeeded_samples_total | jq -r .data.result[].value[1]) -gt 100 ]]'
 
+# Make sure we're proxying writes to the unaggregated namespace
+echo "Wait until data begins being written to remote storage for the unaggregated namespace"
+ATTEMPTS=6 TIMEOUT=2 retry_with_backoff  \
+  '[[ $(curl -sSf 0.0.0.0:9090/api/v1/query?query=database_write_tagged_success\\{namespace=\"unagg\"\\} | jq -r .data.result[0].value[1]) -gt 0 ]]'
+
+# Make sure we're proxying writes to the aggregated namespace
+echo "Wait until data begins being written to remote storage for the aggregated namespace"
+ATTEMPTS=10 TIMEOUT=2 retry_with_backoff  \
+  '[[ $(curl -sSf 0.0.0.0:9090/api/v1/query?query=database_write_tagged_success\\{namespace=\"agg\"\\} | jq -r .data.result[0].value[1]) -gt 0 ]]'
+
 docker-compose -f ${COMPOSE_FILE} down || echo "unable to shutdown containers" # CI fails to stop all containers sometimes

scripts/docker-integration-tests/simple/test.sh

@@ -9,6 +9,14 @@ echo "Run m3dbnode docker container"
 
 CONTAINER_NAME="m3dbnode-version-${REVISION}"
 docker create --name "${CONTAINER_NAME}" -p 9000:9000 -p 9001:9001 -p 9002:9002 -p 9003:9003 -p 9004:9004 -p 7201:7201 "m3dbnode_integration:${REVISION}"
+
+# think of this as a defer func() in golang
+function defer {
+  echo "Remove docker container"
+  docker rm --force "${CONTAINER_NAME}"
+}
+trap defer EXIT
+
 docker start "${CONTAINER_NAME}"
 if [ $? -ne 0 ]; then
   echo "m3dbnode docker failed to start"
@@ -117,10 +125,4 @@ echo "Deleting placement"
 curl -vvvsSf -X DELETE 0.0.0.0:7201/api/v1/placement
 
 echo "Deleting namespace"
-curl -vvvsSf -X DELETE 0.0.0.0:7201/api/v1/namespace/default
-
-echo "Stop docker container"
-docker stop "${CONTAINER_NAME}"
-
-echo "Remove docker container"
-docker rm "${CONTAINER_NAME}"
+curl -vvvsSf -X DELETE 0.0.0.0:7201/api/v1/namespace/default

src/query/api/v1/httpd/handler.go

@@ -103,7 +103,7 @@ func (h *Handler) RegisterRoutes() error {
 
 	// Prometheus remote read/write endpoints
 	promRemoteReadHandler := remote.NewPromReadHandler(h.engine, h.scope.Tagged(remoteSource))
-	promRemoteWriteHandler, err := remote.NewPromWriteHandler(h.storage, nil, h.scope.Tagged(remoteSource))
+	promRemoteWriteHandler, err := remote.NewPromWriteHandler(h.storage, h.downsampler, h.scope.Tagged(remoteSource))
 	if err != nil {
 		return err
 	}

src/query/server/server.go

@@ -58,6 +58,7 @@ import (
 	"github.com/m3db/m3x/pool"
 	xsync "github.com/m3db/m3x/sync"
 	xtime "github.com/m3db/m3x/time"
+	"github.com/uber-go/tally"
 
 	"github.com/pkg/errors"
 	"go.uber.org/zap"
@@ -359,7 +360,7 @@ func newDownsampler(
 		RulesKVStore:          kvStore,
 		AutoMappingRules:      autoMappingRules,
 		ClockOptions:          clock.NewOptions(),
-		InstrumentOptions:     instrumentOpts,
+		InstrumentOptions:     instrumentOpts.SetMetricsScope(tally.NoopScope),
 		TagEncoderOptions:     tagEncoderOptions,
 		TagDecoderOptions:     tagDecoderOptions,
 		TagEncoderPoolOptions: tagEncoderPoolOptions,