etcd_docker 5: Incorporate docker based etcd approach into docker int…

…egration tests. PR 5 for #4144 This PR makes the docker integration tests use containerized etcd. Previously, these relied on M3DB running an embbeded etcd server. There's no inherent need for this, and it opens us up to dependency issues as described in the linked github issue. Note: there are a handful that require multiple servers; these are currently "skipped" (commented). I intend to bring those back at a later date.. commit-id:e67a5172
m3db · Sep 2, 2022 · 4b8bede · 4b8bede
1 parent 197a786
commit 4b8bede
Show file tree

Hide file tree

Showing 41 changed files with 314 additions and 189 deletions.
diff --git a/scripts/docker-integration-tests/aggregator/docker-compose.yml b/scripts/docker-integration-tests/aggregator/docker-compose.yml
@@ -1,17 +1,28 @@
 version: "3.5"
 services:
+  etcd:
+    image: docker.io/bitnami/etcd:3.5
+    expose:
+      - "2379-2380"
+    ports:
+      - "0.0.0.0:2379-2380:2379-2380"
+    environment:
+      - ALLOW_NONE_AUTHENTICATION=yes
+      - ETCD_ADVERTISE_CLIENT_URLS=http://etcd:2379
+    networks:
+      - backend
   dbnode01:
     expose:
       - "9000-9004"
-      - "2379-2380"
       - "7201"
     ports:
       - "0.0.0.0:9000-9004:9000-9004"
-      - "0.0.0.0:2379-2380:2379-2380"
       - "0.0.0.0:7201:7201"
     networks:
       - backend
     image: "m3dbnode_integration:${REVISION}"
+    depends_on:
+      - etcd
   m3coordinator01:
     expose:
       - "7202"
@@ -26,6 +37,8 @@ services:
     image: "m3coordinator_integration:${REVISION}"
     volumes:
       - "./m3coordinator.yml:/etc/m3coordinator/m3coordinator.yml"
+    depends_on:
+      - etcd
   m3aggregator01:
     expose:
       - "6001"
@@ -38,6 +51,8 @@ services:
     image: "m3aggregator_integration:${REVISION}"
     volumes:
       - "./m3aggregator.yml:/etc/m3aggregator/m3aggregator.yml"
+    depends_on:
+      - etcd
   m3aggregator02:
     networks:
       - backend
@@ -46,5 +61,7 @@ services:
     image: "m3aggregator_integration:${REVISION}"
     volumes:
       - "./m3aggregator.yml:/etc/m3aggregator/m3aggregator.yml"
+    depends_on:
+      - etcd
 networks:
-  backend:
+  backend: null
diff --git a/scripts/docker-integration-tests/aggregator/m3aggregator.yml b/scripts/docker-integration-tests/aggregator/m3aggregator.yml
@@ -38,7 +38,7 @@ kvClient:
         autoSyncInterval: 10m
         dialTimeout: 1m
         endpoints:
-          - dbnode01:2379
+          - etcd:2379
 
 runtimeOptions:
   kvConfig:

diff --git a/scripts/docker-integration-tests/aggregator/m3coordinator.yml b/scripts/docker-integration-tests/aggregator/m3coordinator.yml
@@ -24,7 +24,7 @@ clusters:
               autoSyncInterval: 10m
               dialTimeout: 1m
               endpoints:
-                - dbnode01:2379
+                - etcd:2379
 
 downsample:
   rules:

diff --git a/scripts/docker-integration-tests/aggregator/test.sh b/scripts/docker-integration-tests/aggregator/test.sh
@@ -14,6 +14,8 @@ echo "Pull containers required for test"
 docker pull $PROMREMOTECLI_IMAGE
 docker pull $JQ_IMAGE
 
+docker-compose -f ${COMPOSE_FILE} up -d etcd
+
 echo "Run m3dbnode"
 docker-compose -f ${COMPOSE_FILE} up -d dbnode01
 

diff --git a/scripts/docker-integration-tests/aggregator_legacy/docker-compose.yml b/scripts/docker-integration-tests/aggregator_legacy/docker-compose.yml
@@ -1,17 +1,24 @@
 version: "3.5"
 services:
+  etcd:
+    image: docker.io/bitnami/etcd:3.5
+    environment:
+      - ALLOW_NONE_AUTHENTICATION=yes
+      - ETCD_ADVERTISE_CLIENT_URLS=http://etcd:2379
+    networks:
+      - backend
   dbnode01:
     expose:
       - "9000-9004"
-      - "2379-2380"
       - "7201"
     ports:
       - "0.0.0.0:9000-9004:9000-9004"
-      - "0.0.0.0:2379-2380:2379-2380"
       - "0.0.0.0:7201:7201"
     networks:
       - backend
     image: "m3dbnode_integration:${REVISION}"
+    depends_on:
+      - etcd
   m3coordinator01:
     expose:
       - "7202"
@@ -26,6 +33,8 @@ services:
     image: "m3coordinator_integration:${REVISION}"
     volumes:
       - "./m3coordinator.yml:/etc/m3coordinator/m3coordinator.yml"
+    depends_on:
+      - etcd
   m3aggregator01:
     expose:
       - "6001"
@@ -38,6 +47,8 @@ services:
     image: "m3aggregator_integration:${REVISION}"
     volumes:
       - "./m3aggregator.yml:/etc/m3aggregator/m3aggregator.yml"
+    depends_on:
+      - etcd
   m3aggregator02:
     networks:
       - backend
@@ -46,5 +57,7 @@ services:
     image: "m3aggregator_integration:${REVISION}"
     volumes:
       - "./m3aggregator.yml:/etc/m3aggregator/m3aggregator.yml"
+    depends_on:
+      - etcd
 networks:
-  backend:
+  backend: null
diff --git a/scripts/docker-integration-tests/aggregator_legacy/m3aggregator.yml b/scripts/docker-integration-tests/aggregator_legacy/m3aggregator.yml
@@ -57,7 +57,7 @@ kvClient:
     etcdClusters:
       - zone: embedded
         endpoints:
-          - dbnode01:2379
+          - etcd:2379
 
 runtimeOptions:
   kvConfig:

diff --git a/scripts/docker-integration-tests/aggregator_legacy/m3coordinator.yml b/scripts/docker-integration-tests/aggregator_legacy/m3coordinator.yml
@@ -22,7 +22,7 @@ clusters:
           etcdClusters:
             - zone: embedded
               endpoints:
-                - dbnode01:2379
+                - etcd:2379
 
 downsample:
   remoteAggregator:

diff --git a/scripts/docker-integration-tests/aggregator_legacy/test.sh b/scripts/docker-integration-tests/aggregator_legacy/test.sh
@@ -7,6 +7,9 @@ REVISION=$(git rev-parse HEAD)
 COMPOSE_FILE="$M3_PATH"/scripts/docker-integration-tests/aggregator_legacy/docker-compose.yml
 export REVISION
 
+echo "Run etcd"
+docker-compose -f ${COMPOSE_FILE} up -d etcd
+
 echo "Run m3dbnode"
 docker-compose -f ${COMPOSE_FILE} up -d dbnode01
 

diff --git a/scripts/docker-integration-tests/carbon/docker-compose.yml b/scripts/docker-integration-tests/carbon/docker-compose.yml
@@ -1,15 +1,26 @@
 version: "3.5"
 services:
+  etcd:
+    image: docker.io/bitnami/etcd:3.5
+    expose:
+      - "2379-2380"
+    ports:
+      - "0.0.0.0:2379-2380:2379-2380"
+    environment:
+      - ALLOW_NONE_AUTHENTICATION=yes
+      - ETCD_ADVERTISE_CLIENT_URLS=http://etcd:2379
+    networks:
+      - backend
   dbnode01:
     expose:
       - "9000-9004"
-      - "2379-2380"
     ports:
       - "0.0.0.0:9000-9004:9000-9004"
-      - "0.0.0.0:2379-2380:2379-2380"
     networks:
       - backend
     image: "m3dbnode_integration:${REVISION}"
+    depends_on:
+      - etcd
   coordinator01:
     expose:
       - "7201"
@@ -24,5 +35,7 @@ services:
     image: "m3coordinator_integration:${REVISION}"
     volumes:
       - "./:/etc/m3coordinator/"
+    depends_on:
+      - etcd
 networks:
-  backend:
+  backend: null
diff --git a/scripts/docker-integration-tests/carbon/m3coordinator.yml b/scripts/docker-integration-tests/carbon/m3coordinator.yml
@@ -9,7 +9,7 @@ clusters:
           etcdClusters:
             - zone: embedded
               endpoints:
-                - dbnode01:2379
+                - etcd:2379
 
 carbon:
   findResultsIncludeBothExpandableAndLeaf: true

diff --git a/scripts/docker-integration-tests/carbon/test.sh b/scripts/docker-integration-tests/carbon/test.sh
@@ -10,8 +10,7 @@ EXPECTED_PATH=$SCRIPT_PATH/expected
 export REVISION
 
 echo "Run m3dbnode and m3coordinator containers"
-docker-compose -f ${COMPOSE_FILE} up -d dbnode01
-docker-compose -f ${COMPOSE_FILE} up -d coordinator01
+docker-compose -f ${COMPOSE_FILE} up -d
 
 # Think of this as a defer func() in golang
 METRIC_EMIT_PID="-1"
@@ -152,7 +151,7 @@ ATTEMPTS=20 MAX_TIMEOUT=4 TIMEOUT=1 retry_with_backoff "wait_carbon_values_accum
 
 # Now test the max datapoints behavior using max of four datapoints (4x 5s resolution = 20s)
 end=$(date +%s)
-start=$(($end-20)) 
+start=$(($end-20))
 # 1. no max datapoints set, should not adjust number of datapoints coming back
 ATTEMPTS=2 MAX_TIMEOUT=4 TIMEOUT=1 retry_with_backoff "read_carbon 'stat.already-aggregated.foo' 42 $start $end"
 # 2. max datapoints with LTTB, should be an existing value (i.e. 42)

diff --git a/scripts/docker-integration-tests/cold_writes_simple/docker-compose.yml b/scripts/docker-integration-tests/cold_writes_simple/docker-compose.yml
@@ -1,15 +1,26 @@
 version: "3.5"
 services:
+  etcd:
+    image: docker.io/bitnami/etcd:3.5
+    expose:
+      - "2379-2380"
+    ports:
+      - "0.0.0.0:2379-2380:2379-2380"
+    environment:
+      - ALLOW_NONE_AUTHENTICATION=yes
+      - ETCD_ADVERTISE_CLIENT_URLS=http://etcd:2379
+    networks:
+      - backend
   dbnode01:
     expose:
       - "9000-9004"
-      - "2379-2380"
     ports:
       - "0.0.0.0:9000-9004:9000-9004"
-      - "0.0.0.0:2379-2380:2379-2380"
     networks:
       - backend
     image: "m3dbnode_integration:${REVISION}"
+    depends_on:
+      - etcd
   coordinator01:
     expose:
       - "7201"
@@ -24,5 +35,7 @@ services:
     image: "m3coordinator_integration:${REVISION}"
     volumes:
       - "./:/etc/m3coordinator/"
+    depends_on:
+      - etcd
 networks:
-  backend:
+  backend: null
diff --git a/scripts/docker-integration-tests/cold_writes_simple/m3coordinator.yml b/scripts/docker-integration-tests/cold_writes_simple/m3coordinator.yml
@@ -13,4 +13,4 @@ clusters:
           etcdClusters:
             - zone: embedded
               endpoints:
-                - dbnode01:2379
+                - etcd:2379
diff --git a/scripts/docker-integration-tests/cold_writes_simple/test.sh b/scripts/docker-integration-tests/cold_writes_simple/test.sh
@@ -8,9 +8,8 @@ SCRIPT_PATH="$M3_PATH"/scripts/docker-integration-tests/cold_writes_simple
 COMPOSE_FILE=$SCRIPT_PATH/docker-compose.yml
 export REVISION
 
-echo "Run m3dbnode and m3coordinator containers"
-docker-compose -f ${COMPOSE_FILE} up -d --renew-anon-volumes dbnode01
-docker-compose -f ${COMPOSE_FILE} up -d --renew-anon-volumes coordinator01
+echo "Run etcd, m3dbnode and m3coordinator containers"
+docker-compose -f "${COMPOSE_FILE}" up -d --renew-anon-volumes
 
 # Think of this as a defer func() in golang
 function defer {

diff --git a/scripts/docker-integration-tests/coordinator_config_rules/docker-compose.yml b/scripts/docker-integration-tests/coordinator_config_rules/docker-compose.yml
@@ -1,15 +1,26 @@
 version: "3.5"
 services:
+  etcd:
+    image: docker.io/bitnami/etcd:3.5
+    expose:
+      - "2379-2380"
+    ports:
+      - "0.0.0.0:2379-2380:2379-2380"
+    environment:
+      - ALLOW_NONE_AUTHENTICATION=yes
+      - ETCD_ADVERTISE_CLIENT_URLS=http://etcd:2379
+    networks:
+      - backend
   dbnode01:
     expose:
       - "9000-9004"
-      - "2379-2380"
     ports:
       - "0.0.0.0:9000-9004:9000-9004"
-      - "0.0.0.0:2379-2380:2379-2380"
     networks:
       - backend
     image: "m3dbnode_integration:${REVISION}"
+    depends_on:
+      - etcd
   coordinator01:
     expose:
       - "7201"
@@ -24,5 +35,7 @@ services:
     image: "m3coordinator_integration:${REVISION}"
     volumes:
       - "./:/etc/m3coordinator/"
+    depends_on:
+      - etcd
 networks:
-  backend:
+  backend: null
diff --git a/scripts/docker-integration-tests/coordinator_config_rules/m3coordinator.yml b/scripts/docker-integration-tests/coordinator_config_rules/m3coordinator.yml
@@ -9,7 +9,7 @@ clusters:
           etcdClusters:
             - zone: embedded
               endpoints:
-                - dbnode01:2379
+                - etcd:2379
 
 downsample:
   rules:

diff --git a/scripts/docker-integration-tests/coordinator_config_rules/test.sh b/scripts/docker-integration-tests/coordinator_config_rules/test.sh
@@ -16,8 +16,7 @@ docker pull $PROMREMOTECLI_IMAGE
 docker pull $JQ_IMAGE
 
 echo "Run m3dbnode and m3coordinator containers"
-docker-compose -f ${COMPOSE_FILE} up -d dbnode01
-docker-compose -f ${COMPOSE_FILE} up -d coordinator01
+docker-compose -f ${COMPOSE_FILE} up -d
 
 # Think of this as a defer func() in golang
 function defer {

diff --git a/scripts/docker-integration-tests/coordinator_noop/docker-compose.yml b/scripts/docker-integration-tests/coordinator_noop/docker-compose.yml
@@ -1,5 +1,16 @@
 version: "3.5"
 services:
+  etcd:
+    image: docker.io/bitnami/etcd:3.5
+    expose:
+      - "2379-2380"
+    ports:
+      - "0.0.0.0:2379-2380:2379-2380"
+    environment:
+      - ALLOW_NONE_AUTHENTICATION=yes
+      - ETCD_ADVERTISE_CLIENT_URLS=http://etcd:2379
+    networks:
+      - backend
   coordinator01:
     expose:
       - "7201"
@@ -10,33 +21,7 @@ services:
     image: "m3coordinator_integration:${REVISION}"
     volumes:
       - "./m3coordinator.yml:/etc/m3coordinator/m3coordinator.yml"
-  etcd01:
-    expose:
-      - "2379-2380"
-    ports:
-      - "0.0.0.0:2379-2380:2379-2380"
-    networks:
-      - backend
-    image: quay.io/coreos/etcd:v3.4.3
-    command:
-      - "etcd"
-      - "--name"
-      - "etcd01"
-      - "--listen-peer-urls"
-      - "http://0.0.0.0:2380"
-      - "--listen-client-urls"
-      - "http://0.0.0.0:2379"
-      - "--advertise-client-urls"
-      - "http://etcd01:2379"
-      - "--initial-cluster-token"
-      - "etcd-cluster-1"
-      - "--initial-advertise-peer-urls"
-      - "http://etcd01:2380"
-      - "--initial-cluster"
-      - "etcd01=http://etcd01:2380"
-      - "--initial-cluster-state"
-      - "new"
-      - "--data-dir"
-      - "/var/lib/etcd"
+    depends_on:
+      - etcd
 networks:
-  backend:
+  backend: null