From 4b8bede583e82d4cab83c888f68bd1f0b567cb89 Mon Sep 17 00:00:00 2001 From: Andrew Mains Date: Mon, 29 Aug 2022 16:14:48 -0400 Subject: [PATCH] etcd_docker 5: Incorporate docker based etcd approach into docker integration tests. PR 5 for https://github.com/m3db/m3/issues/4144 This PR makes the docker integration tests use containerized etcd. Previously, these relied on M3DB running an embbeded etcd server. There's no inherent need for this, and it opens us up to dependency issues as described in the linked github issue. Note: there are a handful that require multiple servers; these are currently "skipped" (commented). I intend to bring those back at a later date.. commit-id:e67a5172 --- .../aggregator/docker-compose.yml | 23 ++++- .../aggregator/m3aggregator.yml | 2 +- .../aggregator/m3coordinator.yml | 2 +- .../aggregator/test.sh | 2 + .../aggregator_legacy/docker-compose.yml | 19 +++- .../aggregator_legacy/m3aggregator.yml | 2 +- .../aggregator_legacy/m3coordinator.yml | 2 +- .../aggregator_legacy/test.sh | 3 + .../carbon/docker-compose.yml | 19 +++- .../carbon/m3coordinator.yml | 2 +- .../docker-integration-tests/carbon/test.sh | 5 +- .../cold_writes_simple/docker-compose.yml | 19 +++- .../cold_writes_simple/m3coordinator.yml | 2 +- .../cold_writes_simple/test.sh | 5 +- .../docker-compose.yml | 19 +++- .../m3coordinator.yml | 2 +- .../coordinator_config_rules/test.sh | 3 +- .../coordinator_noop/docker-compose.yml | 43 +++------- .../coordinator_noop/m3coordinator.yml | 2 +- .../coordinator_noop/test.sh | 3 +- .../docker-compose.yml | 43 +++------- .../docker-compose-etcd.yml | 15 ++++ .../m3coordinator.Dockerfile | 2 +- .../m3dbnode.Dockerfile | 4 +- .../docker-compose.yml | 53 ++++++------ .../m3aggregator.yml | 2 +- .../m3coordinator-admin.yml | 2 +- .../m3coordinator.yml | 2 +- .../prom_remote_write_backend/test.sh | 2 +- .../prometheus/docker-compose.yml | 21 ++++- .../prometheus/m3coordinator.yml | 2 +- .../repair/docker-compose.yml | 21 ++++- .../repair/m3coordinator.yml | 2 +- .../repair/m3dbnode.yml | 7 +- .../docker-integration-tests/repair/test.sh | 4 +- scripts/docker-integration-tests/run.sh | 86 +++++++++++-------- scripts/docker-integration-tests/setup.sh | 8 +- .../simple_v2_batch_apis/docker-compose.yml | 19 +++- .../simple_v2_batch_apis/m3coordinator.yml | 2 +- .../config/m3dbnode-local-docker-etcd.yml | 11 +++ .../m3coordinator-local-docker-etcd.yml | 16 ++++ 41 files changed, 314 insertions(+), 189 deletions(-) create mode 100644 scripts/docker-integration-tests/docker-compose-etcd.yml create mode 100644 src/dbnode/config/m3dbnode-local-docker-etcd.yml create mode 100644 src/query/config/m3coordinator-local-docker-etcd.yml diff --git a/scripts/docker-integration-tests/aggregator/docker-compose.yml b/scripts/docker-integration-tests/aggregator/docker-compose.yml index c93b41ee25..9ad000907d 100644 --- a/scripts/docker-integration-tests/aggregator/docker-compose.yml +++ b/scripts/docker-integration-tests/aggregator/docker-compose.yml @@ -1,17 +1,28 @@ version: "3.5" services: + etcd: + image: docker.io/bitnami/etcd:3.5 + expose: + - "2379-2380" + ports: + - "0.0.0.0:2379-2380:2379-2380" + environment: + - ALLOW_NONE_AUTHENTICATION=yes + - ETCD_ADVERTISE_CLIENT_URLS=http://etcd:2379 + networks: + - backend dbnode01: expose: - "9000-9004" - - "2379-2380" - "7201" ports: - "0.0.0.0:9000-9004:9000-9004" - - "0.0.0.0:2379-2380:2379-2380" - "0.0.0.0:7201:7201" networks: - backend image: "m3dbnode_integration:${REVISION}" + depends_on: + - etcd m3coordinator01: expose: - "7202" @@ -26,6 +37,8 @@ services: image: "m3coordinator_integration:${REVISION}" volumes: - "./m3coordinator.yml:/etc/m3coordinator/m3coordinator.yml" + depends_on: + - etcd m3aggregator01: expose: - "6001" @@ -38,6 +51,8 @@ services: image: "m3aggregator_integration:${REVISION}" volumes: - "./m3aggregator.yml:/etc/m3aggregator/m3aggregator.yml" + depends_on: + - etcd m3aggregator02: networks: - backend @@ -46,5 +61,7 @@ services: image: "m3aggregator_integration:${REVISION}" volumes: - "./m3aggregator.yml:/etc/m3aggregator/m3aggregator.yml" + depends_on: + - etcd networks: - backend: + backend: null diff --git a/scripts/docker-integration-tests/aggregator/m3aggregator.yml b/scripts/docker-integration-tests/aggregator/m3aggregator.yml index c2f26330bd..1aa1c7c091 100644 --- a/scripts/docker-integration-tests/aggregator/m3aggregator.yml +++ b/scripts/docker-integration-tests/aggregator/m3aggregator.yml @@ -38,7 +38,7 @@ kvClient: autoSyncInterval: 10m dialTimeout: 1m endpoints: - - dbnode01:2379 + - etcd:2379 runtimeOptions: kvConfig: diff --git a/scripts/docker-integration-tests/aggregator/m3coordinator.yml b/scripts/docker-integration-tests/aggregator/m3coordinator.yml index dfc757b8c4..a1e44edda4 100644 --- a/scripts/docker-integration-tests/aggregator/m3coordinator.yml +++ b/scripts/docker-integration-tests/aggregator/m3coordinator.yml @@ -24,7 +24,7 @@ clusters: autoSyncInterval: 10m dialTimeout: 1m endpoints: - - dbnode01:2379 + - etcd:2379 downsample: rules: diff --git a/scripts/docker-integration-tests/aggregator/test.sh b/scripts/docker-integration-tests/aggregator/test.sh index a65b912c8a..a5b44122c8 100755 --- a/scripts/docker-integration-tests/aggregator/test.sh +++ b/scripts/docker-integration-tests/aggregator/test.sh @@ -14,6 +14,8 @@ echo "Pull containers required for test" docker pull $PROMREMOTECLI_IMAGE docker pull $JQ_IMAGE +docker-compose -f ${COMPOSE_FILE} up -d etcd + echo "Run m3dbnode" docker-compose -f ${COMPOSE_FILE} up -d dbnode01 diff --git a/scripts/docker-integration-tests/aggregator_legacy/docker-compose.yml b/scripts/docker-integration-tests/aggregator_legacy/docker-compose.yml index c93b41ee25..76eaac7862 100644 --- a/scripts/docker-integration-tests/aggregator_legacy/docker-compose.yml +++ b/scripts/docker-integration-tests/aggregator_legacy/docker-compose.yml @@ -1,17 +1,24 @@ version: "3.5" services: + etcd: + image: docker.io/bitnami/etcd:3.5 + environment: + - ALLOW_NONE_AUTHENTICATION=yes + - ETCD_ADVERTISE_CLIENT_URLS=http://etcd:2379 + networks: + - backend dbnode01: expose: - "9000-9004" - - "2379-2380" - "7201" ports: - "0.0.0.0:9000-9004:9000-9004" - - "0.0.0.0:2379-2380:2379-2380" - "0.0.0.0:7201:7201" networks: - backend image: "m3dbnode_integration:${REVISION}" + depends_on: + - etcd m3coordinator01: expose: - "7202" @@ -26,6 +33,8 @@ services: image: "m3coordinator_integration:${REVISION}" volumes: - "./m3coordinator.yml:/etc/m3coordinator/m3coordinator.yml" + depends_on: + - etcd m3aggregator01: expose: - "6001" @@ -38,6 +47,8 @@ services: image: "m3aggregator_integration:${REVISION}" volumes: - "./m3aggregator.yml:/etc/m3aggregator/m3aggregator.yml" + depends_on: + - etcd m3aggregator02: networks: - backend @@ -46,5 +57,7 @@ services: image: "m3aggregator_integration:${REVISION}" volumes: - "./m3aggregator.yml:/etc/m3aggregator/m3aggregator.yml" + depends_on: + - etcd networks: - backend: + backend: null diff --git a/scripts/docker-integration-tests/aggregator_legacy/m3aggregator.yml b/scripts/docker-integration-tests/aggregator_legacy/m3aggregator.yml index 569ea9a9c2..e10c7cbe96 100644 --- a/scripts/docker-integration-tests/aggregator_legacy/m3aggregator.yml +++ b/scripts/docker-integration-tests/aggregator_legacy/m3aggregator.yml @@ -57,7 +57,7 @@ kvClient: etcdClusters: - zone: embedded endpoints: - - dbnode01:2379 + - etcd:2379 runtimeOptions: kvConfig: diff --git a/scripts/docker-integration-tests/aggregator_legacy/m3coordinator.yml b/scripts/docker-integration-tests/aggregator_legacy/m3coordinator.yml index 281662faca..d15fd1a78c 100644 --- a/scripts/docker-integration-tests/aggregator_legacy/m3coordinator.yml +++ b/scripts/docker-integration-tests/aggregator_legacy/m3coordinator.yml @@ -22,7 +22,7 @@ clusters: etcdClusters: - zone: embedded endpoints: - - dbnode01:2379 + - etcd:2379 downsample: remoteAggregator: diff --git a/scripts/docker-integration-tests/aggregator_legacy/test.sh b/scripts/docker-integration-tests/aggregator_legacy/test.sh index e101eae186..7f5050005b 100755 --- a/scripts/docker-integration-tests/aggregator_legacy/test.sh +++ b/scripts/docker-integration-tests/aggregator_legacy/test.sh @@ -7,6 +7,9 @@ REVISION=$(git rev-parse HEAD) COMPOSE_FILE="$M3_PATH"/scripts/docker-integration-tests/aggregator_legacy/docker-compose.yml export REVISION +echo "Run etcd" +docker-compose -f ${COMPOSE_FILE} up -d etcd + echo "Run m3dbnode" docker-compose -f ${COMPOSE_FILE} up -d dbnode01 diff --git a/scripts/docker-integration-tests/carbon/docker-compose.yml b/scripts/docker-integration-tests/carbon/docker-compose.yml index 53a28f0b88..a40ac2024a 100644 --- a/scripts/docker-integration-tests/carbon/docker-compose.yml +++ b/scripts/docker-integration-tests/carbon/docker-compose.yml @@ -1,15 +1,26 @@ version: "3.5" services: + etcd: + image: docker.io/bitnami/etcd:3.5 + expose: + - "2379-2380" + ports: + - "0.0.0.0:2379-2380:2379-2380" + environment: + - ALLOW_NONE_AUTHENTICATION=yes + - ETCD_ADVERTISE_CLIENT_URLS=http://etcd:2379 + networks: + - backend dbnode01: expose: - "9000-9004" - - "2379-2380" ports: - "0.0.0.0:9000-9004:9000-9004" - - "0.0.0.0:2379-2380:2379-2380" networks: - backend image: "m3dbnode_integration:${REVISION}" + depends_on: + - etcd coordinator01: expose: - "7201" @@ -24,5 +35,7 @@ services: image: "m3coordinator_integration:${REVISION}" volumes: - "./:/etc/m3coordinator/" + depends_on: + - etcd networks: - backend: + backend: null diff --git a/scripts/docker-integration-tests/carbon/m3coordinator.yml b/scripts/docker-integration-tests/carbon/m3coordinator.yml index 7f33d8701c..b069fbf7b6 100644 --- a/scripts/docker-integration-tests/carbon/m3coordinator.yml +++ b/scripts/docker-integration-tests/carbon/m3coordinator.yml @@ -9,7 +9,7 @@ clusters: etcdClusters: - zone: embedded endpoints: - - dbnode01:2379 + - etcd:2379 carbon: findResultsIncludeBothExpandableAndLeaf: true diff --git a/scripts/docker-integration-tests/carbon/test.sh b/scripts/docker-integration-tests/carbon/test.sh index f1c499a71e..6c811cb3b8 100755 --- a/scripts/docker-integration-tests/carbon/test.sh +++ b/scripts/docker-integration-tests/carbon/test.sh @@ -10,8 +10,7 @@ EXPECTED_PATH=$SCRIPT_PATH/expected export REVISION echo "Run m3dbnode and m3coordinator containers" -docker-compose -f ${COMPOSE_FILE} up -d dbnode01 -docker-compose -f ${COMPOSE_FILE} up -d coordinator01 +docker-compose -f ${COMPOSE_FILE} up -d # Think of this as a defer func() in golang METRIC_EMIT_PID="-1" @@ -152,7 +151,7 @@ ATTEMPTS=20 MAX_TIMEOUT=4 TIMEOUT=1 retry_with_backoff "wait_carbon_values_accum # Now test the max datapoints behavior using max of four datapoints (4x 5s resolution = 20s) end=$(date +%s) -start=$(($end-20)) +start=$(($end-20)) # 1. no max datapoints set, should not adjust number of datapoints coming back ATTEMPTS=2 MAX_TIMEOUT=4 TIMEOUT=1 retry_with_backoff "read_carbon 'stat.already-aggregated.foo' 42 $start $end" # 2. max datapoints with LTTB, should be an existing value (i.e. 42) diff --git a/scripts/docker-integration-tests/cold_writes_simple/docker-compose.yml b/scripts/docker-integration-tests/cold_writes_simple/docker-compose.yml index 53a28f0b88..a40ac2024a 100644 --- a/scripts/docker-integration-tests/cold_writes_simple/docker-compose.yml +++ b/scripts/docker-integration-tests/cold_writes_simple/docker-compose.yml @@ -1,15 +1,26 @@ version: "3.5" services: + etcd: + image: docker.io/bitnami/etcd:3.5 + expose: + - "2379-2380" + ports: + - "0.0.0.0:2379-2380:2379-2380" + environment: + - ALLOW_NONE_AUTHENTICATION=yes + - ETCD_ADVERTISE_CLIENT_URLS=http://etcd:2379 + networks: + - backend dbnode01: expose: - "9000-9004" - - "2379-2380" ports: - "0.0.0.0:9000-9004:9000-9004" - - "0.0.0.0:2379-2380:2379-2380" networks: - backend image: "m3dbnode_integration:${REVISION}" + depends_on: + - etcd coordinator01: expose: - "7201" @@ -24,5 +35,7 @@ services: image: "m3coordinator_integration:${REVISION}" volumes: - "./:/etc/m3coordinator/" + depends_on: + - etcd networks: - backend: + backend: null diff --git a/scripts/docker-integration-tests/cold_writes_simple/m3coordinator.yml b/scripts/docker-integration-tests/cold_writes_simple/m3coordinator.yml index cc33cf4021..dc075a01fe 100644 --- a/scripts/docker-integration-tests/cold_writes_simple/m3coordinator.yml +++ b/scripts/docker-integration-tests/cold_writes_simple/m3coordinator.yml @@ -13,4 +13,4 @@ clusters: etcdClusters: - zone: embedded endpoints: - - dbnode01:2379 + - etcd:2379 diff --git a/scripts/docker-integration-tests/cold_writes_simple/test.sh b/scripts/docker-integration-tests/cold_writes_simple/test.sh index 999ef1b20d..235c870e2b 100755 --- a/scripts/docker-integration-tests/cold_writes_simple/test.sh +++ b/scripts/docker-integration-tests/cold_writes_simple/test.sh @@ -8,9 +8,8 @@ SCRIPT_PATH="$M3_PATH"/scripts/docker-integration-tests/cold_writes_simple COMPOSE_FILE=$SCRIPT_PATH/docker-compose.yml export REVISION -echo "Run m3dbnode and m3coordinator containers" -docker-compose -f ${COMPOSE_FILE} up -d --renew-anon-volumes dbnode01 -docker-compose -f ${COMPOSE_FILE} up -d --renew-anon-volumes coordinator01 +echo "Run etcd, m3dbnode and m3coordinator containers" +docker-compose -f "${COMPOSE_FILE}" up -d --renew-anon-volumes # Think of this as a defer func() in golang function defer { diff --git a/scripts/docker-integration-tests/coordinator_config_rules/docker-compose.yml b/scripts/docker-integration-tests/coordinator_config_rules/docker-compose.yml index 53a28f0b88..a40ac2024a 100644 --- a/scripts/docker-integration-tests/coordinator_config_rules/docker-compose.yml +++ b/scripts/docker-integration-tests/coordinator_config_rules/docker-compose.yml @@ -1,15 +1,26 @@ version: "3.5" services: + etcd: + image: docker.io/bitnami/etcd:3.5 + expose: + - "2379-2380" + ports: + - "0.0.0.0:2379-2380:2379-2380" + environment: + - ALLOW_NONE_AUTHENTICATION=yes + - ETCD_ADVERTISE_CLIENT_URLS=http://etcd:2379 + networks: + - backend dbnode01: expose: - "9000-9004" - - "2379-2380" ports: - "0.0.0.0:9000-9004:9000-9004" - - "0.0.0.0:2379-2380:2379-2380" networks: - backend image: "m3dbnode_integration:${REVISION}" + depends_on: + - etcd coordinator01: expose: - "7201" @@ -24,5 +35,7 @@ services: image: "m3coordinator_integration:${REVISION}" volumes: - "./:/etc/m3coordinator/" + depends_on: + - etcd networks: - backend: + backend: null diff --git a/scripts/docker-integration-tests/coordinator_config_rules/m3coordinator.yml b/scripts/docker-integration-tests/coordinator_config_rules/m3coordinator.yml index 952c9a7ddf..3ca941aa8d 100644 --- a/scripts/docker-integration-tests/coordinator_config_rules/m3coordinator.yml +++ b/scripts/docker-integration-tests/coordinator_config_rules/m3coordinator.yml @@ -9,7 +9,7 @@ clusters: etcdClusters: - zone: embedded endpoints: - - dbnode01:2379 + - etcd:2379 downsample: rules: diff --git a/scripts/docker-integration-tests/coordinator_config_rules/test.sh b/scripts/docker-integration-tests/coordinator_config_rules/test.sh index a1590983c8..48dce08870 100755 --- a/scripts/docker-integration-tests/coordinator_config_rules/test.sh +++ b/scripts/docker-integration-tests/coordinator_config_rules/test.sh @@ -16,8 +16,7 @@ docker pull $PROMREMOTECLI_IMAGE docker pull $JQ_IMAGE echo "Run m3dbnode and m3coordinator containers" -docker-compose -f ${COMPOSE_FILE} up -d dbnode01 -docker-compose -f ${COMPOSE_FILE} up -d coordinator01 +docker-compose -f ${COMPOSE_FILE} up -d # Think of this as a defer func() in golang function defer { diff --git a/scripts/docker-integration-tests/coordinator_noop/docker-compose.yml b/scripts/docker-integration-tests/coordinator_noop/docker-compose.yml index ee8207bd26..edb2a7528c 100644 --- a/scripts/docker-integration-tests/coordinator_noop/docker-compose.yml +++ b/scripts/docker-integration-tests/coordinator_noop/docker-compose.yml @@ -1,5 +1,16 @@ version: "3.5" services: + etcd: + image: docker.io/bitnami/etcd:3.5 + expose: + - "2379-2380" + ports: + - "0.0.0.0:2379-2380:2379-2380" + environment: + - ALLOW_NONE_AUTHENTICATION=yes + - ETCD_ADVERTISE_CLIENT_URLS=http://etcd:2379 + networks: + - backend coordinator01: expose: - "7201" @@ -10,33 +21,7 @@ services: image: "m3coordinator_integration:${REVISION}" volumes: - "./m3coordinator.yml:/etc/m3coordinator/m3coordinator.yml" - etcd01: - expose: - - "2379-2380" - ports: - - "0.0.0.0:2379-2380:2379-2380" - networks: - - backend - image: quay.io/coreos/etcd:v3.4.3 - command: - - "etcd" - - "--name" - - "etcd01" - - "--listen-peer-urls" - - "http://0.0.0.0:2380" - - "--listen-client-urls" - - "http://0.0.0.0:2379" - - "--advertise-client-urls" - - "http://etcd01:2379" - - "--initial-cluster-token" - - "etcd-cluster-1" - - "--initial-advertise-peer-urls" - - "http://etcd01:2380" - - "--initial-cluster" - - "etcd01=http://etcd01:2380" - - "--initial-cluster-state" - - "new" - - "--data-dir" - - "/var/lib/etcd" + depends_on: + - etcd networks: - backend: + backend: null diff --git a/scripts/docker-integration-tests/coordinator_noop/m3coordinator.yml b/scripts/docker-integration-tests/coordinator_noop/m3coordinator.yml index c3c08c0104..6a261ecf26 100644 --- a/scripts/docker-integration-tests/coordinator_noop/m3coordinator.yml +++ b/scripts/docker-integration-tests/coordinator_noop/m3coordinator.yml @@ -23,7 +23,7 @@ clusterManagement: etcdClusters: - zone: embedded endpoints: - - etcd01:2379 + - etcd:2379 tagOptions: idScheme: quoted diff --git a/scripts/docker-integration-tests/coordinator_noop/test.sh b/scripts/docker-integration-tests/coordinator_noop/test.sh index 738ea863d0..0f91ec91da 100755 --- a/scripts/docker-integration-tests/coordinator_noop/test.sh +++ b/scripts/docker-integration-tests/coordinator_noop/test.sh @@ -9,8 +9,7 @@ COMPOSE_FILE=$SCRIPT_PATH/docker-compose.yml export REVISION echo "Run coordinator with no etcd" -docker-compose -f ${COMPOSE_FILE} up -d --renew-anon-volumes coordinator01 -docker-compose -f ${COMPOSE_FILE} up -d --renew-anon-volumes etcd01 +docker-compose -f ${COMPOSE_FILE} up -d --renew-anon-volumes function defer { docker-compose -f ${COMPOSE_FILE} down || echo "unable to shutdown containers" # CI fails to stop all containers sometimes diff --git a/scripts/docker-integration-tests/dedicated_etcd_embedded_coordinator/docker-compose.yml b/scripts/docker-integration-tests/dedicated_etcd_embedded_coordinator/docker-compose.yml index e155eb5334..c39ef36c0c 100644 --- a/scripts/docker-integration-tests/dedicated_etcd_embedded_coordinator/docker-compose.yml +++ b/scripts/docker-integration-tests/dedicated_etcd_embedded_coordinator/docker-compose.yml @@ -1,5 +1,16 @@ version: "3.5" services: + etcd: + image: docker.io/bitnami/etcd:3.5 + expose: + - "2379-2380" + ports: + - "0.0.0.0:2379-2380:2379-2380" + environment: + - ALLOW_NONE_AUTHENTICATION=yes + - ETCD_ADVERTISE_CLIENT_URLS=http://etcd:2379 + networks: + - backend dbnode01: expose: - "9000-9004" @@ -14,33 +25,7 @@ services: - M3DB_HOST_ID=dbnode01 volumes: - "./m3dbnode.yml:/etc/m3dbnode/m3dbnode.yml" - etcd01: - expose: - - "2379-2380" - ports: - - "0.0.0.0:2379-2380:2379-2380" - networks: - - backend - image: quay.io/coreos/etcd:v3.4.3 - command: - - "etcd" - - "--name" - - "etcd01" - - "--listen-peer-urls" - - "http://0.0.0.0:2380" - - "--listen-client-urls" - - "http://0.0.0.0:2379" - - "--advertise-client-urls" - - "http://etcd01:2379" - - "--initial-cluster-token" - - "etcd-cluster-1" - - "--initial-advertise-peer-urls" - - "http://etcd01:2380" - - "--initial-cluster" - - "etcd01=http://etcd01:2380" - - "--initial-cluster-state" - - "new" - - "--data-dir" - - "/var/lib/etcd" + depends_on: + - etcd networks: - backend: + backend: null diff --git a/scripts/docker-integration-tests/docker-compose-etcd.yml b/scripts/docker-integration-tests/docker-compose-etcd.yml new file mode 100644 index 0000000000..90af19b550 --- /dev/null +++ b/scripts/docker-integration-tests/docker-compose-etcd.yml @@ -0,0 +1,15 @@ +version: "3.5" +services: + etcd: + image: docker.io/bitnami/etcd:3.5 + expose: + - "2379-2380" + ports: + - "0.0.0.0:2379-2380:2379-2380" + environment: + - ALLOW_NONE_AUTHENTICATION=yes + networks: + - backend + +networks: + backend: diff --git a/scripts/docker-integration-tests/m3coordinator.Dockerfile b/scripts/docker-integration-tests/m3coordinator.Dockerfile index 0319613977..489030dbea 100644 --- a/scripts/docker-integration-tests/m3coordinator.Dockerfile +++ b/scripts/docker-integration-tests/m3coordinator.Dockerfile @@ -4,7 +4,7 @@ LABEL maintainer="The M3DB Authors " RUN mkdir -p /bin RUN mkdir -p /etc/m3coordinator ADD ./m3coordinator /bin/ -ADD ./m3coordinator-local-etcd.yml /etc/m3coordinator/m3coordinator.yml +ADD ./m3coordinator-local-docker-etcd.yml /etc/m3coordinator/m3coordinator.yml EXPOSE 7201/tcp 7203/tcp diff --git a/scripts/docker-integration-tests/m3dbnode.Dockerfile b/scripts/docker-integration-tests/m3dbnode.Dockerfile index a352ad4bf5..6d430fe803 100644 --- a/scripts/docker-integration-tests/m3dbnode.Dockerfile +++ b/scripts/docker-integration-tests/m3dbnode.Dockerfile @@ -4,9 +4,9 @@ LABEL maintainer="The M3DB Authors " RUN mkdir -p /bin RUN mkdir -p /etc/m3dbnode ADD ./m3dbnode /bin/ -ADD ./m3dbnode-local-etcd.yml /etc/m3dbnode/m3dbnode.yml +ADD ./m3dbnode-local-docker-etcd.yml /etc/m3dbnode/m3dbnode.yml -EXPOSE 2379/tcp 2380/tcp 7201/tcp 7203/tcp 9000-9004/tcp +EXPOSE 7201/tcp 7203/tcp 9000-9004/tcp ENV PANIC_ON_INVARIANT_VIOLATED=true diff --git a/scripts/docker-integration-tests/prom_remote_write_backend/docker-compose.yml b/scripts/docker-integration-tests/prom_remote_write_backend/docker-compose.yml index 6c38ae39de..f89e401db3 100644 --- a/scripts/docker-integration-tests/prom_remote_write_backend/docker-compose.yml +++ b/scripts/docker-integration-tests/prom_remote_write_backend/docker-compose.yml @@ -1,5 +1,16 @@ version: "3.5" services: + etcd: + image: docker.io/bitnami/etcd:3.5 + expose: + - "2379-2380" + ports: + - "0.0.0.0:2379-2380:2379-2380" + environment: + - ALLOW_NONE_AUTHENTICATION=yes + - ETCD_ADVERTISE_CLIENT_URLS=http://etcd:2379 + networks: + - backend m3aggregator01: expose: - "6001" @@ -12,6 +23,8 @@ services: image: "m3aggregator_integration:${REVISION}" volumes: - "./m3aggregator.yml:/etc/m3aggregator/m3aggregator.yml" + depends_on: + - etcd m3aggregator02: expose: - "6002" @@ -24,6 +37,8 @@ services: image: "m3aggregator_integration:${REVISION}" volumes: - "./m3aggregator.yml:/etc/m3aggregator/m3aggregator.yml" + depends_on: + - etcd m3coordinator01: expose: - "7202" @@ -34,6 +49,8 @@ services: image: "m3coordinator_integration:${REVISION}" volumes: - "./:/etc/m3coordinator/" + depends_on: + - etcd coordinatoradmin: expose: - "7201" @@ -44,6 +61,8 @@ services: image: "m3coordinator_integration:${REVISION}" volumes: - "./m3coordinator-admin.yml:/etc/m3coordinator/m3coordinator.yml" + depends_on: + - etcd prometheusraw: expose: - "9090" @@ -60,6 +79,8 @@ services: - "--web.console.libraries=/usr/share/prometheus/console_libraries" - "--web.console.templates=/usr/share/prometheus/consoles" - "--enable-feature=remote-write-receiver" + depends_on: + - etcd prometheusagg: expose: - "9091" @@ -76,33 +97,7 @@ services: - "--web.console.libraries=/usr/share/prometheus/console_libraries" - "--web.console.templates=/usr/share/prometheus/consoles" - "--enable-feature=remote-write-receiver" - etcd01: - expose: - - "2379-2380" - ports: - - "0.0.0.0:2379-2380:2379-2380" - networks: - - backend - image: quay.io/coreos/etcd:v3.4.3 - command: - - "etcd" - - "--name" - - "etcd01" - - "--listen-peer-urls" - - "http://0.0.0.0:2380" - - "--listen-client-urls" - - "http://0.0.0.0:2379" - - "--advertise-client-urls" - - "http://etcd01:2379" - - "--initial-cluster-token" - - "etcd-cluster-1" - - "--initial-advertise-peer-urls" - - "http://etcd01:2380" - - "--initial-cluster" - - "etcd01=http://etcd01:2380" - - "--initial-cluster-state" - - "new" - - "--data-dir" - - "/var/lib/etcd" + depends_on: + - etcd networks: - backend: + backend: null diff --git a/scripts/docker-integration-tests/prom_remote_write_backend/m3aggregator.yml b/scripts/docker-integration-tests/prom_remote_write_backend/m3aggregator.yml index 1d77b0a035..37ac3f2aa4 100644 --- a/scripts/docker-integration-tests/prom_remote_write_backend/m3aggregator.yml +++ b/scripts/docker-integration-tests/prom_remote_write_backend/m3aggregator.yml @@ -40,7 +40,7 @@ kvClient: etcdClusters: - zone: embedded endpoints: - - etcd01:2379 + - etcd:2379 runtimeOptions: kvConfig: diff --git a/scripts/docker-integration-tests/prom_remote_write_backend/m3coordinator-admin.yml b/scripts/docker-integration-tests/prom_remote_write_backend/m3coordinator-admin.yml index c3c08c0104..6a261ecf26 100644 --- a/scripts/docker-integration-tests/prom_remote_write_backend/m3coordinator-admin.yml +++ b/scripts/docker-integration-tests/prom_remote_write_backend/m3coordinator-admin.yml @@ -23,7 +23,7 @@ clusterManagement: etcdClusters: - zone: embedded endpoints: - - etcd01:2379 + - etcd:2379 tagOptions: idScheme: quoted diff --git a/scripts/docker-integration-tests/prom_remote_write_backend/m3coordinator.yml b/scripts/docker-integration-tests/prom_remote_write_backend/m3coordinator.yml index d6c54c8430..935bc8ed5e 100644 --- a/scripts/docker-integration-tests/prom_remote_write_backend/m3coordinator.yml +++ b/scripts/docker-integration-tests/prom_remote_write_backend/m3coordinator.yml @@ -36,7 +36,7 @@ clusterManagement: etcdClusters: - zone: embedded endpoints: - - etcd01:2379 + - etcd:2379 tagOptions: idScheme: quoted diff --git a/scripts/docker-integration-tests/prom_remote_write_backend/test.sh b/scripts/docker-integration-tests/prom_remote_write_backend/test.sh index 208bb88565..7ba4c0435e 100755 --- a/scripts/docker-integration-tests/prom_remote_write_backend/test.sh +++ b/scripts/docker-integration-tests/prom_remote_write_backend/test.sh @@ -20,7 +20,7 @@ docker pull $PROMREMOTECLI_IMAGE trap 'cleanup ${COMPOSE_FILE} ${TEST_SUCCESS}' EXIT echo "Run ETCD" -docker-compose -f "${COMPOSE_FILE}" up -d etcd01 +docker-compose -f "${COMPOSE_FILE}" up -d etcd echo "Run Coordinator in Admin mode" docker-compose -f "${COMPOSE_FILE}" up -d coordinatoradmin diff --git a/scripts/docker-integration-tests/prometheus/docker-compose.yml b/scripts/docker-integration-tests/prometheus/docker-compose.yml index 03b839832b..2bfe100b1e 100644 --- a/scripts/docker-integration-tests/prometheus/docker-compose.yml +++ b/scripts/docker-integration-tests/prometheus/docker-compose.yml @@ -1,17 +1,28 @@ version: "3.5" services: + etcd: + image: docker.io/bitnami/etcd:3.5 + expose: + - "2379-2380" + ports: + - "0.0.0.0:2379-2380:2379-2380" + environment: + - ALLOW_NONE_AUTHENTICATION=yes + - ETCD_ADVERTISE_CLIENT_URLS=http://etcd:2379 + networks: + - backend dbnode01: expose: - "9000-9004" - - "2379-2380" ports: - "0.0.0.0:9000-9004:9000-9004" - - "0.0.0.0:2379-2380:2379-2380" networks: - backend image: "m3dbnode_integration:${REVISION}" volumes: - "./m3dbnode.yml:/etc/m3dbnode/m3dbnode.yml" + depends_on: + - etcd coordinator01: expose: - "7201" @@ -24,6 +35,8 @@ services: image: "m3coordinator_integration:${REVISION}" volumes: - "./:/etc/m3coordinator/" + depends_on: + - etcd prometheus01: expose: - "9090" @@ -34,5 +47,7 @@ services: image: prom/prometheus:latest volumes: - "./prometheus.yml:/etc/prometheus/prometheus.yml" + depends_on: + - etcd networks: - backend: + backend: null diff --git a/scripts/docker-integration-tests/prometheus/m3coordinator.yml b/scripts/docker-integration-tests/prometheus/m3coordinator.yml index 79f849a151..7ebe2792b3 100644 --- a/scripts/docker-integration-tests/prometheus/m3coordinator.yml +++ b/scripts/docker-integration-tests/prometheus/m3coordinator.yml @@ -13,7 +13,7 @@ clusters: etcdClusters: - zone: embedded endpoints: - - dbnode01:2379 + - etcd:2379 query: restrictTags: diff --git a/scripts/docker-integration-tests/repair/docker-compose.yml b/scripts/docker-integration-tests/repair/docker-compose.yml index b91284adcf..bef03881d9 100644 --- a/scripts/docker-integration-tests/repair/docker-compose.yml +++ b/scripts/docker-integration-tests/repair/docker-compose.yml @@ -1,9 +1,19 @@ version: "3.5" services: + etcd: + image: docker.io/bitnami/etcd:3.5 + expose: + - "2379-2380" + ports: + - "0.0.0.0:2379-2380:2379-2380" + environment: + - ALLOW_NONE_AUTHENTICATION=yes + - ETCD_ADVERTISE_CLIENT_URLS=http://etcd:2379 + networks: + - backend dbnode01: expose: - "9000-9004" - - "2379-2380" ports: - "0.0.0.0:9012:9002" - "0.0.0.0:9013:9003" @@ -14,10 +24,11 @@ services: - M3DB_HOST_ID=m3db_local_1 volumes: - "./m3dbnode.yml:/etc/m3dbnode/m3dbnode.yml" + depends_on: + - etcd dbnode02: expose: - "9000-9004" - - "2379-2380" ports: - "0.0.0.0:9022:9002" - "0.0.0.0:9023:9003" @@ -28,6 +39,8 @@ services: - M3DB_HOST_ID=m3db_local_2 volumes: - "./m3dbnode.yml:/etc/m3dbnode/m3dbnode.yml" + depends_on: + - etcd coordinator01: expose: - "7201" @@ -42,5 +55,7 @@ services: image: "m3coordinator_integration:${REVISION}" volumes: - "./:/etc/m3coordinator/" + depends_on: + - etcd networks: - backend: + backend: null diff --git a/scripts/docker-integration-tests/repair/m3coordinator.yml b/scripts/docker-integration-tests/repair/m3coordinator.yml index cc33cf4021..dc075a01fe 100644 --- a/scripts/docker-integration-tests/repair/m3coordinator.yml +++ b/scripts/docker-integration-tests/repair/m3coordinator.yml @@ -13,4 +13,4 @@ clusters: etcdClusters: - zone: embedded endpoints: - - dbnode01:2379 + - etcd:2379 diff --git a/scripts/docker-integration-tests/repair/m3dbnode.yml b/scripts/docker-integration-tests/repair/m3dbnode.yml index 29dcc22d4b..62f3e88cf8 100644 --- a/scripts/docker-integration-tests/repair/m3dbnode.yml +++ b/scripts/docker-integration-tests/repair/m3dbnode.yml @@ -13,12 +13,7 @@ db: etcdClusters: - zone: embedded endpoints: - - dbnode01:2379 - seedNodes: - initialCluster: - - hostID: m3db_local_1 - endpoint: http://dbnode01:2380 - + - etcd:2379 # Enable repairs. repair: enabled: true diff --git a/scripts/docker-integration-tests/repair/test.sh b/scripts/docker-integration-tests/repair/test.sh index ee32d632a9..4cb6e3b4f4 100755 --- a/scripts/docker-integration-tests/repair/test.sh +++ b/scripts/docker-integration-tests/repair/test.sh @@ -9,9 +9,7 @@ COMPOSE_FILE=$SCRIPT_PATH/docker-compose.yml export REVISION echo "Run m3dbnode and m3coordinator containers" -docker-compose -f ${COMPOSE_FILE} up -d --renew-anon-volumes dbnode01 -docker-compose -f ${COMPOSE_FILE} up -d --renew-anon-volumes dbnode02 -docker-compose -f ${COMPOSE_FILE} up -d --renew-anon-volumes coordinator01 +docker-compose -f ${COMPOSE_FILE} up -d --renew-anon-volumes # Think of this as a defer func() in golang function defer { diff --git a/scripts/docker-integration-tests/run.sh b/scripts/docker-integration-tests/run.sh index bead15b207..fa7f2901e3 100755 --- a/scripts/docker-integration-tests/run.sh +++ b/scripts/docker-integration-tests/run.sh @@ -3,45 +3,55 @@ set -ex TESTS=( - scripts/docker-integration-tests/cold_writes_simple/test.sh - scripts/docker-integration-tests/prometheus_replication/test.sh - scripts/docker-integration-tests/carbon/test.sh - scripts/docker-integration-tests/aggregator/test.sh - scripts/docker-integration-tests/aggregator_legacy/test.sh - scripts/docker-integration-tests/query_fanout/test.sh - scripts/docker-integration-tests/repair/test.sh - scripts/docker-integration-tests/replication/test.sh - scripts/docker-integration-tests/multi_cluster_write/test.sh - scripts/docker-integration-tests/coordinator_config_rules/test.sh - scripts/docker-integration-tests/coordinator_noop/test.sh - scripts/docker-integration-tests/prom_remote_write_backend/test.sh + scripts/docker-integration-tests/cold_writes_simple/test.sh + # TODO (amains): This test requires two *separate* etcd clusters, which is a bit harder to setup. + # scripts/docker-integration-tests/prometheus_replication/test.sh + + scripts/docker-integration-tests/carbon/test.sh + scripts/docker-integration-tests/aggregator/test.sh + scripts/docker-integration-tests/aggregator_legacy/test.sh + + # TODO (amains): This test requires two *separate* etcd clusters, which is a bit harder to setup. + # scripts/docker-integration-tests/query_fanout/test.sh + + scripts/docker-integration-tests/repair/test.sh + + # TODO (amains): This test requires two *separate* etcd clusters, which is a bit harder to setup. + # scripts/docker-integration-tests/replication/test.sh + + # TODO (amains): This test requires two *separate* etcd clusters, which is a bit harder to setup. + # scripts/docker-integration-tests/multi_cluster_write/test.sh + + scripts/docker-integration-tests/coordinator_config_rules/test.sh + scripts/docker-integration-tests/coordinator_noop/test.sh + scripts/docker-integration-tests/prom_remote_write_backend/test.sh ) # Some systems, including our default Buildkite hosts, don't come with netcat # installed and we may not have perms to install it. "Install" it in the worst # possible way. if ! command -v nc && [[ "$BUILDKITE" == "true" ]]; then - echo "installing netcat" - NCDIR="$(mktemp -d)" + echo "installing netcat" + NCDIR="$(mktemp -d)" - yumdownloader -y --destdir "$NCDIR" --resolve nc - ( - cd "$NCDIR" - RPM=$(find . -maxdepth 1 -name '*.rpm' | tail -n1) - rpm2cpio "$RPM" | cpio -id - ) + yumdownloader -y --destdir "$NCDIR" --resolve nc + ( + cd "$NCDIR" + RPM=$(find . -maxdepth 1 -name '*.rpm' | tail -n1) + rpm2cpio "$RPM" | cpio -id + ) - export PATH="$PATH:$NCDIR/usr/bin" + export PATH="$PATH:$NCDIR/usr/bin" - function cleanup_nc() { - rm -rf "$NCDIR" - } + function cleanup_nc() { + rm -rf "$NCDIR" + } - trap cleanup_nc EXIT + trap cleanup_nc EXIT fi if [[ -z "$SKIP_SETUP" ]] || [[ "$SKIP_SETUP" == "false" ]]; then - scripts/docker-integration-tests/setup.sh + scripts/docker-integration-tests/setup.sh fi NUM_TESTS=${#TESTS[@]} @@ -50,16 +60,16 @@ MAX_IDX=$(((NUM_TESTS*(BUILDKITE_PARALLEL_JOB+1)/BUILDKITE_PARALLEL_JOB_COUNT)-1 ITER=0 for test in "${TESTS[@]}"; do - if [[ $ITER -ge $MIN_IDX && $ITER -le $MAX_IDX ]]; then - # Ensure all docker containers have been stopped so we don't run into issues - # trying to bind ports. - docker rm -f $(docker ps -aq) 2>/dev/null || true - echo "----------------------------------------------" - echo "running $test" - if ! (export M3_PATH=$(pwd) && $test); then - echo "--- :bk-status-failed: $test FAILED" - exit 1 - fi - fi - ITER="$((ITER+1))" + if [[ $ITER -ge $MIN_IDX && $ITER -le $MAX_IDX ]]; then + # Ensure all docker containers have been stopped so we don't run into issues + # trying to bind ports. + docker rm -f $(docker ps -aq) 2>/dev/null || true + echo "----------------------------------------------" + echo "running $test" + if ! (export M3_PATH=$(pwd) && $test); then + echo "--- :bk-status-failed: $test FAILED" + exit 1 + fi + fi + ITER="$((ITER+1))" done diff --git a/scripts/docker-integration-tests/setup.sh b/scripts/docker-integration-tests/setup.sh index 76659a49a2..d90ecb4967 100755 --- a/scripts/docker-integration-tests/setup.sh +++ b/scripts/docker-integration-tests/setup.sh @@ -15,8 +15,8 @@ mkdir -p ./bin # by keeping all the required files in ./bin, it makes the build context # for docker much smaller -cp ./src/query/config/m3coordinator-local-etcd.yml ./bin -cp ./src/dbnode/config/m3dbnode-local-etcd.yml ./bin +cp ./src/query/config/m3coordinator-local-docker-etcd.yml ./bin +cp ./src/dbnode/config/m3dbnode-local-docker-etcd.yml ./bin cp ./src/aggregator/config/m3aggregator.yml ./bin # build images @@ -26,7 +26,9 @@ function build_image { local svc=$1 echo "creating image for $svc" make ${svc}-linux-amd64 - docker build -t "${svc}_integration:${REVISION}" -f ./scripts/docker-integration-tests/${svc}.Dockerfile ./bin + docker build \ + --no-cache \ + -t "${svc}_integration:${REVISION}" -f ./scripts/docker-integration-tests/${svc}.Dockerfile ./bin } if [[ "$SERVICE" != "" ]]; then diff --git a/scripts/docker-integration-tests/simple_v2_batch_apis/docker-compose.yml b/scripts/docker-integration-tests/simple_v2_batch_apis/docker-compose.yml index bce43d559e..7ed9f8f5ef 100644 --- a/scripts/docker-integration-tests/simple_v2_batch_apis/docker-compose.yml +++ b/scripts/docker-integration-tests/simple_v2_batch_apis/docker-compose.yml @@ -1,15 +1,26 @@ version: "3.5" services: + etcd: + image: docker.io/bitnami/etcd:3.5 + expose: + - "2379-2380" + ports: + - "0.0.0.0:2379-2380:2379-2380" + environment: + - ALLOW_NONE_AUTHENTICATION=yes + - ETCD_ADVERTISE_CLIENT_URLS=http://etcd:2379 + networks: + - backend dbnode01: expose: - "9000-9004" - - "2379-2380" ports: - "0.0.0.0:9000-9004:9000-9004" - - "0.0.0.0:2379-2380:2379-2380" networks: - backend image: "m3dbnode_integration:${REVISION}" + depends_on: + - etcd coordinator01: expose: - "7201" @@ -22,5 +33,7 @@ services: image: "m3coordinator_integration:${REVISION}" volumes: - "./:/etc/m3coordinator/" + depends_on: + - etcd networks: - backend: + backend: null diff --git a/scripts/docker-integration-tests/simple_v2_batch_apis/m3coordinator.yml b/scripts/docker-integration-tests/simple_v2_batch_apis/m3coordinator.yml index 7d5cb12598..945b26e3b5 100644 --- a/scripts/docker-integration-tests/simple_v2_batch_apis/m3coordinator.yml +++ b/scripts/docker-integration-tests/simple_v2_batch_apis/m3coordinator.yml @@ -13,5 +13,5 @@ clusters: etcdClusters: - zone: embedded endpoints: - - dbnode01:2379 + - etcd:2379 useV2BatchAPIs: true diff --git a/src/dbnode/config/m3dbnode-local-docker-etcd.yml b/src/dbnode/config/m3dbnode-local-docker-etcd.yml new file mode 100644 index 0000000000..83650bc894 --- /dev/null +++ b/src/dbnode/config/m3dbnode-local-docker-etcd.yml @@ -0,0 +1,11 @@ +coordinator: {} +"db": + discovery: + "config": + "service": + "etcdClusters": + - "endpoints": ["http://etcd:2379"] + "zone": "embedded" + "service": "m3db" + "zone": "embedded" + "env": "default_env" \ No newline at end of file diff --git a/src/query/config/m3coordinator-local-docker-etcd.yml b/src/query/config/m3coordinator-local-docker-etcd.yml new file mode 100644 index 0000000000..c25dff706c --- /dev/null +++ b/src/query/config/m3coordinator-local-docker-etcd.yml @@ -0,0 +1,16 @@ +clusters: + - namespaces: + - namespace: default + type: unaggregated + retention: 48h + client: + config: + service: + env: default_env + zone: embedded + service: m3db + cacheDir: /var/lib/m3kv + etcdClusters: + - zone: embedded + endpoints: + - http://etcd:2379