feat: Add dev.wait-for.% Make target; use in provisioning tests (#1201

)

This target waits for a list of services to come up. While this is
primarily intended as a developer feature, it should also fix some
intermittent provisioning test failures that we think are race conditions
(where dev.check is called too soon after dev.up).

Also, increase check.sh log output limit to match dev.logs.

.github/workflows/provisioning-tests.yml

@@ -70,11 +70,15 @@ jobs:
       - name: provision
         run:  make dev.provision.${{matrix.services}}
 
-      - name: dev.up
+      - name: "Bring up services"
         run:  make dev.up.${{matrix.services}}
 
-      - name: dev.check
-        run:  make dev.check.${{matrix.services}}
+      - name: "Wait for services to become ready"
+        run: |
+          # Wait a reasonable amount of time for services to come up. If they
+          # don't, then call the checks one more time to ensure that diagnostic
+          # information is printed out. (It's suppressed by wait-for.)
+          timeout 5m make dev.wait-for.${{matrix.services}} || timeout 1m make dev.check.${{matrix.services}}
 
       - name: notify on failure
         if: ${{ failure() && github.ref == 'refs/heads/master' }}

Makefile

@@ -381,6 +381,9 @@ dev.check: dev.check.$(DEFAULT_SERVICES) ## Run checks for the default service s
 dev.check.%:  # Run checks for a given service or set of services.
 	$(WINPTY) bash ./check.sh $*
 
+dev.wait-for.%:  ## Wait for these services to become ready
+	$(WINPTY) bash ./wait-ready.sh $$(echo $* | tr + " ")
+
 dev.validate: ## Print effective Docker Compose config, validating files in COMPOSE_FILE.
 	docker compose config
 
@@ -463,7 +466,7 @@ DB_NAMES_LIST = credentials discovery ecommerce notes registrar xqueue edxapp ed
 _db_copy8_targets = $(addprefix dev.dbcopy8.,$(DB_NAMES_LIST))
 dev.dbcopyall8: ## Copy data from old mysql 5.7 containers into new mysql8 dbs
 	$(MAKE) dev.up.mysql57+mysql80
-	./wait-ready.sh mysql57 mysql80
+	$(MAKE) dev.wait-for.mysql57+mysql80
 	$(MAKE) $(_db_copy8_targets)
 
 dev.dbcopy8.%: ## Copy data from old mysql 5.7 container into a new 8 db

check.sh

@@ -46,7 +46,7 @@ run_check() {
     if bash -c "$cmd"; then  # Run the command itself and check if it succeeded.
         succeeded="$succeeded $check_name"
     else
-        docker compose logs --tail 30 "$service"  # Just show recent logs, not all history
+        docker compose logs --tail 500 "$service"  # Just show recent logs, not all history
         failed="$failed $check_name"
     fi
     set -e  # Re-enable exit-on-error

docs/devstack_interface.rst

@@ -200,6 +200,7 @@ If you want to ensure you are getting the latest listing, simply use ``make help
       dev.up.without-deps.%        Bring up services by themselves.
       dev.up.without-deps.shell.%  Bring up a service by itself + shell into it.
       dev.validate                 Print effective Docker Compose config, validating files in COMPOSE_FILE.
+      dev.wait-for.%               Wait for these services to become ready
       devpi-password               Get the root devpi password for the devpi container.
       docs                         generate Sphinx HTML documentation, including API docs
       hadoop-application-logs-%    View hadoop logs by application Id.

provision.sh

@@ -130,22 +130,22 @@ fi
 
 # Ensure the MySQL5 server is online and usable
 echo "${GREEN}Waiting for MySQL 5.7.${NC}"
-./wait-ready.sh mysql57
+make dev.wait-for.mysql57
 
 # Ensure the MySQL8 server is online and usable
 echo "${GREEN}Waiting for MySQL 8.0.${NC}"
-./wait-ready.sh mysql80
+make dev.wait-for.mysql80
 
 # In the event of a fresh MySQL container, wait a few seconds for the server to restart
 # See https://github.com/docker-library/mysql/issues/245 for why this is necessary.
 sleep 10
 
 echo "${GREEN}Waiting for MySQL 5.7 to restart.${NC}"
-./wait-ready.sh mysql57
+make dev.wait-for.mysql57
 echo -e "${GREEN}MySQL5 ready.${NC}"
 
 echo "${GREEN}Waiting for MySQL 8.0 to restart.${NC}"
-./wait-ready.sh mysql80
+make dev.wait-for.mysql80
 echo -e "${GREEN}MySQL8 ready.${NC}"
 
 # Ensure that the MySQL databases and users are created for all IDAs.
@@ -160,7 +160,7 @@ docker compose exec -T mysql80 bash -e -c "mysql -uroot mysql" < provision-mysql
 if needs_mongo "$to_provision_ordered"; then
 	echo -e "${GREEN}Waiting for MongoDB...${NC}"
 	# mongo container and mongo process/shell inside the container
-	./wait-ready.sh mongo
+	make dev.wait-for.mongo
 	echo -e "${GREEN}MongoDB ready.${NC}"
 	echo -e "${GREEN}Creating MongoDB users...${NC}"
     docker compose exec -T mongo bash -e -c "mongo" < mongo-provision.js