From 466a517f5dfdd4eafd077d5bdd4dcce2339834ef Mon Sep 17 00:00:00 2001 From: Charlie Lye Date: Wed, 27 Sep 2023 11:30:48 +0100 Subject: [PATCH] feat: bootstrap_docker skips build it can pull from ecr. (#2545) As title. --------- Co-authored-by: ludamad --- build-system/scripts/build_local | 48 +++++++++++++++++++-- build-system/scripts/calculate_content_hash | 3 +- build-system/scripts/calculate_image_uri | 1 - 3 files changed, 46 insertions(+), 6 deletions(-) diff --git a/build-system/scripts/build_local b/build-system/scripts/build_local index f95c2413fe9..77c680f1db0 100755 --- a/build-system/scripts/build_local +++ b/build-system/scripts/build_local @@ -63,15 +63,55 @@ for E in "${PROJECTS[@]}"; do fi fi + # TODO: Move everything to aztecprotocol/* + DEPLOY_IMAGE_URI=$ECR_DEPLOY_URL/$REPO:latest + CACHE_IMAGE_URI=$(calculate_image_uri $REPO) + ARR=(${CACHE_IMAGE_URI//:/ }) + REPO_URI=${ARR[0]} + TAG=${ARR[1]} + + GREEN="\033[32m" + BLUE="\033[94m" + YELLOW="\033[93m" + BOLD="\033[1m" + RESET="\033[0m" + echo echo echo - echo "*** Building $PROJECT_DIR_NAME:$DOCKERFILE -> $REPO ***" - echo "*** Context: $PWD" + echo -e "${BOLD}Repository${RESET}: ${BOLD}${BLUE}$REPO${RESET}" + echo -e "${BOLD}Cache Image URI${RESET}: $REPO_URI:$TAG" + echo -e "${BOLD}Deploy Image URI${RESET}: $DEPLOY_IMAGE_URI" + echo -e "${BOLD}Dockerfile${RESET}: $DOCKERFILE" + echo -e "${BOLD}Context${RESET}: $PWD" echo - docker build ${ADDITIONAL_ARGS:-} --build-arg ARG_COMMIT_HASH=$COMMIT_HASH -f $DOCKERFILE -t $ECR_DEPLOY_URL/$REPO:latest . - docker tag $ECR_DEPLOY_URL/$REPO:latest aztecprotocol/$REPO:latest + GREP_PATTERN=$(query_manifest rebuildPatterns $REPO | tr '\n' '|') + # Remove the trailing '|'. + GREP_PATTERN=${GREP_PATTERN%|} + + # Ensure index is updated before we call git diff-index. + git status > /dev/null + + if (git diff-index --name-only HEAD; git ls-files --others --exclude-standard) | grep -qE "$GREP_PATTERN"; then + echo -e "${YELLOW}Project or dependency has local modifications! Building...${RESET}" + docker build ${ADDITIONAL_ARGS:-} --build-arg ARG_COMMIT_HASH=$COMMIT_HASH -f $DOCKERFILE -t $DEPLOY_IMAGE_URI . + else + if docker image ls --format "{{.Repository}}:{{.Tag}}" | grep -q -w $CACHE_IMAGE_URI; then + echo -e "${GREEN}Image exists locally. Tagging as $DEPLOY_IMAGE_URI${RESET}" + docker tag $CACHE_IMAGE_URI $DEPLOY_IMAGE_URI + else + if ecr_login && image_exists $REPO_URI $TAG ; then + docker pull $CACHE_IMAGE_URI + else + docker build ${ADDITIONAL_ARGS:-} --build-arg ARG_COMMIT_HASH=$COMMIT_HASH -f $DOCKERFILE -t $CACHE_IMAGE_URI . + docker tag $CACHE_IMAGE_URI $DEPLOY_IMAGE_URI + fi + fi + fi + + # Retag for aztecprotocol dockerhub. + docker tag $DEPLOY_IMAGE_URI aztecprotocol/$REPO:latest if [ "$PROJECT_DIR_NAME" = "$TARGET_PROJECT" ]; then if [ -n "$LAUNCH" ]; then diff --git a/build-system/scripts/calculate_content_hash b/build-system/scripts/calculate_content_hash index cdc6aa5866c..1d107c69d37 100755 --- a/build-system/scripts/calculate_content_hash +++ b/build-system/scripts/calculate_content_hash @@ -11,9 +11,10 @@ REBUILD_PATTERNS=$(query_manifest rebuildPatterns $REPOSITORY) AWK_PATTERN=$(echo $REBUILD_PATTERNS | sed 's/ /|/g') cd "$(git rev-parse --show-toplevel)" + # an example line is # 100644 da9ae2e020ea7fe3505488bbafb39adc7191559b 0 yarn-project/world-state/tsconfig.json # this format is beneficial as it grabs the hashes from git efficiently # we will next filter by our rebuild patterns # then we pipe the hash portion of each file to git hash-object to produce our content hash -git ls-tree -r $COMMIT_HASH | awk -v pattern="($AWK_PATTERN)" '$4 ~ pattern {print $3}' | git hash-object --stdin +git ls-tree -r $COMMIT_HASH | awk -v pattern="($AWK_PATTERN)" '$4 ~ pattern {print $3}' | git hash-object --stdin \ No newline at end of file diff --git a/build-system/scripts/calculate_image_uri b/build-system/scripts/calculate_image_uri index a7b78a03e66..38038a88319 100755 --- a/build-system/scripts/calculate_image_uri +++ b/build-system/scripts/calculate_image_uri @@ -6,6 +6,5 @@ REPOSITORY=$1 # Assume we might be calling this directly without build system in path export ROOT_PATH=$(git rev-parse --show-toplevel) export PATH="$PATH:$ROOT_PATH/build-system/scripts:$ROOT_PATH/build-system/bin" -export LD_LIBRARY_PATH="$ROOT_PATH/build-system/lib" CONTENT_HASH=$(calculate_content_hash $REPOSITORY) echo "278380418400.dkr.ecr.us-east-2.amazonaws.com/$REPOSITORY:cache-$CONTENT_HASH"