From 5e243772d05fc22a7fbff13ad9f4aa2faaff9d1e Mon Sep 17 00:00:00 2001 From: JGAntunes Date: Fri, 19 Feb 2021 19:40:51 +0000 Subject: [PATCH 01/10] feat: initial work to add support for js workspaces --- run-build-functions.sh | 74 ++++++++++++++++++++++++++++++++++++++++-- 1 file changed, 72 insertions(+), 2 deletions(-) diff --git a/run-build-functions.sh b/run-build-functions.sh index 2a04e62e..848e4084 100755 --- a/run-build-functions.sh +++ b/run-build-functions.sh @@ -40,6 +40,7 @@ mkdir -p $NETLIFY_CACHE_DIR/swift_version # pwd caches mkdir -p $NETLIFY_CACHE_DIR/node_modules +mkdir -p $NETLIFY_CACHE_DIR/js-workspaces mkdir -p $NETLIFY_CACHE_DIR/.bundle mkdir -p $NETLIFY_CACHE_DIR/bower_components mkdir -p $NETLIFY_CACHE_DIR/.venv @@ -102,6 +103,32 @@ run_yarn() { fi + if [ "$NETLIFY_YARN_WORKSPACES" = "true" ] + then + echo "NETLIFY_YARN_WORKSPACES feature flag set" + # Ignore path env var required in order to support yarn 2 projects + if YARN_IGNORE_PATH=1 yarn workspaces info + then + # Extract all the packages and respective locations. .data will be a JSON object like + # { + # "my-package-1": { + # "location": "packages/blog-1", + # "workspaceDependencies": [], + # "mismatchedWorkspaceDependencies": [] + # }, + # (...) + # } + # We need to cache all the node_module dirs, or we'll always be installing them on each run + local package_locations="$(YARN_IGNORE_PATH=1 yarn --json workspaces info | jq -r '.data | fromjson | to_entries | .[].value.location | @sh'| tr -d \')" + restore_js_workspace_cache "$package_locations" + else + echo "No workspace detected" + restore_cwd_cache node_modules "node modules" + fi + else + restore_cwd_cache node_modules "node modules" + fi + echo "Installing NPM modules using Yarn version $(yarn --version)" run_npm_set_temp @@ -130,6 +157,8 @@ run_npm_set_temp() { } run_npm() { + restore_cwd_cache node_modules "node modules" + if [ -n "$NPM_VERSION" ] then if [ "$(npm --version)" != "$NPM_VERSION" ] @@ -475,7 +504,6 @@ install_dependencies() { if [ -f package.json ] then - restore_cwd_cache node_modules "node modules" if [ "$NETLIFY_USE_YARN" = "true" ] || ([ "$NETLIFY_USE_YARN" != "false" ] && [ -f yarn.lock ]) then run_yarn $YARN_VERSION @@ -676,6 +704,11 @@ cache_artifacts() { cache_cwd_directory ".build" "swift build" cache_cwd_directory ".netlify/plugins" "build plugins" + if [ "$NETLIFY_YARN_WORKSPACES" == "true" ] && [ -f package.json ] + then + cache_js_workspaces + fi + if [ -f Cargo.toml ] || [ -f Cargo.lock ] then cache_cwd_directory_fast_copy "target" "rust compile output" @@ -690,7 +723,7 @@ cache_artifacts() { cache_home_directory ".composer" "composer dependencies" cache_home_directory ".homebrew-cache", "homebrew cache" cache_home_directory ".rustup" "rust rustup cache" - + if [ -f Cargo.toml ] || [ -f Cargo.lock ] then cache_home_directory ".cargo/registry" "rust cargo registry cache" @@ -779,6 +812,43 @@ restore_cwd_cache() { move_cache "$NETLIFY_CACHE_DIR/$1" "$PWD/$1" "restoring cached $2" } +# +# Restores node_modules dirs cached for js workspaces +# See https://github.com/netlify/pod-workflow/issues/139/ for more context +# +# Expects: +# $1 array containing package locations relative to the repo's root +restore_js_workspaces_cache() { + local locations=$1 + local cache_dir="$NETLIFY_CACHE_DIR/js-workspaces" + + # Retrieve each workspace node_modules + for location in "${locations[@]}"; do + move_cache "$cache_dir/$location/node_modules" "$NETLIFY_REPO_DIR/$location/node_modules" "restoring workspace $location node modules" + done + # Retrieve hoisted node_modules + move_cache "$cache_dir/node_modules" "$NETLIFY_REPO_DIR/node_modules" "restoring root node modules" + # Keep a record of the workspaces in the project in order to cache them later + echo "${locations[@]}" > "$cache_dir/.workspace_locations" +} + +# +# Caches node_modules dirs from js workspaces. It acts based on the presence of a +# `.workspace_locations` file previously generated in `restore_js_workspaces_cache()` +# +cache_js_workspaces() { + local cache_dir="$NETLIFY_CACHE_DIR/js-workspaces" + if [ -f "$cache_dir/.workspace_locations" ] + then + local locations=($(cat "$cache_dir/.workspace_locations")) + for location in "${locations[@]}"; do + move_cache "$NETLIFY_REPO_DIR/$location/node_modules" "$cache_dir/$location/node_modules" "saving workspace $location node modules" + done + # Retrieve hoisted node_modules + move_cache "$NETLIFY_REPO_DIR/node_modules" "$cache_dir/node_modules" "saving root node modules" + fi +} + cache_cwd_directory() { move_cache "$PWD/$1" "$NETLIFY_CACHE_DIR/$1" "saving $2" } From 620c1ceda06cd10d96ee216efd71d1dc3a76fd60 Mon Sep 17 00:00:00 2001 From: JGAntunes Date: Mon, 22 Feb 2021 13:31:46 +0000 Subject: [PATCH 02/10] fix(ws-cache): addressing some typos and moving the caching call before the regular node_modules cache --- run-build-functions.sh | 23 ++++++++++++----------- 1 file changed, 12 insertions(+), 11 deletions(-) diff --git a/run-build-functions.sh b/run-build-functions.sh index 848e4084..69f209cd 100755 --- a/run-build-functions.sh +++ b/run-build-functions.sh @@ -119,8 +119,8 @@ run_yarn() { # (...) # } # We need to cache all the node_module dirs, or we'll always be installing them on each run - local package_locations="$(YARN_IGNORE_PATH=1 yarn --json workspaces info | jq -r '.data | fromjson | to_entries | .[].value.location | @sh'| tr -d \')" - restore_js_workspace_cache "$package_locations" + local package_locations=($(YARN_IGNORE_PATH=1 yarn --json workspaces info | jq -r '.data | fromjson | to_entries | .[].value.location | @sh'| tr -d \')) + restore_js_workspaces_cache "${package_locations[@]}" else echo "No workspace detected" restore_cwd_cache node_modules "node modules" @@ -697,6 +697,11 @@ install_dependencies() { # cache_artifacts() { echo "Caching artifacts" + if [ "$NETLIFY_YARN_WORKSPACES" == "true" ] && [ -f package.json ] + then + cache_js_workspaces + fi + cache_cwd_directory ".bundle" "ruby gems" cache_cwd_directory "bower_components" "bower components" cache_cwd_directory "node_modules" "node modules" @@ -704,11 +709,6 @@ cache_artifacts() { cache_cwd_directory ".build" "swift build" cache_cwd_directory ".netlify/plugins" "build plugins" - if [ "$NETLIFY_YARN_WORKSPACES" == "true" ] && [ -f package.json ] - then - cache_js_workspaces - fi - if [ -f Cargo.toml ] || [ -f Cargo.lock ] then cache_cwd_directory_fast_copy "target" "rust compile output" @@ -817,9 +817,9 @@ restore_cwd_cache() { # See https://github.com/netlify/pod-workflow/issues/139/ for more context # # Expects: -# $1 array containing package locations relative to the repo's root +# $@ each argument should be a package location relative to the repo's root restore_js_workspaces_cache() { - local locations=$1 + local locations=("$@") local cache_dir="$NETLIFY_CACHE_DIR/js-workspaces" # Retrieve each workspace node_modules @@ -827,7 +827,7 @@ restore_js_workspaces_cache() { move_cache "$cache_dir/$location/node_modules" "$NETLIFY_REPO_DIR/$location/node_modules" "restoring workspace $location node modules" done # Retrieve hoisted node_modules - move_cache "$cache_dir/node_modules" "$NETLIFY_REPO_DIR/node_modules" "restoring root node modules" + move_cache "$cache_dir/node_modules" "$NETLIFY_REPO_DIR/node_modules" "restoring workspace root node modules" # Keep a record of the workspaces in the project in order to cache them later echo "${locations[@]}" > "$cache_dir/.workspace_locations" } @@ -842,10 +842,11 @@ cache_js_workspaces() { then local locations=($(cat "$cache_dir/.workspace_locations")) for location in "${locations[@]}"; do + mkdir -p "$cache_dir/$location" move_cache "$NETLIFY_REPO_DIR/$location/node_modules" "$cache_dir/$location/node_modules" "saving workspace $location node modules" done # Retrieve hoisted node_modules - move_cache "$NETLIFY_REPO_DIR/node_modules" "$cache_dir/node_modules" "saving root node modules" + move_cache "$NETLIFY_REPO_DIR/node_modules" "$cache_dir/node_modules" "saving workspace root node modules" fi } From 4a61af74ac00197885d8507263e60260bb45ae9f Mon Sep 17 00:00:00 2001 From: JGAntunes Date: Mon, 22 Feb 2021 19:06:23 +0000 Subject: [PATCH 03/10] fix(ws-cache): using a tmp dir for workspace locations file & better handling of node_modules cache move out --- run-build-functions.sh | 52 ++++++++++++++++++++++++++++-------------- 1 file changed, 35 insertions(+), 17 deletions(-) diff --git a/run-build-functions.sh b/run-build-functions.sh index 69f209cd..aec7b99a 100755 --- a/run-build-functions.sh +++ b/run-build-functions.sh @@ -60,6 +60,10 @@ mkdir -p $NETLIFY_CACHE_DIR/.gimme_cache/gocache mkdir -p $NETLIFY_CACHE_DIR/.homebrew-cache mkdir -p $NETLIFY_CACHE_DIR/.cargo + +# tmp dir +NETLIFY_TMP_DIR=$(mktemp -d) + : ${YARN_FLAGS=""} : ${NPM_FLAGS=""} : ${BUNDLER_FLAGS=""} @@ -106,7 +110,8 @@ run_yarn() { if [ "$NETLIFY_YARN_WORKSPACES" = "true" ] then echo "NETLIFY_YARN_WORKSPACES feature flag set" - # Ignore path env var required in order to support yarn 2 projects + # YARN_IGNORE_PATH will ignore the presence of a local yarn executable (i.e. yarn 2) and default + # to using the global one (which, for now, is alwasy yarn 1.x). See https://yarnpkg.com/configuration/yarnrc#ignorePath if YARN_IGNORE_PATH=1 yarn workspaces info then # Extract all the packages and respective locations. .data will be a JSON object like @@ -697,14 +702,17 @@ install_dependencies() { # cache_artifacts() { echo "Caching artifacts" - if [ "$NETLIFY_YARN_WORKSPACES" == "true" ] && [ -f package.json ] - then - cache_js_workspaces - fi cache_cwd_directory ".bundle" "ruby gems" cache_cwd_directory "bower_components" "bower components" - cache_cwd_directory "node_modules" "node modules" + + if [ "$NETLIFY_YARN_WORKSPACES" == "true" ] + then + cache_node_modules + else + cache_cwd_directory "node_modules" "node modules" + fi + cache_cwd_directory ".venv" "python virtualenv" cache_cwd_directory ".build" "swift build" cache_cwd_directory ".netlify/plugins" "build plugins" @@ -829,7 +837,20 @@ restore_js_workspaces_cache() { # Retrieve hoisted node_modules move_cache "$cache_dir/node_modules" "$NETLIFY_REPO_DIR/node_modules" "restoring workspace root node modules" # Keep a record of the workspaces in the project in order to cache them later - echo "${locations[@]}" > "$cache_dir/.workspace_locations" + echo "${locations[@]}" > "$NETLIFY_TMP_DIR/.workspace_locations" +} + +# +# Caches node_modules dirs for a js project. Either detects the presence of js workspaces +# via the `.workspace_locations` file, or looks at the node_modules in the cwd. +# +cache_node_modules() { + if [ -f "$NETLIFY_TMP_DIR/.workspace_locations" ] + then + cache_js_workspaces + else + cache_cwd_directory "node_modules" "node modules" + fi } # @@ -838,16 +859,13 @@ restore_js_workspaces_cache() { # cache_js_workspaces() { local cache_dir="$NETLIFY_CACHE_DIR/js-workspaces" - if [ -f "$cache_dir/.workspace_locations" ] - then - local locations=($(cat "$cache_dir/.workspace_locations")) - for location in "${locations[@]}"; do - mkdir -p "$cache_dir/$location" - move_cache "$NETLIFY_REPO_DIR/$location/node_modules" "$cache_dir/$location/node_modules" "saving workspace $location node modules" - done - # Retrieve hoisted node_modules - move_cache "$NETLIFY_REPO_DIR/node_modules" "$cache_dir/node_modules" "saving workspace root node modules" - fi + local locations=($(cat "$NETLIFY_TMP_DIR/.workspace_locations")) + for location in "${locations[@]}"; do + mkdir -p "$cache_dir/$location" + move_cache "$NETLIFY_REPO_DIR/$location/node_modules" "$cache_dir/$location/node_modules" "saving workspace $location node modules" + done + # Retrieve hoisted node_modules + move_cache "$NETLIFY_REPO_DIR/node_modules" "$cache_dir/node_modules" "saving workspace root node modules" } cache_cwd_directory() { From ef73c8409763fd955bc04f8e1c1d1cb58e99a2b1 Mon Sep 17 00:00:00 2001 From: JGAntunes Date: Wed, 24 Feb 2021 19:47:50 +0000 Subject: [PATCH 04/10] fix(ws-cache): correctly handle dirs with spaces --- run-build-functions.sh | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/run-build-functions.sh b/run-build-functions.sh index aec7b99a..8c13ee8f 100755 --- a/run-build-functions.sh +++ b/run-build-functions.sh @@ -114,6 +114,7 @@ run_yarn() { # to using the global one (which, for now, is alwasy yarn 1.x). See https://yarnpkg.com/configuration/yarnrc#ignorePath if YARN_IGNORE_PATH=1 yarn workspaces info then + local package_locations # Extract all the packages and respective locations. .data will be a JSON object like # { # "my-package-1": { @@ -124,7 +125,7 @@ run_yarn() { # (...) # } # We need to cache all the node_module dirs, or we'll always be installing them on each run - local package_locations=($(YARN_IGNORE_PATH=1 yarn --json workspaces info | jq -r '.data | fromjson | to_entries | .[].value.location | @sh'| tr -d \')) + mapfile -t package_locations <<< "$(YARN_IGNORE_PATH=1 yarn --json workspaces info | jq -r '.data | fromjson | to_entries | .[].value.location')" restore_js_workspaces_cache "${package_locations[@]}" else echo "No workspace detected" @@ -788,11 +789,11 @@ cache_artifacts() { move_cache() { local src=$1 local dst=$2 - if [ -d $src ] + if [ -d "$src" ] then echo "Started $3" - rm -rf $dst - mv $src $dst + rm -rf "$dst" + mv "$src" "$dst" echo "Finished $3" fi } @@ -837,7 +838,7 @@ restore_js_workspaces_cache() { # Retrieve hoisted node_modules move_cache "$cache_dir/node_modules" "$NETLIFY_REPO_DIR/node_modules" "restoring workspace root node modules" # Keep a record of the workspaces in the project in order to cache them later - echo "${locations[@]}" > "$NETLIFY_TMP_DIR/.workspace_locations" + printf '%s\n' "${locations[@]}" > "$NETLIFY_TMP_DIR/.workspace_locations" } # @@ -859,7 +860,9 @@ cache_node_modules() { # cache_js_workspaces() { local cache_dir="$NETLIFY_CACHE_DIR/js-workspaces" - local locations=($(cat "$NETLIFY_TMP_DIR/.workspace_locations")) + local locations + mapfile -t locations <<< "$(cat "$NETLIFY_TMP_DIR/.workspace_locations")" + for location in "${locations[@]}"; do mkdir -p "$cache_dir/$location" move_cache "$NETLIFY_REPO_DIR/$location/node_modules" "$cache_dir/$location/node_modules" "saving workspace $location node modules" From 70c8a36e04ac440911ec776386d80536063768c6 Mon Sep 17 00:00:00 2001 From: JGAntunes Date: Thu, 25 Feb 2021 15:13:00 +0000 Subject: [PATCH 05/10] fix(ws-cache): use a variable instead of tmp dir to keep ws locations --- run-build-functions.sh | 18 +++++++----------- 1 file changed, 7 insertions(+), 11 deletions(-) diff --git a/run-build-functions.sh b/run-build-functions.sh index 8c13ee8f..32a48711 100755 --- a/run-build-functions.sh +++ b/run-build-functions.sh @@ -60,10 +60,6 @@ mkdir -p $NETLIFY_CACHE_DIR/.gimme_cache/gocache mkdir -p $NETLIFY_CACHE_DIR/.homebrew-cache mkdir -p $NETLIFY_CACHE_DIR/.cargo - -# tmp dir -NETLIFY_TMP_DIR=$(mktemp -d) - : ${YARN_FLAGS=""} : ${NPM_FLAGS=""} : ${BUNDLER_FLAGS=""} @@ -838,30 +834,30 @@ restore_js_workspaces_cache() { # Retrieve hoisted node_modules move_cache "$cache_dir/node_modules" "$NETLIFY_REPO_DIR/node_modules" "restoring workspace root node modules" # Keep a record of the workspaces in the project in order to cache them later - printf '%s\n' "${locations[@]}" > "$NETLIFY_TMP_DIR/.workspace_locations" + NETLIFY_JS_WORKSPACE_LOCATIONS=$(printf '%s\n' "${locations[@]}") } # # Caches node_modules dirs for a js project. Either detects the presence of js workspaces -# via the `.workspace_locations` file, or looks at the node_modules in the cwd. +# via the `NETLIFY_JS_WORKSPACE_LOCATIONS` variable, or looks at the node_modules in the cwd. # cache_node_modules() { - if [ -f "$NETLIFY_TMP_DIR/.workspace_locations" ] + if [ -z "$NETLIFY_JS_WORKSPACE_LOCATIONS" ] then - cache_js_workspaces - else cache_cwd_directory "node_modules" "node modules" + else + cache_js_workspaces fi } # # Caches node_modules dirs from js workspaces. It acts based on the presence of a -# `.workspace_locations` file previously generated in `restore_js_workspaces_cache()` +# `NETLIFY_JS_WORKSPACE_LOCATIONS` variable previously set in `restore_js_workspaces_cache()` # cache_js_workspaces() { local cache_dir="$NETLIFY_CACHE_DIR/js-workspaces" local locations - mapfile -t locations <<< "$(cat "$NETLIFY_TMP_DIR/.workspace_locations")" + mapfile -t locations <<< "$NETLIFY_JS_WORKSPACE_LOCATIONS" for location in "${locations[@]}"; do mkdir -p "$cache_dir/$location" From 32f7b0df2c181f9dfeb4f654b1115ead254e46cf Mon Sep 17 00:00:00 2001 From: JGAntunes Date: Fri, 26 Feb 2021 13:32:18 +0000 Subject: [PATCH 06/10] fix(ws-cache): no need for intermediate variables for ws locations --- run-build-functions.sh | 15 +++++++-------- 1 file changed, 7 insertions(+), 8 deletions(-) diff --git a/run-build-functions.sh b/run-build-functions.sh index 32a48711..1ca8780c 100755 --- a/run-build-functions.sh +++ b/run-build-functions.sh @@ -824,17 +824,17 @@ restore_cwd_cache() { # Expects: # $@ each argument should be a package location relative to the repo's root restore_js_workspaces_cache() { - local locations=("$@") + # Keep a record of the workspaces in the project in order to cache them later + NETLIFY_JS_WORKSPACE_LOCATIONS=("$@") + local cache_dir="$NETLIFY_CACHE_DIR/js-workspaces" # Retrieve each workspace node_modules - for location in "${locations[@]}"; do + for location in "${NETLIFY_JS_WORKSPACE_LOCATIONS[@]}"; do move_cache "$cache_dir/$location/node_modules" "$NETLIFY_REPO_DIR/$location/node_modules" "restoring workspace $location node modules" done # Retrieve hoisted node_modules move_cache "$cache_dir/node_modules" "$NETLIFY_REPO_DIR/node_modules" "restoring workspace root node modules" - # Keep a record of the workspaces in the project in order to cache them later - NETLIFY_JS_WORKSPACE_LOCATIONS=$(printf '%s\n' "${locations[@]}") } # @@ -842,7 +842,8 @@ restore_js_workspaces_cache() { # via the `NETLIFY_JS_WORKSPACE_LOCATIONS` variable, or looks at the node_modules in the cwd. # cache_node_modules() { - if [ -z "$NETLIFY_JS_WORKSPACE_LOCATIONS" ] + # Check the number of workspace locations detected + if [ "${#NETLIFY_JS_WORKSPACE_LOCATIONS[@]}" -eq 0 ] then cache_cwd_directory "node_modules" "node modules" else @@ -856,10 +857,8 @@ cache_node_modules() { # cache_js_workspaces() { local cache_dir="$NETLIFY_CACHE_DIR/js-workspaces" - local locations - mapfile -t locations <<< "$NETLIFY_JS_WORKSPACE_LOCATIONS" - for location in "${locations[@]}"; do + for location in "${NETLIFY_JS_WORKSPACE_LOCATIONS[@]}"; do mkdir -p "$cache_dir/$location" move_cache "$NETLIFY_REPO_DIR/$location/node_modules" "$cache_dir/$location/node_modules" "saving workspace $location node modules" done From 14b1ea8cb00c87c1b649b344b5ebd6f94683aedc Mon Sep 17 00:00:00 2001 From: JGAntunes Date: Fri, 26 Feb 2021 13:54:36 +0000 Subject: [PATCH 07/10] chore(ws-cache): move cache_dir to the upper scope --- run-build-functions.sh | 22 ++++++++++++---------- 1 file changed, 12 insertions(+), 10 deletions(-) diff --git a/run-build-functions.sh b/run-build-functions.sh index 1ca8780c..ef316371 100755 --- a/run-build-functions.sh +++ b/run-build-functions.sh @@ -39,8 +39,10 @@ mkdir -p $NETLIFY_CACHE_DIR/ruby_version mkdir -p $NETLIFY_CACHE_DIR/swift_version # pwd caches +NETLIFY_JS_WORKSPACES_CACHE_DIR="$NETLIFY_CACHE_DIR/js-workspaces" + +mkdir -p $NETLIFY_JS_WORKSPACES_CACHE_DIR mkdir -p $NETLIFY_CACHE_DIR/node_modules -mkdir -p $NETLIFY_CACHE_DIR/js-workspaces mkdir -p $NETLIFY_CACHE_DIR/.bundle mkdir -p $NETLIFY_CACHE_DIR/bower_components mkdir -p $NETLIFY_CACHE_DIR/.venv @@ -827,14 +829,14 @@ restore_js_workspaces_cache() { # Keep a record of the workspaces in the project in order to cache them later NETLIFY_JS_WORKSPACE_LOCATIONS=("$@") - local cache_dir="$NETLIFY_CACHE_DIR/js-workspaces" - # Retrieve each workspace node_modules for location in "${NETLIFY_JS_WORKSPACE_LOCATIONS[@]}"; do - move_cache "$cache_dir/$location/node_modules" "$NETLIFY_REPO_DIR/$location/node_modules" "restoring workspace $location node modules" + move_cache "$NETLIFY_JS_WORKSPACES_CACHE_DIR/$location/node_modules" \ + "$NETLIFY_REPO_DIR/$location/node_modules" \ + "restoring workspace $location node modules" done # Retrieve hoisted node_modules - move_cache "$cache_dir/node_modules" "$NETLIFY_REPO_DIR/node_modules" "restoring workspace root node modules" + move_cache "$NETLIFY_JS_WORKSPACES_CACHE_DIR/node_modules" "$NETLIFY_REPO_DIR/node_modules" "restoring workspace root node modules" } # @@ -856,14 +858,14 @@ cache_node_modules() { # `NETLIFY_JS_WORKSPACE_LOCATIONS` variable previously set in `restore_js_workspaces_cache()` # cache_js_workspaces() { - local cache_dir="$NETLIFY_CACHE_DIR/js-workspaces" - for location in "${NETLIFY_JS_WORKSPACE_LOCATIONS[@]}"; do - mkdir -p "$cache_dir/$location" - move_cache "$NETLIFY_REPO_DIR/$location/node_modules" "$cache_dir/$location/node_modules" "saving workspace $location node modules" + mkdir -p "$NETLIFY_JS_WORKSPACES_CACHE_DIR/$location" + move_cache "$NETLIFY_REPO_DIR/$location/node_modules" \ + "$NETLIFY_JS_WORKSPACES_CACHE_DIR/$location/node_modules" \ + "saving workspace $location node modules" done # Retrieve hoisted node_modules - move_cache "$NETLIFY_REPO_DIR/node_modules" "$cache_dir/node_modules" "saving workspace root node modules" + move_cache "$NETLIFY_REPO_DIR/node_modules" "$NETLIFY_JS_WORKSPACES_CACHE_DIR/node_modules" "saving workspace root node modules" } cache_cwd_directory() { From 073a3db99dd3523a392c1966071442f0f0bf1cdc Mon Sep 17 00:00:00 2001 From: JGAntunes Date: Fri, 26 Feb 2021 15:59:41 +0000 Subject: [PATCH 08/10] chore(ws-cache): don't call yarn workspaces twice --- run-build-functions.sh | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/run-build-functions.sh b/run-build-functions.sh index ef316371..18633b83 100755 --- a/run-build-functions.sh +++ b/run-build-functions.sh @@ -108,9 +108,11 @@ run_yarn() { if [ "$NETLIFY_YARN_WORKSPACES" = "true" ] then echo "NETLIFY_YARN_WORKSPACES feature flag set" + local workspace_output # YARN_IGNORE_PATH will ignore the presence of a local yarn executable (i.e. yarn 2) and default # to using the global one (which, for now, is alwasy yarn 1.x). See https://yarnpkg.com/configuration/yarnrc#ignorePath - if YARN_IGNORE_PATH=1 yarn workspaces info + workspace_output=$(YARN_IGNORE_PATH=1 yarn workspaces info) + if $? then local package_locations # Extract all the packages and respective locations. .data will be a JSON object like @@ -123,7 +125,7 @@ run_yarn() { # (...) # } # We need to cache all the node_module dirs, or we'll always be installing them on each run - mapfile -t package_locations <<< "$(YARN_IGNORE_PATH=1 yarn --json workspaces info | jq -r '.data | fromjson | to_entries | .[].value.location')" + mapfile -t package_locations <<< "$workspace_output" restore_js_workspaces_cache "${package_locations[@]}" else echo "No workspace detected" From 8ab88aec42aabe2ec9ecb9681fc0c0a699189d87 Mon Sep 17 00:00:00 2001 From: JGAntunes Date: Fri, 26 Feb 2021 17:30:06 +0000 Subject: [PATCH 09/10] fix(ws-cache): ended up messing the jq command :facepalm: --- run-build-functions.sh | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/run-build-functions.sh b/run-build-functions.sh index 18633b83..d8ad5335 100755 --- a/run-build-functions.sh +++ b/run-build-functions.sh @@ -109,10 +109,12 @@ run_yarn() { then echo "NETLIFY_YARN_WORKSPACES feature flag set" local workspace_output + local workspace_exit_code # YARN_IGNORE_PATH will ignore the presence of a local yarn executable (i.e. yarn 2) and default # to using the global one (which, for now, is alwasy yarn 1.x). See https://yarnpkg.com/configuration/yarnrc#ignorePath - workspace_output=$(YARN_IGNORE_PATH=1 yarn workspaces info) - if $? + workspace_output="$(YARN_IGNORE_PATH=1 yarn workspaces --json info )" + workspace_exit_code=$? + if [ $workspace_exit_code -eq 0 ] then local package_locations # Extract all the packages and respective locations. .data will be a JSON object like @@ -125,7 +127,7 @@ run_yarn() { # (...) # } # We need to cache all the node_module dirs, or we'll always be installing them on each run - mapfile -t package_locations <<< "$workspace_output" + mapfile -t package_locations <<< "$(echo "$workspace_output" | jq -r '.data | fromjson | to_entries | .[].value.location')" restore_js_workspaces_cache "${package_locations[@]}" else echo "No workspace detected" From c4e36419b96d31d7cb401b07877a6b8d1c52f3c8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jo=C3=A3o=20Antunes?= Date: Mon, 1 Mar 2021 17:43:32 +0000 Subject: [PATCH 10/10] chore: typo in comment run-build-functions.sh Co-authored-by: ehmicky --- run-build-functions.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/run-build-functions.sh b/run-build-functions.sh index d8ad5335..c807eeec 100755 --- a/run-build-functions.sh +++ b/run-build-functions.sh @@ -111,7 +111,7 @@ run_yarn() { local workspace_output local workspace_exit_code # YARN_IGNORE_PATH will ignore the presence of a local yarn executable (i.e. yarn 2) and default - # to using the global one (which, for now, is alwasy yarn 1.x). See https://yarnpkg.com/configuration/yarnrc#ignorePath + # to using the global one (which, for now, is always yarn 1.x). See https://yarnpkg.com/configuration/yarnrc#ignorePath workspace_output="$(YARN_IGNORE_PATH=1 yarn workspaces --json info )" workspace_exit_code=$? if [ $workspace_exit_code -eq 0 ]