From 3d9f6172613d6b8227421b36d6c12adeebaa119c Mon Sep 17 00:00:00 2001 From: Marco Ippolito Date: Wed, 26 Apr 2023 12:06:22 +0200 Subject: [PATCH 1/9] tools: automate icu-small update --- .github/workflows/tools.yml | 17 +++++++++ tools/dep_updaters/update-icu.sh | 63 ++++++++++++++++++++++++++++++++ tools/icu/shrink-icu-src.py | 0 3 files changed, 80 insertions(+) create mode 100755 tools/dep_updaters/update-icu.sh mode change 100644 => 100755 tools/icu/shrink-icu-src.py diff --git a/.github/workflows/tools.yml b/.github/workflows/tools.yml index 30f9b4fcebceb8..3a095f69f20abf 100644 --- a/.github/workflows/tools.yml +++ b/.github/workflows/tools.yml @@ -24,6 +24,7 @@ on: - doc - eslint - googletest + - icu - libuv - lint-md-dependencies - llhttp @@ -36,6 +37,9 @@ on: - undici - uvwasi +env: + PYTHON_VERSION: '3.11' + permissions: contents: read @@ -252,11 +256,24 @@ jobs: cat temp-output tail -n1 temp-output | grep "NEW_VERSION=" >> "$GITHUB_ENV" || true rm temp-output + - id: icu + subsystem: deps + label: dependencies, test + run: | + ./tools/dep_updaters/update-icu.sh > temp-output + cat temp-output + tail -n1 temp-output | grep "NEW_VERSION=" >> "$GITHUB_ENV" || true + rm temp-output steps: - uses: actions/checkout@ac593985615ec2ede58e132d2e21d2b1cbd6127c # v3.3.0 if: github.event_name == 'schedule' || inputs.id == 'all' || inputs.id == matrix.id with: persist-credentials: false + - name: Set up Python ${{ env.PYTHON_VERSION }} + if: matrix.id == 'icu' && (github.event_name == 'schedule' || inputs.id == 'all' || inputs.id == matrix.id) + uses: actions/setup-python@d27e3f3d7c64b4bbf8e4abfb9b63b83e846e0435 # v4.5.0 + with: + python-version: ${{ env.PYTHON_VERSION }} - run: ${{ matrix.run }} if: github.event_name == 'schedule' || inputs.id == 'all' || inputs.id == matrix.id env: diff --git a/tools/dep_updaters/update-icu.sh b/tools/dep_updaters/update-icu.sh new file mode 100755 index 00000000000000..282aeb8ec953e0 --- /dev/null +++ b/tools/dep_updaters/update-icu.sh @@ -0,0 +1,63 @@ +#!/bin/sh +set -e +# Shell script to update icu in the source tree to a specific version + +BASE_DIR=$(cd "$(dirname "$0")/../.." && pwd) +DEPS_DIR="$BASE_DIR/deps" +TOOLS_DIR="$BASE_DIR/tools" + +[ -z "$NODE" ] && NODE="$BASE_DIR/out/Release/node" +[ -x "$NODE" ] || NODE=$(command -v node) + +NEW_VERSION="$("$NODE" --input-type=module <<'EOF' +const res = await fetch('https://api.github.com/repos/unicode-org/icu/releases/latest'); +if (!res.ok) throw new Error(`FetchError: ${res.status} ${res.statusText}`, { cause: res }); +const { tag_name } = await res.json(); +console.log(tag_name.replace('release-', '')); +EOF +)" + +ICU_VERSION_H="$DEPS_DIR/icu-small/source/common/unicode/uvernum.h" +CURRENT_MAJOR_VERSION=$(grep "#define U_ICU_VERSION_MAJOR_NUM" "$ICU_VERSION_H" | cut -d ' ' -f3 | tr -d '\r') +CURRENT_MINOR_VERSION=$(grep "#define U_ICU_VERSION_MINOR_NUM" "$ICU_VERSION_H" | cut -d ' ' -f3 | tr -d '\r') + +CURRENT_VERSION="$CURRENT_MAJOR_VERSION-$CURRENT_MINOR_VERSION" + +echo "Comparing $NEW_VERSION with $CURRENT_VERSION" + +if [ "$NEW_VERSION" = "$CURRENT_VERSION" ]; then + echo "Skipped because icu is on the latest version." + exit 0 +fi + +NEW_MAJOR_VERSION=$(echo "$NEW_VERSION" | cut -d '-' -f1) +NEW_MINOR_VERSION=$(echo "$NEW_VERSION" | cut -d '-' -f2) + +NEW_VERSION_TGZ="https://github.com/unicode-org/icu/releases/download/release-${NEW_VERSION}/icu4c-${NEW_MAJOR_VERSION}_${NEW_MINOR_VERSION}-src.tgz" + +./configure --with-intl=full-icu --with-icu-source="$NEW_VERSION_TGZ" + +"$TOOLS_DIR/icu/shrink-icu-src.py" + +rm -rf "$DEPS_DIR/icu" + +CHECKSUM=$(curl -s "$NEW_VERSION_TGZ" | md5sum | cut -d ' ' -f1) + +sed -i '' -e "s|\"url\": \"\(.*\)\".*|\"url\": \"$NEW_VERSION_TGZ\",|" "$TOOLS_DIR/icu/current_ver.dep" + +sed -i '' -e "s|\"md5\": \"\(.*\)\".*|\"md5\": \"$CHECKSUM\"|" "$TOOLS_DIR/icu/current_ver.dep" + +rm -rf out "$DEPS_DIR/icu" "$DEPS_DIR/icu4c*" + +echo "All done!" +echo "" +echo "Please git add icu, commit the new version:" +echo "" +echo "$ git add -A deps/icu-small" +echo "$ git add tools/icu/current_ver.dep" +echo "$ git commit -m \"deps: update icu to $NEW_VERSION\"" +echo "" + +# The last line of the script should always print the new version, +# as we need to add it to $GITHUB_ENV variable. +echo "NEW_VERSION=$NEW_VERSION" diff --git a/tools/icu/shrink-icu-src.py b/tools/icu/shrink-icu-src.py old mode 100644 new mode 100755 From 503ffb7d71d8c4ebfbb747a11196419b28fd5432 Mon Sep 17 00:00:00 2001 From: Marco Ippolito Date: Wed, 26 Apr 2023 18:03:27 +0200 Subject: [PATCH 2/9] fix: replace - with . --- tools/dep_updaters/update-icu.sh | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/tools/dep_updaters/update-icu.sh b/tools/dep_updaters/update-icu.sh index 282aeb8ec953e0..c9b654581947aa 100755 --- a/tools/dep_updaters/update-icu.sh +++ b/tools/dep_updaters/update-icu.sh @@ -13,15 +13,13 @@ NEW_VERSION="$("$NODE" --input-type=module <<'EOF' const res = await fetch('https://api.github.com/repos/unicode-org/icu/releases/latest'); if (!res.ok) throw new Error(`FetchError: ${res.status} ${res.statusText}`, { cause: res }); const { tag_name } = await res.json(); -console.log(tag_name.replace('release-', '')); +console.log(tag_name.replace('release-', '').replace('-','.')); EOF )" ICU_VERSION_H="$DEPS_DIR/icu-small/source/common/unicode/uvernum.h" -CURRENT_MAJOR_VERSION=$(grep "#define U_ICU_VERSION_MAJOR_NUM" "$ICU_VERSION_H" | cut -d ' ' -f3 | tr -d '\r') -CURRENT_MINOR_VERSION=$(grep "#define U_ICU_VERSION_MINOR_NUM" "$ICU_VERSION_H" | cut -d ' ' -f3 | tr -d '\r') -CURRENT_VERSION="$CURRENT_MAJOR_VERSION-$CURRENT_MINOR_VERSION" +CURRENT_VERSION="$(grep "#define U_ICU_VERSION " "$ICU_VERSION_H" | cut -d'"' -f2)" echo "Comparing $NEW_VERSION with $CURRENT_VERSION" @@ -30,8 +28,8 @@ if [ "$NEW_VERSION" = "$CURRENT_VERSION" ]; then exit 0 fi -NEW_MAJOR_VERSION=$(echo "$NEW_VERSION" | cut -d '-' -f1) -NEW_MINOR_VERSION=$(echo "$NEW_VERSION" | cut -d '-' -f2) +NEW_MAJOR_VERSION=$(echo "$NEW_VERSION" | cut -d '.' -f1) +NEW_MINOR_VERSION=$(echo "$NEW_VERSION" | cut -d '.' -f2) NEW_VERSION_TGZ="https://github.com/unicode-org/icu/releases/download/release-${NEW_VERSION}/icu4c-${NEW_MAJOR_VERSION}_${NEW_MINOR_VERSION}-src.tgz" From 0299d5178a1cf50c65a406b3c1dce685a0034b8c Mon Sep 17 00:00:00 2001 From: Marco Ippolito Date: Thu, 27 Apr 2023 10:02:04 +0200 Subject: [PATCH 3/9] fix: md5 from source --- tools/dep_updaters/update-icu.sh | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/tools/dep_updaters/update-icu.sh b/tools/dep_updaters/update-icu.sh index c9b654581947aa..ad0df62837d747 100755 --- a/tools/dep_updaters/update-icu.sh +++ b/tools/dep_updaters/update-icu.sh @@ -31,17 +31,22 @@ fi NEW_MAJOR_VERSION=$(echo "$NEW_VERSION" | cut -d '.' -f1) NEW_MINOR_VERSION=$(echo "$NEW_VERSION" | cut -d '.' -f2) -NEW_VERSION_TGZ="https://github.com/unicode-org/icu/releases/download/release-${NEW_VERSION}/icu4c-${NEW_MAJOR_VERSION}_${NEW_MINOR_VERSION}-src.tgz" -./configure --with-intl=full-icu --with-icu-source="$NEW_VERSION_TGZ" +NEW_VERSION_TGZ="icu4c-${NEW_MAJOR_VERSION}_${NEW_MINOR_VERSION}-src.tgz" + +NEW_VERSION_TGZ_URL="https://github.com/unicode-org/icu/releases/download/release-${NEW_MAJOR_VERSION}-${NEW_MINOR_VERSION}/$NEW_VERSION_TGZ" + +NEW_VERSION_MD5="https://github.com/unicode-org/icu/releases/download/release-${NEW_MAJOR_VERSION}-${NEW_MINOR_VERSION}/icu4c-${NEW_MAJOR_VERSION}_${NEW_MINOR_VERSION}-src.md5" + +./configure --with-intl=full-icu --with-icu-source="$NEW_VERSION_TGZ_URL" "$TOOLS_DIR/icu/shrink-icu-src.py" rm -rf "$DEPS_DIR/icu" -CHECKSUM=$(curl -s "$NEW_VERSION_TGZ" | md5sum | cut -d ' ' -f1) +CHECKSUM=$(curl -sL "$NEW_VERSION_MD5" | grep "$NEW_VERSION_TGZ" | grep -v "\.asc$" | awk '{print $1}') -sed -i '' -e "s|\"url\": \"\(.*\)\".*|\"url\": \"$NEW_VERSION_TGZ\",|" "$TOOLS_DIR/icu/current_ver.dep" +sed -i '' -e "s|\"url\": \"\(.*\)\".*|\"url\": \"$NEW_VERSION_TGZ_URL\",|" "$TOOLS_DIR/icu/current_ver.dep" sed -i '' -e "s|\"md5\": \"\(.*\)\".*|\"md5\": \"$CHECKSUM\"|" "$TOOLS_DIR/icu/current_ver.dep" From c82d718c5e88bd47abdbb52871b5ba19e68a7eac Mon Sep 17 00:00:00 2001 From: Marco Ippolito Date: Thu, 27 Apr 2023 10:49:07 +0200 Subject: [PATCH 4/9] fix: comparing checksum --- tools/dep_updaters/update-icu.sh | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/tools/dep_updaters/update-icu.sh b/tools/dep_updaters/update-icu.sh index ad0df62837d747..0bf0bc05d1dfc0 100755 --- a/tools/dep_updaters/update-icu.sh +++ b/tools/dep_updaters/update-icu.sh @@ -46,6 +46,15 @@ rm -rf "$DEPS_DIR/icu" CHECKSUM=$(curl -sL "$NEW_VERSION_MD5" | grep "$NEW_VERSION_TGZ" | grep -v "\.asc$" | awk '{print $1}') +GENERATED_CHECKSUM=$( curl -sL "$NEW_VERSION_TGZ_URL" | md5sum | cut -d ' ' -f1) + +echo "Comparing checksums: deposited $CHECKSUM with $GENERATED_CHECKSUM" + +if [ "$CHECKSUM" != "$GENERATED_CHECKSUM" ]; then + echo "Skipped because checksums do not match." + exit 0 +fi + sed -i '' -e "s|\"url\": \"\(.*\)\".*|\"url\": \"$NEW_VERSION_TGZ_URL\",|" "$TOOLS_DIR/icu/current_ver.dep" sed -i '' -e "s|\"md5\": \"\(.*\)\".*|\"md5\": \"$CHECKSUM\"|" "$TOOLS_DIR/icu/current_ver.dep" From 84c60773df8e2aa3c0c632394b475f4e9d27d075 Mon Sep 17 00:00:00 2001 From: Marco Ippolito Date: Thu, 27 Apr 2023 11:04:49 +0200 Subject: [PATCH 5/9] fix: mention the script in the maintaining guide --- doc/contributing/maintaining/maintaining-icu.md | 3 +++ 1 file changed, 3 insertions(+) diff --git a/doc/contributing/maintaining/maintaining-icu.md b/doc/contributing/maintaining/maintaining-icu.md index a8ce43d3a3b4aa..64ee5006c58274 100644 --- a/doc/contributing/maintaining/maintaining-icu.md +++ b/doc/contributing/maintaining/maintaining-icu.md @@ -101,6 +101,9 @@ Node.js is built. ## How to upgrade ICU +> The script `tools/dep_updaters/update-icu.sh` automates +this process. + * Make sure your Node.js workspace is clean (`git status` should be sufficient). * Configure Node.js with the specific [ICU version](http://site.icu-project.org/download) From e794280b4f5ad7fd166f6b6dcdda8a468c09b438 Mon Sep 17 00:00:00 2001 From: Marco Ippolito Date: Thu, 27 Apr 2023 12:05:29 +0200 Subject: [PATCH 6/9] fix: lint --- doc/contributing/maintaining/maintaining-icu.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/contributing/maintaining/maintaining-icu.md b/doc/contributing/maintaining/maintaining-icu.md index 64ee5006c58274..00992258ae2611 100644 --- a/doc/contributing/maintaining/maintaining-icu.md +++ b/doc/contributing/maintaining/maintaining-icu.md @@ -102,7 +102,7 @@ Node.js is built. ## How to upgrade ICU > The script `tools/dep_updaters/update-icu.sh` automates -this process. +> this process. * Make sure your Node.js workspace is clean (`git status` should be sufficient). From b915e511d75c1ddc55c415c0d354b5e7124db9b9 Mon Sep 17 00:00:00 2001 From: Marco Ippolito Date: Fri, 28 Apr 2023 15:39:45 +0200 Subject: [PATCH 7/9] fix: update timezone.txt --- tools/dep_updaters/update-icu.sh | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/tools/dep_updaters/update-icu.sh b/tools/dep_updaters/update-icu.sh index 0bf0bc05d1dfc0..a7dc66ec8f5a46 100755 --- a/tools/dep_updaters/update-icu.sh +++ b/tools/dep_updaters/update-icu.sh @@ -59,6 +59,16 @@ sed -i '' -e "s|\"url\": \"\(.*\)\".*|\"url\": \"$NEW_VERSION_TGZ_URL\",|" "$TOO sed -i '' -e "s|\"md5\": \"\(.*\)\".*|\"md5\": \"$CHECKSUM\"|" "$TOOLS_DIR/icu/current_ver.dep" +# Get latest tz version (hacky, it should use an api) + +TZ_VERSION=$(curl -s "https://data.iana.org/time-zones/releases/" | sed -n 's/.*href="\([^"]*\).*/\1/p'| grep '^tzdb-.*tar.lz.asc' | tail -1 | sed -e 's/tzdb-\(.*\)\.tar.lz.asc/\1/') + +TXT_VERSION="$BASE_DIR/test/fixtures/tz-version.txt" + +echo "Latest tz version is $TZ_VERSION" + +echo "$TZ_VERSION" > "$TXT_VERSION" + rm -rf out "$DEPS_DIR/icu" "$DEPS_DIR/icu4c*" echo "All done!" From 12a7f5019ca16395ea93db437242e5f37961909a Mon Sep 17 00:00:00 2001 From: Marco Ippolito Date: Tue, 2 May 2023 08:50:23 +0200 Subject: [PATCH 8/9] fix: remove tz update --- tools/dep_updaters/update-icu.sh | 10 ---------- 1 file changed, 10 deletions(-) diff --git a/tools/dep_updaters/update-icu.sh b/tools/dep_updaters/update-icu.sh index a7dc66ec8f5a46..0bf0bc05d1dfc0 100755 --- a/tools/dep_updaters/update-icu.sh +++ b/tools/dep_updaters/update-icu.sh @@ -59,16 +59,6 @@ sed -i '' -e "s|\"url\": \"\(.*\)\".*|\"url\": \"$NEW_VERSION_TGZ_URL\",|" "$TOO sed -i '' -e "s|\"md5\": \"\(.*\)\".*|\"md5\": \"$CHECKSUM\"|" "$TOOLS_DIR/icu/current_ver.dep" -# Get latest tz version (hacky, it should use an api) - -TZ_VERSION=$(curl -s "https://data.iana.org/time-zones/releases/" | sed -n 's/.*href="\([^"]*\).*/\1/p'| grep '^tzdb-.*tar.lz.asc' | tail -1 | sed -e 's/tzdb-\(.*\)\.tar.lz.asc/\1/') - -TXT_VERSION="$BASE_DIR/test/fixtures/tz-version.txt" - -echo "Latest tz version is $TZ_VERSION" - -echo "$TZ_VERSION" > "$TXT_VERSION" - rm -rf out "$DEPS_DIR/icu" "$DEPS_DIR/icu4c*" echo "All done!" From 15ff4c572aa6a2a04b15ba07d4bf57bc72962c3b Mon Sep 17 00:00:00 2001 From: Marco Ippolito Date: Tue, 2 May 2023 09:02:24 +0200 Subject: [PATCH 9/9] fix: use version as whole --- tools/dep_updaters/update-icu.sh | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/tools/dep_updaters/update-icu.sh b/tools/dep_updaters/update-icu.sh index 0bf0bc05d1dfc0..1a5a57853f477c 100755 --- a/tools/dep_updaters/update-icu.sh +++ b/tools/dep_updaters/update-icu.sh @@ -28,15 +28,15 @@ if [ "$NEW_VERSION" = "$CURRENT_VERSION" ]; then exit 0 fi -NEW_MAJOR_VERSION=$(echo "$NEW_VERSION" | cut -d '.' -f1) -NEW_MINOR_VERSION=$(echo "$NEW_VERSION" | cut -d '.' -f2) +DASHED_NEW_VERSION=$(echo "$NEW_VERSION" | sed 's/\./-/g') +LOW_DASHED_NEW_VERSION=$(echo "$NEW_VERSION" | sed 's/\./_/g') -NEW_VERSION_TGZ="icu4c-${NEW_MAJOR_VERSION}_${NEW_MINOR_VERSION}-src.tgz" +NEW_VERSION_TGZ="icu4c-${LOW_DASHED_NEW_VERSION}-src.tgz" -NEW_VERSION_TGZ_URL="https://github.com/unicode-org/icu/releases/download/release-${NEW_MAJOR_VERSION}-${NEW_MINOR_VERSION}/$NEW_VERSION_TGZ" +NEW_VERSION_TGZ_URL="https://github.com/unicode-org/icu/releases/download/release-${DASHED_NEW_VERSION}/$NEW_VERSION_TGZ" -NEW_VERSION_MD5="https://github.com/unicode-org/icu/releases/download/release-${NEW_MAJOR_VERSION}-${NEW_MINOR_VERSION}/icu4c-${NEW_MAJOR_VERSION}_${NEW_MINOR_VERSION}-src.md5" +NEW_VERSION_MD5="https://github.com/unicode-org/icu/releases/download/release-${DASHED_NEW_VERSION}/icu4c-${LOW_DASHED_NEW_VERSION}-src.md5" ./configure --with-intl=full-icu --with-icu-source="$NEW_VERSION_TGZ_URL"