From 47547bd6ffce4c68e3cdac854975d3c24faa5834 Mon Sep 17 00:00:00 2001 From: Thomas Barber Date: Fri, 13 Sep 2024 13:02:54 +0000 Subject: [PATCH 1/3] Adding build script options Add options to allow building each stage of the build script in turn to allow better integration with continuous integration pipelines. --- build.sh | 30 +++++++++++++++++++++++++----- taintfox_mozconfig_ubuntu | 5 ++++- 2 files changed, 29 insertions(+), 6 deletions(-) diff --git a/build.sh b/build.sh index 3a780107a906d..9b8ae81615289 100644 --- a/build.sh +++ b/build.sh @@ -31,6 +31,9 @@ NO_CLOBBER= MAKE_GIT_COMMIT= SKIP_PREPARATION= SKIP_BOOTSTRAP= +SKIP_PATCHING= +SKIP_BUILD= +SKIP_PACKAGE= RESET_GIT_REPO= _determine_obj_dir() { @@ -150,24 +153,35 @@ main() { if [ -z "$SKIP_PREPARATION" ]; then _prepare_foxhound fi - if [ -n "$WITH_PLAYWRIGHT_INTEGRATION" ] && [ -z "$SKIP_PREPARATION" ]; then + if [ -n "$WITH_PLAYWRIGHT_INTEGRATION" ] && [ -z "$SKIP_PATCHING" ]; then _check_foxhound_repo_state _prepare_playwright _patch_foxhound fi _determine_obj_dir _status "Determined MOZ_OBJDIR as: $FOXHOUND_OBJ_DIR" - _build_foxhound - _package_foxhound + if [ -z "$SKIP_BUILD" ]; then + _build_foxhound + fi + if [ -z "$SKIP_PACKAGE" ]; then + _package_foxhound + fi } _help() { echo "Builds project foxhound" echo - echo "Syntax: build.sh [-h|c|p|b|r|g|s]" + echo "Syntax: build.sh [-h|c|p|b|r|g|s|t|u|v]" + echo + echo "For example, to build from scratch with playwright:" + echo "> bash build.sh -p" + echo echo "options:" echo "c Does not clobber the build prior to building." echo "s Skip the preparation phase (i.e., avoid applying playwright patches again)." + echo "t Skip applying the playwright patches" + echo "u Skip the build itself" + echo "v Skip the packaging stage" echo "b Skip './mach bootstrap' (This can help if the binaries are not available for download anymore)." echo "r Resets the Git repository prior to building. This will delete any (uncommitted) changes you made!" echo "p Builds with playwright integration." @@ -176,7 +190,7 @@ _help() { echo } -while getopts "hpcsrbg" option; do +while getopts "hpcstuvrbg" option; do case $option in h) _help @@ -187,6 +201,12 @@ while getopts "hpcsrbg" option; do RESET_GIT_REPO=1;; s) SKIP_PREPARATION=1;; + t) + SKIP_PATCHING=1;; + u) + SKIP_BUILD=1;; + v) + SKIP_PACKAGE=1;; c) NO_CLOBBER=1;; g) diff --git a/taintfox_mozconfig_ubuntu b/taintfox_mozconfig_ubuntu index f25937aad636d..371fbe67c1943 100644 --- a/taintfox_mozconfig_ubuntu +++ b/taintfox_mozconfig_ubuntu @@ -36,9 +36,12 @@ export MOZ_PACKAGE_JSSHELL=1 ac_add_options --with-app-name=foxhound mk_add_options MOZ_APP_NAME=foxhound mk_add_options MOZ_OBJDIR=@TOPSRCDIR@/obj-tf-release -ac_add_options --disable-tests +#ac_add_options --disable-tests ac_add_options --disable-sandbox ac_add_options --without-wasm-sandboxed-libraries +ac_add_options --disable-crashreporter +ac_add_options --disable-backgroundtasks +ac_add_options --enable-release # # OPTIMIZED BUILD WITH DEBUG SYMBOLS From d632d92e5c19dfc04f271e2a3204dd0da04c453a Mon Sep 17 00:00:00 2001 From: Thomas Barber Date: Fri, 13 Sep 2024 14:37:12 +0000 Subject: [PATCH 2/3] Create versioned playwright zip --- build.sh | 3 +++ 1 file changed, 3 insertions(+) diff --git a/build.sh b/build.sh index 9b8ae81615289..66230fa83c9b9 100644 --- a/build.sh +++ b/build.sh @@ -137,6 +137,9 @@ _package_foxhound() { fi pushd "${FOXHOUND_OBJ_DIR}/dist" || exit 1 zip -r foxhound_linux.zip foxhound + if [ -n "$WITH_PLAYWRIGHT_INTEGRATION" ]; then + cp foxhound_linux.zip foxhound_linux_${PLAYWRIGHT_VERSION}.zip + fi _status "Zip located at '$(pwd || true)/foxhound_linux.zip', done!" popd > /dev/null || exit 1 popd > /dev/null || exit 1 From 2f1f7e111fa2a44742f0a316f536e1acf8170f49 Mon Sep 17 00:00:00 2001 From: David Klein Date: Tue, 8 Oct 2024 16:54:59 +0200 Subject: [PATCH 3/3] Variosu improvements to build script - Added back support to skip preparation phase. - Added dry run option - Added option to skip the git repo check Should align it with the SAP build pipeline. --- build.sh | 123 +++++++++++++++++++++++++++++++++++++------------------ 1 file changed, 84 insertions(+), 39 deletions(-) diff --git a/build.sh b/build.sh index 66230fa83c9b9..96cd3d9f3674e 100644 --- a/build.sh +++ b/build.sh @@ -35,6 +35,7 @@ SKIP_PATCHING= SKIP_BUILD= SKIP_PACKAGE= RESET_GIT_REPO= +DRY_RUN= _determine_obj_dir() { if [ ! -d "${FOXHOUND_OBJ_DIR}" ]; then @@ -49,7 +50,7 @@ _determine_obj_dir() { _make_git_commit() { pushd "${FOXHOUND_DIR}" > /dev/null || _die "Can't change into foxhound dir: ${FOXHOUND_DIR}" - NEW_BRANCHNAME="pw-build-$(date +%s)" + NEW_BRANCHNAME="pw-build-$(date +%s)" git switch -c "${NEW_BRANCHNAME}" || _die "Can't create new Git branch called: ${NEW_BRANCHNAME}" git add . git commit --author="Foxhound builder " --message="Added Playwright support to Foxhound." @@ -64,15 +65,20 @@ _prepare_playwright() { . "${CURRENT_DIR}/.PLAYWRIGHT_VERSION" _status "Set playwright version to: ${PLAYWRIGHT_VERSION}" popd > /dev/null || exit 1 - pushd "${PLAYWRIGHT_DIR}" > /dev/null || _die "Can't change into playwright dir: ${PLAYWRIGHT_DIR}" - git fetch --all || _die "Git fetch failed for Playwright" - git checkout "${PLAYWRIGHT_VERSION}" || _die "Can't checkout playwright version ${PLAYWRIGHT_VERSION}.." - popd > /dev/null || exit 1 + _status "Fetching Playwright and checking out ${PLAYWRIGHT_VERSION} branch" + if [ -z "$DRY_RUN" ]; then + pushd "${PLAYWRIGHT_DIR}" > /dev/null || _die "Can't change into playwright dir: ${PLAYWRIGHT_DIR}" + git fetch --all || _die "Git fetch failed for Playwright" + git checkout "${PLAYWRIGHT_VERSION}" || _die "Can't checkout playwright version ${PLAYWRIGHT_VERSION}.." + popd > /dev/null || exit 1 + fi } _checkout_playwright() { _status "Cloning playwright into ${PLAYWRIGHT_DIR}" - git clone https://github.com/microsoft/playwright.git "${PLAYWRIGHT_DIR}" || _die "Cloning playwright failed!" + if [ -z "$DRY_RUN" ]; then + git clone https://github.com/microsoft/playwright.git "${PLAYWRIGHT_DIR}" || _die "Cloning playwright failed!" + fi } _check_foxhound_repo_state() { @@ -86,23 +92,33 @@ _prepare_foxhound() { _status "Preparing the foxhound build environment" if [ -d "./juggler" ]; then _status "Deleting stale juggler" - rm -rf "${FOXHOUND_DIR}/juggler" + if [ -z "$DRY_RUN" ]; then + rm -rf "${FOXHOUND_DIR}/juggler" + fi fi if [ ! -f "${FOXHOUND_DIR}/.mozconfig" ]; then _status "Setting default mozconfig from Ubuntu profile" - cp "${FOXHOUND_DIR}/taintfox_mozconfig_ubuntu" "${FOXHOUND_DIR}/.mozconfig" + if [ -z "$DRY_RUN" ]; then + cp "${FOXHOUND_DIR}/taintfox_mozconfig_ubuntu" "${FOXHOUND_DIR}/.mozconfig" + fi fi if [ -n "$RESET_GIT_REPO" ]; then _status "Resetting Git repository" - git reset --hard HEAD + if [ -z "$DRY_RUN" ]; then + git reset --hard HEAD + fi fi if [ -z "$NO_CLOBBER" ]; then _status "Clobbering the build environment" - ./mach --no-interactive clobber + if [ -z "$DRY_RUN" ]; then + ./mach --no-interactive clobber + fi fi if [ -z "$SKIP_BOOTSTRAP" ]; then _status "Preparing environment via './mach bootstrap'" - ./mach --no-interactive bootstrap --no-system-changes --application-choice=browser || _die "Bootstrapping failed! You can install dependencies manually and skip this step via the '-b' flag" + if [ -z "$DRY_RUN" ]; then + ./mach --no-interactive bootstrap --no-system-changes --application-choice=browser || _die "Bootstrapping failed! You can install dependencies manually and skip this step via the '-b' flag" + fi fi popd > /dev/null || exit 1 } @@ -110,38 +126,54 @@ _prepare_foxhound() { _patch_foxhound() { _status "Patching foxhound for playwright integration" pushd "${FOXHOUND_DIR}" > /dev/null || _die "Can't change into foxhound dir: ${FOXHOUND_DIR}" - cp -r "${PLAYWRIGHT_DIR}/browser_patches/firefox/juggler" "juggler" - git apply --verbose --index --whitespace=nowarn --recount "${PLAYWRIGHT_DIR}/browser_patches/firefox/patches"/* || _die "Playwright patches failed to apply." + if [ -z "$DRY_RUN" ]; then + cp -r "${PLAYWRIGHT_DIR}/browser_patches/firefox/juggler" "juggler" + git apply --verbose --index --whitespace=nowarn --recount "${PLAYWRIGHT_DIR}/browser_patches/firefox/patches"/* || _die "Playwright patches failed to apply." + fi if [ -n "$MAKE_GIT_COMMIT" ]; then _status "Creating Git commit" - _make_git_commit + if [ -z "$DRY_RUN" ]; then + _make_git_commit + fi fi popd > /dev/null || exit 1 } _build_foxhound() { pushd "${FOXHOUND_DIR}" > /dev/null || _die "Can't change into foxhound dir: ${FOXHOUND_DIR}" - ./mach build || _die "./mach build error" + _status "Starting Foxhound build.. This can take a while" + if [ -z "$DRY_RUN" ]; then + ./mach build || _die "./mach build error" + fi popd > /dev/null || exit 1 } _package_foxhound() { pushd "${FOXHOUND_DIR}" > /dev/null || _die "Can't change into foxhound dir: ${FOXHOUND_DIR}" _status "Packaging foxhound.." - ./mach package || _die "./mach package error" - if [ -n "$WITH_PLAYWRIGHT_INTEGRATION" ]; then - mkdir -p "${FOXHOUND_OBJ_DIR}/dist/foxhound/defaults/pref" - cp "${PLAYWRIGHT_DIR}/browser_patches/firefox/preferences/playwright.cfg" "${FOXHOUND_OBJ_DIR}/dist/foxhound/" - cp "${PLAYWRIGHT_DIR}/browser_patches/firefox/preferences/00-playwright-prefs.js" "${FOXHOUND_OBJ_DIR}/dist/foxhound/defaults/pref/" + if [ -z "$DRY_RUN" ]; then + ./mach package || _die "./mach package error" fi - pushd "${FOXHOUND_OBJ_DIR}/dist" || exit 1 - zip -r foxhound_linux.zip foxhound + if [ -n "$WITH_PLAYWRIGHT_INTEGRATION" ]; then - cp foxhound_linux.zip foxhound_linux_${PLAYWRIGHT_VERSION}.zip + _status "Applying playwright preferences" + if [ -z "$DRY_RUN" ]; then + mkdir -p "${FOXHOUND_OBJ_DIR}/dist/foxhound/defaults/pref" + cp "${PLAYWRIGHT_DIR}/browser_patches/firefox/preferences/playwright.cfg" "${FOXHOUND_OBJ_DIR}/dist/foxhound/" + cp "${PLAYWRIGHT_DIR}/browser_patches/firefox/preferences/00-playwright-prefs.js" "${FOXHOUND_OBJ_DIR}/dist/foxhound/defaults/pref/" + fi + fi + _status "Creating Foxhound zip" + if [ -z "$DRY_RUN" ]; then + pushd "${FOXHOUND_OBJ_DIR}/dist" || exit 1 + zip -r foxhound_linux.zip foxhound + if [ -n "$WITH_PLAYWRIGHT_INTEGRATION" ]; then + cp foxhound_linux.zip "foxhound_linux_${PLAYWRIGHT_VERSION}.zip" + fi + _status "Zip located at '$(pwd || true)/foxhound_linux.zip', done!" + popd > /dev/null || exit 1 fi - _status "Zip located at '$(pwd || true)/foxhound_linux.zip', done!" - popd > /dev/null || exit 1 popd > /dev/null || exit 1 } @@ -156,8 +188,11 @@ main() { if [ -z "$SKIP_PREPARATION" ]; then _prepare_foxhound fi - if [ -n "$WITH_PLAYWRIGHT_INTEGRATION" ] && [ -z "$SKIP_PATCHING" ]; then - _check_foxhound_repo_state + if [[ -n "$WITH_PLAYWRIGHT_INTEGRATION" && ( -z "$SKIP_PREPARATION" && -z "$SKIP_PATCHING") ]]; then + if [ -z "$SKIP_GIT_CHECK" ]; then + _check_foxhound_repo_state + fi + _prepare_playwright _patch_foxhound fi @@ -174,7 +209,7 @@ main() { _help() { echo "Builds project foxhound" echo - echo "Syntax: build.sh [-h|c|p|b|r|g|s|t|u|v]" + echo "Syntax: build.sh [-c|s|t|u|v|b|r|p|g|n|h]" echo echo "For example, to build from scratch with playwright:" echo "> bash build.sh -p" @@ -189,20 +224,29 @@ _help() { echo "r Resets the Git repository prior to building. This will delete any (uncommitted) changes you made!" echo "p Builds with playwright integration." echo "g Create a Git commit with the playwright patches." + echo "n Dry run. Only print the step the script would perform." echo "h Print this Help." echo + echo + echo "Environment variables:" + echo + echo "These are meant to be used if you know what you are doing and can lead to states that are difficult to revert. Use with caution!" + echo + echo "SKIP_GIT_CHECK:" + echo "If set to any value this skips the check whether the git repository is dirty." + echo "Warning: When used together with Playwright support this can lead to a state where untangling the Playwright patches from your changes to commit them is very cumbersome!" } -while getopts "hpcstuvrbg" option; do - case $option in +while getopts "hpcstuvrbgn" option; do + case "$option" in h) _help exit;; - b) + b) SKIP_BOOTSTRAP=1;; - r) + r) RESET_GIT_REPO=1;; - s) + s) SKIP_PREPARATION=1;; t) SKIP_PATCHING=1;; @@ -210,15 +254,16 @@ while getopts "hpcstuvrbg" option; do SKIP_BUILD=1;; v) SKIP_PACKAGE=1;; - c) + c) NO_CLOBBER=1;; - g) + g) MAKE_GIT_COMMIT=1;; - p) + p) WITH_PLAYWRIGHT_INTEGRATION=1;; - \?) - echo "Error: Invalid option" - exit;; + n) + DRY_RUN=1;; + \?) + _die "Error: Invalid option: $option";; esac done