From 6b1cbb2a733fe8c9be0455de314618446b6444b6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tobias=20Kn=C3=B6ppler?= <6317548+theCalcaholic@users.noreply.github.com> Date: Sun, 28 Apr 2024 18:27:26 +0200 Subject: [PATCH 01/14] build-LXD.sh,system_tests.py,build-lxd.yml: Add support for incus MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Tobias Knöppler <6317548+theCalcaholic@users.noreply.github.com> --- .github/workflows/build-lxd.yml | 130 +++++++++++++++++++++----------- build/build-LXD.sh | 36 +++++---- build/buildlib.sh | 37 --------- tests/system_tests.py | 17 ++++- 4 files changed, 118 insertions(+), 102 deletions(-) diff --git a/.github/workflows/build-lxd.yml b/.github/workflows/build-lxd.yml index 9914bc3fc..90a159fba 100644 --- a/.github/workflows/build-lxd.yml +++ b/.github/workflows/build-lxd.yml @@ -34,18 +34,22 @@ jobs: runs-on: ubuntu-latest outputs: runner_label: ${{ steps.script.outputs.runner_label }} + lxc_cmd: ${{ steps.script.outputs.lxc_cmd }} steps: - name: script id: script run: | if [[ "${LXD_ARCH}" == "arm64" ]] then + LXC_CMD="incus" RUNNER_LABEL="ubuntu-20.04-arm64" else + LXC_CMD="lxc" RUNNER_LABEL="ubuntu-20.04" fi - echo "runner_label=$RUNNER_LABEL" | tee -a $GITHUB_OUTPUT + echo "runner_label=$RUNNER_LABEL" | tee -a $GITHUB_OUTPUT + echo "lxc_cmd=$LXC_CMD" | tee -a $GITHUB_OUTPUT build-current: needs: - determine-runner @@ -57,14 +61,16 @@ jobs: VERSION: "${{ inputs.git_ref || github.head_ref || github.ref_name }}" ARTIFACT_NAME: "${{ github.run_id }}-lxd-${{ inputs.arch || 'x86' }}-image" LXD_ARCH: "${{ inputs.arch || 'x86' }}" + LXC: "${{ needs.determine-runner.outputs.lxc_cmd }}" steps: - name: Checkout code uses: actions/checkout@v3 with: ref: "${{ env.VERSION }}" - name: Cleanup lxd - run: test -z "$(lxc profile device show default | grep eth0)" || lxc profile device remove default eth0 + run: test -z "$("$LXC" profile device show default | grep eth0)" || "$LXC" profile device remove default eth0 - uses: whywaita/setup-lxd@v1 + if: ${{ needs.determine-runner.outputs.lxc_cmd == 'lxc' }} continue-on-error: true with: lxd_version: latest/stable @@ -73,6 +79,8 @@ jobs: sudo iptables -I DOCKER-USER -i lxdbr0 -j ACCEPT sudo iptables -I DOCKER-USER -o lxdbr0 -m conntrack --ctstate RELATED,ESTABLISHED -j ACCEPT - name: Build LXD image + env: + USE_INCUS: "${{ needs.determine-runner.outputs.lxc_cmd == 'incus' && 'yes' || 'no' }}" run: | ./build/build-LXD.sh - name: Pack LXD image @@ -80,7 +88,7 @@ jobs: run: | . ./build/buildlib.sh ARTIFACT_FILE="NextcloudPi_LXD_${LXD_ARCH:-x86}_${VERSION//\//_}" - lxc image export -q ncp/"${version}" "output/${ARTIFACT_FILE}" + "$LXC" image export -q ncp/"${version}" "output/${ARTIFACT_FILE}" echo "artifact_file=${ARTIFACT_FILE}.tar.gz" >> $GITHUB_OUTPUT - name: upload LXD image to artifact store uses: actions/upload-artifact@v3 @@ -90,6 +98,8 @@ jobs: if-no-files-found: error build-previous: + needs: + - determine-runner runs-on: [ ubuntu-20.04 ] if: ${{ inputs.arch == 'x86' || inputs.arch == '' }} outputs: @@ -100,6 +110,7 @@ jobs: VERSION: "${{ inputs.git_ref || github.head_ref || github.ref_name }}" ARTIFACT_NAME: "${{ github.run_id }}-lxd-${{ inputs.arch || 'x86' }}-image-previous" LXD_ARCH: "${{ inputs.arch || 'x86' }}" + LXC: "${{ needs.determine-runner.outputs.lxc_cmd }}" steps: - name: Checkout code uses: actions/checkout@v3 @@ -134,6 +145,7 @@ jobs: echo "VERSION=$version" >> "$GITHUB_ENV" echo "previous_version=${version}" >> $GITHUB_OUTPUT - uses: whywaita/setup-lxd@v1 + if: ${{ needs.determine-runner.outputs.lxc_cmd == 'lxc' }} continue-on-error: true with: lxd_version: latest/stable @@ -143,6 +155,8 @@ jobs: sudo iptables -I DOCKER-USER -o lxdbr0 -m conntrack --ctstate RELATED,ESTABLISHED -j ACCEPT - name: Build LXD image + env: + USE_INCUS: "${{ needs.determine-runner.outputs.lxc_cmd == 'incus' && 'yes' || 'no' }}" run: | ./build/build-LXD.sh - name: Pack LXD image @@ -150,7 +164,7 @@ jobs: run: | . ./build/buildlib.sh ARTIFACT_FILE="NextcloudPi_LXD_${LXD_ARCH:-x86}_${VERSION//\//_}" - lxc image export -q ncp/"${version}" "output/${ARTIFACT_FILE}" + "${LXC}" image export -q ncp/"${version}" "output/${ARTIFACT_FILE}" echo "artifact_file=${ARTIFACT_FILE}.tar.gz" >> $GITHUB_OUTPUT - name: upload LXD image to artifact store uses: actions/upload-artifact@v3 @@ -161,6 +175,7 @@ jobs: test-update: needs: + - determine-runner - build-previous runs-on: [ubuntu-20.04] outputs: @@ -169,8 +184,10 @@ jobs: env: VERSION: "${{ inputs.git_ref || github.ref }}" ARTIFACT_NAME: "${{ needs.build-previous.outputs.artifact_name }}" + LXC: "${{ needs.determine-runner.outputs.lxc_cmd }}" steps: - uses: whywaita/setup-lxd@v1 + if: ${{ needs.determine-runner.outputs.lxc_cmd == 'lxc' }} continue-on-error: true with: lxd_version: latest/stable @@ -203,43 +220,43 @@ jobs: - name: Launch ncp container run: | set -x - lxc delete -q -f ncp || true - lxc image import -q "./${{ needs.build-previous.outputs.artifact_file }}" --alias "ncp/update" - systemd-run --user --scope -p "Delegate=yes" lxc launch -q "ncp/update" ncp - lxc exec ncp -- bash -c 'while [ "$(systemctl is-system-running 2>/dev/null)" != "running" ] && [ "$(systemctl is-system-running 2>/dev/null)" != "degraded" ]; do :; done' - lxc exec ncp -- rm -f /opt/ncdata/data/nextcloud.log + "$LXC" delete -q -f ncp || true + "$LXC" image import -q "./${{ needs.build-previous.outputs.artifact_file }}" --alias "ncp/update" + systemd-run --user --scope -p "Delegate=yes" "$LXC" launch -q "ncp/update" ncp + "$LXC" exec ncp -- bash -c 'while [ "$(systemctl is-system-running 2>/dev/null)" != "running" ] && [ "$(systemctl is-system-running 2>/dev/null)" != "degraded" ]; do :; done' + "$LXC" exec ncp -- rm -f /opt/ncdata/data/nextcloud.log sleep 30 - ip="$(lxc list -c n4 -f csv | grep '^ncp' | cut -d ',' -f2)" + ip="$("$LXC" list -c n4 -f csv | grep '^ncp' | cut -d ',' -f2)" ip="${ip/% *}" echo "${ip} nextcloudpi.local" | sudo tee /etc/hosts - name: Activate and Test LXD Image working-directory: ./tests run: | - lxc exec ncp -- bash -c 'tail -f /var/log/ncp.log' |& awk '{ print "NCP::" $0 }' & + "$LXC" exec ncp -- bash -c 'tail -f /var/log/ncp.log' |& awk '{ print "NCP::" $0 }' & python activation_tests.py --no-gui "nextcloudpi.local" 443 4443 || { echo "Activation test failed!" echo "Geckodriver logs:" tail -n 20 geckodriver.log >&2 || true echo "================" echo "ncp.log: " - lxc exec ncp -- "tail -n20 /var/log/ncp.log" || true + "$LXC" exec ncp -- "tail -n20 /var/log/ncp.log" || true exit 1 } python system_tests.py --non-interactive || { echo "System test failed!" exit 1 } - python nextcloud_tests.py --no-gui "nextcloudpi.local" 443 4443 || { + python nextcloud_tests.py --no-gui --skip-release-check "nextcloudpi.local" 443 4443 || { echo "Nextcloud test failed!" echo "Geckodriver logs:" tail -n 20 geckodriver.log >&2 || true echo "================" echo "ncp.log: " - lxc exec ncp -- "tail -n20 /var/log/ncp.log" || true + "$LXC" exec ncp -- "tail -n20 /var/log/ncp.log" || true echo "================" echo "nextcloud log: " - datadir="$(lxc exec ncp -- ncc config:system:get datadirectory)" - lxc exec ncp -- cat "$datadir/nextcloud.log" || true + datadir="$("$LXC" exec ncp -- ncc config:system:get datadirectory)" + "$LXC" exec ncp -- cat "$datadir/nextcloud.log" || true exit 1 } @@ -250,42 +267,61 @@ jobs: BRANCH="${BRANCH/refs\/tags\//}" if [[ "$BRANCH" =~ "refs/pull/"* ]] then - UPDATE_ARGS=("${{ github.base_ref }}" "$VERSION") + UPDATE_ARGS=("${{ github.head_ref }}" "$VERSION") else UPDATE_ARGS=("$BRANCH") fi echo "VERSION=${VERSION}" >> "$GITHUB_ENV" echo "Running update ${{ needs.build-previous.outputs.previous_version }} -> ${VERSION}" - current_nc_version="$(lxc exec ncp -- ncc status | grep "version:" | awk '{ print $3 }')" + current_nc_version="$("$LXC" exec ncp -- ncc status | grep "version:" | awk '{ print $3 }')" latest_nc_version="$(cat etc/ncp.cfg | jq -r '.nextcloud_version')" - lxc exec ncp -- bash -c "DBG=x ncp-update ${UPDATE_ARGS[*]}" - lxc exec ncp -- /usr/local/bin/ncc status + "$LXC" exec ncp -- bash -c "DBG=x ncp-update ${UPDATE_ARGS[*]}" + "$LXC" exec ncp -- /usr/local/bin/ncc status if [[ "$current_nc_version" =~ "$latest_nc_version".* ]] then echo "Nextcloud is up to date - skipping NC update test." else - lxc exec ncp -- bash -c "DBG=x ncp-update-nc ${latest_nc_version?}" + "$LXC" exec ncp -- bash -c "DBG=x ncp-update-nc ${latest_nc_version?}" fi - lxc exec ncp -- rm -f /opt/ncdata/data/nextcloud.log + "$LXC" exec ncp -- rm -f /opt/ncdata/data/nextcloud.log - lxc stop ncp + "$LXC" stop ncp - name: Relaunch container run: | set -x - systemd-run --user --scope -p "Delegate=yes" lxc start ncp - lxc exec ncp -- bash -c 'while [ "$(systemctl is-system-running 2>/dev/null)" != "running" ] && [ "$(systemctl is-system-running 2>/dev/null)" != "degraded" ]; do :; done' + systemd-run --user --scope -p "Delegate=yes" "$LXC" start ncp + "$LXC" exec ncp -- bash -c 'while [ "$(systemctl is-system-running 2>/dev/null)" != "running" ] && [ "$(systemctl is-system-running 2>/dev/null)" != "degraded" ]; do :; done' sleep 30 - ip="$(lxc list -c n4 -f csv | grep '^ncp' | cut -d ',' -f2)" + ip="$("$LXC" list -c n4 -f csv | grep '^ncp' | cut -d ',' -f2)" ip="${ip/% *}" echo "${ip} nextcloudpi.local" | sudo tee /etc/hosts - name: Test LXD Image working-directory: ./tests run: | lxc exec ncp -- bash -c 'tail -f /var/log/ncp.log' |& awk '{ print "NCP::" $0 }' & + "$LXC" exec ncp -- bash -c 'tail -f /var/log/ncp.log' |& awk '{ print "NCP::" $0 }' & + python system_tests.py --non-interactive || { + echo "System test failed!" + exit 1 + } + python nextcloud_tests.py --no-gui --skip-release-check "nextcloudpi.local" 443 4443 || { + echo "Nextcloud test failed!" + echo "Geckodriver logs:" + tail -n 20 geckodriver.log >&2 || true + echo "================" + echo "ncp.log: " + "$LXC" exec ncp -- "tail -n20 /var/log/ncp.log" || true + echo "================" + echo "nextcloud log: " + datadir="$("$LXC" exec ncp -- ncc config:system:get datadirectory)" + "$LXC" exec ncp -- cat "$datadir/nextcloud.log" || true + exit 1 + } + python system_tests.py --non-interactive || { echo "System test failed!" exit 1 @@ -296,14 +332,14 @@ jobs: tail -n 20 geckodriver.log >&2 || true echo "================" echo "ncp.log: " - lxc exec ncp -- "tail -n20 /var/log/ncp.log" || true + "$LXC" exec ncp -- "tail -n20 /var/log/ncp.log" || true echo "================" echo "nextcloud log: " - datadir="$(lxc exec ncp -- ncc config:system:get datadirectory)" - lxc exec ncp -- cat "$datadir/nextcloud.log" || true + datadir="$("$LXC" exec ncp -- ncc config:system:get datadirectory)" + "$LXC" exec ncp -- cat "$datadir/nextcloud.log" || true exit 1 } - lxc stop ncp + "$LXC" stop ncp test-fresh-install: needs: @@ -314,14 +350,16 @@ jobs: VERSION: "${{ inputs.git_ref || github.head_ref || github.ref_name }}" ARTIFACT_NAME: ${{ needs.build-current.outputs.artifact_name }} ARTIFACT_FILE: ${{ needs.build-current.outputs.artifact_file }} + LXC: "${{ needs.determine-runner.outputs.lxc_cmd }}" steps: - name: Checkout code uses: actions/checkout@v3 with: ref: "${{ env.VERSION }}" - name: Cleanup lxd - run: test -z "$(lxc profile device show default | grep eth0)" || lxc profile device remove default eth0 + run: test -z "$("$LXC" profile device show default | grep eth0)" || "$LXC" profile device remove default eth0 - uses: whywaita/setup-lxd@v1 + if: ${{ needs.determine-runner.outputs.lxc_cmd == 'lxc' }} continue-on-error: true with: lxd_version: latest/stable @@ -347,38 +385,38 @@ jobs: - name: Remove old lxd images if: ${{ inputs.arch == 'arm64' }} run: | - for img in $(lxc image list -c f -f csv) + for img in $("$LXC" image list -c f -f csv) do - lxc image delete "$img" + "$LXC" image delete "$img" done - name: Launch ncp container run: | set -x . ./build/buildlib.sh - lxc delete -q -f ncp || true - lxc image import -q "./${ARTIFACT_FILE?}" --alias "ncp/test" || true - systemd-run --user --scope -p "Delegate=yes" lxc launch -q "ncp/test" ncp || \ - sudo systemd-run --scope -p "Delegate=yes" lxc launch -q "ncp/test" ncp - lxc exec ncp -- bash -c 'while [ "$(systemctl is-system-running 2>/dev/null)" != "running" ] && [ "$(systemctl is-system-running 2>/dev/null)" != "degraded" ]; do :; done' + "$LXC" delete -q -f ncp || true + "$LXC" image import -q "./${ARTIFACT_FILE?}" --alias "ncp/test" || true + systemd-run --user --scope -p "Delegate=yes" "$LXC" launch -q "ncp/test" ncp || \ + sudo systemd-run --scope -p "Delegate=yes" "$LXC" launch -q "ncp/test" ncp + "$LXC" exec ncp -- bash -c 'while [ "$(systemctl is-system-running 2>/dev/null)" != "running" ] && [ "$(systemctl is-system-running 2>/dev/null)" != "degraded" ]; do :; done' sleep 30 - ip="$(lxc list -c n4 -f csv | grep '^ncp' | cut -d ',' -f2)" + ip="$("$LXC" list -c n4 -f csv | grep '^ncp' | cut -d ',' -f2)" ip="${ip/% *}" echo "${ip} nextcloudpi.local" | sudo tee /etc/hosts - name: Test LXD Image working-directory: ./tests run: | - lxc exec ncp -- bash -c 'tail -f /var/log/ncp.log' |& awk '{ print "NCP::" $0 }' & + "$LXC" exec ncp -- bash -c 'tail -f /var/log/ncp.log' |& awk '{ print "NCP::" $0 }' & python activation_tests.py --no-gui "nextcloudpi.local" 443 4443 || { echo "Activation test failed!" echo "Geckodriver logs:" tail -n 20 geckodriver.log >&2 || true echo "================" echo "ncp.log: " - lxc exec ncp -- "tail -n20 /var/log/ncp.log" || true + "$LXC" exec ncp -- "tail -n20 /var/log/ncp.log" || true echo "================" echo "nextcloud log: " - datadir="$(lxc exec ncp -- ncc config:system:get datadirectory)" - lxc exec ncp -- cat "$datadir/nextcloud.log" || true + datadir="$("$LXC" exec ncp -- ncc config:system:get datadirectory)" + "$LXC" exec ncp -- cat "$datadir/nextcloud.log" || true exit 1 } python system_tests.py --non-interactive || { @@ -391,11 +429,11 @@ jobs: tail -n 20 geckodriver.log >&2 || true echo "================" echo "ncp.log: " - lxc exec ncp -- "tail -n20 /var/log/ncp.log" || true + "$LXC" exec ncp -- "tail -n20 /var/log/ncp.log" || true echo "================" echo "nextcloud log: " - datadir="$(lxc exec ncp -- ncc config:system:get datadirectory)" - lxc exec ncp -- cat "$datadir/nextcloud.log" || true + datadir="$("$LXC" exec ncp -- ncc config:system:get datadirectory)" + "$LXC" exec ncp -- cat "$datadir/nextcloud.log" || true exit 1 } lxc stop ncp diff --git a/build/build-LXD.sh b/build/build-LXD.sh index 41a8ed947..770752c68 100755 --- a/build/build-LXD.sh +++ b/build/build-LXD.sh @@ -31,15 +31,20 @@ prepare_dirs # tmp cache output ## BUILD NCP -lxc delete -f ncp 2>/dev/null || true -LXC_CREATE=(lxc init -p default) +debian_version="$(. etc/library.sh > /dev/null 2>&1; echo "${RELEASE%%-security}")" + +LXC_CMD=lxc +[[ "$USE_INCUS" == "yes" ]] && LXC_CMD=incus + +$LXC_CMD delete -f ncp 2>/dev/null || true +LXC_CREATE=($LXC_CMD init -p default) [[ -n "$LXD_EXTRA_PROFILE" ]] && LXC_CREATE+=(-p "$LXD_EXTRA_PROFILE") if [[ -n "$LXD_ARCH" ]] && [[ "$LXD_ARCH" != "x86" ]] then echo "Building for architecture: $LXD_ARCH" - LXC_CREATE+=("images:debian/bullseye/$LXD_ARCH") + LXC_CREATE+=("images:debian/${debian_version}/$LXD_ARCH") else - LXC_CREATE+=('images:debian/bullseye') + LXC_CREATE+=("images:debian/${debian_version}") fi LXC_CREATE+=(ncp) "${LXC_CREATE[@]}" @@ -56,19 +61,20 @@ LXC_CREATE+=(ncp) # fi #fi -systemd-run --user --scope -p "Delegate=yes" lxc start ncp -q || \ -sudo systemd-run --scope -p "Delegate=yes" lxc start ncp -q -lxc config device add ncp buildcode disk source="$(pwd)" path=/build -lxc exec ncp -- bash -c 'while [ "$(systemctl is-system-running 2>/dev/null)" != "running" ] && [ "$(systemctl is-system-running 2>/dev/null)" != "degraded" ]; do :; done' -lxc exec ncp -- bash -c 'CODE_DIR=/build DBG=x bash /build/install.sh' -lxc exec ncp -- bash -c 'source /build/etc/library.sh; run_app_unsafe /build/post-inst.sh' -lxc exec ncp -- bash -c "echo '$(basename "$IMG")' > /usr/local/etc/ncp-baseimage" -lxc stop ncp -lxc config device remove ncp buildcode -lxc publish -q ncp -f --alias ncp/"${version}" +set -x +systemd-run --user --scope -p "Delegate=yes" $LXC_CMD start ncp -q || \ +sudo systemd-run --scope -p "Delegate=yes" $LXC_CMD start ncp -q +$LXC_CMD config device add ncp buildcode disk source="$(pwd)" path=/build +$LXC_CMD exec ncp -- bash -c 'while [ "$(systemctl is-system-running 2>/dev/null)" != "running" ] && [ "$(systemctl is-system-running 2>/dev/null)" != "degraded" ]; do :; done' +$LXC_CMD exec ncp -- bash -c 'CODE_DIR=/build DBG=x bash /build/install.sh' +$LXC_CMD exec ncp -- bash -c 'source /build/etc/library.sh; run_app_unsafe /build/post-inst.sh' +$LXC_CMD exec ncp -- bash -c "echo '$(basename "$IMG")' > /usr/local/etc/ncp-baseimage" +$LXC_CMD stop ncp +$LXC_CMD config device remove ncp buildcode +$LXC_CMD publish -q ncp -f --alias ncp/"${version}" ## pack -[[ " $* " =~ .*" --pack ".* ]] && lxc image export -q ncp/"${version}" "$TAR" +[[ " $* " =~ .*" --pack ".* ]] && $LXC_CMD image export -q ncp/"${version}" "$TAR" exit 0 diff --git a/build/buildlib.sh b/build/buildlib.sh index 5870ab8f5..5398cf555 100644 --- a/build/buildlib.sh +++ b/build/buildlib.sh @@ -335,43 +335,6 @@ function generate_changelog() sed 's|* \[tag: |\n[|' > changelog.md } -function upload_ftp() -{ - local IMGNAME="$1" - echo -e "\n\e[1m[ Upload FTP ]\e[0m" - echo "* $IMGNAME..." - [[ -f torrent/"$IMGNAME"/"$IMGNAME".tar.bz2 ]] || { echo "No image file found, abort"; return 1; } - [[ "$FTPPASS" == "" ]] && { echo "No FTPPASS variable found, skip upload"; return 0; } - - cd torrent - - ftp -np ftp.ownyourbits.com < CompletedProcess: return False set_cohorte_id(99) - handle_error(run(pre_cmd + ['/usr/local/bin/ncp-check-version'], stdout=PIPE, stderr=PIPE)) - result = handle_error(run(pre_cmd + ['cat', '/var/run/.ncp-latest-version'], stdout=PIPE, stderr=PIPE)) + chk_version = handle_error(run(pre_cmd + ['/usr/local/bin/ncp-check-version'], stdout=PIPE, stderr=PIPE)) + try: + result = handle_error(run(pre_cmd + ['cat', '/var/run/.ncp-latest-version'], stdout=PIPE, stderr=PIPE)) + except ProcessExecutionException as e: + print("stderr:", chk_version.stderr) + print("stdout:", chk_version.stdout) + raise e if 'v99.99.99' not in result.stdout: print(f"{tc.red}error{tc.normal} Expected latest detected version to be v99.99.99, was {result.stdout}") return False @@ -291,7 +300,7 @@ def set_cohorte_id(cohorte_id: int) -> CompletedProcess: # detect if we are running this in a LXC instance try: - lxc_running = run(['lxc', 'info', 'ncp'], stdout=PIPE, check = True) + lxc_running = run([lxc_command, 'info', 'ncp'], stdout=PIPE, check = True) except: lxc_running = False @@ -319,7 +328,7 @@ def set_cohorte_id(cohorte_id: int) -> CompletedProcess: # LXC method elif lxc_running: print( tc.brown + "* local LXC instance detected" + tc.normal) - pre_cmd = ['lxc', 'exec', 'ncp', '--'] + pre_cmd = [lxc_command, 'exec', 'ncp', '--'] elif systemd_container_running: pre_cmd = ['systemd-run', '--wait', '-P', '--machine=ncp'] From 6b18cdcc1a7e77f103e262a596243936017a7905 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tobias=20Kn=C3=B6ppler?= <6317548+theCalcaholic@users.noreply.github.com> Date: Sun, 28 Apr 2024 18:31:16 +0200 Subject: [PATCH 02/14] nc-nextcloud.sh,install.sh,lamp.sh: Fix build process for debian 12 MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Tobias Knöppler <6317548+theCalcaholic@users.noreply.github.com> --- .github/workflows/build-lxd.yml | 1 - bin/ncp/CONFIG/nc-nextcloud.sh | 4 ++-- etc/ncp.cfg | 2 +- install.sh | 7 ++++++- lamp.sh | 11 ++--------- 5 files changed, 11 insertions(+), 14 deletions(-) diff --git a/.github/workflows/build-lxd.yml b/.github/workflows/build-lxd.yml index 90a159fba..5061610d2 100644 --- a/.github/workflows/build-lxd.yml +++ b/.github/workflows/build-lxd.yml @@ -302,7 +302,6 @@ jobs: - name: Test LXD Image working-directory: ./tests run: | - lxc exec ncp -- bash -c 'tail -f /var/log/ncp.log' |& awk '{ print "NCP::" $0 }' & "$LXC" exec ncp -- bash -c 'tail -f /var/log/ncp.log' |& awk '{ print "NCP::" $0 }' & python system_tests.py --non-interactive || { echo "System test failed!" diff --git a/bin/ncp/CONFIG/nc-nextcloud.sh b/bin/ncp/CONFIG/nc-nextcloud.sh index 635e4869f..d2d1cc793 100644 --- a/bin/ncp/CONFIG/nc-nextcloud.sh +++ b/bin/ncp/CONFIG/nc-nextcloud.sh @@ -29,8 +29,8 @@ install() $APTINSTALL lbzip2 iputils-ping jq wget # NOTE: php-smbclient in sury but not in Debian sources, we'll use the binary version # https://docs.nextcloud.com/server/latest/admin_manual/configuration_files/external_storage/smb.html - $APTINSTALL -t $RELEASE smbclient exfat-fuse exfat-utils # for external storage - $APTINSTALL -t $RELEASE exfat-fuse exfat-utils # for external storage + $APTINSTALL -t $RELEASE smbclient exfat-fuse exfatprogs # for external storage + $APTINSTALL -t $RELEASE exfat-fuse exfatprogs # for external storage $APTINSTALL -t $RELEASE php${PHPVER}-exif # for gallery $APTINSTALL -t $RELEASE php${PHPVER}-bcmath # for LDAP $APTINSTALL -t $RELEASE php${PHPVER}-gmp # for bookmarks diff --git a/etc/ncp.cfg b/etc/ncp.cfg index ef1dddc94..bbe7137c0 100644 --- a/etc/ncp.cfg +++ b/etc/ncp.cfg @@ -1,5 +1,5 @@ { "nextcloud_version": "28.0.4", "php_version": "8.1", - "release": "bullseye" + "release": "bookworm" } diff --git a/install.sh b/install.sh index 4502df305..24c051ecd 100644 --- a/install.sh +++ b/install.sh @@ -30,7 +30,7 @@ type mysqld &>/dev/null && mysql -e 'use nextcloud' &>/dev/null && { echo "The ' # get dependencies apt-get update -apt-get install --no-install-recommends -y git ca-certificates sudo lsb-release wget +DEBIAN_FRONTEND=noninteractive apt-get install --no-install-recommends -y git ca-certificates sudo lsb-release wget jq # get install code if [[ "${CODE_DIR}" == "" ]]; then @@ -63,6 +63,11 @@ cp etc/ncp.cfg /usr/local/etc/ cp -r etc/ncp-templates /usr/local/etc/ install_app lamp.sh +if [[ -d "/run/systemd/system" ]] && is_lxc +then + DEBIAN_FRONTEND=noninteractive apt-get install --no-install-recommends -y systemd-resolved + systemctl enable systemd-resolved +fi install_app bin/ncp/CONFIG/nc-nextcloud.sh run_app_unsafe bin/ncp/CONFIG/nc-nextcloud.sh rm /usr/local/etc/ncp-config.d/nc-nextcloud.cfg # armbian overlay is ro diff --git a/lamp.sh b/lamp.sh index aadff56ce..7a1cdf134 100644 --- a/lamp.sh +++ b/lamp.sh @@ -37,13 +37,6 @@ install() apache2ctl -V || true # Create systemd users to keep uids persistent between containers - id -u systemd-resolve || { - addgroup --quiet --system systemd-journal - adduser --quiet -u 180 --system --group --no-create-home --home /run/systemd \ - --gecos "systemd Network Management" systemd-network - adduser --quiet -u 181 --system --group --no-create-home --home /run/systemd \ - --gecos "systemd Resolver" systemd-resolve - } install_with_shadow_workaround --no-install-recommends systemd $APTINSTALL -t $RELEASE php${PHPVER} php${PHPVER}-curl php${PHPVER}-gd php${PHPVER}-fpm php${PHPVER}-cli php${PHPVER}-opcache \ php${PHPVER}-mbstring php${PHPVER}-xml php${PHPVER}-zip php${PHPVER}-fileinfo php${PHPVER}-ldap \ @@ -67,7 +60,7 @@ install() install_template apache2/http2.conf.sh /etc/apache2/conf-available/http2.conf --defaults - # CONFIGURE PHP7 + # CONFIGURE PHP ########################################## install_template "php/opcache.ini.sh" "/etc/php/${PHPVER}/mods-available/opcache.ini" --defaults @@ -97,7 +90,7 @@ install() # launch mariadb if not already running if ! [[ -f /run/mysqld/mysqld.pid ]]; then echo "Starting mariaDB" - mysqld & + sudo -u mysql mysqld & fi # wait for mariadb From cc69c2bf2a634bfb3667db16402f9b524b6b50e2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tobias=20Kn=C3=B6ppler?= <6317548+theCalcaholic@users.noreply.github.com> Date: Sun, 28 Apr 2024 18:37:24 +0200 Subject: [PATCH 03/14] /etc/ncp-dist-upgrade*: Implement dist-upgrade for bullseye->bookworm and add integration tests for it MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Tobias Knöppler <6317548+theCalcaholic@users.noreply.github.com> --- .../create-test-instance-bullseye/action.yml | 35 ++ .../actions/create-test-instance/action.yml | 2 +- .github/workflows/build-lxd.yml | 169 ++++++ .github/workflows/build-sd-images.yml | 525 +++++++++++++++++- .github/workflows/release.yml | 35 +- .github/workflows/vm-tests.yml | 445 +++++++++++---- bin/ncp-dist-upgrade | 160 +----- bin/ncp-dist-upgrade.d/debian-10.sh | 137 +++++ bin/ncp-dist-upgrade.d/debian-11.sh | 79 +++ ncp.sh | 2 +- tests/nextcloud_tests.py | 16 +- 11 files changed, 1357 insertions(+), 248 deletions(-) create mode 100644 .github/actions/create-test-instance-bullseye/action.yml create mode 100644 bin/ncp-dist-upgrade.d/debian-10.sh create mode 100644 bin/ncp-dist-upgrade.d/debian-11.sh diff --git a/.github/actions/create-test-instance-bullseye/action.yml b/.github/actions/create-test-instance-bullseye/action.yml new file mode 100644 index 000000000..07a8bff2e --- /dev/null +++ b/.github/actions/create-test-instance-bullseye/action.yml @@ -0,0 +1,35 @@ +name: Create Test VM +description: Create NCP instance for testing in the Hetzner cloud +inputs: + version: + description: version (git rev / tag / branch) to install + required: true + uid: + description: A unique ID for labeling/naming generated resources + required: true + hcloud_token: + description: A auth token for Hetzner cloud + required: true + server_type: + description: Server type to use for hetzner servers + required: true + default: "cx11" + +outputs: + server_address: + description: Adress of the test instance + snapshot_id: + description: ID of the generated postinstall snapshot + test_server_id: + description: ID of the created test server +runs: + using: docker + image: docker://thecalcaholic/ncp-test-automation:bullseye + + env: + HCLOUD_TOKEN: ${{ inputs.hcloud_token }} + UID: ${{ inputs.uid }} + SERVER_TYPE: ${{ inputs.server_type }} + args: + - /ncp-test-automation/bin/actions/create-test-instance.sh + - ${{ inputs.version }} \ No newline at end of file diff --git a/.github/actions/create-test-instance/action.yml b/.github/actions/create-test-instance/action.yml index 0894a397a..8229415db 100644 --- a/.github/actions/create-test-instance/action.yml +++ b/.github/actions/create-test-instance/action.yml @@ -24,7 +24,7 @@ outputs: description: ID of the created test server runs: using: docker - image: docker://thecalcaholic/ncp-test-automation + image: docker://thecalcaholic/ncp-test-automation:bookworm env: HCLOUD_TOKEN: ${{ inputs.hcloud_token }} diff --git a/.github/workflows/build-lxd.yml b/.github/workflows/build-lxd.yml index 5061610d2..f84fe0786 100644 --- a/.github/workflows/build-lxd.yml +++ b/.github/workflows/build-lxd.yml @@ -321,6 +321,175 @@ jobs: exit 1 } + test-dist-upgrade: + needs: + - determine-runner + runs-on: [ubuntu-20.04] + env: + VERSION: "${{ inputs.git_ref || github.ref }}" + ARTIFACT_NAME: "${{ needs.build-previous.outputs.artifact_name }}" + LXC: "${{ needs.determine-runner.outputs.lxc_cmd }}" + PREVIOUS_IMAGE_URL: "https://github.com/nextcloud/nextcloudpi/releases/download/v1.53.2/NextcloudPi_LXD_x86_v1.53.2.tar.gz" + steps: + - uses: whywaita/setup-lxd@v1 + if: ${{ needs.determine-runner.outputs.lxc_cmd == 'lxc' }} + continue-on-error: true + with: + lxd_version: latest/stable + - name: Fix LXD + run: | + sudo iptables -I DOCKER-USER -i lxdbr0 -j ACCEPT + sudo iptables -I DOCKER-USER -o lxdbr0 -m conntrack --ctstate RELATED,ESTABLISHED -j ACCEPT + - name: Checkout code + uses: actions/checkout@v3 + with: + ref: "${{ env.VERSION }}" + - name: Setup Firefox + continue-on-error: true + id: setup-firefox-browser-action + uses: browser-actions/setup-firefox@latest + - name: Setup Firefox from packages + if: ${{ steps.setup-firefox-browser-action.outcome == 'failure' }} + run: | + sudo apt-get install -y --no-install-recommends firefox + - name: Setup GeckoDriver + uses: ChlodAlejandro/setup-geckodriver@latest + with: + token: ${{ secrets.GITHUB_TOKEN }} + - name: Setup Selenium + run: pip install selenium + - name: download LXD image + run: | + wget -qO ./ncp.tar.gz "${PREVIOUS_IMAGE_URL?}" + - name: Launch ncp container + run: | + set -x + "$LXC" delete -q -f ncp || true + "$LXC" image import -q "./ncp.tar.gz" --alias "ncp/update" + systemd-run --user --scope -p "Delegate=yes" "$LXC" launch -q "ncp/update" ncp + "$LXC" exec ncp -- bash -c 'while [ "$(systemctl is-system-running 2>/dev/null)" != "running" ] && [ "$(systemctl is-system-running 2>/dev/null)" != "degraded" ]; do :; done' + "$LXC" exec ncp -- rm -f /opt/ncdata/data/nextcloud.log + sleep 30 + ip="$("$LXC" list -c n4 -f csv | grep '^ncp' | cut -d ',' -f2)" + ip="${ip/% *}" + echo "${ip} nextcloudpi.local" | sudo tee /etc/hosts + - name: Activate and Test LXD Image + working-directory: ./tests + run: | + "$LXC" exec ncp -- bash -c 'tail -f /var/log/ncp.log' |& awk '{ print "NCP::" $0 }' & + python activation_tests.py --no-gui "nextcloudpi.local" 443 4443 || { + echo "Activation test failed!" + echo "Geckodriver logs:" + tail -n 20 geckodriver.log >&2 || true + echo "================" + echo "ncp.log: " + "$LXC" exec ncp -- "tail -n20 /var/log/ncp.log" || true + exit 1 + } + python system_tests.py --non-interactive --skip-update-test || { + echo "System test failed!" + exit 1 + } + python nextcloud_tests.py --no-gui --skip-release-check "nextcloudpi.local" 443 4443 || { + echo "Nextcloud test failed!" + echo "Geckodriver logs:" + tail -n 20 geckodriver.log >&2 || true + echo "================" + echo "ncp.log: " + "$LXC" exec ncp -- "tail -n20 /var/log/ncp.log" || true + echo "================" + echo "nextcloud log: " + datadir="$("$LXC" exec ncp -- ncc config:system:get datadirectory)" + "$LXC" exec ncp -- cat "$datadir/nextcloud.log" || true + exit 1 + } + + - name: Update ncp + run: | + set -ex + BRANCH="${VERSION/refs\/heads\//}" + BRANCH="${BRANCH/refs\/tags\//}" + if [[ "$BRANCH" =~ "refs/pull/"* ]] + then + UPDATE_ARGS=("${{ github.head_ref }}" "$VERSION") + else + UPDATE_ARGS=("$BRANCH") + fi + echo "VERSION=${VERSION}" >> "$GITHUB_ENV" + echo "Running update to ${VERSION}" + + current_nc_version="$("$LXC" exec ncp -- ncc status | grep "version:" | awk '{ print $3 }')" + latest_nc_version="$(cat etc/ncp.cfg | jq -r '.nextcloud_version')" + + "$LXC" exec ncp -- bash -c "DBG=x ncp-update ${UPDATE_ARGS[*]}" + "$LXC" exec ncp -- /usr/local/bin/ncc status + + if [[ "$current_nc_version" =~ "$latest_nc_version".* ]] + then + echo "Nextcloud is up to date - skipping NC update test." + else + "$LXC" exec ncp -- bash -c "DBG=x ncp-update-nc ${latest_nc_version?}" + fi + + "$LXC" exec ncp -- rm -f /opt/ncdata/data/nextcloud.log + + "$LXC" stop ncp + - name: Relaunch container + run: | + set -x + systemd-run --user --scope -p "Delegate=yes" "$LXC" start ncp + "$LXC" exec ncp -- bash -c 'while [ "$(systemctl is-system-running 2>/dev/null)" != "running" ] && [ "$(systemctl is-system-running 2>/dev/null)" != "degraded" ]; do :; done' + sleep 30 + ip="$("$LXC" list -c n4 -f csv | grep '^ncp' | cut -d ',' -f2)" + ip="${ip/% *}" + echo "${ip} nextcloudpi.local" | sudo tee /etc/hosts + - name: Test LXD Image + working-directory: ./tests + run: | + "$LXC" exec ncp -- bash -c 'tail -f /var/log/ncp.log' |& awk '{ print "NCP::" $0 }' & + python system_tests.py --non-interactive --skip-update-test || { + echo "System test failed!" + exit 1 + } + python nextcloud_tests.py --no-gui --skip-release-check "nextcloudpi.local" 443 4443 || { + echo "Nextcloud test failed!" + echo "Geckodriver logs:" + tail -n 20 geckodriver.log >&2 || true + echo "================" + echo "ncp.log: " + "$LXC" exec ncp -- "tail -n20 /var/log/ncp.log" || true + echo "================" + echo "nextcloud log: " + datadir="$("$LXC" exec ncp -- ncc config:system:get datadirectory)" + "$LXC" exec ncp -- cat "$datadir/nextcloud.log" || true + exit 1 + } + - name: NCP distupgrade + id: distupgrade + run: | + set -x + "$LXC" exec ncp -- cat /etc/os-release | grep 'VERSION_ID="11"' || { + echo "can't upgrade from Debian $("$LXC" exec ncp -- cat /etc/os-release | grep VERSION_ID=)" + exit 1 + } + "$LXC" exec ncp -- bash -c "DEBIAN_FRONTEND=noninteractive ncp-dist-upgrade" + + "$LXC" exec ncp -- rm -f /opt/ncdata/data/nextcloud.log + + "$LXC" stop ncp + - name: Relaunch container + run: | + set -x + systemd-run --user --scope -p "Delegate=yes" "$LXC" start ncp + "$LXC" exec ncp -- bash -c 'while [ "$(systemctl is-system-running 2>/dev/null)" != "running" ] && [ "$(systemctl is-system-running 2>/dev/null)" != "degraded" ]; do :; done' + sleep 30 + ip="$("$LXC" list -c n4 -f csv | grep '^ncp' | cut -d ',' -f2)" + ip="${ip/% *}" + echo "${ip} nextcloudpi.local" | sudo tee /etc/hosts + - name: Test LXD Image + working-directory: ./tests + run: | + "$LXC" exec ncp -- bash -c 'tail -f /var/log/ncp.log' |& awk '{ print "NCP::" $0 }' & python system_tests.py --non-interactive || { echo "System test failed!" exit 1 diff --git a/.github/workflows/build-sd-images.yml b/.github/workflows/build-sd-images.yml index c37b77997..5bc96d61e 100644 --- a/.github/workflows/build-sd-images.yml +++ b/.github/workflows/build-sd-images.yml @@ -186,6 +186,14 @@ jobs: sudo wget -nv https://github.com/multiarch/qemu-user-static/releases/latest/download/qemu-arm-static -O raspbian_root/usr/bin/qemu-arm-static sudo chmod +x raspbian_root/usr/bin/qemu-{arm,aarch64}-static echo 'Mutex posixsem' | sudo tee -a raspbian_root/etc/apache2/mods-available/ssl.conf + echo 'ignore-warnings ARM64-COW-BUG' | sudo tee -a raspbian_root/etc/redis/redis.conf + sudo mkdir -p raspbian_root/etc/systemd/system/redis-server.service.d + echo '[Service]' | sudo tee raspbian_root/etc/systemd/system/redis-server.service.d/ncp.conf + echo 'PrivateUsers=false' | sudo tee -a raspbian_root/etc/systemd/system/redis-server.service.d/ncp.conf + + sudo mkdir -p raspbian_root/etc/systemd/system/php8.1-fpm.service.d + echo '[Service]' | sudo tee raspbian_root/etc/systemd/system/php8.1-fpm.service.d/ncp.conf + echo 'ExecStartPre=mkdir -p /var/run/php' | sudo tee -a raspbian_root/etc/systemd/system/php8.1-fpm.service.d/ncp.conf - name: Test image id: test run: | @@ -308,8 +316,9 @@ jobs: echo -e "${LOG_DIAG} ncp.log: " "${CONTAINER_CMD[@]}" -q ncp /bin/bash -c "tail -n20 /var/log/ncp.log" |& awk "{ print \"${LOG_DIAG} \" \$0 }" || true echo "================" - echo "Nextcloud log: " - "${CONTAINER_CMD[@]}" -q ncp cat /opt/ncdata/data/nextcloud.log + echo "${LOG_DIAG} Nextcloud log: " + "${CONTAINER_CMD[@]}" -q ncp /bin/bash -c 'ls -l /opt/ncdata/data/nextcloud.log' |& awk "{ print \"${LOG_DIAG} \" \$0 }" || true + "${CONTAINER_CMD[@]}" -q ncp /bin/bash -c 'cat /opt/ncdata/data/nextcloud.log' |& awk "{ print \"${LOG_DIAG} \" \$0 }" || true sleep 12 continue } @@ -323,3 +332,515 @@ jobs: echo -e "${LOG_CICD} Nextcloud test failed in all attempts!" exit 1 } + +# test-distupgrade: +# runs-on: ubuntu-20.04 +# env: +# VERSION: "${{ inputs.git_ref || github.ref }}" +# LOG_GUEST: "\\033[1;34mGUEST::\\033[0m" +# LOG_NCP: "\\033[1;36m~NCP::\\033[0m" +# LOG_CICD: "\\033[1;35mCICD::\\033[0m" +# LOG_TEST: "\\033[1;33mTEST::\\033[0m" +# LOG_DIAG: "\\033[1;31mDIAG::\\033[0m" +# PREVIOUS_VERSION: "v1.53.2" +# defaults: +# run: +# shell: bash +# steps: +# - name: Set up QEMU +# uses: docker/setup-qemu-action@v3 +# - name: Checkout code +# uses: actions/checkout@v3 +# with: +# ref: "${{ env.VERSION }}" +# - name: Download previous image +# id: download-previous-image +# run: | +# set -x +# mkdir -p output +# cd output +# wgetrc=0 +# wgeterr="$(wget -O ./ncp.zip "https://github.com/nextcloud/nextcloudpi/releases/download/${PREVIOUS_VERSION?}/NextcloudPi_${{ inputs.board_name }}_${PREVIOUS_VERSION}.zip" 2>&1)" || wgetrc=$? +# if [[ $wgetrc -ne 0 ]] +# then +# if echo "$wgeterr" | grep '404 Not Found' +# then +# echo "Board not found in previous release - skipping." +# echo "skipped=true" >> "$GITHUB_OUTPUT" +# exit 0 +# else +# echo "$wgeterr" +# exit $wgetrc +# fi +# fi +# echo "skipped=false" >> "$GITHUB_OUTPUT" +# unzip ncp.zip +# rm ncp.zip +# mv NextcloudPi_${{ inputs.board_name }}_${PREVIOUS_VERSION}.img ./ncp.img +# echo "ARTIFACT_FILE=ncp.img" >> "$GITHUB_ENV" +# - name: Prepare test +# if: ${{ steps.download-previous-image.outputs.skipped == 'false' }} +# run: | +# set -x +# mv output/${ARTIFACT_FILE?} ncp.img +# sudo apt-get install -y systemd-container +# sudo pip install selenium +# sudo rm -rf raspbian_root +# . ./build/buildlib.sh +# mount_raspbian "ncp.img" +# sudo cat raspbian_root/etc/machine-id +# sudo systemd-id128 new | sudo tee ./raspbian_root/etc/machine-id +# sudo wget -nv https://github.com/multiarch/qemu-user-static/releases/latest/download/qemu-aarch64-static -O raspbian_root/usr/bin/qemu-aarch64-static +# sudo wget -nv https://github.com/multiarch/qemu-user-static/releases/latest/download/qemu-arm-static -O raspbian_root/usr/bin/qemu-arm-static +# sudo chmod +x raspbian_root/usr/bin/qemu-{arm,aarch64}-static +# echo 'Mutex posixsem' | sudo tee -a raspbian_root/etc/apache2/mods-available/ssl.conf +# echo 'ignore-warnings ARM64-COW-BUG' | sudo tee -a raspbian_root/etc/redis/redis.conf +# sudo mkdir -p raspbian_root/etc/systemd/system/redis-server.service.d +# echo '[Service]' | sudo tee raspbian_root/etc/systemd/system/redis-server.service.d/ncp.conf +# echo 'PrivateUsers=false' | sudo tee -a raspbian_root/etc/systemd/system/redis-server.service.d/ncp.conf +# - name: Test and activate image +# if: ${{ steps.download-previous-image.outputs.skipped == 'false' }} +# id: test +# run: | +# +# log_err() { +# rc="${1?}" +# msg="${2?}" +# echo -e "${LOG_DIAG} $msg" >&2 +# return $rc +# } +# +# sudo systemd-nspawn --boot -D ./raspbian_root/ -M ncp --hostname=nextcloudpi |& awk "{ print \"${LOG_GUEST} \" \$0 }" & +# sleep 60 +# +# CONTAINER_CMD=(sudo systemd-run --machine=ncp -P --wait) +# +# success=false +# for attempt in {1..30} +# do +# echo -e "${LOG_CICD} == Wait until container network is available (attempt $attempt/30) ==" +# ip="$("${CONTAINER_CMD[@]}" bash -c '. /usr/local/etc/library.sh > /dev/null; get_ip')" +# [[ -n "$ip" ]] && curl -k "https://$ip/activate/" > /dev/null || { sleep 6; continue; } +# success=true +# break +# done +# sudo cat ./raspbian_root/var/log/ncp.log |& awk "{ print \"${LOG_NCP} \" \$0 }" +# sudo tail -n 0 -f ./raspbian_root/var/log/ncp.log |& awk "{ print \"${LOG_NCP} \" \$0 }" & +# +# [[ "$success" == "true" ]] || { +# echo -e "${LOG_CICD} Could not reach container. Aborting..." +# exit 1 +# } +# +# attempt=0 +# success=false +# for attempt in {1..150} +# do +# echo -e "${LOG_CICD} Waiting for container startup (attempt $attempt/150)..." +# "${CONTAINER_CMD[@]}" journalctl -eu redis.service || true +# "${CONTAINER_CMD[@]}" systemctl status php8.1-fpm.service || true +# redis_pw="$("${CONTAINER_CMD[@]}" bash -c ". /usr/local/etc/library.sh; get_nc_config_value 'redis\"][\"password'")" \ +# && redis_socket="$("${CONTAINER_CMD[@]}" bash -c ". /usr/local/etc/library.sh; get_nc_config_value 'redis\"][\"host'")" \ +# || log_err $? "Error retrieving redis credentials" || true +# if { "${CONTAINER_CMD[@]}" -q ncc status |& awk "{ print \"${LOG_DIAG} \" \$0 }" || log_err $? "ncc status check failed"; } \ +# && { [[ "$("${CONTAINER_CMD[@]}" ncc maintenance:mode)" =~ .*disabled.* ]] || log_err $? "Maintenance mode is enabled or could not be retrieved"; } \ +# && { "${CONTAINER_CMD[@]}" redis-cli -s "$redis_socket" -a "$redis_pw" set redisready yes |& awk "{ print \"${LOG_DIAG} \" \$0 }" || log_err $? "Failed to set redis variable"; } \ +# && { "${CONTAINER_CMD[@]}" redis-cli -s "$redis_socket" -a "$redis_pw" get redisready |& awk "{ print \"${LOG_DIAG} \" \$0 }" || log_err $? "Failed to read redis variable"; } +# then +# echo -e "${LOG_CICD} Startup successful" +# success=true +# break +# fi +# attempt=$((attempt + 1)) +# sleep 5 +# done +# +# [[ "$success" == "true" ]] || { +# echo -e "${LOG_CICD} Timeout reached." +# "${CONTAINER_CMD[@]}" -q systemctl status mysql |& awk "{ print \"${LOG_DIAG} \" \$0 }" +# "${CONTAINER_CMD[@]}" -q systemctl status redis |& awk "{ print \"${LOG_DIAG} \" \$0 }" +# "${CONTAINER_CMD[@]}" -q systemctl status 'php*-fpm' |& awk "{ print \"${LOG_DIAG} \" \$0 }" +# "${CONTAINER_CMD[@]}" -q systemctl status apache2 |& awk "{ print \"${LOG_DIAG} \" \$0 }" +# "${CONTAINER_CMD[@]}" -q ncp-diag |& awk "{ print \"${LOG_DIAG} \" \$0 }" +# exit 1 +# } +# +# set -x +# set +e +# +# success=false +# for attempt in {1..5} +# do +# echo -e "${LOG_CICD} == Activation Tests (attempt $attempt/5) ==" +# python tests/activation_tests.py -t 300 --no-gui "$ip" 443 4443 |& awk "{ print \"${LOG_TEST} \" \$0 }" +# [[ ${PIPESTATUS[0]} -eq 0 ]] || { +# echo -e "${LOG_CICD} Activation test failed!" +# echo -e "${LOG_DIAG} Geckodriver logs:" +# tail -n 20 geckodriver.log >&2 |& awk "{ print \"${LOG_DIAG} \" \$0 }" || true +# echo "================" +# echo -e "${LOG_DIAG} mysql: " +# "${CONTAINER_CMD[@]}" -q ncp-diag |& awk "{ print \"${LOG_DIAG} \" \$0 }" || true +# "${CONTAINER_CMD[@]}" -q systemctl status mysql |& awk "{ print \"${LOG_DIAG} \" \$0 }" || true +# sleep 12 +# continue +# } +# success=true +# break +# done +# [[ "$success" == "true" ]] || { +# echo -e "${LOG_CICD} Activation test failed in all attempts!" +# exit 1 +# } +# +# success=false +# for attempt in {1..5} +# do +# echo -e "${LOG_CICD} == System Tests (attempt $attempt/5) ==" +# sudo python tests/system_tests.py --non-interactive --skip-update-test |& awk "{ print \"${LOG_TEST} \" \$0 }" +# [[ ${PIPESTATUS[0]} -eq 0 ]] || { +# echo -e "${LOG_CICD} System test failed!" +# sleep 12 +# continue +# } +# success=true +# break +# done +# [[ "$success" == "true" ]] || { +# echo -e "${LOG_CICD} System test failed in all attempts!" +# exit 1 +# } +# +# success=false +# for attempt in {1..5} +# do +# echo -e "${LOG_CICD} == Nextcloud Tests (attempt $attempt/5) ==" +# python tests/nextcloud_tests.py --no-gui --skip-release-check "$ip" 443 4443 |& awk "{ print \"${LOG_TEST} \" \$0 }" +# [[ ${PIPESTATUS[0]} -eq 0 ]] || { +# echo -e "${LOG_CICD} Nextcloud test failed!" +# echo -e "{$LOG_DIAG} Geckodriver logs:" +# tail -n 20 geckodriver.log >&2 |& awk "{ print \"${LOG_DIAG} \" \$0 }" || true +# echo -e "${LOG_CICD} ================" +# echo -e "${LOG_DIAG} ncp.log: " +# "${CONTAINER_CMD[@]}" -q ncp /bin/bash -c "tail -n20 /var/log/ncp.log" |& awk "{ print \"${LOG_DIAG} \" \$0 }" || true +# echo "================" +# echo "${LOG_DIAG} Nextcloud log: " +# "${CONTAINER_CMD[@]}" -q ncp /bin/bash -c 'ls -l /opt/ncdata/data/nextcloud.log' |& awk "{ print \"${LOG_DIAG} \" \$0 }" || true +# "${CONTAINER_CMD[@]}" -q ncp /bin/bash -c 'cat /opt/ncdata/data/nextcloud.log' |& awk "{ print \"${LOG_DIAG} \" \$0 }" || true +# sleep 12 +# continue +# } +# success=true +# break +# done +# +# [[ "$success" == "true" ]] || { +# echo -e "${LOG_CICD} Nextcloud test failed in all attempts!" +# exit 1 +# } +# - name: Update NCP +# if: ${{ steps.download-previous-image.outputs.skipped == 'false' }} +# run: | +# set -ex +# +# CONTAINER_CMD=(sudo systemd-run --machine=ncp -P --wait) +# +# BRANCH="${VERSION/refs\/heads\//}" +# BRANCH="${BRANCH/refs\/tags\//}" +# if [[ "$BRANCH" =~ "refs/pull/"* ]] +# then +# UPDATE_ARGS=("${{ github.head_ref }}" "$VERSION") +# else +# UPDATE_ARGS=("$BRANCH") +# fi +# current_nc_version="$("${CONTAINER_CMD[@]}" ncc status | grep "version:" | awk '{ print $3 }')" +# latest_nc_version="$(cat etc/ncp.cfg | jq -r '.nextcloud_version')" +# +# echo "Updating from $PREVIOUS_VERSION to $VERSION" +# +# "${CONTAINER_CMD[@]}" bash -c "DBG=x ncp-update ${UPDATE_ARGS[*]}" +# "${CONTAINER_CMD[@]}" /usr/local/bin/ncc status +# # "${CONTAINER_CMD[@]}" bash -c 'curl https://download.nextcloud.com/server/releases/nextcloud-28.0.4.tar.bz2 > /var/www/nextcloud-28.0.4.tar.bz2' +# k0nKat1Nation +# +# # if [[ "$current_nc_version" =~ "$latest_nc_version".* ]] +# # then +# # echo "Nextcloud is up to date - skipping NC update test." +# # else +# # "${CONTAINER_CMD[@]}" bash -c "DBG=x ncp-update-nc ${latest_nc_version?}" +# # fi +# +# sudo machinectl terminate ncp +# sudo rm -f ./raspbian_root/opt/ncdata/data/nextcloud.log +# - name: Test image after update +# if: ${{ steps.download-previous-image.outputs.skipped == 'false' }} +# run: | +# +# log_err() { +# rc="${1?}" +# msg="${2?}" +# echo -e "${LOG_DIAG} $msg" >&2 +# return $rc +# } +# +# sudo systemd-nspawn --boot -D ./raspbian_root/ -M ncp --hostname=nextcloudpi |& awk "{ print \"${LOG_GUEST} \" \$0 }" & +# sleep 60 +# +# CONTAINER_CMD=(sudo systemd-run --machine=ncp -P --wait) +# +# success=false +# for attempt in {1..30} +# do +# echo -e "${LOG_CICD} == Wait until container network is available (attempt $attempt/30) ==" +# ip="$("${CONTAINER_CMD[@]}" bash -c '. /usr/local/etc/library.sh > /dev/null; get_ip')" +# [[ -n "$ip" ]] && curl -k "https://$ip/activate/" > /dev/null || { sleep 6; continue; } +# success=true +# break +# done +# sudo cat ./raspbian_root/var/log/ncp.log |& awk "{ print \"${LOG_NCP} \" \$0 }" +# sudo tail -n 0 -f ./raspbian_root/var/log/ncp.log |& awk "{ print \"${LOG_NCP} \" \$0 }" & +# +# [[ "$success" == "true" ]] || { +# echo -e "${LOG_CICD} Could not reach container. Aborting..." +# exit 1 +# } +# +# attempt=0 +# success=false +# for attempt in {1..150} +# do +# echo -e "${LOG_CICD} Waiting for container startup (attempt $attempt/150)..." +# "${CONTAINER_CMD[@]}" journalctl -eu redis.service || true +# "${CONTAINER_CMD[@]}" systemctl status php8.1-fpm.service || true +# redis_pw="$("${CONTAINER_CMD[@]}" bash -c ". /usr/local/etc/library.sh; get_nc_config_value 'redis\"][\"password'")" \ +# && redis_socket="$("${CONTAINER_CMD[@]}" bash -c ". /usr/local/etc/library.sh; get_nc_config_value 'redis\"][\"host'")" \ +# || log_err $? "Error retrieving redis credentials" || true +# if { "${CONTAINER_CMD[@]}" -q ncc status |& awk "{ print \"${LOG_DIAG} \" \$0 }" || log_err $? "ncc status check failed"; } \ +# && { [[ "$("${CONTAINER_CMD[@]}" ncc maintenance:mode)" =~ .*disabled.* ]] || log_err $? "Maintenance mode is enabled or could not be retrieved"; } \ +# && { "${CONTAINER_CMD[@]}" redis-cli -s "$redis_socket" -a "$redis_pw" set redisready yes |& awk "{ print \"${LOG_DIAG} \" \$0 }" || log_err $? "Failed to set redis variable"; } \ +# && { "${CONTAINER_CMD[@]}" redis-cli -s "$redis_socket" -a "$redis_pw" get redisready |& awk "{ print \"${LOG_DIAG} \" \$0 }" || log_err $? "Failed to read redis variable"; } +# then +# echo -e "${LOG_CICD} Startup successful" +# success=true +# break +# fi +# attempt=$((attempt + 1)) +# sleep 5 +# done +# +# [[ "$success" == "true" ]] || { +# echo -e "${LOG_CICD} Timeout reached." +# "${CONTAINER_CMD[@]}" -q systemctl status mysql |& awk "{ print \"${LOG_DIAG} \" \$0 }" +# "${CONTAINER_CMD[@]}" -q systemctl status redis |& awk "{ print \"${LOG_DIAG} \" \$0 }" +# "${CONTAINER_CMD[@]}" -q systemctl status 'php*-fpm' |& awk "{ print \"${LOG_DIAG} \" \$0 }" +# "${CONTAINER_CMD[@]}" -q systemctl status apache2 |& awk "{ print \"${LOG_DIAG} \" \$0 }" +# "${CONTAINER_CMD[@]}" -q ncp-diag |& awk "{ print \"${LOG_DIAG} \" \$0 }" +# exit 1 +# } +# +# set -x +# set +e +# +# success=false +# for attempt in {1..5} +# do +# echo -e "${LOG_CICD} == System Tests (attempt $attempt/5) ==" +# sudo python tests/system_tests.py --non-interactive --skip-update-test |& awk "{ print \"${LOG_TEST} \" \$0 }" +# [[ ${PIPESTATUS[0]} -eq 0 ]] || { +# echo -e "${LOG_CICD} System test failed!" +# sleep 12 +# continue +# } +# success=true +# break +# done +# [[ "$success" == "true" ]] || { +# echo -e "${LOG_CICD} System test failed in all attempts!" +# exit 1 +# } +# +# success=false +# for attempt in {1..5} +# do +# echo -e "${LOG_CICD} == Nextcloud Tests (attempt $attempt/5) ==" +# python tests/nextcloud_tests.py --no-gui --skip-release-check "$ip" 443 4443 |& awk "{ print \"${LOG_TEST} \" \$0 }" +# [[ ${PIPESTATUS[0]} -eq 0 ]] || { +# echo -e "${LOG_CICD} Nextcloud test failed!" +# echo -e "{$LOG_DIAG} Geckodriver logs:" +# tail -n 20 geckodriver.log >&2 |& awk "{ print \"${LOG_DIAG} \" \$0 }" || true +# echo -e "${LOG_CICD} ================" +# echo -e "${LOG_DIAG} ncp.log: " +# "${CONTAINER_CMD[@]}" -q ncp /bin/bash -c "tail -n20 /var/log/ncp.log" |& awk "{ print \"${LOG_DIAG} \" \$0 }" || true +# echo "================" +# echo "${LOG_DIAG} Nextcloud log: " +# "${CONTAINER_CMD[@]}" -q ncp /bin/bash -c 'ls -l /opt/ncdata/data/nextcloud.log' |& awk "{ print \"${LOG_DIAG} \" \$0 }" || true +# "${CONTAINER_CMD[@]}" -q ncp /bin/bash -c 'cat /opt/ncdata/data/nextcloud.log' |& awk "{ print \"${LOG_DIAG} \" \$0 }" || true +# sleep 12 +# continue +# } +# success=true +# break +# done +# +# [[ "$success" == "true" ]] || { +# echo -e "${LOG_CICD} Nextcloud test failed in all attempts!" +# exit 1 +# } +# +# - name: Run dist-upgrade +# if: ${{ steps.download-previous-image.outputs.skipped == 'false' }} +# run: | +# set -ex +# +# source ./library.sh +# +# echo "Updating from $PREVIOUS_VERSION to $VERSION +# +# CONTAINER_CMD=(sudo systemd-run --machine=ncp -P --wait) +# +# sudo grep 'VERSION="11"' ./raspbian_root/etc/os-release || { +# echo "Can't dist-upgrade from debian version $(sudo grep 'Version=' ./raspbian_root/etc/os-release)" +# exit 1 +# } +# "${CONTAINER_CMD[@]}" DBG=x ncp-dist-upgrade "$VERSION" +# +# sudo machinectl terminate ncp +# sudo rm -f ./raspbian_root/opt/ncdata/data/nextcloud.log +# +# - name: Test image after dist-upgrade +# if: ${{ steps.download-previous-image.outputs.skipped == 'false' }} +# run: | +# +# log_err() { +# rc="${1?}" +# msg="${2?}" +# echo -e "${LOG_DIAG} $msg" >&2 +# return $rc +# } +# +# sudo systemd-nspawn --boot -D ./raspbian_root/ -M ncp --hostname=nextcloudpi |& awk "{ print \"${LOG_GUEST} \" \$0 }" & +# sleep 60 +# +# CONTAINER_CMD=(sudo systemd-run --machine=ncp -P --wait) +# +# success=false +# for attempt in {1..30} +# do +# echo -e "${LOG_CICD} == Wait until container network is available (attempt $attempt/30) ==" +# ip="$("${CONTAINER_CMD[@]}" bash -c '. /usr/local/etc/library.sh > /dev/null; get_ip')" +# [[ -n "$ip" ]] && curl -k "https://$ip/activate/" > /dev/null || { sleep 6; continue; } +# success=true +# break +# done +# sudo cat ./raspbian_root/var/log/ncp.log |& awk "{ print \"${LOG_NCP} \" \$0 }" +# sudo tail -n 0 -f ./raspbian_root/var/log/ncp.log |& awk "{ print \"${LOG_NCP} \" \$0 }" & +# +# [[ "$success" == "true" ]] || { +# echo -e "${LOG_CICD} Could not reach container. Aborting..." +# exit 1 +# } +# +# attempt=0 +# success=false +# for attempt in {1..150} +# do +# echo -e "${LOG_CICD} Waiting for container startup (attempt $attempt/150)..." +# "${CONTAINER_CMD[@]}" journalctl -eu redis.service || true +# "${CONTAINER_CMD[@]}" systemctl status php8.1-fpm.service || true +# redis_pw="$("${CONTAINER_CMD[@]}" bash -c ". /usr/local/etc/library.sh; get_nc_config_value 'redis\"][\"password'")" \ +# && redis_socket="$("${CONTAINER_CMD[@]}" bash -c ". /usr/local/etc/library.sh; get_nc_config_value 'redis\"][\"host'")" \ +# || log_err $? "Error retrieving redis credentials" || true +# if { "${CONTAINER_CMD[@]}" -q ncc status |& awk "{ print \"${LOG_DIAG} \" \$0 }" || log_err $? "ncc status check failed"; } \ +# && { [[ "$("${CONTAINER_CMD[@]}" ncc maintenance:mode)" =~ .*disabled.* ]] || log_err $? "Maintenance mode is enabled or could not be retrieved"; } \ +# && { "${CONTAINER_CMD[@]}" redis-cli -s "$redis_socket" -a "$redis_pw" set redisready yes |& awk "{ print \"${LOG_DIAG} \" \$0 }" || log_err $? "Failed to set redis variable"; } \ +# && { "${CONTAINER_CMD[@]}" redis-cli -s "$redis_socket" -a "$redis_pw" get redisready |& awk "{ print \"${LOG_DIAG} \" \$0 }" || log_err $? "Failed to read redis variable"; } +# then +# echo -e "${LOG_CICD} Startup successful" +# success=true +# break +# fi +# attempt=$((attempt + 1)) +# sleep 5 +# done +# +# [[ "$success" == "true" ]] || { +# echo -e "${LOG_CICD} Timeout reached." +# "${CONTAINER_CMD[@]}" -q systemctl status mysql |& awk "{ print \"${LOG_DIAG} \" \$0 }" +# "${CONTAINER_CMD[@]}" -q systemctl status redis |& awk "{ print \"${LOG_DIAG} \" \$0 }" +# "${CONTAINER_CMD[@]}" -q systemctl status 'php*-fpm' |& awk "{ print \"${LOG_DIAG} \" \$0 }" +# "${CONTAINER_CMD[@]}" -q systemctl status apache2 |& awk "{ print \"${LOG_DIAG} \" \$0 }" +# "${CONTAINER_CMD[@]}" -q ncp-diag |& awk "{ print \"${LOG_DIAG} \" \$0 }" +# exit 1 +# } +# +# set -x +# set +e +# +# success=false +# for attempt in {1..5} +# do +# echo -e "${LOG_CICD} == Activation Tests (attempt $attempt/5) ==" +# python tests/activation_tests.py -t 300 --no-gui "$ip" 443 4443 |& awk "{ print \"${LOG_TEST} \" \$0 }" +# [[ ${PIPESTATUS[0]} -eq 0 ]] || { +# echo -e "${LOG_CICD} Activation test failed!" +# echo -e "${LOG_DIAG} Geckodriver logs:" +# tail -n 20 geckodriver.log >&2 |& awk "{ print \"${LOG_DIAG} \" \$0 }" || true +# echo "================" +# echo -e "${LOG_DIAG} mysql: " +# "${CONTAINER_CMD[@]}" -q ncp-diag |& awk "{ print \"${LOG_DIAG} \" \$0 }" || true +# "${CONTAINER_CMD[@]}" -q systemctl status mysql |& awk "{ print \"${LOG_DIAG} \" \$0 }" || true +# sleep 12 +# continue +# } +# success=true +# break +# done +# [[ "$success" == "true" ]] || { +# echo -e "${LOG_CICD} Activation test failed in all attempts!" +# exit 1 +# } +# +# success=false +# for attempt in {1..5} +# do +# echo -e "${LOG_CICD} == System Tests (attempt $attempt/5) ==" +# sudo python tests/system_tests.py --non-interactive |& awk "{ print \"${LOG_TEST} \" \$0 }" +# [[ ${PIPESTATUS[0]} -eq 0 ]] || { +# echo -e "${LOG_CICD} System test failed!" +# sleep 12 +# continue +# } +# success=true +# break +# done +# [[ "$success" == "true" ]] || { +# echo -e "${LOG_CICD} System test failed in all attempts!" +# exit 1 +# } +# +# success=false +# for attempt in {1..5} +# do +# echo -e "${LOG_CICD} == Nextcloud Tests (attempt $attempt/5) ==" +# python tests/nextcloud_tests.py --no-gui "$ip" 443 4443 |& awk "{ print \"${LOG_TEST} \" \$0 }" +# [[ ${PIPESTATUS[0]} -eq 0 ]] || { +# echo -e "${LOG_CICD} Nextcloud test failed!" +# echo -e "{$LOG_DIAG} Geckodriver logs:" +# tail -n 20 geckodriver.log >&2 |& awk "{ print \"${LOG_DIAG} \" \$0 }" || true +# echo -e "${LOG_CICD} ================" +# echo -e "${LOG_DIAG} ncp.log: " +# "${CONTAINER_CMD[@]}" -q ncp /bin/bash -c "tail -n20 /var/log/ncp.log" |& awk "{ print \"${LOG_DIAG} \" \$0 }" || true +# echo "================" +# echo "${LOG_DIAG} Nextcloud log: " +# "${CONTAINER_CMD[@]}" -q ncp /bin/bash -c 'ls -l /opt/ncdata/data/nextcloud.log' |& awk "{ print \"${LOG_DIAG} \" \$0 }" || true +# "${CONTAINER_CMD[@]}" -q ncp /bin/bash -c 'cat /opt/ncdata/data/nextcloud.log' |& awk "{ print \"${LOG_DIAG} \" \$0 }" || true +# sleep 12 +# continue +# } +# success=true +# break +# done +# +# [[ "$success" == "true" ]] || { +# echo -e "${LOG_CICD} Nextcloud test failed in all attempts!" +# exit 1 +# } diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index b5d40ad1a..1c9a2cc9b 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -89,13 +89,22 @@ jobs: git_ref: "${{ inputs.git_ref || github.head_ref || github.ref_name }}" secrets: inherit - raspberrypi: + raspberrypi-4: if: ${{ inputs.sd-images || ( github.event_name != 'workflow_dispatch' && !startsWith(github.ref_name, 'docker-') ) }} uses: ./.github/workflows/build-sd-images.yml with: git_ref: "${{ inputs.git_ref || github.head_ref || github.ref_name }}" - board_id: raspberrypi - board_name: RaspberryPi + board_id: rpi4b + board_name: RaspberryPi4 + secrets: inherit + + raspberrypi-5: + if: ${{ inputs.sd-images || ( github.event_name != 'workflow_dispatch' && !startsWith(github.ref_name, 'docker-') ) }} + uses: ./.github/workflows/build-sd-images.yml + with: + git_ref: "${{ inputs.git_ref || github.head_ref || github.ref_name }}" + board_id: rpi5b + board_name: RaspberryPi5 secrets: inherit # TODO: Fix 32bit armbian images @@ -207,19 +216,31 @@ jobs: artifact_file: "${{ needs.lxd-arm64.outputs.lxc_artifact_file }}" dry_run: ${{ (!inputs.release && github.event_name == 'workflow_dispatch') || github.ref_type != 'tag' || !(github.ref_protected || startsWith(github.ref, 'refs/tags/v')) }} - raspberrypi-release: + raspberrypi-4-release: needs: - - raspberrypi + - raspberrypi-4 - github-release if: ${{ inputs.sd-images || github.event_name != 'workflow_dispatch' }} uses: ./.github/workflows/publish-image.yml with: git_ref: "${{ inputs.git_ref || github.head_ref || github.ref_name }}" - artifact_id: "${{ needs.raspberrypi.outputs.artifact_name }}" - artifact_file: "${{ needs.raspberrypi.outputs.artifact_file }}" + artifact_id: "${{ needs.raspberrypi-4.outputs.artifact_name }}" + artifact_file: "${{ needs.raspberrypi-4.outputs.artifact_file }}" dry_run: ${{ (!inputs.release && github.event_name == 'workflow_dispatch') || github.ref_type != 'tag' || !(github.ref_protected || startsWith(github.ref, 'refs/tags/v')) }} secrets: inherit + raspberrypi-5-release: + needs: + - raspberrypi-5 + - github-release + if: ${{ inputs.sd-images || github.event_name != 'workflow_dispatch' }} + uses: ./.github/workflows/publish-image.yml + with: + git_ref: "${{ inputs.git_ref || github.head_ref || github.ref_name }}" + artifact_id: "${{ needs.raspberrypi-5.outputs.artifact_name }}" + artifact_file: "${{ needs.raspberrypi-5.outputs.artifact_file }}" + dry_run: ${{ (!inputs.release && github.event_name == 'workflow_dispatch') || github.ref_type != 'tag' || !(github.ref_protected || startsWith(github.ref, 'refs/tags/v')) }} + secrets: inherit odroidxu4-release: needs: - odroidxu4 diff --git a/.github/workflows/vm-tests.yml b/.github/workflows/vm-tests.yml index ee8258939..4d61f12b7 100644 --- a/.github/workflows/vm-tests.yml +++ b/.github/workflows/vm-tests.yml @@ -18,53 +18,124 @@ on: - master jobs: - setup-installation-test-instance: + installation-test: runs-on: ubuntu-latest + container: + image: thecalcaholic/ncp-test-automation:bookworm + env: + HCLOUD_TOKEN: "${{ secrets.TEST_AUTOMATION_HCLOUD_API_TOKEN }}" + UID: "${{ github.run_id }}-install" outputs: server_address: ${{ steps.create-test-instance.outputs.server_address }} snapshot_id: ${{ steps.create-test-instance.outputs.snapshot_id }} test_server_id: ${{ steps.create-test-instance.outputs.test_server_id }} version: ${{ env.VERSION }} + test_result: ${{ steps.final_test.outputs.test_result }} + ssh_artifact_name: ${{ env.SSH_ARTIFACT_NAME }} env: VERSION: "${{ inputs.git_ref || github.head_ref || github.ref_name }}" + HOME: /root + SSH_ARTIFACT_NAME: "${{ github.run_id }}-install-ssh" + UID: "${{ github.run_id }}-install" + defaults: + run: + shell: bash steps: - uses: actions/checkout@v3 - - run: | + with: + path: /home/runner/actions-runner/_work/nextcloudpi/nextcloudpi + - name: Generate ssh keypair + working-directory: /__w/nextcloudpi/nextcloudpi + run: | set -e - mkdir -p ./.ssh + mkdir -p .ssh ssh-keygen -t ed25519 -f ".ssh/automation_ssh_key" + . /ncp-test-automation/bin/entrypoint.sh - name: upload ssh private key to artifact store uses: actions/upload-artifact@v3 with: - name: ${{ github.run_id }}-install-ssh-privkey - path: .ssh + name: "${{ env.SSH_ARTIFACT_NAME }}" + path: /__w/nextcloudpi/nextcloudpi/.ssh if-no-files-found: error - id: create-test-instance uses: ./.github/actions/create-test-instance with: version: ${{ env.VERSION }} - uid: "${{ github.run_id }}-install" + uid: "${{ env.UID }}" hcloud_token: ${{ secrets.TEST_AUTOMATION_HCLOUD_API_TOKEN }} server_type: "cx11" + - name: set instance variables + run: | + echo "SERVER_ADDRESS=${{ steps.create-test-instance.outputs.server_address }}" >> "$GITHUB_ENV" + echo "SNAPSHOT_ID=${{ steps.create-test-instance.outputs.snapshot_id }}" >> "$GITHUB_ENV" + - name: Test postinstall VM + id: final_test + working-directory: /ncp-test-automation/bin + run: | + set -e + echo "Setup ssh" + chmod 0600 /__w/nextcloudpi/nextcloudpi/.ssh/automation_ssh_key + eval "$(ssh-agent)" + ssh-add /__w/nextcloudpi/nextcloudpi/.ssh/automation_ssh_key + + source ./library.sh + + trap '[ $? -eq 0 ] || echo "test_result=failure" >> "$GITHUB_OUTPUT"; terminate-ssh-port-forwarding "${SERVER_ADDRESS}"' EXIT 1 2 + + setup-ssh-port-forwarding "$SERVER_ADDRESS" + + echo "Run integration tests" + ssh -o "StrictHostKeyChecking=no" -o "UserKnownHostsFile=/dev/null" "root@${SERVER_ADDRESS}" cat /usr/local/etc/instance.cfg + set -x + test-ncp-instance -a -f "$SNAPSHOT_ID" -b "${VERSION}" --systemtest-args "--skip-update-test" "root@${SERVER_ADDRESS}" "localhost" "8443" "9443" || { + + echo "Integration tests failed" + echo "Here are the last lines of ncp-install.log:" + echo "===========================================" + ssh "${SSH_OPTIONS[@]}" "root@${SERVER_ADDRESS}" tail /var/log/ncp-install.log; + echo "===========================================" + echo "and ncp.log:" + echo "===========================================" + ssh "${SSH_OPTIONS[@]}" "root@${SERVER_ADDRESS}" tail /var/log/ncp.log; + echo "===========================================" + exit 1 + } + echo "test_result=success" >> "$GITHUB_OUTPUT"; - setup-update-test-instance: + update-test: runs-on: ubuntu-latest + container: + image: thecalcaholic/ncp-test-automation:bullseye + env: + HCLOUD_TOKEN: "${{ secrets.TEST_AUTOMATION_HCLOUD_API_TOKEN }}" + UID: "${{ github.run_id }}-update" + defaults: + run: + shell: bash outputs: server_address: ${{ steps.create-test-instance.outputs.server_address }} snapshot_id: ${{ steps.create-test-instance.outputs.snapshot_id }} test_server_id: ${{ steps.create-test-instance.outputs.test_server_id }} previous_version: ${{ steps.find-version.outputs.previous_version }} version: ${{ env.VERSION }} + test_result: ${{ steps.final_test.outputs.test_result }} + ssh_artifact_name: ${{ env.SSH_ARTIFACT_NAME }} env: VERSION: "${{ inputs.git_ref || github.head_ref || github.ref_name }}" + HOME: /root + SSH_ARTIFACT_NAME: "${{ github.run_id }}-update-ssh" + UID: "${{ github.run_id }}-update" steps: - uses: actions/checkout@v3 with: fetch-depth: 0 + path: /__w/nextcloudpi/nextcloudpi - name: find reference version + working-directory: /__w/nextcloudpi/nextcloudpi shell: bash id: find-version run: | + chown -R "$(id -u):$(id -g)" . set -e if [[ -n "${{ github.base_ref }}" ]] then @@ -82,62 +153,44 @@ jobs: version="${version%-*-*}" fi echo "Previous version is '$version'" - echo "previous_version=${version}" >> $GITHUB_OUTPUT + echo "PREVIOUS_VERSION=${version}" >> "$GITHUB_ENV" - name: Generate ssh key run: | set -x - mkdir -p ./.ssh - ssh-keygen -t ed25519 -f ".ssh/automation_ssh_key" + mkdir -p /__w/nextcloudpi/nextcloudpi/.ssh + ssh-keygen -t ed25519 -f "/__w/nextcloudpi/nextcloudpi/.ssh/automation_ssh_key" + . /ncp-test-automation/bin/entrypoint.sh - name: upload ssh private key to artifact store uses: actions/upload-artifact@v3 with: - name: ${{ github.run_id }}-update-ssh-privkey - path: .ssh + name: "${{ env.SSH_ARTIFACT_NAME }}" + path: /__w/nextcloudpi/nextcloudpi/.ssh if-no-files-found: error - id: create-test-instance uses: ./.github/actions/create-test-instance with: - version: "${{ steps.find-version.outputs.previous_version }}" - uid: "${{ github.run_id }}-update" + version: "${{ env.PREVIOUS_VERSION }}" + uid: "${{ env.UID }}" hcloud_token: ${{ secrets.TEST_AUTOMATION_HCLOUD_API_TOKEN }} server_type: "cx11" - - run-installation-test: - needs: - - setup-installation-test-instance - runs-on: ubuntu-latest - - container: - image: thecalcaholic/ncp-test-automation:latest - env: - HCLOUD_TOKEN: "${{ secrets.TEST_AUTOMATION_HCLOUD_API_TOKEN }}" - UID: "${{ github.run_id }}-install" - env: - VERSION: ${{ needs.setup-installation-test-instance.outputs.version }} - SERVER_ADDRESS: "${{ needs.setup-installation-test-instance.outputs.server_address }}" - SNAPSHOT_ID: "${{ needs.setup-installation-test-instance.outputs.snapshot_id }}" - HOME: /root - defaults: - run: - shell: bash - steps: + - name: Set instance variables + run: | + echo "SERVER_ADDRESS=${{ steps.create-test-instance.outputs.server_address }}" >> "$GITHUB_ENV" + echo "SNAPSHOT_ID=${{ steps.create-test-instance.outputs.snapshot_id }}" >> "$GITHUB_ENV" - uses: actions/checkout@v3 with: repository: 'theCalcaholic/ncp-test-automation' - - name: download ssh private key from artifact store - uses: actions/download-artifact@v3 - with: - name: ${{ github.run_id }}-install-ssh-privkey - path: .ssh - - name: Test postinstall VM + ref: "bullseye" + path: /__w/nextcloudpi/nextcloudpi/ncp-test-automation + - name: Activate and Test postinstall VM + working-directory: /__w/nextcloudpi/nextcloudpi/ncp-test-automation/bin run: | set -e echo "Setup ssh" - chmod 0600 ./.ssh/automation_ssh_key + chmod 0600 /__w/nextcloudpi/nextcloudpi/.ssh/automation_ssh_key eval "$(ssh-agent)" - ssh-add ./.ssh/automation_ssh_key + ssh-add /__w/nextcloudpi/nextcloudpi/.ssh/automation_ssh_key - cd bin source ./library.sh trap 'terminate-ssh-port-forwarding "${SERVER_ADDRESS}"' EXIT 1 2 @@ -146,8 +199,8 @@ jobs: echo "Run integration tests" ssh -o "StrictHostKeyChecking=no" -o "UserKnownHostsFile=/dev/null" "root@${SERVER_ADDRESS}" cat /usr/local/etc/instance.cfg - test-ncp-instance -a -f "$SNAPSHOT_ID" -b "${VERSION}" --systemtest-args "--skip-update-test" "root@${SERVER_ADDRESS}" "localhost" "8443" "9443" || { - + test-ncp-instance -a -f "$SNAPSHOT_ID" -b "${VERSION}" --systemtest-args "--skip-update-test" --nc-test-args "--skip-release-check" "root@${SERVER_ADDRESS}" "localhost" "8443" "9443" || { + echo "Integration tests failed" echo "Here are the last lines of ncp-install.log:" echo "===========================================" @@ -159,44 +212,123 @@ jobs: echo "===========================================" exit 1 } + - name: perform update + working-directory: /__w/nextcloudpi/nextcloudpi/ncp-test-automation/bin + run: | + set -e + + echo "Setup ssh" + chmod 0600 /__w/nextcloudpi/nextcloudpi/.ssh/automation_ssh_key + eval "$(ssh-agent)" + ssh-add /__w/nextcloudpi/nextcloudpi/.ssh/automation_ssh_key + + source ./library.sh - run-update-test: - needs: - - setup-update-test-instance - runs-on: ubuntu-latest + echo "Updating from $PREVIOUS_VERSION to $VERSION" + ssh-keygen -f "$HOME/.ssh/known_hosts" -R "${SERVER_ADDRESS}" 2> /dev/null || true + ssh "${SSH_OPTIONS[@]}" "root@${SERVER_ADDRESS}" "ncp-update '$VERSION'" + - name: Run integration tests after update + id: final_test + working-directory: /__w/nextcloudpi/nextcloudpi/ncp-test-automation/bin + run: | + set -e + + echo "Setup ssh" + eval "$(ssh-agent)" + ssh-add /__w/nextcloudpi/nextcloudpi/.ssh/automation_ssh_key + + source ./library.sh + + trap '[ $? -eq 0 ] || echo "test_result=failure" >> "$GITHUB_OUTPUT"; terminate-ssh-port-forwarding "${SERVER_ADDRESS}"' EXIT 1 2 + + echo "Run integration tests" + setup-ssh-port-forwarding "$SERVER_ADDRESS" + NC_TEST_ARGS=() + ssh "${SSH_OPTIONS[@]}" "root@${SERVER_ADDRESS}" cat /etc/os-release | grep VERSION_ID=12 || NC_TEST_ARGS+=("--skip-release-check") + set -x + test-ncp-instance -f "$SNAPSHOT_ID" -b "${VERSION}" --nc-test-args "$NC_TEST_ARGS" --systemtest-args "--skip-update-test" "root@${SERVER_ADDRESS}" "localhost" "8443" "9443" || { + + echo "Integration tests failed" + echo "Here are the last lines of ncp-install.log:" + echo "===========================================" + ssh "${SSH_OPTIONS[@]}" "root@${SERVER_ADDRESS}" tail /var/log/ncp-install.log; + echo "===========================================" + echo "and ncp.log:" + echo "===========================================" + ssh "${SSH_OPTIONS[@]}" "root@${SERVER_ADDRESS}" tail /var/log/ncp.log; + echo "===========================================" + exit 1 + } + + echo "test_result=success" >> "$GITHUB_OUTPUT"; + dist-upgrade-test: + runs-on: ubuntu-latest container: - image: thecalcaholic/ncp-test-automation:latest + image: thecalcaholic/ncp-test-automation:bullseye env: HCLOUD_TOKEN: "${{ secrets.TEST_AUTOMATION_HCLOUD_API_TOKEN }}" - UID: "${{ github.run_id }}-update" - env: - PREVIOUS_VERSION: ${{ needs.setup-update-test-instance.outputs.previous_version }} - VERSION: ${{ needs.setup-update-test-instance.outputs.version }} - SERVER_ADDRESS: "${{ needs.setup-update-test-instance.outputs.server_address }}" - SNAPSHOT_ID: "${{ needs.setup-update-test-instance.outputs.snapshot_id }}" - HOME: /root + UID: "${{ github.run_id }}-distupgrade" defaults: run: shell: bash + outputs: + test_result: ${{ steps.final_test.outputs.test_result }} + ssh_artifact_name: ${{ env.SSH_ARTIFACT_NAME }} + server_address: ${{ steps.create-test-instance.outputs.server_address }} + snapshot_id: ${{ steps.create-test-instance.outputs.snapshot_id }} + test_server_id: ${{ steps.create-test-instance.outputs.test_server_id }} + previous_version: ${{ env.PREVIOUS_VERSION }} + version: ${{ env.VERSION }} + env: + PREVIOUS_VERSION: "v1.53.3" + VERSION: "${{ inputs.git_ref || github.head_ref || github.ref_name }}" + HOME: /root + SSH_ARTIFACT_NAME: "${{ github.run_id }}-distupgrade-ssh" + UID: "${{ github.run_id }}-distupgrade" steps: - uses: actions/checkout@v3 with: - repository: 'theCalcaholic/ncp-test-automation' - - name: download ssh private key from artifact store - uses: actions/download-artifact@v3 + fetch-depth: 0 + path: /__w/nextcloudpi/nextcloudpi + - name: Generate ssh key + run: | + set -x + chown -R "$(id -u):$(id -g)" /__w/nextcloudpi/nextcloudpi + mkdir -p /__w/nextcloudpi/nextcloudpi/.ssh + ssh-keygen -t ed25519 -f "/__w/nextcloudpi/nextcloudpi/.ssh/automation_ssh_key" + . /ncp-test-automation/bin/entrypoint.sh + - name: upload ssh private key to artifact store + uses: actions/upload-artifact@v3 + with: + name: "${{ env.SSH_ARTIFACT_NAME }}" + path: /__w/nextcloudpi/nextcloudpi/.ssh + if-no-files-found: error + - id: create-test-instance + uses: ./.github/actions/create-test-instance-bullseye with: - name: ${{ github.run_id }}-update-ssh-privkey - path: .ssh + version: "${{ env.PREVIOUS_VERSION }}" + uid: "${{ env.UID }}" + hcloud_token: ${{ secrets.TEST_AUTOMATION_HCLOUD_API_TOKEN }} + server_type: "cx11" + - name: Set instance variables + run: | + echo "SERVER_ADDRESS=${{ steps.create-test-instance.outputs.server_address }}" >> "$GITHUB_ENV" + echo "SNAPSHOT_ID=${{ steps.create-test-instance.outputs.snapshot_id }}" >> "$GITHUB_ENV" + - uses: actions/checkout@v3 + with: + repository: 'theCalcaholic/ncp-test-automation' + ref: "bullseye" + path: /__w/nextcloudpi/nextcloudpi/ncp-test-automation - name: Activate and Test postinstall VM + working-directory: /__w/nextcloudpi/nextcloudpi/ncp-test-automation/bin run: | set -e echo "Setup ssh" - chmod 0600 ./.ssh/automation_ssh_key + chmod 0600 /__w/nextcloudpi/nextcloudpi/.ssh/automation_ssh_key eval "$(ssh-agent)" - ssh-add ./.ssh/automation_ssh_key + ssh-add /__w/nextcloudpi/nextcloudpi/.ssh/automation_ssh_key - cd bin source ./library.sh trap 'terminate-ssh-port-forwarding "${SERVER_ADDRESS}"' EXIT 1 2 @@ -205,7 +337,7 @@ jobs: echo "Run integration tests" ssh -o "StrictHostKeyChecking=no" -o "UserKnownHostsFile=/dev/null" "root@${SERVER_ADDRESS}" cat /usr/local/etc/instance.cfg - test-ncp-instance -a -f "$SNAPSHOT_ID" -b "${VERSION}" --systemtest-args "--skip-update-test" "root@${SERVER_ADDRESS}" "localhost" "8443" "9443" || { + test-ncp-instance -a -f "$SNAPSHOT_ID" -b "${VERSION}" --systemtest-args "--skip-update-test" --nc-test-args "--skip-release-check" "root@${SERVER_ADDRESS}" "localhost" "8443" "9443" || { echo "Integration tests failed" echo "Here are the last lines of ncp-install.log:" @@ -219,32 +351,86 @@ jobs: exit 1 } - name: perform update + working-directory: /__w/nextcloudpi/nextcloudpi/ncp-test-automation/bin run: | set -e echo "Setup ssh" - chmod 0600 ./.ssh/automation_ssh_key + chmod 0600 /__w/nextcloudpi/nextcloudpi/.ssh/automation_ssh_key eval "$(ssh-agent)" - ssh-add ./.ssh/automation_ssh_key + ssh-add /__w/nextcloudpi/nextcloudpi/.ssh/automation_ssh_key - . ./bin/library.sh + source ./library.sh echo "Updating from $PREVIOUS_VERSION to $VERSION" ssh-keygen -f "$HOME/.ssh/known_hosts" -R "${SERVER_ADDRESS}" 2> /dev/null || true ssh "${SSH_OPTIONS[@]}" "root@${SERVER_ADDRESS}" "ncp-update '$VERSION'" - - name: Run integration tests + - name: Run integration tests after update + working-directory: /__w/nextcloudpi/nextcloudpi/ncp-test-automation/bin run: | set -e echo "Setup ssh" eval "$(ssh-agent)" - ssh-add ./.ssh/automation_ssh_key + ssh-add /__w/nextcloudpi/nextcloudpi/.ssh/automation_ssh_key - cd bin source ./library.sh trap 'terminate-ssh-port-forwarding "${SERVER_ADDRESS}"' EXIT 1 2 + echo "Run integration tests" + setup-ssh-port-forwarding "$SERVER_ADDRESS" + NC_TEST_ARGS=() + ssh "${SSH_OPTIONS[@]}" "root@${SERVER_ADDRESS}" cat /etc/os-release | grep VERSION_ID=12 || NC_TEST_ARGS+=("--skip-release-check") + set -x + test-ncp-instance -f "$SNAPSHOT_ID" -b "${VERSION}" --nc-test-args "$NC_TEST_ARGS" --systemtest-args "--skip-update-test" "root@${SERVER_ADDRESS}" "localhost" "8443" "9443" || { + + echo "Integration tests failed" + echo "Here are the last lines of ncp-install.log:" + echo "===========================================" + ssh "${SSH_OPTIONS[@]}" "root@${SERVER_ADDRESS}" tail /var/log/ncp-install.log; + echo "===========================================" + echo "and ncp.log:" + echo "===========================================" + ssh "${SSH_OPTIONS[@]}" "root@${SERVER_ADDRESS}" tail /var/log/ncp.log; + echo "===========================================" + exit 1 + } + + - name: NCP distupgrade + id: distupgrade + working-directory: /__w/nextcloudpi/nextcloudpi/ncp-test-automation/bin + run: | + set -e + + echo "Setup ssh" + eval "$(ssh-agent)" + ssh-add /__w/nextcloudpi/nextcloudpi/.ssh/automation_ssh_key + + source ./library.sh + + ssh "${SSH_OPTIONS[@]}" "root@${SERVER_ADDRESS}" cat /etc/os-release | grep 'VERSION_ID="11"' || { + echo "Can't upgrade from Debian $(ssh "${SSH_OPTIONS[@]}" "root@${SERVER_ADDRESS}" grep 'VERSION_ID=' /etc/os-release)" + echo "skipped=yes" | tee -a $GITHUB_OUTPUT + exit 1 + } + ssh "${SSH_OPTIONS[@]}" "root@${SERVER_ADDRESS}" DEBIAN_FRONTEND=noninteractive ncp-dist-upgrade + echo "skipped=no" | tee -a $GITHUB_OUTPUT + + - name: Run integration tests after dist-upgrade + id: final_test + working-directory: /ncp-test-automation/bin + run: | + set -e + + echo "Setup ssh" + eval "$(ssh-agent)" + ssh-add /__w/nextcloudpi/nextcloudpi/.ssh/automation_ssh_key + + source ./library.sh + + trap '[ $? -eq 0 ] || echo "test_result=failure" >> "$GITHUB_OUTPUT"; terminate-ssh-port-forwarding "${SERVER_ADDRESS}"' EXIT 1 2 + echo "Run integration tests" setup-ssh-port-forwarding "$SERVER_ADDRESS" @@ -255,18 +441,20 @@ jobs: echo "===========================================" ssh "${SSH_OPTIONS[@]}" "root@${SERVER_ADDRESS}" tail /var/log/ncp-install.log; echo "===========================================" - echo "and ncp.log:" + echo "ncp.log:" echo "===========================================" ssh "${SSH_OPTIONS[@]}" "root@${SERVER_ADDRESS}" tail /var/log/ncp.log; echo "===========================================" + echo "nextcloud.log:" + ssh "${SSH_OPTIONS[@]}" "root@${SERVER_ADDRESS}" tail /opt/ncdata/data/nextcloud.log; exit 1 } + echo "test_result=success" >> "$GITHUB_OUTPUT" install-postactivation-snapshot: if: ${{ always() }} needs: - - setup-installation-test-instance - - run-installation-test + - installation-test runs-on: ubuntu-latest container: image: thecalcaholic/ncp-test-automation:latest @@ -275,28 +463,28 @@ jobs: env: TEST_TYPE: install - SERVER_ADDRESS: ${{ needs.setup-installation-test-instance.outputs.server_address }} - TEST_RESULT: ${{ needs.setup-installation-test-instance.result }} - TEST_SERVER_ID: ${{ needs.setup-installation-test-instance.outputs.test_server_id }} - VERSION: ${{ needs.setup-installation-test-instance.outputs.version }} - + SERVER_ADDRESS: ${{ needs.installation-test.outputs.server_address }} + TEST_RESULT: ${{ needs.installation-test.test_result }} + TEST_SERVER_ID: ${{ needs.installation-test.outputs.test_server_id }} + VERSION: ${{ needs.installation-test.outputs.version }} + SSH_ARTIFACT_NAME: "${{ needs.installation-test.outputs.ssh_artifact_name }}" UID: ${{ github.run_id }}-install steps: - name: download ssh private key from artifact store uses: actions/download-artifact@v3 if: ${{ contains('success|failure', env.TEST_RESULT) }} with: - name: ${{ github.run_id }}-${{ env.TEST_TYPE }}-ssh-privkey + name: ${{ env.SSH_ARTIFACT_NAME }} path: /github/workspace/.ssh - name: Shutdown server if: ${{ contains('success|failure', env.TEST_RESULT) }} run: | - chmod 0600 /github/workspace/.ssh/automation_ssh_key - export SSH_PUBLIC_KEY="$(cat /github/workspace/.ssh/automation_ssh_key.pub)" + chmod 0600 /github/workspace/.ssh/automation_ssh_key + export SSH_PUBLIC_KEY="$(cat /github/workspace/.ssh/automation_ssh_key.pub)" bash /ncp-test-automation/bin/entrypoint.sh eval "$(ssh-agent)" - ssh-add /github/workspace/.ssh/automation_ssh_key - + ssh-add /github/workspace/.ssh/automation_ssh_key + ssh -o "StrictHostKeyChecking=no" -o "UserKnownHostsFile=/dev/null" "root@${SERVER_ADDRESS?}" <&2; exit 1; } - -APTINSTALL="apt-get install -y --no-install-recommends" -export DEBIAN_FRONTEND=noninteractive - -echo " ->>> ATTENTION <<< -This is a dangerous process that is only guaranteed to work properly if you -have not made manual changes in the system. Backup the SD card first and -proceed at your own risk. - -Note that this is not a requirement for NCP to continue working properly. -The current distribution will keep receiving updates for some time. - -Do you want to continue? [y/N]" - -read key -[[ "$key" == y ]] || exit 0 - -source /usr/local/etc/library.sh # sets NCPCFG RELEASE PHPVER -old_cfg="${NCPCFG}" - -trap "echo 'Something went wrong. Fix it and try again'" EXIT - -save_maintenance_mode - -# Fix grub-pc issue in VM -if apt show grub-pc-bin &>/dev/null; then - $APTINSTALL grub -fi - -apt-get update -apt-get upgrade -y - -# remove old PHP version -set +e -apt-get purge -y php${PHPVER} php${PHPVER}-curl php${PHPVER}-gd php${PHPVER}-fpm php${PHPVER}-cli php${PHPVER}-opcache \ - php${PHPVER}-mbstring php${PHPVER}-xml php${PHPVER}-zip php${PHPVER}-fileinfo php${PHPVER}-ldap \ - php${PHPVER}-intl php${PHPVER}-bz2 php${PHPVER}-json -apt-get purge -y php${PHPVER}-mysql -apt-get purge -y php${PHPVER}-redis -apt-get purge -y php${PHPVER}-exif -apt-get purge -y php${PHPVER}-bcmath -apt-get purge -y php${PHPVER}-gmp -apt-get purge -y php${PHPVER}-imagick set -e -# update sources -sed -i 's/buster/bullseye/g' /etc/apt/sources.list -sed -i 's/buster/bullseye/g' /etc/apt/sources.list.d/* || true -sed -i 's/bullseye\/updates/bullseye-security/g' /etc/apt/sources.list -rm -f /etc/apt/sources.list.d/php.list - -# fix DHCP systemd service command https://forums.raspberrypi.com/viewtopic.php?t=320383 in raspbian -if [[ -f /usr/bin/raspi-config ]]; then - sed -i 's|ExecStart=/usr/lib/dhcpcd5/dhcpcd -q -w|ExecStart=/usr/sbin/dhcpcd -q -w|g' /etc/systemd/system/dhcpcd.service.d/wait.conf +[[ "$UID" -eq 0 ]] || { + echo "ERROR: Must be run as root (try sudo ncp-dist-upgrade)" + exit 1 +} + +. /etc/os-release + +if [[ "$VERSION_ID" -eq 10 ]] +then + UPGRADE_CMD=(bash /usr/local/bin/ncp-dist-upgrade.d/debian-10.sh) +elif [[ "$VERSION_ID" -eq 11 ]] +then + UPGRADE_CMD=(bash /usr/local/bin/ncp-dist-upgrade.d/debian-11.sh) +else + echo "No dist-upgrade available for OS version: Debian ${VERSION}." + exit 0 fi -# install latest distro -apt-get update -apt-get dist-upgrade -y - -# install latest PHP version -release_new=$(jq -r '.release' < "${new_cfg}") -# the default repo in bullseye is bullseye-security - use bullseye if it is not available -grep -Eh '^deb ' /etc/apt/sources.list | grep 'bullseye-security' > /dev/null && release_new="${release_new}-security" -php_ver_new=$(jq -r '.php_version' < "${new_cfg}") -# PHP 8.1 is only supported via the -[[ "$php_ver_new" != 8.1 ]] || php_ver_new=7.4 - -$APTINSTALL -t ${release_new} php${php_ver_new} php${php_ver_new}-curl php${php_ver_new}-gd php${php_ver_new}-fpm php${php_ver_new}-cli php${php_ver_new}-opcache \ - php${php_ver_new}-mbstring php${php_ver_new}-xml php${php_ver_new}-zip php${php_ver_new}-fileinfo php${php_ver_new}-ldap \ - php${php_ver_new}-intl php${php_ver_new}-bz2 php${php_ver_new}-json - -$APTINSTALL php${php_ver_new}-mysql -$APTINSTALL -t ${release_new} php${php_ver_new}-redis - -$APTINSTALL -t ${release_new} smbclient exfat-fuse exfat-utils -sleep 2 # avoid systemd thinking that PHP is in a crash/restart loop -$APTINSTALL -t ${release_new} php${php_ver_new}-exif -sleep 2 # avoid systemd thinking that PHP is in a crash/restart loop -$APTINSTALL -t ${release_new} php${php_ver_new}-bcmath -sleep 2 # avoid systemd thinking that PHP is in a crash/restart loop -$APTINSTALL -t ${release_new} php${php_ver_new}-gmp -#$APTINSTALL -t ${release_new} imagemagick php${php_ver_new}-imagick ghostscript - -# Reinstall prometheus-node-exporter, specifically WITH install-recommends to include collectors on bullseye and later -{ dpkg -l | grep '^ii.*prometheus-node-exporter' >/dev/null && apt-get install -y prometheus-node-exporter-collectors; } || true - -apt-get autoremove -y -apt-get clean - -cat > /etc/php/${php_ver_new}/fpm/conf.d/90-ncp.ini <&2; exit 1; } + +APTINSTALL="apt-get install -y --no-install-recommends" +export DEBIAN_FRONTEND=noninteractive + +echo " +>>> ATTENTION <<< +This is a dangerous process that is only guaranteed to work properly if you +have not made manual changes in the system. Backup the SD card first and +proceed at your own risk. + +Note that this is not a requirement for NCP to continue working properly. +The current distribution will keep receiving updates for some time. + +Do you want to continue? [y/N]" + +read key +[[ "$key" == y ]] || exit 0 + +source /usr/local/etc/library.sh # sets NCPCFG RELEASE PHPVER +old_cfg="${NCPCFG}" + +trap "echo 'Something went wrong. Fix it and try again'" EXIT + +save_maintenance_mode + +# Fix grub-pc issue in VM +if apt show grub-pc-bin &>/dev/null; then + $APTINSTALL grub +fi + +apt-get update +apt-get upgrade -y + +# remove old PHP version +set +e +apt-get purge -y php${PHPVER} php${PHPVER}-curl php${PHPVER}-gd php${PHPVER}-fpm php${PHPVER}-cli php${PHPVER}-opcache \ + php${PHPVER}-mbstring php${PHPVER}-xml php${PHPVER}-zip php${PHPVER}-fileinfo php${PHPVER}-ldap \ + php${PHPVER}-intl php${PHPVER}-bz2 php${PHPVER}-json +apt-get purge -y php${PHPVER}-mysql +apt-get purge -y php${PHPVER}-redis +apt-get purge -y php${PHPVER}-exif +apt-get purge -y php${PHPVER}-bcmath +apt-get purge -y php${PHPVER}-gmp +apt-get purge -y php${PHPVER}-imagick +set -e + +# update sources +sed -i 's/buster/bullseye/g' /etc/apt/sources.list +sed -i 's/buster/bullseye/g' /etc/apt/sources.list.d/* || true +sed -i 's/bullseye\/updates/bullseye-security/g' /etc/apt/sources.list +rm -f /etc/apt/sources.list.d/php.list + +# fix DHCP systemd service command https://forums.raspberrypi.com/viewtopic.php?t=320383 in raspbian +if [[ -f /usr/bin/raspi-config ]]; then + sed -i 's|ExecStart=/usr/lib/dhcpcd5/dhcpcd -q -w|ExecStart=/usr/sbin/dhcpcd -q -w|g' /etc/systemd/system/dhcpcd.service.d/wait.conf +fi + +# install latest distro +apt-get update +apt-get dist-upgrade -y + +# install latest PHP version +release_new=$(jq -r '.release' < "${new_cfg}") +# the default repo in bullseye is bullseye-security - use bullseye if it is not available +grep -Eh '^deb ' /etc/apt/sources.list | grep 'bullseye-security' > /dev/null && release_new="${release_new}-security" +php_ver_new=$(jq -r '.php_version' < "${new_cfg}") +# PHP 8.1 is only supported via the +[[ "$php_ver_new" != 8.1 ]] || php_ver_new=7.4 + +$APTINSTALL -t ${release_new} php${php_ver_new} php${php_ver_new}-curl php${php_ver_new}-gd php${php_ver_new}-fpm php${php_ver_new}-cli php${php_ver_new}-opcache \ + php${php_ver_new}-mbstring php${php_ver_new}-xml php${php_ver_new}-zip php${php_ver_new}-fileinfo php${php_ver_new}-ldap \ + php${php_ver_new}-intl php${php_ver_new}-bz2 php${php_ver_new}-json + +$APTINSTALL php${php_ver_new}-mysql +$APTINSTALL -t ${release_new} php${php_ver_new}-redis + +$APTINSTALL -t ${release_new} smbclient exfat-fuse exfat-utils +sleep 2 # avoid systemd thinking that PHP is in a crash/restart loop +$APTINSTALL -t ${release_new} php${php_ver_new}-exif +sleep 2 # avoid systemd thinking that PHP is in a crash/restart loop +$APTINSTALL -t ${release_new} php${php_ver_new}-bcmath +sleep 2 # avoid systemd thinking that PHP is in a crash/restart loop +$APTINSTALL -t ${release_new} php${php_ver_new}-gmp +#$APTINSTALL -t ${release_new} imagemagick php${php_ver_new}-imagick ghostscript + +# Reinstall prometheus-node-exporter, specifically WITH install-recommends to include collectors on bullseye and later +{ dpkg -l | grep '^ii.*prometheus-node-exporter' >/dev/null && apt-get install -y prometheus-node-exporter-collectors; } || true + +apt-get autoremove -y +apt-get clean + +cat > /etc/php/${php_ver_new}/fpm/conf.d/90-ncp.ini <&2; exit 1; } + +echo " +>>> ATTENTION <<< +This is a dangerous process that is only guaranteed to work properly if you +have not made manual changes in the system. Backup the SD card first and +proceed at your own risk. + +Note that this is not a requirement for NCP to continue working properly. +The current distribution will keep receiving updates for some time. + +Do you want to continue? [y/N]" + +if [[ "${DEBIAN_FRONTEND:-}" == "noninteractive" ]] || ! [[ -t 0 ]] +then + echo "Noninteractive environment detected. Automatically proceeding in 30 seconds..." + sleep 30 +else + read -n1 -r key + [[ "${key,,}" == y ]] || exit 0 +fi + +export DEBIAN_FRONTEND=noninteractive + +source /usr/local/etc/library.sh +is_more_recent_than "${PHPVER}.0" "8.0.0" || { + echo "You still have PHP version ${PHPVER} installed. Please update to the latest supported version of nextcloud (which will also update your PHP version) before proceeding with the distribution upgrade." + echo "Exiting." + exit 1 +} +save_maintenance_mode + +# Perform dist-upgrade + +apt-get update && apt-get upgrade -y +for aptlist in /etc/apt/sources.list /etc/apt/sources.list.d/php.list /etc/apt/sources.list.d/armbian.list +do + [ -f "$aptlist" ] && sed -i -e "s/bullseye/bookworm/g" "$aptlist" +done +for aptlist in /etc/apt/sources.list.d/*.list +do + [[ "$aptlist" =~ "/etc/apt/sources.list.d/"(php|armbian)".list" ]] || continue + echo "Disabling repositories from \"$aptlist\"" + sed -i -e "s/deb/#deb/g" "$aptlist" +done +apt-get update && apt-get upgrade -y --without-new-pkgs +if is_lxc +then + # Required to avoid breakage of /etc/resolv.conf + apt-get install -y --no-install-recommends systemd-resolved && systemctl enable --now systemd-resolved +fi +apt-get full-upgrade -y +sudo apt-get --purge autoremove -y + +apt-get install -y --no-install-recommends exfatprogs + +#mkdir -p /etc/systemd/system/php8.1-fpm.service.d +#echo '[Service]' > /etc/systemd/system/php8.1-fpm.service.d/ncp.conf +#echo 'ExecStartPre=mkdir -p /var/run/php' >> /etc/systemd/system/php8.1-fpm.service.d/ncp.conf +#[[ "$INIT_SYSTEM" != "systemd" ]] || { systemctl daemon-reload && systemctl restart php8.1-fpm; } + +restore_maintenance_mode +cfg="$(jq "." "$NCPCFG")" +cfg="$(jq ".release = \"bookworm\"" <<<"$cfg")" +echo "$cfg" > "$NCPCFG" +rm -f /etc/update-motd.d/30ncp-dist-upgrade + +echo "Update to Debian 12 (bookworm) successful." + +is_active_app unattended-upgrades && { + echo "Setting up unattended upgrades..." + run_app unattended-upgrades || true + echo "done." +} \ No newline at end of file diff --git a/ncp.sh b/ncp.sh index bbe57b6a5..6233cf639 100644 --- a/ncp.sh +++ b/ncp.sh @@ -22,7 +22,7 @@ install() { # NCP-CONFIG apt-get update - $APTINSTALL git dialog whiptail jq file lsb-release + $APTINSTALL git dialog whiptail jq file lsb-release tmux mkdir -p "$CONFDIR" "$BINDIR" # This has changed, pi user no longer exists by default, the user needs to create it with Raspberry Pi imager diff --git a/tests/nextcloud_tests.py b/tests/nextcloud_tests.py index b2e4cfc4b..5be92aced 100755 --- a/tests/nextcloud_tests.py +++ b/tests/nextcloud_tests.py @@ -140,7 +140,7 @@ def close_first_run_wizard(driver: WebDriver): time.sleep(3) -def test_nextcloud(IP: str, nc_port: str, driver: WebDriver): +def test_nextcloud(IP: str, nc_port: str, driver: WebDriver, skip_release_check: bool): """ Login and assert admin page checks""" test = Test() test.new("nextcloud page") @@ -263,8 +263,11 @@ def test_nextcloud(IP: str, nc_port: str, driver: WebDriver): expected['ncp_version'] = True elif 'php version' in divs[0].text.lower() and divs[1].text == ncp_cfg['php_version']: expected['php_version'] = True - elif 'debian release' in divs[0].text.lower() and divs[1].text == ncp_cfg['release']: - expected['debian_release'] = True + elif 'debian release' in divs[0].text.lower(): + if divs[1].text == ncp_cfg['release'] or skip_release_check: + expected['debian_release'] = True + else: + print(f"{tc.yellow}{divs[1].text} != {ncp_cfg['release']}") failed = list(map(lambda item: item[0], filter(lambda item: not item[1], expected.items()))) test.check(len(failed) == 0, f"checks failed for admin section: [{', '.join(failed)}]") except Exception as e: @@ -302,11 +305,12 @@ def test_nextcloud(IP: str, nc_port: str, driver: WebDriver): # parse options try: - opts, args = getopt.getopt(sys.argv[1:], 'hn', ['help', 'new', 'no-gui']) + opts, args = getopt.getopt(sys.argv[1:], 'hn', ['help', 'new', 'no-gui', 'skip-release-check']) except getopt.GetoptError: usage() sys.exit(2) + skip_release_check = False options = webdriver.FirefoxOptions() for opt, arg in opts: if opt in ('-h', '--help'): @@ -317,6 +321,8 @@ def test_nextcloud(IP: str, nc_port: str, driver: WebDriver): os.unlink(test_cfg) elif opt == '--no-gui': options.add_argument("-headless") + elif opt == '--skip-release-check': + skip_release_check = True else: usage() sys.exit(2) @@ -355,7 +361,7 @@ def test_nextcloud(IP: str, nc_port: str, driver: WebDriver): driver = webdriver.Firefox(options=options) failed=False try: - test_nextcloud(IP, nc_port, driver) + test_nextcloud(IP, nc_port, driver, skip_release_check) except Exception as e: print(e) print(traceback.format_exc()) From 3548fea9ea5bbdcb8d4d57588f2812fb363ce33b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tobias=20Kn=C3=B6ppler?= <6317548+theCalcaholic@users.noreply.github.com> Date: Sun, 28 Apr 2024 18:40:05 +0200 Subject: [PATCH 04/14] armbian.sh: Disable autologin, require root password and use passwd -l instead of nologin shell for root login prevention MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Tobias Knöppler <6317548+theCalcaholic@users.noreply.github.com> --- build/armbian/armbian.sh | 6 ++++++ ncp.sh | 2 +- updates/1.54.0.sh | 10 ++++++++++ 3 files changed, 17 insertions(+), 1 deletion(-) create mode 100644 updates/1.54.0.sh diff --git a/build/armbian/armbian.sh b/build/armbian/armbian.sh index 554c31cf2..985121062 100644 --- a/build/armbian/armbian.sh +++ b/build/armbian/armbian.sh @@ -39,6 +39,12 @@ run_app_unsafe post-inst.sh # disable SSH by default, it can be enabled through ncp-web systemctl disable ssh +# disable armbian autologin +rm -f /etc/systemd/system/getty@.service.d/override.conf +rm -f /etc/systemd/system/serial-getty@.service.d/override.conf +rm -f /root/.not_logged_in_yet +sed -i 's|^root::|root:x:|' /etc/passwd + basename "$IMG" | tee /usr/local/etc/ncp-baseimage cd - diff --git a/ncp.sh b/ncp.sh index 6233cf639..b907c74ac 100644 --- a/ncp.sh +++ b/ncp.sh @@ -82,7 +82,7 @@ EOF echo -e "$WEBPASSWD\n$WEBPASSWD" | passwd "$WEBADMIN" is_docker || is_lxc || { chsh -s /usr/sbin/nologin "$WEBADMIN" - chsh -s /usr/sbin/nologin root + passwd -l root } ## NCP LAUNCHER diff --git a/updates/1.54.0.sh b/updates/1.54.0.sh new file mode 100644 index 000000000..04626894f --- /dev/null +++ b/updates/1.54.0.sh @@ -0,0 +1,10 @@ +#!/usr/bin/env bash + +if getent passwd "$LOGNAME" | grep -e 'root' | grep -e '/usr/sbin/nologin' +then + chsh -s /bin/bash root + passwd -l root +fi + +apt-get update +apt-get install --no-install-recommends -y tmux From 339fb0506375d5cc04e9bf87ac96a3806a9b5c67 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tobias=20Kn=C3=B6ppler?= <6317548+theCalcaholic@users.noreply.github.com> Date: Sun, 28 Apr 2024 18:43:19 +0200 Subject: [PATCH 05/14] dnsmasq.sh: Fix dns resolv workaround for hosts using resolvconf instead of systemd-resolved MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Tobias Knöppler <6317548+theCalcaholic@users.noreply.github.com> --- bin/ncp/NETWORKING/dnsmasq.sh | 44 ++++++++++++++--------------------- 1 file changed, 18 insertions(+), 26 deletions(-) diff --git a/bin/ncp/NETWORKING/dnsmasq.sh b/bin/ncp/NETWORKING/dnsmasq.sh index 9e5ce3a54..4f93c89f8 100644 --- a/bin/ncp/NETWORKING/dnsmasq.sh +++ b/bin/ncp/NETWORKING/dnsmasq.sh @@ -16,38 +16,30 @@ install() apt-get install --no-install-recommends -y dnsmasq rc=0 service dnsmasq status > /dev/null 2>&1 || rc=$? - ! is_docker && [[ $rc -eq 3 ]] && ! [[ "$INIT_SYSTEM" =~ ^("chroot"|"unknown")$ ]] && { + [[ $rc -eq 3 ]] && ! [[ "$INIT_SYSTEM" =~ ^("chroot"|"unknown")$ ]] && command -v systemd-resolve > /dev/null || { echo "Applying workaround for dnsmasq bug (compare issue #1446)" - service systemd-resolved stop || true - service dnsmasq start - service dnsmasq status + mkdir -p /etc/systemd/resolved.conf.d + if systemctl status systemd-resolved + then + cat < /etc/systemd/resolved.conf.d/nostublistener.conf +[Resolve] +DNSStubListener=no +EOF + [[ "$INIT_SYSTEM" != "systemd" ]] || systemctl restart systemd-resolved + else + systemctl stop resolvconf + systemctl start dnsmasq + systemctl status dnsmasq + fi +# service systemd-resolved stop || true + systemctl start dnsmasq + systemctl status dnsmasq } service dnsmasq stop - [[ "$INIT_SYSTEM" == "systemd" ]] && service systemd-resolved start || true + [[ "$INIT_SYSTEM" != "systemd" ]] || systemctl start systemd-resolved || systemctl start resolvconf update-rc.d dnsmasq disable || rm /etc/systemd/system/multi-user.target.wants/dnsmasq.service - [[ "$DOCKERBUILD" == 1 ]] && { - cat > /etc/services-available.d/100dnsmasq < Date: Sun, 28 Apr 2024 18:44:46 +0200 Subject: [PATCH 06/14] ncp.sh,nc-passwd.sh: Add activation hint in tty login screen MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Tobias Knöppler <6317548+theCalcaholic@users.noreply.github.com> --- bin/ncp/CONFIG/nc-passwd.sh | 1 + ncp.sh | 4 ++++ update.sh | 4 +--- 3 files changed, 6 insertions(+), 3 deletions(-) diff --git a/bin/ncp/CONFIG/nc-passwd.sh b/bin/ncp/CONFIG/nc-passwd.sh index e015117da..2f85b53fc 100644 --- a/bin/ncp/CONFIG/nc-passwd.sh +++ b/bin/ncp/CONFIG/nc-passwd.sh @@ -29,6 +29,7 @@ configure() a2dissite ncp-activation a2ensite ncp 001-nextcloud apachectl -k graceful + sed -i -e 's/^NCP is not activated yet.*$//' /etc/issue # Trusted Domain (local/public IP), also configures notify_push bash /usr/local/bin/nextcloud-domain.sh diff --git a/ncp.sh b/ncp.sh index b907c74ac..8813c60cd 100644 --- a/ncp.sh +++ b/ncp.sh @@ -244,6 +244,10 @@ EOF #!/bin/bash /usr/local/bin/ncp-check-updates EOF + + echo ' +NCP is not activated yet. Please enter https://nextcloudpi.local or this instance'"'"'s local IP address in your webbrowser to complete activation. You can find detailed instructions at https://nextcloudpi.com/activate +' >> /etc/issue chmod a+x /etc/update-motd.d/* ## HOSTNAME AND mDNS diff --git a/update.sh b/update.sh index 78e9fed2b..88a5e65c5 100755 --- a/update.sh +++ b/update.sh @@ -201,9 +201,7 @@ check_distro "$NCPCFG" && check_distro etc/ncp.cfg || { cfg="$(jq '.release = "'$release_new'"' <<<"$cfg")" echo "$cfg" > /usr/local/etc/ncp-recommended.cfg - [[ -f /.dockerenv ]] && \ - msg="Update to $release_new available. Get the latest container to upgrade" || \ - msg="Update to $release_new available. Type 'sudo ncp-dist-upgrade' to upgrade" + msg="Update to $release_new available. Type 'sudo ncp-dist-upgrade' to upgrade" echo "${msg}" notify_admin "New distribution available" "${msg}" wall "${msg}" From d00f870161157040f273f2d96a75b0a2c3494507 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tobias=20Kn=C3=B6ppler?= <6317548+theCalcaholic@users.noreply.github.com> Date: Sun, 28 Apr 2024 18:45:31 +0200 Subject: [PATCH 07/14] Remove ownyourbits.com in various places MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Tobias Knöppler <6317548+theCalcaholic@users.noreply.github.com> --- bin/ncp/CONFIG/nc-init.sh | 6 +++--- bin/ncp/SECURITY/fail2ban.sh | 2 +- etc/ncp-config.d/dnsmasq.cfg | 4 ++-- etc/ncp-config.d/letsencrypt.cfg | 6 +++--- etc/ncp-config.d/nc-trusted-domains.cfg | 4 ++-- etc/ncp-config.d/no-ip.cfg | 4 ++-- ncp-app/appinfo/info.xml | 2 +- 7 files changed, 14 insertions(+), 14 deletions(-) diff --git a/bin/ncp/CONFIG/nc-init.sh b/bin/ncp/CONFIG/nc-init.sh index 96f182504..e6768af50 100644 --- a/bin/ncp/CONFIG/nc-init.sh +++ b/bin/ncp/CONFIG/nc-init.sh @@ -111,8 +111,8 @@ EOF # email ncc config:system:set mail_smtpmode --value="sendmail" ncc config:system:set mail_smtpauthtype --value="LOGIN" - ncc config:system:set mail_from_address --value="admin" - ncc config:system:set mail_domain --value="ownyourbits.com" + ncc config:system:set mail_from_address --value="noreply" + ncc config:system:set mail_domain --value="nextcloudpi.com" # Fix NCP theme [[ -e /usr/local/etc/logo ]] && { @@ -129,7 +129,7 @@ EOF mysql nextcloud < 0.0.2 agpl - nachoparker + Tobias Knöppler NextcloudPi tools https://github.com/nextcloud/nextcloudpi/issues From d36ba71438db4b6ec36d739479729cb29a4605e4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tobias=20Kn=C3=B6ppler?= <6317548+theCalcaholic@users.noreply.github.com> Date: Sun, 28 Apr 2024 20:08:12 +0200 Subject: [PATCH 08/14] .vm-tests.yml: Use fixed previous version and create-test-instance-bullseye in update test MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Tobias Knöppler <6317548+theCalcaholic@users.noreply.github.com> --- .github/workflows/release.yml | 82 +--------------------------------- .github/workflows/vm-tests.yml | 6 ++- 2 files changed, 5 insertions(+), 83 deletions(-) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 1c9a2cc9b..ec61ff9a7 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -16,11 +16,6 @@ on: description: 'Build/test LXD image?' required: false default: true - docker: - type: boolean - description: 'Build/test docker images?' - required: false - default: true installer: type: boolean description: 'Run curl-installer/upgrade tests?' @@ -28,16 +23,14 @@ on: default: true release: type: boolean - description: 'Release images to GH and docker registry' + description: 'Release images to GH' required: false default: false push: tags: - "v*" - - "docker-v*" branches: - master - - docker-stable permissions: contents: write @@ -58,30 +51,6 @@ jobs: arch: 'arm64' secrets: inherit - docker-x86: - if: ${{ inputs.docker || ( github.event_name != 'workflow_dispatch' && startsWith(github.ref_name, 'docker-') ) }} - uses: ./.github/workflows/build-docker.yml - with: - git_ref: "${{ inputs.git_ref || github.head_ref || github.ref_name }}" - arch: "x86" - secrets: inherit - - docker-armhf: - if: ${{ inputs.docker || ( github.event_name != 'workflow_dispatch' && startsWith(github.ref_name, 'docker-') ) }} - uses: ./.github/workflows/build-docker.yml - with: - git_ref: "${{ inputs.git_ref || github.head_ref || github.ref_name }}" - arch: "armhf" - secrets: inherit - - docker-arm64: - if: ${{ inputs.docker || ( github.event_name != 'workflow_dispatch' && startsWith(github.ref_name, 'docker-') ) }} - uses: ./.github/workflows/build-docker.yml - with: - git_ref: "${{ inputs.git_ref || github.head_ref || github.ref_name }}" - arch: "arm64" - secrets: inherit - test-curl-installer: if: ${{ inputs.installer || ( github.event_name != 'workflow_dispatch' && !startsWith(github.ref_name, 'docker-') ) }} uses: ./.github/workflows/vm-tests.yml @@ -326,55 +295,6 @@ jobs: dry_run: ${{ (!inputs.release && github.event_name == 'workflow_dispatch') || github.ref_type != 'tag' || !(github.ref_protected || startsWith(github.ref, 'refs/tags/v')) }} secrets: inherit - docker-release: - needs: - - docker-x86 - - docker-armhf - - docker-arm64 - if: ${{ (inputs.release || github.event_name != 'workflow_dispatch') && github.ref_type == 'tag' && (github.ref_protected || startsWith(github.ref, 'refs/tags/docker-v')) }} - runs-on: ubuntu-latest - steps: - - name: Login to DockerHub - uses: docker/login-action@v1 - with: - username: ${{ secrets.DOCKERHUB_USERNAME }} - password: ${{ secrets.DOCKERHUB_TOKEN }} - - - name: Checkout code - uses: actions/checkout@v3 - - - name: Create manifest and push as tag to docker hub - run: | - . ./build/buildlib.sh - version="${version#docker-}" - - for arch in x86 armhf arm64 - do - docker pull "thecalcaholic/ncp-internal-${arch}:${{ github.run_id }}" - docker tag "thecalcaholic/ncp-internal-${arch}:${{ github.run_id }}" "ownyourbits/nextcloudpi-${arch}:${version?}" - docker tag "ownyourbits/nextcloudpi-${arch}:${version?}" "ownyourbits/nextcloudpi-${arch}:latest" - docker push "ownyourbits/nextcloudpi-${arch}:${version?}" - [[ "$version" =~ ^(docker-)?v[0-9]+'.'[0-9]+'.'[0-9]+$ ]] && docker push "ownyourbits/nextcloudpi-${arch}:latest" - done - - docker manifest create ownyourbits/nextcloudpi:${version?} \ - ownyourbits/nextcloudpi-armhf:${version?} \ - ownyourbits/nextcloudpi-x86:${version?} \ - ownyourbits/nextcloudpi-arm64:${version?} - docker manifest push ownyourbits/nextcloudpi:${version?} - - - name: Create manifest and push as latest to docker hub - run: | - [[ "${{ github.ref }}" =~ ^'refs/tags/'(docker-)?'v'[0-9]+'.'[0-9]+'.'[0-9]+$ ]] || { - echo "not tagging latest - is a pre-/beta-release" - exit 0 - } - docker manifest create ownyourbits/nextcloudpi:latest \ - ownyourbits/nextcloudpi-armhf:latest \ - ownyourbits/nextcloudpi-x86:latest \ - ownyourbits/nextcloudpi-arm64:latest - docker manifest push ownyourbits/nextcloudpi:latest - github-release: if: ${{ github.event_name == 'workflow_dispatch' || !startsWith(github.ref_name, 'docker-') }} needs: diff --git a/.github/workflows/vm-tests.yml b/.github/workflows/vm-tests.yml index 4d61f12b7..d01a97bfd 100644 --- a/.github/workflows/vm-tests.yml +++ b/.github/workflows/vm-tests.yml @@ -153,7 +153,9 @@ jobs: version="${version%-*-*}" fi echo "Previous version is '$version'" - echo "PREVIOUS_VERSION=${version}" >> "$GITHUB_ENV" + #TODO: Revert to dynamically found version + #echo "PREVIOUS_VERSION=${version}" >> "$GITHUB_ENV" + echo "PREVIOUS_VERSION=v1.53.3" >> "$GITHUB_ENV" - name: Generate ssh key run: | set -x @@ -167,7 +169,7 @@ jobs: path: /__w/nextcloudpi/nextcloudpi/.ssh if-no-files-found: error - id: create-test-instance - uses: ./.github/actions/create-test-instance + uses: ./.github/actions/create-test-instance-bullseye with: version: "${{ env.PREVIOUS_VERSION }}" uid: "${{ env.UID }}" From 7c82963363fef5cb5f67f1cf01c7889ac8d00688 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tobias=20Kn=C3=B6ppler?= <6317548+theCalcaholic@users.noreply.github.com> Date: Sun, 28 Apr 2024 22:52:54 +0200 Subject: [PATCH 09/14] install.sh,ncp.sh,1.54.0.sh: Disable root login via ssh MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Tobias Knöppler <6317548+theCalcaholic@users.noreply.github.com> --- bin/ncp-dist-upgrade | 2 +- install.sh | 11 +++++++++++ ncp.sh | 1 + updates/1.54.0.sh | 3 ++- 4 files changed, 15 insertions(+), 2 deletions(-) diff --git a/bin/ncp-dist-upgrade b/bin/ncp-dist-upgrade index 54e23c06b..2721fe896 100755 --- a/bin/ncp-dist-upgrade +++ b/bin/ncp-dist-upgrade @@ -2,7 +2,7 @@ set -e -[[ "$UID" -eq 0 ]] || { +[[ "${EUID}" -eq 0 ]] || { echo "ERROR: Must be run as root (try sudo ncp-dist-upgrade)" exit 1 } diff --git a/install.sh b/install.sh index 24c051ecd..b36c94e4a 100644 --- a/install.sh +++ b/install.sh @@ -28,6 +28,17 @@ export PATH="/usr/local/sbin:/usr/sbin:/sbin:${PATH}" type mysqld &>/dev/null && echo ">>> WARNING: existing mysqld configuration will be changed <<<" type mysqld &>/dev/null && mysql -e 'use nextcloud' &>/dev/null && { echo "The 'nextcloud' database already exists. Aborting"; exit 1; } +[[ "$DEBIAN_FRONTEND" == "noninteractive" ]] || { + echo "WARNING: This installer will disable SSH login for the root user and reset its password. +If you need to login with root, you should make sure, you have a root session open that you can use, +to revert these changes afterwards (set PermitRootLogin to 'yes' in /etc/ssh/sshd_config and run passwd as root)." + for i in {1..10} + do + echo "Continuing in $((30-(3*i)))s (press Ctrl+C to abort)..." + sleep 3 + done +} + # get dependencies apt-get update DEBIAN_FRONTEND=noninteractive apt-get install --no-install-recommends -y git ca-certificates sudo lsb-release wget jq diff --git a/ncp.sh b/ncp.sh index 8813c60cd..4eff4a150 100644 --- a/ncp.sh +++ b/ncp.sh @@ -83,6 +83,7 @@ EOF is_docker || is_lxc || { chsh -s /usr/sbin/nologin "$WEBADMIN" passwd -l root + sed -i -e 's/^PermitRootLogin.*$/PermitRootLogin No/' /etc/ssh/sshd_config } ## NCP LAUNCHER diff --git a/updates/1.54.0.sh b/updates/1.54.0.sh index 04626894f..a3dcfba20 100644 --- a/updates/1.54.0.sh +++ b/updates/1.54.0.sh @@ -1,9 +1,10 @@ #!/usr/bin/env bash -if getent passwd "$LOGNAME" | grep -e 'root' | grep -e '/usr/sbin/nologin' +if getent passwd "root" | grep -e '/usr/sbin/nologin' then chsh -s /bin/bash root passwd -l root + sed -i -e 's/^PermitRootLogin.*$/PermitRootLogin No/' /etc/ssh/sshd_config fi apt-get update From 97e27883fe594427d51f25b68d31b3347cd94bdc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tobias=20Kn=C3=B6ppler?= <6317548+theCalcaholic@users.noreply.github.com> Date: Sun, 28 Apr 2024 23:12:42 +0200 Subject: [PATCH 10/14] Add support for Nextcloud 28.0.5 MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Tobias Knöppler <6317548+theCalcaholic@users.noreply.github.com> --- etc/ncp-config.d/nc-nextcloud.cfg | 2 +- etc/ncp.cfg | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/etc/ncp-config.d/nc-nextcloud.cfg b/etc/ncp-config.d/nc-nextcloud.cfg index ef57d9171..85dc839c4 100644 --- a/etc/ncp-config.d/nc-nextcloud.cfg +++ b/etc/ncp-config.d/nc-nextcloud.cfg @@ -9,7 +9,7 @@ { "id": "VER", "name": "Version", - "value": "28.0.4" + "value": "28.0.5" }, { "id": "MAXFILESIZE", diff --git a/etc/ncp.cfg b/etc/ncp.cfg index bbe7137c0..c638b358b 100644 --- a/etc/ncp.cfg +++ b/etc/ncp.cfg @@ -1,5 +1,5 @@ { - "nextcloud_version": "28.0.4", + "nextcloud_version": "28.0.5", "php_version": "8.1", "release": "bookworm" } From 219c6da34aa25137c3dd4ac6ee3a017cc6e16566 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tobias=20Kn=C3=B6ppler?= <6317548+theCalcaholic@users.noreply.github.com> Date: Sun, 28 Apr 2024 23:13:41 +0200 Subject: [PATCH 11/14] Enable canary group for staging releases MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Tobias Knöppler <6317548+theCalcaholic@users.noreply.github.com> --- bin/ncp-check-version | 2 -- 1 file changed, 2 deletions(-) diff --git a/bin/ncp-check-version b/bin/ncp-check-version index dc499ad65..ddee1071a 100755 --- a/bin/ncp-check-version +++ b/bin/ncp-check-version @@ -17,8 +17,6 @@ cd "$TEMPDIR" || exit 1 VER=$( git describe --always --tags | grep -oP "v\d+\.\d+\.\d+" ) canary="$(. /usr/local/etc/library.sh; find_app_param ncp-community.sh CANARY)" -# TODO: Remove temporary canary override after staged rollouts test -canary="no" [[ "$canary" != "yes" ]] && [[ -f "/usr/local/etc/instance.cfg" ]] && { cohorte_id="$(jq .cohorteId /usr/local/etc/instance.cfg)" From 82ccc2abb8f714d4892f313414f2b4cf000ded2f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tobias=20Kn=C3=B6ppler?= <6317548+theCalcaholic@users.noreply.github.com> Date: Sun, 28 Apr 2024 23:16:48 +0200 Subject: [PATCH 12/14] Enable v1.54.0 for 10% of users MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Tobias Knöppler <6317548+theCalcaholic@users.noreply.github.com> --- staged_rollouts/v1.54.0.txt | 10 ++++++++++ 1 file changed, 10 insertions(+) create mode 100644 staged_rollouts/v1.54.0.txt diff --git a/staged_rollouts/v1.54.0.txt b/staged_rollouts/v1.54.0.txt new file mode 100644 index 000000000..9db674ceb --- /dev/null +++ b/staged_rollouts/v1.54.0.txt @@ -0,0 +1,10 @@ +39 +21 +52 +77 +71 +80 +90 +29 +6 +62 \ No newline at end of file From 2d1345f6df2982b6fcd7c44449f438bb0cb3fd0d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tobias=20Kn=C3=B6ppler?= <6317548+theCalcaholic@users.noreply.github.com> Date: Mon, 29 Apr 2024 00:40:01 +0200 Subject: [PATCH 13/14] library.sh: Remove invalid trusted_proxy MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Tobias Knöppler <6317548+theCalcaholic@users.noreply.github.com> --- .github/workflows/build-lxd.yml | 1 - etc/library.sh | 2 +- updates/1.54.0.sh | 6 ++++++ 3 files changed, 7 insertions(+), 2 deletions(-) diff --git a/.github/workflows/build-lxd.yml b/.github/workflows/build-lxd.yml index f84fe0786..c25cdb214 100644 --- a/.github/workflows/build-lxd.yml +++ b/.github/workflows/build-lxd.yml @@ -327,7 +327,6 @@ jobs: runs-on: [ubuntu-20.04] env: VERSION: "${{ inputs.git_ref || github.ref }}" - ARTIFACT_NAME: "${{ needs.build-previous.outputs.artifact_name }}" LXC: "${{ needs.determine-runner.outputs.lxc_cmd }}" PREVIOUS_IMAGE_URL: "https://github.com/nextcloud/nextcloudpi/releases/download/v1.53.2/NextcloudPi_LXD_x86_v1.53.2.tar.gz" steps: diff --git a/etc/library.sh b/etc/library.sh index b95b56ebf..a00b8a5fd 100644 --- a/etc/library.sh +++ b/etc/library.sh @@ -156,7 +156,7 @@ function set-nc-domain() if is_ncp_activated && is_app_enabled notify_push; then ncc config:system:set trusted_proxies 11 --value="127.0.0.1" ncc config:system:set trusted_proxies 12 --value="::1" - ncc config:system:set trusted_proxies 13 --value="${domain}" +# ncc config:system:set trusted_proxies 13 --value="${domain}" ncc config:system:set trusted_proxies 14 --value="$(dig +short "${domain}")" sleep 5 # this seems to be required in the VM for some reason. We get `http2 error: protocol error` after ncp-upgrade-nc for try in {1..5} diff --git a/updates/1.54.0.sh b/updates/1.54.0.sh index a3dcfba20..105877884 100644 --- a/updates/1.54.0.sh +++ b/updates/1.54.0.sh @@ -7,5 +7,11 @@ then sed -i -e 's/^PermitRootLogin.*$/PermitRootLogin No/' /etc/ssh/sshd_config fi +for i in {10..15} +do + proxy="$(ncc config:system:get trusted_proxies "$i")" + [[ -z "$proxy" ]] || python3 -c "import ipaddress; ipaddress.ip_address('${proxy}')" || ncc config:system:delete trusted_proxies "$i" +done + apt-get update apt-get install --no-install-recommends -y tmux From 7ade3ad6b39f4b714a5494c22426f84da1ce1516 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Sun, 28 Apr 2024 23:29:06 +0000 Subject: [PATCH 14/14] build(deps-dev): bump express from 4.18.2 to 4.19.2 in /ncp-app Bumps [express](https://github.com/expressjs/express) from 4.18.2 to 4.19.2. - [Release notes](https://github.com/expressjs/express/releases) - [Changelog](https://github.com/expressjs/express/blob/master/History.md) - [Commits](https://github.com/expressjs/express/compare/4.18.2...4.19.2) --- updated-dependencies: - dependency-name: express dependency-type: indirect ... Signed-off-by: dependabot[bot] --- ncp-app/package-lock.json | 32 ++++++++++++++++---------------- 1 file changed, 16 insertions(+), 16 deletions(-) diff --git a/ncp-app/package-lock.json b/ncp-app/package-lock.json index dccd68a06..0971a4f08 100644 --- a/ncp-app/package-lock.json +++ b/ncp-app/package-lock.json @@ -5328,14 +5328,14 @@ "peer": true }, "node_modules/body-parser": { - "version": "1.20.1", - "resolved": "https://registry.npmjs.org/body-parser/-/body-parser-1.20.1.tgz", - "integrity": "sha512-jWi7abTbYwajOytWCQc37VulmWiRae5RyTpaCyDcS5/lMdtwSz5lOpDE67srw/HYe35f1z3fDQw+3txg7gNtWw==", + "version": "1.20.2", + "resolved": "https://registry.npmjs.org/body-parser/-/body-parser-1.20.2.tgz", + "integrity": "sha512-ml9pReCu3M61kGlqoTm2umSXTlRTuGTx0bfYj+uIUKKYycG5NtSbeetV3faSU6R7ajOPw0g/J1PvK4qNy7s5bA==", "dev": true, "peer": true, "dependencies": { "bytes": "3.1.2", - "content-type": "~1.0.4", + "content-type": "~1.0.5", "debug": "2.6.9", "depd": "2.0.0", "destroy": "1.2.0", @@ -5343,7 +5343,7 @@ "iconv-lite": "0.4.24", "on-finished": "2.4.1", "qs": "6.11.0", - "raw-body": "2.5.1", + "raw-body": "2.5.2", "type-is": "~1.6.18", "unpipe": "1.0.0" }, @@ -6131,9 +6131,9 @@ "integrity": "sha512-Kvp459HrV2FEJ1CAsi1Ku+MY3kasH19TFykTz2xWmMeq6bk2NU3XXvfJ+Q61m0xktWwt+1HSYf3JZsTms3aRJg==" }, "node_modules/cookie": { - "version": "0.5.0", - "resolved": "https://registry.npmjs.org/cookie/-/cookie-0.5.0.tgz", - "integrity": "sha512-YZ3GUyn/o8gfKJlnlX7g7xq4gyO6OSuhGPKaaGssGB2qgDUS0gPgtTvoyZLTt9Ab6dC4hfc9dV5arkvc/OCmrw==", + "version": "0.6.0", + "resolved": "https://registry.npmjs.org/cookie/-/cookie-0.6.0.tgz", + "integrity": "sha512-U71cyTamuh1CRNCfpGY6to28lxvNwPG4Guz/EVjgf3Jmzv0vlDp1atT9eS5dDjMYHucpHbWns6Lwf3BKz6svdw==", "dev": true, "peer": true, "engines": { @@ -8197,18 +8197,18 @@ } }, "node_modules/express": { - "version": "4.18.2", - "resolved": "https://registry.npmjs.org/express/-/express-4.18.2.tgz", - "integrity": "sha512-5/PsL6iGPdfQ/lKM1UuielYgv3BUoJfz1aUwU9vHZ+J7gyvwdQXFEBIEIaxeGf0GIcreATNyBExtalisDbuMqQ==", + "version": "4.19.2", + "resolved": "https://registry.npmjs.org/express/-/express-4.19.2.tgz", + "integrity": "sha512-5T6nhjsT+EOMzuck8JjBHARTHfMht0POzlA60WV2pMD3gyXw2LZnZ+ueGdNxG+0calOJcWKbpFcuzLZ91YWq9Q==", "dev": true, "peer": true, "dependencies": { "accepts": "~1.3.8", "array-flatten": "1.1.1", - "body-parser": "1.20.1", + "body-parser": "1.20.2", "content-disposition": "0.5.4", "content-type": "~1.0.4", - "cookie": "0.5.0", + "cookie": "0.6.0", "cookie-signature": "1.0.6", "debug": "2.6.9", "depd": "2.0.0", @@ -13445,9 +13445,9 @@ } }, "node_modules/raw-body": { - "version": "2.5.1", - "resolved": "https://registry.npmjs.org/raw-body/-/raw-body-2.5.1.tgz", - "integrity": "sha512-qqJBtEyVgS0ZmPGdCFPWJ3FreoqvG4MVQln/kCgF7Olq95IbOp0/BWyMwbdtn4VTvkM8Y7khCQ2Xgk/tcrCXig==", + "version": "2.5.2", + "resolved": "https://registry.npmjs.org/raw-body/-/raw-body-2.5.2.tgz", + "integrity": "sha512-8zGqypfENjCIqGhgXToC8aB2r7YrBX+AQAfIPs/Mlk+BtPTztOvTS01NRW/3Eh60J+a48lt8qsCzirQ6loCVfA==", "dev": true, "peer": true, "dependencies": {