Redirector update (cronjob) #2584
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Redirector update (cronjob) | |
| on: | |
| repository_dispatch: | |
| types: ["Redirector update"] | |
| schedule: | |
| - cron: "00 8 * * *" | |
| workflow_dispatch: | |
| jobs: | |
| Check: | |
| name: "Check membership" | |
| runs-on: Linux | |
| steps: | |
| - name: "Check membership" | |
| uses: armbian/actions/team-check@main | |
| with: | |
| ORG_MEMBERS: ${{ secrets.ORG_MEMBERS }} | |
| GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} | |
| TEAM: "Release manager" | |
| ################################################################################################### | |
| Test-debs-beta: | |
| name: "Check debs-beta mirrors" | |
| needs: Check | |
| outputs: | |
| matrix: ${{steps.json.outputs.JSON_CONTENT}} | |
| runs-on: Linux | |
| steps: | |
| - name: Get primary mirrors from database | |
| id: json | |
| run: | | |
| echo 'JSON_CONTENT<<EOF' >> $GITHUB_OUTPUT | |
| curl -H "Authorization: Token ${{ secrets.NETBOX_TOKEN }}" -H "Accept: application/json; indent=4" \ | |
| "https://stuff.armbian.com/netbox/api/virtualization/virtual-machines/?limit=500&name__empty=false&tag=debs-beta&status=active" \ | |
| | jq '.results[] | .name,.custom_fields["download_path_debs"],.id' | sed "s|null|beta|" | sed "s/\"//g" \ | |
| | xargs -n3 -d'\n' | sed -e 's/\s\+/\//' | sed "s/ /,/" | jq -cnR '[inputs | select(length>0)]' | jq >> $GITHUB_OUTPUT | |
| echo 'EOF' >> $GITHUB_OUTPUT | |
| Test-debs: | |
| name: "Check debs mirrors" | |
| needs: Check | |
| outputs: | |
| matrix: ${{steps.json.outputs.JSON_CONTENT}} | |
| runs-on: Linux | |
| steps: | |
| - name: Get primary mirrors from database | |
| id: json | |
| run: | | |
| echo 'JSON_CONTENT<<EOF' >> $GITHUB_OUTPUT | |
| curl -H "Authorization: Token ${{ secrets.NETBOX_TOKEN }}" -H "Accept: application/json; indent=4" \ | |
| "https://stuff.armbian.com/netbox/api/virtualization/virtual-machines/?limit=500&name__empty=false&tag=debs&status=active" \ | |
| | jq '.results[] | .name,.custom_fields["download_path_debs"],.id' | sed "s|null|apt|" | sed "s/\"//g" \ | |
| | xargs -n3 -d'\n' | sed -e 's/\s\+/\//' | sed "s/ /,/" | jq -cnR '[inputs | select(length>0)]' | jq >> $GITHUB_OUTPUT | |
| echo 'EOF' >> $GITHUB_OUTPUT | |
| Test-archive: | |
| name: "Check archive mirrors" | |
| needs: Check | |
| outputs: | |
| matrix: ${{steps.json.outputs.JSON_CONTENT}} | |
| runs-on: Linux | |
| steps: | |
| - name: Get primary mirrors from database | |
| id: json | |
| run: | | |
| echo 'JSON_CONTENT<<EOF' >> $GITHUB_OUTPUT | |
| curl -H "Authorization: Token ${{ secrets.NETBOX_TOKEN }}" -H "Accept: application/json; indent=4" \ | |
| "https://stuff.armbian.com/netbox/api/virtualization/virtual-machines/?limit=500&name__empty=false&tag=archive&status=active" \ | |
| | jq '.results[] | .name,.custom_fields["download_path_debs"],.id' | sed "s|null|archive|" | sed "s/\"//g" \ | |
| | xargs -n3 -d'\n' | sed -e 's/\s\+/\//' | sed "s/ /,/" | jq -cnR '[inputs | select(length>0)]' | jq >> $GITHUB_OUTPUT | |
| echo 'EOF' >> $GITHUB_OUTPUT | |
| Test-images: | |
| name: "Check images mirrors" | |
| needs: Check | |
| outputs: | |
| matrix: ${{steps.json.outputs.JSON_CONTENT}} | |
| runs-on: Linux | |
| steps: | |
| - name: Get primary mirrors from database | |
| id: json | |
| run: | | |
| echo 'JSON_CONTENT<<EOF' >> $GITHUB_OUTPUT | |
| curl -H "Authorization: Token ${{ secrets.NETBOX_TOKEN }}" -H "Accept: application/json; indent=4" \ | |
| "https://stuff.armbian.com/netbox/api/virtualization/virtual-machines/?limit=500&name__empty=false&tag=images&status=active" \ | |
| | jq '.results[] | .name,.custom_fields["download_path_images"],.id' | sed "s|null|dl|" | sed "s/\"//g" \ | |
| | xargs -n3 -d'\n' | sed -e 's/\s\+/\//' | sed "s/ /,/" | jq -cnR '[inputs | select(length>0)]' | jq >> $GITHUB_OUTPUT | |
| echo 'EOF' >> $GITHUB_OUTPUT | |
| ################################################################################################### | |
| Ping-debs-beta: | |
| name: "Check debs-beta" | |
| runs-on: Linux | |
| needs: Test-debs-beta | |
| outputs: | |
| matrix: ${{needs.Test-debs-beta.outputs.matrix}} | |
| if: ${{ needs.Test-debs-beta.outputs.matrix != '[]' && needs.Test-debs-beta.outputs.matrix != '' }} | |
| timeout-minutes: 60 | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| node: ${{fromJson(needs.Test-debs-beta.outputs.matrix)}} | |
| steps: | |
| - name: "Check" | |
| run: | | |
| SERVER_URL=$(echo "${{ matrix.node }}" | cut -d"," -f1) | |
| SERVER_ID=$(echo "${{ matrix.node }}" | cut -d"," -f2) | |
| ORIGIN=$(timeout 30 curl --retry 5 -s https://rsync.armbian.com/beta/control || true) | |
| CHECK=$(timeout 30 curl --retry 5 -s https://${SERVER_URL}/control || true) | |
| rm -rf downloads/* | |
| if [[ "$ORIGIN" == "$CHECK" ]]; then | |
| mkdir -p downloads | |
| echo $SERVER_URL > downloads/$SERVER_ID | |
| fi | |
| - name: Upload JSON | |
| uses: actions/upload-artifact@v3 | |
| with: | |
| name: debs-beta | |
| path: downloads | |
| if-no-files-found: ignore | |
| Ping-debs: | |
| name: "Check debs" | |
| runs-on: Linux | |
| needs: Test-debs | |
| outputs: | |
| matrix: ${{needs.Test-debs.outputs.matrix}} | |
| if: ${{ needs.Test-debs.outputs.matrix != '[]' && needs.Test-debs.outputs.matrix != '' }} | |
| timeout-minutes: 60 | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| node: ${{fromJson(needs.Test-debs.outputs.matrix)}} | |
| steps: | |
| - name: "Check" | |
| run: | | |
| SERVER_URL=$(echo "${{ matrix.node }}" | cut -d"," -f1) | |
| SERVER_ID=$(echo "${{ matrix.node }}" | cut -d"," -f2) | |
| ORIGIN=$(timeout 30 curl --retry 5 -s https://rsync.armbian.com/apt/control || true) | |
| CHECK=$(timeout 30 curl --retry 5 -s https://${SERVER_URL}/control || true) | |
| rm -rf downloads/* | |
| if [[ "$ORIGIN" == "$CHECK" ]]; then | |
| mkdir -p downloads | |
| echo $SERVER_URL > downloads/$SERVER_ID | |
| fi | |
| - name: Upload JSON | |
| uses: actions/upload-artifact@v3 | |
| with: | |
| name: debs | |
| path: downloads | |
| if-no-files-found: ignore | |
| Ping-archive: | |
| name: "Check archive" | |
| runs-on: Linux | |
| needs: Test-archive | |
| outputs: | |
| matrix: ${{needs.Test-archive.outputs.matrix}} | |
| if: ${{ needs.Test-archive.outputs.matrix != '[]' && needs.Test-archive.outputs.matrix != '' }} | |
| timeout-minutes: 60 | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| node: ${{fromJson(needs.Test-archive.outputs.matrix)}} | |
| steps: | |
| - name: "Check" | |
| run: | | |
| SERVER_URL=$(echo "${{ matrix.node }}" | cut -d"," -f1) | |
| SERVER_ID=$(echo "${{ matrix.node }}" | cut -d"," -f2) | |
| ORIGIN=$(timeout 30 curl --retry 5 -s https://rsync.armbian.com/archive/control || true) | |
| CHECK=$(timeout 30 curl --retry 5 -s https://${SERVER_URL}/control || true) | |
| rm -rf downloads/* | |
| if [[ "$ORIGIN" == "$CHECK" ]]; then | |
| mkdir -p downloads | |
| echo $SERVER_URL > downloads/$SERVER_ID | |
| fi | |
| - name: Upload JSON | |
| uses: actions/upload-artifact@v3 | |
| with: | |
| name: archive | |
| path: downloads | |
| if-no-files-found: ignore | |
| Ping-images: | |
| name: "Check images" | |
| runs-on: Linux | |
| needs: Test-images | |
| outputs: | |
| matrix: ${{needs.Test-images.outputs.matrix}} | |
| if: ${{ needs.Test-images.outputs.matrix != '[]' && needs.Test-images.outputs.matrix != '' }} | |
| timeout-minutes: 60 | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| node: ${{fromJson(needs.Test-images.outputs.matrix)}} | |
| steps: | |
| - name: "Check" | |
| run: | | |
| SERVER_URL=$(echo "${{ matrix.node }}" | cut -d"," -f1) | |
| SERVER_ID=$(echo "${{ matrix.node }}" | cut -d"," -f2) | |
| ORIGIN=$(timeout 30 curl --retry 5 -s https://rsync.armbian.com/dl/control || true) | |
| CHECK=$(timeout 30 curl --retry 5 -s https://${SERVER_URL}/control || true) | |
| rm -rf downloads/* | |
| if [[ "$ORIGIN" == "$CHECK" ]]; then | |
| mkdir -p downloads | |
| echo $SERVER_URL > downloads/$SERVER_ID | |
| fi | |
| - name: Upload JSON | |
| uses: actions/upload-artifact@v3 | |
| with: | |
| name: images | |
| path: downloads | |
| if-no-files-found: ignore | |
| ################################################################################################### | |
| Prepare-debs-beta: | |
| name: "Generate debs-beta YAML for re-director" | |
| needs: Ping-debs-beta | |
| runs-on: Linux | |
| steps: | |
| - name: "Clean" | |
| run: | | |
| rm -rf downloads/* | |
| - name: "Download artifacts" | |
| uses: actions/download-artifact@v3 | |
| with: | |
| name: debs-beta | |
| path: downloads | |
| - name: "Check" | |
| run: | | |
| ls -1 downloads/ | sed ':a; N; s/\n/ /; ta' | |
| echo "failoverserver=$(ls -1 downloads/ | sed ':a; N; s/\n/ /; ta') " >> $GITHUB_ENV | |
| - name: Test debs | |
| uses: armbian/actions/make-yaml-redirector@main | |
| with: | |
| variant: debs-beta | |
| failoverserver: "${{ env.failoverserver }}" | |
| port: 8083 | |
| geodb: /scripts/redirect-config/GeoLite2-City.mmdb | |
| asndb: /scripts/redirect-config/GeoLite2-ASN.mmdb | |
| dl_map: /scripts/redirect-config/all-images.json | |
| reloadKey: NLnC4QNF862QumEpt7zwVPZAEvqAz6cZ | |
| netbox: ${{ secrets.NETBOX_TOKEN }} | |
| Prepare-debs: | |
| name: "Generate debs YAML for re-director" | |
| needs: Ping-debs | |
| runs-on: Linux | |
| steps: | |
| - name: "Clean" | |
| run: | | |
| rm -rf downloads/* | |
| - name: "Download artifacts" | |
| uses: actions/download-artifact@v3 | |
| with: | |
| name: debs | |
| path: downloads | |
| - name: "Check" | |
| run: | | |
| ls -1 downloads/ | sed ':a; N; s/\n/ /; ta' | |
| echo "failoverserver=$(ls -1 downloads/ | sed ':a; N; s/\n/ /; ta') " >> $GITHUB_ENV | |
| - name: Test debs | |
| uses: armbian/actions/make-yaml-redirector@main | |
| with: | |
| variant: debs | |
| failoverserver: "${{ env.failoverserver }}" | |
| geodb: /scripts/redirect-config/GeoLite2-City.mmdb | |
| asndb: /scripts/redirect-config/GeoLite2-ASN.mmdb | |
| dl_map: /scripts/redirect-config/all-images.json | |
| reloadKey: NLnC4QNF862QumEpt7zwVPZAEvqAz6cZ | |
| netbox: ${{ secrets.NETBOX_TOKEN }} | |
| Prepare-archive: | |
| name: "Generate archive YAML for re-director" | |
| needs: Ping-archive | |
| runs-on: Linux | |
| steps: | |
| - name: "Clean" | |
| run: | | |
| rm -rf downloads/* | |
| - name: "Download artifacts" | |
| uses: actions/download-artifact@v3 | |
| with: | |
| name: archive | |
| path: downloads | |
| - name: "Check" | |
| run: | | |
| ls -1 downloads/ | sed ':a; N; s/\n/ /; ta' | |
| echo "failoverserver=$(ls -1 downloads/ | sed ':a; N; s/\n/ /; ta') " >> $GITHUB_ENV | |
| - name: Test archive | |
| uses: armbian/actions/make-yaml-redirector@main | |
| with: | |
| variant: archive | |
| failoverserver: "${{ env.failoverserver }}" | |
| port: 8082 | |
| geodb: /scripts/redirect-config/GeoLite2-City.mmdb | |
| asndb: /scripts/redirect-config/GeoLite2-ASN.mmdb | |
| dl_map: /scripts/redirect-config/all-images.json | |
| reloadKey: NLnC4QNF862QumEpt7zwVPZAEvqAz6cZ | |
| netbox: ${{ secrets.NETBOX_TOKEN }} | |
| Prepare-images: | |
| name: "Generate images YAML for re-director" | |
| needs: Ping-images | |
| runs-on: Linux | |
| steps: | |
| - name: "Clean" | |
| run: | | |
| rm -rf downloads/* | |
| - name: "Download artifacts" | |
| uses: actions/download-artifact@v3 | |
| with: | |
| name: images | |
| path: downloads | |
| - name: "Check" | |
| run: | | |
| ls -1 downloads/ | sed ':a; N; s/\n/ /; ta' | |
| echo "failoverserver=$(ls -1 downloads/ | sed ':a; N; s/\n/ /; ta') " >> $GITHUB_ENV | |
| - name: Test images | |
| uses: armbian/actions/make-yaml-redirector@main | |
| with: | |
| variant: images | |
| failoverserver: "${{ env.failoverserver }}" | |
| port: 8081 | |
| geodb: /scripts/redirect-config/GeoLite2-City.mmdb | |
| asndb: /scripts/redirect-config/GeoLite2-ASN.mmdb | |
| dl_map: /scripts/redirect-config/all-images.json | |
| reloadKey: NLnC4QNF862QumEpt7zwVPZAEvqAz6cZ | |
| netbox: ${{ secrets.NETBOX_TOKEN }} | |
| ################################################################################################### | |
| Make: | |
| name: "Upload" | |
| needs: [ Prepare-debs,Prepare-debs-beta,Prepare-archive,Prepare-images ] | |
| runs-on: Linux | |
| steps: | |
| - name: "Clean" | |
| run: | | |
| rm -rf downloads/* | |
| - uses: robinraju/release-downloader@v1.11 | |
| with: | |
| repository: "P3TERX/GeoLite.mmdb" | |
| latest: true | |
| fileName: "*.mmdb" | |
| - name: "Download artifacts" | |
| uses: actions/download-artifact@v3 | |
| with: | |
| name: redirector-config | |
| path: downloads | |
| - name: "Check" | |
| run: | | |
| ls -l downloads | |
| ls -l *.mmdb | |
| - uses: geekyeggo/delete-artifact@v5 | |
| with: | |
| name: | | |
| redirector-config | |
| debs | |
| debs-beta | |
| archive | |
| images | |
| failOnError: false | |
| - name: Install SSH key | |
| uses: shimataro/ssh-key-action@v2 | |
| with: | |
| key: ${{ secrets.KEY_UPLOAD }} | |
| known_hosts: ${{ secrets.KNOWN_HOSTS_ARMBIAN_UPLOAD }} | |
| if_key_exists: replace | |
| # To do: add runner to redirector, but on clean install. Until then upload files. | |
| - name: Upload files | |
| run: | | |
| rsync -e "ssh -p ${{ secrets.HOST_UPLOAD_PORT }}" -arvc --remove-source-files \ | |
| downloads/dlrouter-archive.yaml \ | |
| downloads/dlrouter-debs-beta.yaml \ | |
| downloads/dlrouter-images.yaml \ | |
| downloads/dlrouter-debs.yaml \ | |
| ${{ secrets.HOST_UPLOAD_USER }}@${{ secrets.HOST_UPLOAD }}:/incoming/json |