Skip to content
Redirector update (cronjob)

Redirector update #2589

Sign in to view logs

Redirector update

Redirector update #2589

Workflow file for this run

.github/workflows/redirector-update.yml at c1b9f19
name: Redirector update (cronjob)
on:
repository_dispatch:
types: ["Redirector update"]
schedule:
- cron: "00 8 * * *"
workflow_dispatch:
jobs:
Check:
name: "Check membership"
runs-on: Linux
steps:
- name: "Check membership"
uses: armbian/actions/team-check@main
with:
ORG_MEMBERS: ${{ secrets.ORG_MEMBERS }}
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
TEAM: "Release manager"
###################################################################################################
Test-debs-beta:
name: "Check debs-beta mirrors"
needs: Check
outputs:
matrix: ${{steps.json.outputs.JSON_CONTENT}}
runs-on: Linux
steps:
- name: Get primary mirrors from database
id: json
run: |
echo 'JSON_CONTENT<<EOF' >> $GITHUB_OUTPUT
curl -H "Authorization: Token ${{ secrets.NETBOX_TOKEN }}" -H "Accept: application/json; indent=4" \
"https://stuff.armbian.com/netbox/api/virtualization/virtual-machines/?limit=500&name__empty=false&tag=debs-beta&status=active" \
| jq '.results[] | .name,.custom_fields["download_path_debs"],.id' | sed "s|null|beta|" | sed "s/\"//g" \
| xargs -n3 -d'\n' | sed -e 's/\s\+/\//' | sed "s/ /,/" | jq -cnR '[inputs | select(length>0)]' | jq >> $GITHUB_OUTPUT
echo 'EOF' >> $GITHUB_OUTPUT
Test-debs:
name: "Check debs mirrors"
needs: Check
outputs:
matrix: ${{steps.json.outputs.JSON_CONTENT}}
runs-on: Linux
steps:
- name: Get primary mirrors from database
id: json
run: |
echo 'JSON_CONTENT<<EOF' >> $GITHUB_OUTPUT
curl -H "Authorization: Token ${{ secrets.NETBOX_TOKEN }}" -H "Accept: application/json; indent=4" \
"https://stuff.armbian.com/netbox/api/virtualization/virtual-machines/?limit=500&name__empty=false&tag=debs&status=active" \
| jq '.results[] | .name,.custom_fields["download_path_debs"],.id' | sed "s|null|apt|" | sed "s/\"//g" \
| xargs -n3 -d'\n' | sed -e 's/\s\+/\//' | sed "s/ /,/" | jq -cnR '[inputs | select(length>0)]' | jq >> $GITHUB_OUTPUT
echo 'EOF' >> $GITHUB_OUTPUT
Test-archive:
name: "Check archive mirrors"
needs: Check
outputs:
matrix: ${{steps.json.outputs.JSON_CONTENT}}
runs-on: Linux
steps:
- name: Get primary mirrors from database
id: json
run: |
echo 'JSON_CONTENT<<EOF' >> $GITHUB_OUTPUT
curl -H "Authorization: Token ${{ secrets.NETBOX_TOKEN }}" -H "Accept: application/json; indent=4" \
"https://stuff.armbian.com/netbox/api/virtualization/virtual-machines/?limit=500&name__empty=false&tag=archive&status=active" \
| jq '.results[] | .name,.custom_fields["download_path_debs"],.id' | sed "s|null|archive|" | sed "s/\"//g" \
| xargs -n3 -d'\n' | sed -e 's/\s\+/\//' | sed "s/ /,/" | jq -cnR '[inputs | select(length>0)]' | jq >> $GITHUB_OUTPUT
echo 'EOF' >> $GITHUB_OUTPUT
Test-images:
name: "Check images mirrors"
needs: Check
outputs:
matrix: ${{steps.json.outputs.JSON_CONTENT}}
runs-on: Linux
steps:
- name: Get primary mirrors from database
id: json
run: |
echo 'JSON_CONTENT<<EOF' >> $GITHUB_OUTPUT
curl -H "Authorization: Token ${{ secrets.NETBOX_TOKEN }}" -H "Accept: application/json; indent=4" \
"https://stuff.armbian.com/netbox/api/virtualization/virtual-machines/?limit=500&name__empty=false&tag=images&status=active" \
| jq '.results[] | .name,.custom_fields["download_path_images"],.id' | sed "s|null|dl|" | sed "s/\"//g" \
| xargs -n3 -d'\n' | sed -e 's/\s\+/\//' | sed "s/ /,/" | jq -cnR '[inputs | select(length>0)]' | jq >> $GITHUB_OUTPUT
echo 'EOF' >> $GITHUB_OUTPUT
###################################################################################################
Ping-debs-beta:
name: "Check debs-beta"
runs-on: Linux
needs: Test-debs-beta
outputs:
matrix: ${{needs.Test-debs-beta.outputs.matrix}}
if: ${{ needs.Test-debs-beta.outputs.matrix != '[]' && needs.Test-debs-beta.outputs.matrix != '' }}
timeout-minutes: 60
strategy:
fail-fast: false
matrix:
node: ${{fromJson(needs.Test-debs-beta.outputs.matrix)}}
steps:
- name: "Check"
run: |
SERVER_URL=$(echo "${{ matrix.node }}" | cut -d"," -f1)
SERVER_ID=$(echo "${{ matrix.node }}" | cut -d"," -f2)
ORIGIN=$(timeout 30 curl --retry 5 -s https://rsync.armbian.com/beta/control || true)
CHECK=$(timeout 30 curl --retry 5 -s https://${SERVER_URL}/control || true)
rm -rf downloads/*
if [[ "$ORIGIN" == "$CHECK" ]]; then
mkdir -p downloads
echo $SERVER_URL > downloads/$SERVER_ID
fi
- name: Upload JSON
uses: actions/upload-artifact@v3
with:
name: debs-beta
path: downloads
if-no-files-found: ignore
Ping-debs:
name: "Check debs"
runs-on: Linux
needs: Test-debs
outputs:
matrix: ${{needs.Test-debs.outputs.matrix}}
if: ${{ needs.Test-debs.outputs.matrix != '[]' && needs.Test-debs.outputs.matrix != '' }}
timeout-minutes: 60
strategy:
fail-fast: false
matrix:
node: ${{fromJson(needs.Test-debs.outputs.matrix)}}
steps:
- name: "Check"
run: |
SERVER_URL=$(echo "${{ matrix.node }}" | cut -d"," -f1)
SERVER_ID=$(echo "${{ matrix.node }}" | cut -d"," -f2)
ORIGIN=$(timeout 30 curl --retry 5 -s https://rsync.armbian.com/apt/control || true)
CHECK=$(timeout 30 curl --retry 5 -s https://${SERVER_URL}/control || true)
rm -rf downloads/*
if [[ "$ORIGIN" == "$CHECK" ]]; then
mkdir -p downloads
echo $SERVER_URL > downloads/$SERVER_ID
fi
- name: Upload JSON
uses: actions/upload-artifact@v3
with:
name: debs
path: downloads
if-no-files-found: ignore
Ping-archive:
name: "Check archive"
runs-on: Linux
needs: Test-archive
outputs:
matrix: ${{needs.Test-archive.outputs.matrix}}
if: ${{ needs.Test-archive.outputs.matrix != '[]' && needs.Test-archive.outputs.matrix != '' }}
timeout-minutes: 60
strategy:
fail-fast: false
matrix:
node: ${{fromJson(needs.Test-archive.outputs.matrix)}}
steps:
- name: "Check"
run: |
SERVER_URL=$(echo "${{ matrix.node }}" | cut -d"," -f1)
SERVER_ID=$(echo "${{ matrix.node }}" | cut -d"," -f2)
ORIGIN=$(timeout 30 curl --retry 5 -s https://rsync.armbian.com/archive/control || true)
CHECK=$(timeout 30 curl --retry 5 -s https://${SERVER_URL}/control || true)
rm -rf downloads/*
if [[ "$ORIGIN" == "$CHECK" ]]; then
mkdir -p downloads
echo $SERVER_URL > downloads/$SERVER_ID
fi
- name: Upload JSON
uses: actions/upload-artifact@v3
with:
name: archive
path: downloads
if-no-files-found: ignore
Ping-images:
name: "Check images"
runs-on: Linux
needs: Test-images
outputs:
matrix: ${{needs.Test-images.outputs.matrix}}
if: ${{ needs.Test-images.outputs.matrix != '[]' && needs.Test-images.outputs.matrix != '' }}
timeout-minutes: 60
strategy:
fail-fast: false
matrix:
node: ${{fromJson(needs.Test-images.outputs.matrix)}}
steps:
- name: "Check"
run: |
SERVER_URL=$(echo "${{ matrix.node }}" | cut -d"," -f1)
SERVER_ID=$(echo "${{ matrix.node }}" | cut -d"," -f2)
ORIGIN=$(timeout 30 curl --retry 5 -s https://rsync.armbian.com/dl/control || true)
CHECK=$(timeout 30 curl --retry 5 -s https://${SERVER_URL}/control || true)
rm -rf downloads/*
if [[ "$ORIGIN" == "$CHECK" ]]; then
mkdir -p downloads
echo $SERVER_URL > downloads/$SERVER_ID
fi
- name: Upload JSON
uses: actions/upload-artifact@v3
with:
name: images
path: downloads
if-no-files-found: ignore
###################################################################################################
Prepare-debs-beta:
name: "Generate debs-beta YAML for re-director"
needs: Ping-debs-beta
runs-on: Linux
steps:
- name: "Clean"
run: |
rm -rf downloads/*
- name: "Download artifacts"
uses: actions/download-artifact@v3
with:
name: debs-beta
path: downloads
- name: "Check"
run: |
ls -1 downloads/ | sed ':a; N; s/\n/ /; ta'
echo "failoverserver=$(ls -1 downloads/ | sed ':a; N; s/\n/ /; ta') " >> $GITHUB_ENV
- name: Test debs
uses: armbian/actions/make-yaml-redirector@main
with:
variant: debs-beta
failoverserver: "${{ env.failoverserver }}"
port: 8083
geodb: /scripts/redirect-config/GeoLite2-City.mmdb
asndb: /scripts/redirect-config/GeoLite2-ASN.mmdb
dl_map: /scripts/redirect-config/all-images.json
reloadKey: NLnC4QNF862QumEpt7zwVPZAEvqAz6cZ
netbox: ${{ secrets.NETBOX_TOKEN }}
Prepare-debs:
name: "Generate debs YAML for re-director"
needs: Ping-debs
runs-on: Linux
steps:
- name: "Clean"
run: |
rm -rf downloads/*
- name: "Download artifacts"
uses: actions/download-artifact@v3
with:
name: debs
path: downloads
- name: "Check"
run: |
ls -1 downloads/ | sed ':a; N; s/\n/ /; ta'
echo "failoverserver=$(ls -1 downloads/ | sed ':a; N; s/\n/ /; ta') " >> $GITHUB_ENV
- name: Test debs
uses: armbian/actions/make-yaml-redirector@main
with:
variant: debs
failoverserver: "${{ env.failoverserver }}"
geodb: /scripts/redirect-config/GeoLite2-City.mmdb
asndb: /scripts/redirect-config/GeoLite2-ASN.mmdb
dl_map: /scripts/redirect-config/all-images.json
reloadKey: NLnC4QNF862QumEpt7zwVPZAEvqAz6cZ
netbox: ${{ secrets.NETBOX_TOKEN }}
Prepare-archive:
name: "Generate archive YAML for re-director"
needs: Ping-archive
runs-on: Linux
steps:
- name: "Clean"
run: |
rm -rf downloads/*
- name: "Download artifacts"
uses: actions/download-artifact@v3
with:
name: archive
path: downloads
- name: "Check"
run: |
ls -1 downloads/ | sed ':a; N; s/\n/ /; ta'
echo "failoverserver=$(ls -1 downloads/ | sed ':a; N; s/\n/ /; ta') " >> $GITHUB_ENV
- name: Test archive
uses: armbian/actions/make-yaml-redirector@main
with:
variant: archive
failoverserver: "${{ env.failoverserver }}"
port: 8082
geodb: /scripts/redirect-config/GeoLite2-City.mmdb
asndb: /scripts/redirect-config/GeoLite2-ASN.mmdb
dl_map: /scripts/redirect-config/all-images.json
reloadKey: NLnC4QNF862QumEpt7zwVPZAEvqAz6cZ
netbox: ${{ secrets.NETBOX_TOKEN }}
Prepare-images:
name: "Generate images YAML for re-director"
needs: Ping-images
runs-on: Linux
steps:
- name: "Clean"
run: |
rm -rf downloads/*
- name: "Download artifacts"
uses: actions/download-artifact@v3
with:
name: images
path: downloads
- name: "Check"
run: |
ls -1 downloads/ | sed ':a; N; s/\n/ /; ta'
echo "failoverserver=$(ls -1 downloads/ | sed ':a; N; s/\n/ /; ta') " >> $GITHUB_ENV
- name: Test images
uses: armbian/actions/make-yaml-redirector@main
with:
variant: images
failoverserver: "${{ env.failoverserver }}"
port: 8081
geodb: /scripts/redirect-config/GeoLite2-City.mmdb
asndb: /scripts/redirect-config/GeoLite2-ASN.mmdb
dl_map: /scripts/redirect-config/all-images.json
reloadKey: NLnC4QNF862QumEpt7zwVPZAEvqAz6cZ
netbox: ${{ secrets.NETBOX_TOKEN }}
###################################################################################################
Make:
name: "Upload"
needs: [ Prepare-debs,Prepare-debs-beta,Prepare-archive,Prepare-images ]
runs-on: Linux
steps:
- name: "Clean"
run: |
rm -rf downloads/*
- uses: robinraju/release-downloader@v1.11
with:
repository: "P3TERX/GeoLite.mmdb"
latest: true
fileName: "*.mmdb"
- name: "Download artifacts"
uses: actions/download-artifact@v3
with:
name: redirector-config
path: downloads
- name: "Check"
run: |
ls -l downloads
ls -l *.mmdb
- uses: geekyeggo/delete-artifact@v5
with:
name: |
redirector-config
debs
debs-beta
archive
images
failOnError: false
- name: Install SSH key
uses: shimataro/ssh-key-action@v2
with:
key: ${{ secrets.KEY_UPLOAD }}
known_hosts: ${{ secrets.KNOWN_HOSTS_ARMBIAN_UPLOAD }}
if_key_exists: replace
# To do: add runner to redirector, but on clean install. Until then upload files.
- name: Upload files
run: |
rsync -e "ssh -p ${{ secrets.HOST_UPLOAD_PORT }}" -arvc --remove-source-files \
downloads/dlrouter-archive.yaml \
downloads/dlrouter-debs-beta.yaml \
downloads/dlrouter-images.yaml \
downloads/dlrouter-debs.yaml \
${{ secrets.HOST_UPLOAD_USER }}@${{ secrets.HOST_UPLOAD }}:/incoming/json