Generate v2ray routing rules #176
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Generate v2ray routing rules | |
| on: | |
| workflow_dispatch: | |
| inputs: | |
| PRE_RELEASE: | |
| description: "Set as pre-release" | |
| required: false | |
| default: "false" | |
| schedule: | |
| - cron: "0 1 * * *" | |
| jobs: | |
| build: | |
| runs-on: ubuntu-latest | |
| steps: | |
| - name: Checkout Iran-v2ray-rules repository | |
| uses: actions/checkout@v4 | |
| - name: Checkout domain-list-community repository | |
| uses: actions/checkout@v4 | |
| with: | |
| repository: v2fly/domain-list-community | |
| path: v2ray-geosite | |
| - name: Setup Go | |
| uses: actions/setup-go@v4 | |
| with: | |
| go-version: "1.21" | |
| cache-dependency-path: v2ray-geosite/go.sum | |
| - name: Set ENV variables | |
| run: | | |
| echo "RELEASE_NAME=$(date +%Y%m%d%H%M)" >> $GITHUB_ENV | |
| echo "TAG_NAME=$(date +%Y%m%d%H%M)" >> $GITHUB_ENV | |
| echo "RELEASE_DATE=$(date +'%F %T %Z')" >> $GITHUB_ENV | |
| - name: Install dependencies | |
| run: | | |
| sudo apt-get update | |
| sudo apt-get install dos2unix idn2 | |
| - name: Create release directory | |
| run: mkdir release | |
| - name: Get messengers IP list | |
| run: | | |
| curl -sSL https://raw.githubusercontent.com/Chocolate4U/ito.gov.ir-Mirror/main/data/Messengers.csv | awk -F"," '{print $2}' | sed '1d' > messengers-ip.txt | |
| - name: Get domestic CDNs IP list | |
| run: | | |
| curl --connect-timeout 15 -sSL https://www.arvancloud.ir/en/ips.txt -o arvancloud-ip.txt || curl -sSL https://raw.githubusercontent.com/Chocolate4U/Iran-v2ray-rules/release/text/arvancloud.txt -o arvancloud-ip.txt | |
| curl --connect-timeout 15 -sSL https://api.derak.cloud/public/ipv4 -o derakcloud-ipv4.txt || curl -sSL https://github.com/Chocolate4U/Iran-v2ray-rules/blob/release/text/derakcloud.txt -o derakcloud-ipv4.txt | |
| curl --connect-timeout 15 -sSL https://api.derak.cloud/public/ipv6 -o derakcloud-ipv6.txt || curl -sSL https://github.com/Chocolate4U/Iran-v2ray-rules/blob/release/text/derakcloud.txt -o derakcloud-ipv6.txt | |
| curl --connect-timeout 15 -sSL https://ips.f95.com/ip.txt -o iranserver-ip.txt || curl -sSL https://raw.githubusercontent.com/Chocolate4U/Iran-v2ray-rules/release/text/iranserver.txt -o iranserver-ip.txt | |
| curl --connect-timeout 15 -sSL https://parspack.com/cdnips.txt -o parspack-ip.txt || curl -sSL https://raw.githubusercontent.com/Chocolate4U/Iran-v2ray-rules/release/text/parspack.txt -o parspack-ip.txt | |
| - name: Get GeoLite2 | |
| env: | |
| LICENSE_KEY: ${{ secrets.MAXMIND_GEOLITE2_LICENSE }} | |
| run: | | |
| curl -sSL "https://download.maxmind.com/app/geoip_download?edition_id=GeoLite2-Country-CSV&license_key=${LICENSE_KEY}&suffix=zip" -o GeoLite2-Country-CSV.zip | |
| unzip GeoLite2-Country-CSV.zip | |
| rm -f GeoLite2-Country-CSV.zip | |
| mv GeoLite2* geolite2 | |
| - name: Generate ir domains list | |
| run: | | |
| curl -sSL https://github.com/bootmortis/iran-hosted-domains/releases/latest/download/domains.txt | grep -Ev ".+\.ir$" | sed '1 a\ir\nxn--mgba3a4f16a' | LC_ALL=C sort -u > ir.txt | |
| curl -sSLO https://raw.githubusercontent.com/Chocolate4U/Iran-v2ray-rules/redundant/redundant-domains.txt | |
| comm -23 ir.txt redundant-domains.txt > ir-lite.txt | |
| echo "TOTAL_IR=$(wc -l < ir.txt)" >> $GITHUB_ENV | |
| echo "TOTAL_IR_LITE=$(wc -l < ir-lite.txt)" >> $GITHUB_ENV | |
| mv ir.txt ir-lite.txt release | |
| - name: Generate ads list | |
| run: | | |
| curl -sSL https://raw.githubusercontent.com/MasterKia/PersianBlocker/main/PersianBlockerHosts.txt | sed -e 's/^\(|\|\*\|\.\|\-\|0\.0\.0\.0\|127\.0\.0\.1\)*//g' -e 's/\^.*$//g' -e '/!\|?\|@\|#\|\*\|_\|\\\|\/\|\[\|]\|\[\|\([0-9]\{1,3\}\.\)\{3\}[0-9]\{1,3\}/d' -e '/\.$/d' -e '/^\s*$/d' | awk '{$1=$1};1' | dos2unix | idn2 --no-alabelroundtrip --no-tr46 | LC_ALL=C sort -u > ads.txt | |
| echo "TOTAL_IR_ADS=$(wc -l < ads.txt)" >> $GITHUB_ENV | |
| - name: Generate category-ads-all list | |
| run: | | |
| curl -sSL https://raw.githubusercontent.com/MasterKia/PersianBlocker/main/PersianBlockerHosts.txt > category-ads-all-raw.txt | |
| curl -sSL https://raw.githubusercontent.com/hagezi/dns-blocklists/main/wildcard/light-onlydomains.txt >> category-ads-all-raw.txt | |
| curl -sSL https://pgl.yoyo.org/adservers/serverlist.php?hostformat=nohtml >> category-ads-all-raw.txt | |
| curl -sSL https://raw.githubusercontent.com/jerryn70/GoodbyeAds/master/Extension/GoodbyeAds-Samsung-AdBlock.txt >> category-ads-all-raw.txt | |
| curl -sSL https://raw.githubusercontent.com/jerryn70/GoodbyeAds/master/Extension/GoodbyeAds-Xiaomi-Extension.txt >> category-ads-all-raw.txt | |
| cat category-ads-all-raw.txt | sed -e 's/^\(|\|\*\|\.\|\-\|0\.0\.0\.0\|127\.0\.0\.1\)*//g' -e 's/\^.*$//g' -e '/!\|?\|@\|#\|\*\|_\|\\\|\/\|\[\|]\|\[\|\([0-9]\{1,3\}\.\)\{3\}[0-9]\{1,3\}/d' -e '/\.$/d' -e '/^\s*$/d' | awk '{$1=$1};1' | dos2unix | idn2 --no-alabelroundtrip --no-tr46 | LC_ALL=C sort -u > category-ads-all-temp.txt | |
| curl -sSL https://raw.githubusercontent.com/hagezi/dns-blocklists/main/whitelist.txt > whitelist-raw.txt | |
| curl -sSL https://raw.githubusercontent.com/hagezi/dns-blocklists/main/whitelist-referral.txt >> whitelist-raw.txt | |
| curl -sSL https://raw.githubusercontent.com/AdguardTeam/AdGuardSDNSFilter/master/Filters/exclusions.txt >> whitelist-raw.txt | |
| curl -sSL https://raw.githubusercontent.com/AdguardTeam/AdGuardSDNSFilter/master/Filters/exceptions.txt >> whitelist-raw.txt | |
| cat whitelist-raw.txt | sed -e 's/^\(|\|@\|\*\|\.\|\-\|0\.0\.0\.0\|127\.0\.0\.1\)*//g' -e 's/\^.*$//g' -e '/!\|?\|@\|#\|\*\|_\|\\\|\/\|\[\|]\|\[\|\([0-9]\{1,3\}\.\)\{3\}[0-9]\{1,3\}/d' -e '/\.$/d' -e '/^\s*$/d' | awk '{$1=$1};1' | dos2unix | idn2 --no-alabelroundtrip --no-tr46 | LC_ALL=C sort -u > whitelist-temp.txt | |
| comm -23 category-ads-all-temp.txt whitelist-temp.txt > category-ads-all-temp-temp.txt | |
| sed -e 's/^/\./' -e 's/\./\\./g' -e 's/\-/\\-/g' -e 's/$/\$/' category-ads-all-temp-temp.txt > category-ads-all-sub.txt | |
| cat category-ads-all-temp-temp.txt | LC_ALL=C grep -f category-ads-all-sub.txt | LC_ALL=C sort -u > category-ads-all-redundant-sub.txt | |
| comm -23 category-ads-all-temp-temp.txt category-ads-all-redundant-sub.txt > category-ads-all.txt | |
| echo "TOTAL_ADS=$(wc -l < category-ads-all.txt)" >> $GITHUB_ENV | |
| rm -f category-ads-all-raw.txt whitelist-raw.txt category-ads-all-temp.txt whitelist-temp.txt | |
| mv ads.txt category-ads-all.txt release | |
| - name: Generate malware list | |
| run: | | |
| curl -sSL https://malware-filter.gitlab.io/malware-filter/urlhaus-filter-dnscrypt-blocked-names-online.txt | sed -e 's/^\(|\|@\|\*\|\.\|\-\|0\.0\.0\.0\|127\.0\.0\.1\)*//g' -e 's/\^.*$//g' -e '/!\|?\|@\|#\|\*\|_\|\\\|\/\|\[\|]\|\[\|\([0-9]\{1,3\}\.\)\{3\}[0-9]\{1,3\}/d' -e '/\.$/d' -e '/^\s*$/d' | awk '{$1=$1};1' | dos2unix | idn2 --no-alabelroundtrip --no-tr46 | LC_ALL=C sort -u > malware-temp.txt | |
| curl -sSL https://malware-filter.gitlab.io/malware-filter/urlhaus-filter-dnscrypt-blocked-ips-online.txt | sed '/#/d' > malware-ip.txt | |
| sed -e 's/^/\./' -e 's/\./\\./g' -e 's/\-/\\-/g' -e 's/$/\$/' malware-temp.txt > malware-sub.txt | |
| cat malware-temp.txt | LC_ALL=C grep -f malware-sub.txt | LC_ALL=C sort -u > malware-redundant-sub.txt | |
| comm -23 malware-temp.txt malware-redundant-sub.txt > malware.txt | |
| echo "TOTAL_MALWARE=$(wc -l < malware.txt)" >> $GITHUB_ENV | |
| mv malware.txt release | |
| - name: Generate phishing list | |
| run: | | |
| curl -sSL https://malware-filter.gitlab.io/malware-filter/phishing-filter-dnscrypt-blocked-names.txt | sed -e 's/^\(|\|@\|\*\|\.\|\-\|0\.0\.0\.0\|127\.0\.0\.1\)*//g' -e 's/\^.*$//g' -e '/!\|?\|@\|#\|\*\|_\|\\\|\/\|\[\|]\|\[\|\([0-9]\{1,3\}\.\)\{3\}[0-9]\{1,3\}/d' -e '/\.$/d' -e '/^\s*$/d' | awk '{$1=$1};1' | dos2unix | idn2 --no-alabelroundtrip --no-tr46 | LC_ALL=C sort -u > phishing-temp.txt | |
| curl -sSL https://malware-filter.gitlab.io/malware-filter/phishing-filter-dnscrypt-blocked-ips.txt | sed '/#/d' > phishing-ip.txt | |
| sed -e 's/^/\./' -e 's/\./\\./g' -e 's/\-/\\-/g' -e 's/$/\$/' phishing-temp.txt > phishing-sub.txt | |
| cat phishing-temp.txt | LC_ALL=C grep -f phishing-sub.txt | LC_ALL=C sort -u > phishing-redundant-sub.txt | |
| comm -23 phishing-temp.txt phishing-redundant-sub.txt > phishing.txt | |
| echo "TOTAL_PHISHING=$(wc -l < phishing.txt)" >> $GITHUB_ENV | |
| mv phishing.txt release | |
| - name: Generate cryptominers domains list | |
| run: | | |
| curl -sSL https://raw.githubusercontent.com/hoshsadiq/adblock-nocoin-list/master/hosts.txt | sed -e 's/^\(|\|\*\|\.\|\-\|0\.0\.0\.0\|127\.0\.0\.1\)*//g' -e 's/\^.*$//g' -e '/!\|?\|@\|#\|\*\|_\|\\\|\/\|\[\|]\|\[\|\([0-9]\{1,3\}\.\)\{3\}[0-9]\{1,3\}/d' -e '/\.$/d' -e '/^\s*$/d' | awk '{$1=$1};1' | dos2unix | idn2 --no-alabelroundtrip --no-tr46 | LC_ALL=C sort -u > cryptominers-temp.txt | |
| sed -e 's/^/\./' -e 's/\./\\./g' -e 's/\-/\\-/g' -e 's/$/\$/' cryptominers-temp.txt > cryptominers-sub.txt | |
| cat cryptominers-temp.txt | LC_ALL=C grep -f cryptominers-sub.txt | LC_ALL=C sort -u > cryptominers-redundant-sub.txt | |
| comm -23 cryptominers-temp.txt cryptominers-redundant-sub.txt > cryptominers.txt | |
| echo "TOTAL_CRYPTO=$(wc -l < cryptominers.txt)" >> $GITHUB_ENV | |
| mv cryptominers.txt release | |
| - name: Generate Social Media domains list | |
| run: | | |
| curl -sSL https://raw.githubusercontent.com/StevenBlack/hosts/master/alternates/social-only/hosts | sed -e 's/^\(|\|\*\|\.\|\-\|0\.0\.0\.0 \|127\.0\.0\.1 \)*//g' -e 's/\^.*$//g' -e 's/^\(www\.\)*//g' -e '/!\|?\|@\|#\|\*\|_\|\\\|\/\|\[\|]\|\[\|\([0-9]\{1,3\}\.\)\{3\}[0-9]\{1,3\}/d' -e '/\.$/d' -e '/^\s*$/d' | awk '{$1=$1};1' | dos2unix | idn2 --no-alabelroundtrip --no-tr46 | LC_ALL=C sort -u > social-temp.txt | |
| sed -e 's/^/\./' -e 's/\./\\./g' -e 's/\-/\\-/g' -e 's/$/\$/' social-temp.txt > social-sub.txt | |
| cat social-temp.txt | LC_ALL=C grep -f social-sub.txt | LC_ALL=C sort -u > social-redundant-sub.txt | |
| comm -23 social-temp.txt social-redundant-sub.txt > social.txt | |
| echo "TOTAL_SOCIAL=$(wc -l < social.txt)" >> $GITHUB_ENV | |
| mv social.txt release | |
| - name: Generate nsfw domains list | |
| run: | | |
| curl -sSL https://raw.githubusercontent.com/StevenBlack/hosts/master/alternates/gambling-porn-only/hosts | sed -e 's/^\(|\|\*\|\.\|\-\|0\.0\.0\.0 \|127\.0\.0\.1 \)*//g' -e 's/\^.*$//g' -e 's/^\(www\.\)*//g' -e '/!\|?\|@\|#\|\*\|_\|\\\|\/\|\[\|]\|\[\|\([0-9]\{1,3\}\.\)\{3\}[0-9]\{1,3\}/d' -e '/\.$/d' -e '/^\s*$/d' | awk '{$1=$1};1' | dos2unix | idn2 --no-alabelroundtrip --no-tr46 | LC_ALL=C sort -u > nsfw-temp.txt | |
| sed -e 's/^/\./' -e 's/\./\\./g' -e 's/\-/\\-/g' -e 's/$/\$/' nsfw-temp.txt > nsfw-sub.txt | |
| cat nsfw-temp.txt | LC_ALL=C grep -f nsfw-sub.txt | LC_ALL=C sort -u > nsfw-redundant-sub.txt | |
| comm -23 nsfw-temp.txt nsfw-redundant-sub.txt > nsfw.txt | |
| echo "TOTAL_NSFW=$(wc -l < nsfw.txt)" >> $GITHUB_ENV | |
| mv nsfw.txt release | |
| - name: Generate geoip.dat, geoip-lite.dat, security-ip.dat, Country.mmdb, Country-lite.mmdb and Security-ip.mmdb files | |
| run: | | |
| go install -v github.com/Loyalsoldier/geoip@latest | |
| $(go env GOPATH)/bin/geoip -c config.json | |
| cp output/dat/geoip.dat output/dat/geoip-lite.dat output/dat/security-ip.dat release | |
| cp output/maxmind/Country.mmdb output/maxmind/Country-lite.mmdb output/maxmind/Security-ip.mmdb release | |
| cp -fpPR output/text release | |
| - name: Verify Country.mmdb, Country-lite.mmdb and Security-ip.mmdb files | |
| run: | | |
| go install -v github.com/maxmind/mmdbverify@latest | |
| $(go env GOPATH)/bin/mmdbverify -file release/Country.mmdb | |
| $(go env GOPATH)/bin/mmdbverify -file release/Country-lite.mmdb | |
| $(go env GOPATH)/bin/mmdbverify -file release/Security-ip.mmdb | |
| - name: Generate geosite.dat, geosite-lite.dat and security.dat files | |
| run: | | |
| cd v2ray-geosite | |
| cp ../release/ir.txt data/ir | |
| cp ../release/ads.txt data/ads | |
| cp ../release/category-ads-all.txt data/category-ads-all | |
| cp ../release/malware.txt data/malware | |
| cp ../release/phishing.txt data/phishing | |
| cp ../release/cryptominers.txt data/cryptominers | |
| cp ../release/social.txt data/social | |
| cp ../release/nsfw.txt data/nsfw | |
| go mod edit -go=1.21 | |
| go get -u | |
| go mod tidy | |
| go run ./ --datapath=data --outputdir=../release --outputname=geosite.dat | |
| mkdir datalite | |
| cp ../release/ir-lite.txt datalite/ir | |
| cp ../release/ads.txt datalite/ads | |
| go run ./ --datapath=datalite --outputdir=../release --outputname=geosite-lite.dat | |
| mkdir security | |
| cp ../release/category-ads-all.txt security/category-ads-all | |
| cp ../release/malware.txt security/malware | |
| cp ../release/phishing.txt security/phishing | |
| cp ../release/cryptominers.txt security/cryptominers | |
| go run ./ --datapath=security --outputdir=../release --outputname=security.dat | |
| - name: Generate sha256sum | |
| run: | | |
| sha256sum release/geoip.dat > release/geoip.dat.sha256sum | |
| sha256sum release/geoip-lite.dat > release/geoip-lite.dat.sha256sum | |
| sha256sum release/security-ip.dat > release/security-ip.dat.sha256sum | |
| sha256sum release/Country.mmdb > release/Country.mmdb.sha256sum | |
| sha256sum release/Country-lite.mmdb > release/Country-lite.mmdb.sha256sum | |
| sha256sum release/Security-ip.mmdb > release/Security-ip.mmdb.sha256sum | |
| sha256sum release/geosite.dat > release/geosite.dat.sha256sum | |
| sha256sum release/geosite-lite.dat > release/geosite-lite.dat.sha256sum | |
| sha256sum release/security.dat > release/security.dat.sha256sum | |
| - name: Generate Release Notes | |
| run: | | |
| echo "* Updated on ${{ env.RELEASE_DATE }}" > RELEASE_NOTES | |
| echo "* IR Domains (Excluding .ir Domains) -> ${{ env.TOTAL_IR }}" >> RELEASE_NOTES | |
| echo "* Active IR Domains (Excluding .ir Domains, Used in geosite-lite) -> ${{ env.TOTAL_IR_LITE }}" >> RELEASE_NOTES | |
| echo "* IR Ad Domains -> ${{ env.TOTAL_IR_ADS }}" >> RELEASE_NOTES | |
| echo "* All Ad Domains -> ${{ env.TOTAL_ADS }}" >> RELEASE_NOTES | |
| echo "* Malware Domains -> ${{ env.TOTAL_MALWARE }}" >> RELEASE_NOTES | |
| echo "* Phishing Domains -> ${{ env.TOTAL_PHISHING }}" >> RELEASE_NOTES | |
| echo "* CryptoMiner Domains -> ${{ env.TOTAL_CRYPTO }}" >> RELEASE_NOTES | |
| echo "* Social Media Domains -> ${{ env.TOTAL_SOCIAL }}" >> RELEASE_NOTES | |
| echo "* NSFW Domains -> ${{ env.TOTAL_NSFW }}" >> RELEASE_NOTES | |
| - name: Push assets to release branch | |
| if: ${{ inputs.PRE_RELEASE == 'false' }} | |
| run: | | |
| cd release || exit 1 | |
| git init | |
| git config --local user.name "github-actions[bot]" | |
| git config --local user.email "41898282+github-actions[bot]@users.noreply.github.com" | |
| git checkout -b release | |
| git add . | |
| git commit -m "${{ env.RELEASE_NAME }}" | |
| git remote add origin "https://${{ github.actor }}:${{ secrets.GITHUB_TOKEN }}@github.com/${{ github.repository }}" | |
| git push -f origin release | |
| - name: Purge jsDelivr CDN cache | |
| if: ${{ inputs.PRE_RELEASE == 'false' }} | |
| run: | | |
| cd release || exit 1 | |
| for file in $(ls); do | |
| curl -i "https://purge.jsdelivr.net/gh/${{ github.repository }}@release/${file}" | |
| done | |
| - name: Release and upload assets | |
| uses: softprops/action-gh-release@v1 | |
| with: | |
| name: ${{ env.RELEASE_NAME }} | |
| tag_name: ${{ env.TAG_NAME }} | |
| body_path: RELEASE_NOTES | |
| draft: false | |
| prerelease: ${{ inputs.PRE_RELEASE }} | |
| files: | | |
| release/*.dat | |
| release/*.mmdb | |
| release/*.sha256sum | |
| env: | |
| GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} | |
| - name: Clean older releases | |
| if: ${{ inputs.PRE_RELEASE == 'false' }} | |
| uses: dev-drprasad/delete-older-releases@v0.3.2 | |
| with: | |
| keep_latest: 7 | |
| delete_tags: true | |
| env: | |
| GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} |