Skip to content

Generate v2ray routing rules #438

Generate v2ray routing rules

Generate v2ray routing rules #438

Workflow file for this run

name: Generate v2ray routing rules
on:
workflow_dispatch:
inputs:
PRE_RELEASE:
description: "Set as pre-release"
required: false
type: boolean
default: false
schedule:
- cron: "0 1 * * *"
jobs:
build:
runs-on: ubuntu-latest
steps:
- name: Checkout VN-v2ray-rules repository
uses: actions/checkout@v4
- name: Checkout domain-list-community repository
uses: actions/checkout@v4
with:
repository: v2fly/domain-list-community
path: v2ray-geosite
- name: Setup Go
uses: actions/setup-go@v4
with:
go-version: "1.23.0"
cache-dependency-path: v2ray-geosite/go.sum
- name: Set ENV variables
run: |
echo "RELEASE_NAME=$(date +%Y%m%d%H%M)" >> $GITHUB_ENV
echo "TAG_NAME=$(date +%Y%m%d%H%M)" >> $GITHUB_ENV
echo "RELEASE_DATE=$(date +'%F %T %Z')" >> $GITHUB_ENV
- name: Install dependencies
run: |
sudo apt-get update
sudo apt-get install dos2unix idn2
- name: Create release directory
run: mkdir release
- name: Get domestic CDNs IP list
run: |
curl --connect-timeout 15 -sSL https://public-dns.info/nameserver/vn.txt -o vnserver-ip.txt || curl -sSL https://raw.githubusercontent.com/vuong2023/vn-v2ray-rules/main/vn.txt -o vnserver-ip.txt
- name: Get GeoLite2
env:
LICENSE_KEY: ${{ secrets.MAXMIND_GEOLITE2_LICENSE }}
run: |
curl -sSL "https://download.maxmind.com/app/geoip_download?edition_id=GeoLite2-Country-CSV&license_key=${LICENSE_KEY}&suffix=zip" -o GeoLite2-Country-CSV.zip
unzip GeoLite2-Country-CSV.zip
rm -f GeoLite2-Country-CSV.zip
mv GeoLite2* geolite2
- name: Generate vn domains list
run: |
curl -sSL https://raw.githubusercontent.com/vuong2023/vn-v2ray-rules/main/domains.txt | grep -Ev ".+\.vn$" | sed '1 a\vn\nvn' | LC_ALL=C sort -u > vn.txt
curl -sSLO https://raw.githubusercontent.com/vuong2023/vn-v2ray-rules/redundant/redundant-domains.txt
comm -23 vn.txt redundant-domains.txt > vn-lite.txt
echo "TOTAL_VN=$(wc -l < vn.txt)" >> $GITHUB_ENV
echo "TOTAL_VN_LITE=$(wc -l < vn-lite.txt)" >> $GITHUB_ENV
mv vn.txt vn-lite.txt release
- name: Generate ads list
run: |
curl -sSL https://raw.githubusercontent.com/bigdargon/hostsVN/master/source/adservers.txt | sed -e 's/^\(|\|\*\|\.\|\-\|0\.0\.0\.0\|127\.0\.0\.1\)*//g' -e 's/\^.*$//g' -e '/!\|?\|@\|#\|\*\|_\|\\\|\/\|\[\|]\|\[\|\([0-9]\{1,3\}\.\)\{3\}[0-9]\{1,3\}/d' -e '/\.$/d' -e '/^\s*$/d' | awk '{$1=$1};1' | dos2unix | idn2 --no-alabelroundtrip --no-tr46 | LC_ALL=C sort -u > ads.txt
echo "TOTAL_VN_ADS=$(wc -l < ads.txt)" >> $GITHUB_ENV
- name: Generate category-ads-all list
run: |
curl -sSL https://raw.githubusercontent.com/bigdargon/hostsVN/master/source/adservers.txt > category-ads-all-raw.txt
curl -sSL https://raw.githubusercontent.com/hagezi/dns-blocklists/main/wildcard/light-onlydomains.txt >> category-ads-all-raw.txt
curl -sSL https://pgl.yoyo.org/adservers/serverlist.php?hostformat=nohtml >> category-ads-all-raw.txt
curl -sSL https://raw.githubusercontent.com/jerryn70/GoodbyeAds/master/Extension/GoodbyeAds-Samsung-AdBlock.txt >> category-ads-all-raw.txt
curl -sSL https://raw.githubusercontent.com/jerryn70/GoodbyeAds/master/Extension/GoodbyeAds-Xiaomi-Extension.txt >> category-ads-all-raw.txt
curl -sSL https://raw.githubusercontent.com/bigdargon/hostsVN/master/source/hosts-VN.txt >> category-ads-all-raw.txt
curl -sSL https://raw.githubusercontent.com/bigdargon/hostsVN/master/source/adservers-all.txt >> category-ads-all-raw.txt
curl -sSL https://raw.githubusercontent.com/bigdargon/hostsVN/master/extensions/adult/filter.txt >> category-ads-all-raw.txt
curl -sSL https://raw.githubusercontent.com/bigdargon/hostsVN/master/extensions/threat/filter-VN.txt >> category-ads-all-raw.txt
curl -sSL https://raw.githubusercontent.com/bigdargon/hostsVN/master/extensions/adult/hosts >> category-ads-all-raw.txt
curl -sSL https://raw.githubusercontent.com/bigdargon/hostsVN/master/extensions/gambling/filter-VN.txt >> category-ads-all-raw.txt
curl -sSL https://raw.githubusercontent.com/bigdargon/hostsVN/master/extensions/threat/filter-VN.txt >> category-ads-all-raw.txt
curl -sSL https://raw.githubusercontent.com/StevenBlack/hosts/master/data/hostsVN/hosts | sed -e 's/^\(|\|\*\|\.\|\-\|0\.0\.0\.0\|127\.0\.0\.1\)*//g' -e 's/\^.*$//g' -e '/!\|?\|@\|#\|\*\|_\|\\\|\/\|\[\|]\|\[\|\([0-9]\{1,3\}\.\)\{3\}[0-9]\{1,3\}/d' -e '/\.$/d' -e '/^\s*$/d' | awk '{$1=$1};1' | dos2unix | idn2 --no-alabelroundtrip --no-tr46 | LC_ALL=C sort -u > category-ads-all-raw.txt
cat category-ads-all-raw.txt | sed -e 's/^\(|\|\*\|\.\|\-\|0\.0\.0\.0\|127\.0\.0\.1\)*//g' -e 's/\^.*$//g' -e '/!\|?\|@\|#\|\*\|_\|\\\|\/\|\[\|]\|\[\|\([0-9]\{1,3\}\.\)\{3\}[0-9]\{1,3\}/d' -e '/\.$/d' -e '/^\s*$/d' | awk '{$1=$1};1' | dos2unix | idn2 --no-alabelroundtrip --no-tr46 | LC_ALL=C sort -u > category-ads-all-temp.txt
curl -sSL https://raw.githubusercontent.com/hagezi/dns-blocklists/main/whitelist.txt >> whitelist-raw.txt
curl -sSL https://raw.githubusercontent.com/hagezi/dns-blocklists/main/whitelist-referral.txt >> whitelist-raw.txt
curl -sSL https://raw.githubusercontent.com/hagezi/dns-blocklists/main/domains/pro.plus.txt >> whitelist-raw.txt
curl -sSL https://raw.githubusercontent.com/bigdargon/hostsVN/master/filters/adservers-all.txt >> whitelist-raw.txt
curl -sSL https://raw.githubusercontent.com/AdguardTeam/AdGuardSDNSFilter/master/Filters/exclusions.txt >> whitelist-raw.txt
curl -sSL https://raw.githubusercontent.com/AdguardTeam/AdGuardSDNSFilter/master/Filters/exceptions.txt >> whitelist-raw.txt
cat whitelist-raw.txt | sed -e 's/^\(|\|@\|\*\|\.\|\-\|0\.0\.0\.0\|127\.0\.0\.1\)*//g' -e 's/\^.*$//g' -e '/!\|?\|@\|#\|\*\|_\|\\\|\/\|\[\|]\|\[\|\([0-9]\{1,3\}\.\)\{3\}[0-9]\{1,3\}/d' -e '/\.$/d' -e '/^\s*$/d' | awk '{$1=$1};1' | dos2unix | idn2 --no-alabelroundtrip --no-tr46 | LC_ALL=C sort -u > whitelist-temp.txt
comm -23 category-ads-all-temp.txt whitelist-temp.txt > category-ads-all-temp-temp.txt
sed -e 's/^/\./' -e 's/\./\\./g' -e 's/\-/\\-/g' -e 's/$/\$/' category-ads-all-temp-temp.txt > category-ads-all-sub.txt
cat category-ads-all-temp-temp.txt | LC_ALL=C grep -f category-ads-all-sub.txt | LC_ALL=C sort -u > category-ads-all-redundant-sub.txt
comm -23 category-ads-all-temp-temp.txt category-ads-all-redundant-sub.txt > category-ads-all.txt
echo "TOTAL_ADS=$(wc -l < category-ads-all.txt)" >> $GITHUB_ENV
rm -f category-ads-all-raw.txt whitelist-raw.txt category-ads-all-temp.txt whitelist-temp.txt
mv ads.txt category-ads-all.txt release
- name: Generate malware list
run: |
curl -sSL https://malware-filter.gitlab.io/malware-filter/urlhaus-filter-dnscrypt-blocked-names-online.txt | sed -e 's/^\(|\|@\|\*\|\.\|\-\|0\.0\.0\.0\|127\.0\.0\.1\)*//g' -e 's/\^.*$//g' -e '/!\|?\|@\|#\|\*\|_\|\\\|\/\|\[\|]\|\[\|\([0-9]\{1,3\}\.\)\{3\}[0-9]\{1,3\}/d' -e '/\.$/d' -e '/^\s*$/d' | awk '{$1=$1};1' | dos2unix | idn2 --no-alabelroundtrip --no-tr46 | LC_ALL=C sort -u > malware-temp.txt
curl -sSL https://malware-filter.gitlab.io/malware-filter/vn-badsite-filter-domains.txt | sed -e 's/^\(|\|@\|\*\|\.\|\-\|0\.0\.0\.0\|127\.0\.0\.1\)*//g' -e 's/\^.*$//g' -e '/!\|?\|@\|#\|\*\|_\|\\\|\/\|\[\|]\|\[\|\([0-9]\{1,3\}\.\)\{3\}[0-9]\{1,3\}/d' -e '/\.$/d' -e '/^\s*$/d' | awk '{$1=$1};1' | dos2unix | idn2 --no-alabelroundtrip --no-tr46 | LC_ALL=C sort -u > malware-temp.txt
curl -sSL https://malware-filter.gitlab.io/malware-filter/urlhaus-filter-dnscrypt-blocked-ips-online.txt | sed '/#/d' > malware-ip.txt
curl -sSL https://malware-filter.gitlab.io/malware-filter/vn-badsite-filter-dnscrypt-blocked-ips.txt | sed '/#/d' > malware-ip.txt
curl -sSL https://raw.githubusercontent.com/bigdargon/hostsVN/master/extensions/ip/list | sed '/#/d' > malware-ip.txt
sed -e 's/^/\./' -e 's/\./\\./g' -e 's/\-/\\-/g' -e 's/$/\$/' malware-temp.txt > malware-sub.txt
cat malware-temp.txt | LC_ALL=C grep -f malware-sub.txt | LC_ALL=C sort -u > malware-redundant-sub.txt
comm -23 malware-temp.txt malware-redundant-sub.txt > malware.txt
echo "TOTAL_MALWARE=$(wc -l < malware.txt)" >> $GITHUB_ENV
mv malware.txt release
- name: Generate phishing list
run: |
curl -sSL https://malware-filter.gitlab.io/malware-filter/phishing-filter-dnscrypt-blocked-names.txt | sed -e 's/^\(|\|@\|\*\|\.\|\-\|0\.0\.0\.0\|127\.0\.0\.1\)*//g' -e 's/\^.*$//g' -e '/!\|?\|@\|#\|\*\|_\|\\\|\/\|\[\|]\|\[\|\([0-9]\{1,3\}\.\)\{3\}[0-9]\{1,3\}/d' -e '/\.$/d' -e '/^\s*$/d' | awk '{$1=$1};1' | dos2unix | idn2 --no-alabelroundtrip --no-tr46 | LC_ALL=C sort -u > phishing-temp.txt
curl -sSL https://malware-filter.gitlab.io/malware-filter/phishing-filter-dnscrypt-blocked-ips.txt | sed '/#/d' > phishing-ip.txt
sed -e 's/^/\./' -e 's/\./\\./g' -e 's/\-/\\-/g' -e 's/$/\$/' phishing-temp.txt > phishing-sub.txt
cat phishing-temp.txt | LC_ALL=C grep -f phishing-sub.txt | LC_ALL=C sort -u > phishing-redundant-sub.txt
comm -23 phishing-temp.txt phishing-redundant-sub.txt > phishing.txt
echo "TOTAL_PHISHING=$(wc -l < phishing.txt)" >> $GITHUB_ENV
mv phishing.txt release
- name: Generate cryptominers domains list
run: |
curl -sSL https://raw.githubusercontent.com/hoshsadiq/adblock-nocoin-list/master/hosts.txt | sed -e 's/^\(|\|\*\|\.\|\-\|0\.0\.0\.0\|127\.0\.0\.1\)*//g' -e 's/\^.*$//g' -e '/!\|?\|@\|#\|\*\|_\|\\\|\/\|\[\|]\|\[\|\([0-9]\{1,3\}\.\)\{3\}[0-9]\{1,3\}/d' -e '/\.$/d' -e '/^\s*$/d' | awk '{$1=$1};1' | dos2unix | idn2 --no-alabelroundtrip --no-tr46 | LC_ALL=C sort -u > cryptominers-temp.txt
sed -e 's/^/\./' -e 's/\./\\./g' -e 's/\-/\\-/g' -e 's/$/\$/' cryptominers-temp.txt > cryptominers-sub.txt
cat cryptominers-temp.txt | LC_ALL=C grep -f cryptominers-sub.txt | LC_ALL=C sort -u > cryptominers-redundant-sub.txt
comm -23 cryptominers-temp.txt cryptominers-redundant-sub.txt > cryptominers.txt
echo "TOTAL_CRYPTO=$(wc -l < cryptominers.txt)" >> $GITHUB_ENV
mv cryptominers.txt release
- name: Generate Social Media domains list
run: |
curl -sSL https://raw.githubusercontent.com/StevenBlack/hosts/master/alternates/social-only/hosts | sed -e 's/^\(|\|\*\|\.\|\-\|0\.0\.0\.0 \|127\.0\.0\.1 \)*//g' -e 's/\^.*$//g' -e 's/^\(www\.\)*//g' -e '/!\|?\|@\|#\|\*\|_\|\\\|\/\|\[\|]\|\[\|\([0-9]\{1,3\}\.\)\{3\}[0-9]\{1,3\}/d' -e '/\.$/d' -e '/^\s*$/d' | awk '{$1=$1};1' | dos2unix | idn2 --no-alabelroundtrip --no-tr46 | LC_ALL=C sort -u > social-temp.txt
sed -e 's/^/\./' -e 's/\./\\./g' -e 's/\-/\\-/g' -e 's/$/\$/' social-temp.txt > social-sub.txt
cat social-temp.txt | LC_ALL=C grep -f social-sub.txt | LC_ALL=C sort -u > social-redundant-sub.txt
comm -23 social-temp.txt social-redundant-sub.txt > social.txt
echo "TOTAL_SOCIAL=$(wc -l < social.txt)" >> $GITHUB_ENV
mv social.txt release
- name: Generate nsfw domains list
run: |
curl -sSL https://raw.githubusercontent.com/StevenBlack/hosts/master/alternates/gambling-porn-only/hosts | sed -e 's/^\(|\|\*\|\.\|\-\|0\.0\.0\.0 \|127\.0\.0\.1 \)*//g' -e 's/\^.*$//g' -e 's/^\(www\.\)*//g' -e '/!\|?\|@\|#\|\*\|_\|\\\|\/\|\[\|]\|\[\|\([0-9]\{1,3\}\.\)\{3\}[0-9]\{1,3\}/d' -e '/\.$/d' -e '/^\s*$/d' | awk '{$1=$1};1' | dos2unix | idn2 --no-alabelroundtrip --no-tr46 | LC_ALL=C sort -u > nsfw-temp.txt
sed -e 's/^/\./' -e 's/\./\\./g' -e 's/\-/\\-/g' -e 's/$/\$/' nsfw-temp.txt > nsfw-sub.txt
cat nsfw-temp.txt | LC_ALL=C grep -f nsfw-sub.txt | LC_ALL=C sort -u > nsfw-redundant-sub.txt
comm -23 nsfw-temp.txt nsfw-redundant-sub.txt > nsfw.txt
echo "TOTAL_NSFW=$(wc -l < nsfw.txt)" >> $GITHUB_ENV
mv nsfw.txt release
- name: Generate sex domain list
run: |
curl -sSL https://raw.githubusercontent.com/bigdargon/hostsVN/master/extensions/source/adult-VN.txt | sed -e 's/^\(|\|@\|\*\|\.\|\-\|0\.0\.0\.0\|127\.0\.0\.1\)*//g' -e 's/\^.*$//g' -e '/!\|?\|@\|#\|\*\|_\|\\\|\/\|\[\|]\|\[\|\([0-9]\{1,3\}\.\)\{3\}[0-9]\{1,3\}/d' -e '/\.$/d' -e '/^\s*$/d' | awk '{$1=$1};1' | dos2unix | idn2 --no-alabelroundtrip --no-tr46 | LC_ALL=C sort -u > sex-temp.txt
curl -sSL https://raw.githubusercontent.com/bigdargon/hostsVN/master/extensions/source/adult.txt | sed '/#/d' >> sex-temp.txt
sed -e 's/^/\./' -e 's/\./\\./g' -e 's/\-/\\-/g' -e 's/$/\$/' sex-temp.txt > sex-sub.txt
sort -u sex-temp.txt > sorted-sex-temp.txt
grep -F -f sex-sub.txt sorted-sex-temp.txt | sort -u > sex-redundant-sub.txt
comm -23 sorted-sex-temp.txt sex-redundant-sub.txt > sex.txt
echo "TOTAL_SEX=$(wc -l < sex.txt)" >> $GITHUB_ENV
mv sex.txt release
- name: Generate geoip.dat, geoip-lite.dat, security-ip.dat, Country.mmdb, Country-lite.mmdb and Security-ip.mmdb files
run: |
go install -v github.com/Loyalsoldier/geoip@latest
$(go env GOPATH)/bin/geoip -c config.json
cp output/dat/geoip.dat output/dat/geoip-lite.dat output/dat/security-ip.dat release
cp output/maxmind/Country.mmdb output/maxmind/Country-lite.mmdb output/maxmind/Security-ip.mmdb release
cp -fpPR output/text release
- name: Verify Country.mmdb, Country-lite.mmdb and Security-ip.mmdb files
run: |
go install -v github.com/maxmind/mmdbverify@latest
$(go env GOPATH)/bin/mmdbverify -file release/Country.mmdb
$(go env GOPATH)/bin/mmdbverify -file release/Country-lite.mmdb
$(go env GOPATH)/bin/mmdbverify -file release/Security-ip.mmdb
- name: Generate geosite.dat, geosite-lite.dat and security.dat files
run: |
cd v2ray-geosite
cp ../release/vn.txt data/vn
cp ../release/ads.txt data/ads
cp ../release/category-ads-all.txt data/category-ads-all
cp ../release/malware.txt data/malware
cp ../release/phishing.txt data/phishing
cp ../release/cryptominers.txt data/cryptominers
cp ../release/social.txt data/social
cp ../release/nsfw.txt data/nsfw
cp ../release/sex.txt data/sex
go mod edit -go=1.21
go get -u
go mod tidy
go run ./ --datapath=data --outputdir=../release --outputname=geosite.dat
mkdir datalite
cp ../release/vn-lite.txt datalite/vn
cp ../release/ads.txt datalite/ads
go run ./ --datapath=datalite --outputdir=../release --outputname=geosite-lite.dat
mkdir security
cp ../release/category-ads-all.txt security/category-ads-all
cp ../release/malware.txt security/malware
cp ../release/phishing.txt security/phishing
cp ../release/cryptominers.txt security/cryptominers
go run ./ --datapath=security --outputdir=../release --outputname=security.dat
- name: Generate sha256sum
run: |
sha256sum release/geoip.dat > release/geoip.dat.sha256sum
sha256sum release/geoip-lite.dat > release/geoip-lite.dat.sha256sum
sha256sum release/security-ip.dat > release/security-ip.dat.sha256sum
sha256sum release/Country.mmdb > release/Country.mmdb.sha256sum
sha256sum release/Country-lite.mmdb > release/Country-lite.mmdb.sha256sum
sha256sum release/Security-ip.mmdb > release/Security-ip.mmdb.sha256sum
sha256sum release/geosite.dat > release/geosite.dat.sha256sum
sha256sum release/geosite-lite.dat > release/geosite-lite.dat.sha256sum
sha256sum release/security.dat > release/security.dat.sha256sum
- name: Generate Release Notes
run: |
echo "* Updated on ${{ env.RELEASE_DATE }}" > RELEASE_NOTES
echo "* VN Domains (Excluding .vn Domains) -> ${{ env.TOTAL_VN }}" >> RELEASE_NOTES
echo "* Active VN Domains (Excluding .vn Domains, Used in geosite-lite) -> ${{ env.TOTAL_VN_LITE }}" >> RELEASE_NOTES
echo "* VN Ad Domains -> ${{ env.TOTAL_VN_ADS }}" >> RELEASE_NOTES
echo "* All Ad Domains -> ${{ env.TOTAL_ADS }}" >> RELEASE_NOTES
echo "* Malware Domains -> ${{ env.TOTAL_MALWARE }}" >> RELEASE_NOTES
echo "* Phishing Domains -> ${{ env.TOTAL_PHISHING }}" >> RELEASE_NOTES
echo "* CryptoMiner Domains -> ${{ env.TOTAL_CRYPTO }}" >> RELEASE_NOTES
echo "* Social Media Domains -> ${{ env.TOTAL_SOCIAL }}" >> RELEASE_NOTES
echo "* NSFW Domains -> ${{ env.TOTAL_NSFW }}" >> RELEASE_NOTES
echo "* SEX Domains -> ${{ env.TOTAL_SEX }}" >> RELEASE_NOTES
- name: Push assets to release branch
if: ${{ !inputs.PRE_RELEASE }}
run: |
cd release || exit 1
git init
git config --local user.name "github-actions[bot]"
git config --local user.email "41898282+github-actions[bot]@users.noreply.github.com"
git checkout -b release
git add .
git commit -m "${{ env.RELEASE_NAME }}"
git remote add origin "https://${{ github.actor }}:${{ secrets.GITHUB_TOKEN }}@github.com/${{ github.repository }}"
git push -f origin release
- name: Purge jsDelivr CDN cache
if: ${{ !inputs.PRE_RELEASE }}
run: |
cd release || exit 1
for file in $(ls); do
curl -i "https://purge.jsdelivr.net/gh/${{ github.repository }}@release/${file}"
done
- name: Release and upload assets
uses: softprops/action-gh-release@v1
with:
name: ${{ env.RELEASE_NAME }}
tag_name: ${{ env.TAG_NAME }}
body_path: RELEASE_NOTES
draft: false
prerelease: ${{ inputs.PRE_RELEASE }}
files: |
release/*.dat
release/*.mmdb
release/*.sha256sum
env:
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
- name: Clean older releases
if: ${{ !inputs.PRE_RELEASE }}
uses: dev-drprasad/delete-older-releases@v0.3.2
with:
keep_latest: 7
delete_tags: true
env:
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}