Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Add domain list community categories, new files and other improvements #11

Merged
merged 2 commits into from
Oct 30, 2023
Merged

Add domain list community categories, new files and other improvements #11

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
03fa5ca
feat: integrate domain-list-community data
Chocolate4U Oct 30, 2023
017d75a
feat: add new rule files
Chocolate4U Oct 30, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
108 changes: 71 additions & 37 deletions .github/workflows/release.yml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2,29 +2,34 @@ name: Generate v2ray routing rules

on:
workflow_dispatch:
inputs:
PRE_RELEASE:
description: "Set as pre-release"
required: false
default: "false"
schedule:
- cron: "0 1 * * *"

jobs:
build:
runs-on: ubuntu-latest

steps:
- name: Checkout Iran-v2ray-rules repository
uses: actions/checkout@v4
- name: Checkout v2ray-geosite repository

- name: Checkout domain-list-community repository
uses: actions/checkout@v4
with:
repository: Chocolate4U/v2ray-geosite
repository: v2fly/domain-list-community
path: v2ray-geosite

- name: Setup Go
uses: actions/setup-go@v4
with:
go-version: '1.21'
go-version: "1.21"
cache-dependency-path: v2ray-geosite/go.sum

- name: Set ENV variables
run: |
echo "RELEASE_NAME=$(date +%Y%m%d%H%M)" >> $GITHUB_ENV
Expand All @@ -34,8 +39,7 @@ jobs:
- name: Install dependencies
run: |
sudo apt-get update
sudo apt-get install dos2unix
sudo apt-get install idn2
sudo apt-get install dos2unix idn2

- name: Create release directory
run: mkdir release
Expand Down Expand Up @@ -63,21 +67,16 @@ jobs:

- name: Generate ir domains list
run: |
curl -sSL https://github.com/bootmortis/iran-hosted-domains/releases/latest/download/domains.txt | grep -Ev ".+\.ir$" | sed '1 a\ir\nxn--mgba3a4f16a' | sort -u > ir.txt
curl -sSL https://github.com/bootmortis/iran-hosted-domains/releases/latest/download/domains.txt | grep -Ev ".+\.ir$" | sed '1 a\ir\nxn--mgba3a4f16a' | LC_ALL=C sort -u > ir.txt
curl -sSLO https://raw.githubusercontent.com/Chocolate4U/Iran-v2ray-rules/redundant/redundant-domains.txt
comm -23 ir.txt redundant-domains.txt > ir-lite.txt
echo "TOTAL_IR=$(wc -l < ir.txt)" >> $GITHUB_ENV
echo "TOTAL_IR_LITE=$(wc -l < ir-lite.txt)" >> $GITHUB_ENV
mv ir.txt ir-lite.txt release

- name: Get cn tld list
run: |
curl -sSL https://raw.githubusercontent.com/Loyalsoldier/v2ray-rules-dat/release/direct-tld-list.txt -o cn.txt
mv cn.txt release

- name: Generate ads list
run: |
curl -sSL https://raw.githubusercontent.com/MasterKia/PersianBlocker/main/PersianBlockerHosts.txt | sed -e 's/^\(|\|\*\|\.\|\-\|0\.0\.0\.0\|127\.0\.0\.1\)*//g' -e 's/\^.*$//g' -e '/!\|?\|@\|#\|\*\|_\|\\\|\/\|\[\|]\|\[\|\([0-9]\{1,3\}\.\)\{3\}[0-9]\{1,3\}/d' -e '/\.$/d' -e '/^\s*$/d' | awk '{$1=$1};1' | dos2unix | idn2 --no-alabelroundtrip --no-tr46 | sort -u > ads.txt
curl -sSL https://raw.githubusercontent.com/MasterKia/PersianBlocker/main/PersianBlockerHosts.txt | sed -e 's/^\(|\|\*\|\.\|\-\|0\.0\.0\.0\|127\.0\.0\.1\)*//g' -e 's/\^.*$//g' -e '/!\|?\|@\|#\|\*\|_\|\\\|\/\|\[\|]\|\[\|\([0-9]\{1,3\}\.\)\{3\}[0-9]\{1,3\}/d' -e '/\.$/d' -e '/^\s*$/d' | awk '{$1=$1};1' | dos2unix | idn2 --no-alabelroundtrip --no-tr46 | LC_ALL=C sort -u > ads.txt
echo "TOTAL_IR_ADS=$(wc -l < ads.txt)" >> $GITHUB_ENV

- name: Generate category-ads-all list
Expand All @@ -87,95 +86,127 @@ jobs:
curl -sSL https://pgl.yoyo.org/adservers/serverlist.php?hostformat=nohtml >> category-ads-all-raw.txt
curl -sSL https://raw.githubusercontent.com/jerryn70/GoodbyeAds/master/Extension/GoodbyeAds-Samsung-AdBlock.txt >> category-ads-all-raw.txt
curl -sSL https://raw.githubusercontent.com/jerryn70/GoodbyeAds/master/Extension/GoodbyeAds-Xiaomi-Extension.txt >> category-ads-all-raw.txt
cat category-ads-all-raw.txt | sed -e 's/^\(|\|\*\|\.\|\-\|0\.0\.0\.0\|127\.0\.0\.1\)*//g' -e 's/\^.*$//g' -e '/!\|?\|@\|#\|\*\|_\|\\\|\/\|\[\|]\|\[\|\([0-9]\{1,3\}\.\)\{3\}[0-9]\{1,3\}/d' -e '/\.$/d' -e '/^\s*$/d' | awk '{$1=$1};1' | dos2unix | idn2 --no-alabelroundtrip --no-tr46 | sort -u > category-ads-all-temp.txt
cat category-ads-all-raw.txt | sed -e 's/^\(|\|\*\|\.\|\-\|0\.0\.0\.0\|127\.0\.0\.1\)*//g' -e 's/\^.*$//g' -e '/!\|?\|@\|#\|\*\|_\|\\\|\/\|\[\|]\|\[\|\([0-9]\{1,3\}\.\)\{3\}[0-9]\{1,3\}/d' -e '/\.$/d' -e '/^\s*$/d' | awk '{$1=$1};1' | dos2unix | idn2 --no-alabelroundtrip --no-tr46 | LC_ALL=C sort -u > category-ads-all-temp.txt

curl -sSL https://raw.githubusercontent.com/hagezi/dns-blocklists/main/whitelist.txt > whitelist-raw.txt
curl -sSL https://raw.githubusercontent.com/hagezi/dns-blocklists/main/whitelist-referral.txt >> whitelist-raw.txt
curl -sSL https://raw.githubusercontent.com/AdguardTeam/AdGuardSDNSFilter/master/Filters/exclusions.txt >> whitelist-raw.txt
curl -sSL https://raw.githubusercontent.com/AdguardTeam/AdGuardSDNSFilter/master/Filters/exceptions.txt >> whitelist-raw.txt
cat whitelist-raw.txt | sed -e 's/^\(|\|@\|\*\|\.\|\-\|0\.0\.0\.0\|127\.0\.0\.1\)*//g' -e 's/\^.*$//g' -e '/!\|?\|@\|#\|\*\|_\|\\\|\/\|\[\|]\|\[\|\([0-9]\{1,3\}\.\)\{3\}[0-9]\{1,3\}/d' -e '/\.$/d' -e '/^\s*$/d' | awk '{$1=$1};1' | dos2unix | idn2 --no-alabelroundtrip --no-tr46 | sort -u > whitelist-temp.txt
cat whitelist-raw.txt | sed -e 's/^\(|\|@\|\*\|\.\|\-\|0\.0\.0\.0\|127\.0\.0\.1\)*//g' -e 's/\^.*$//g' -e '/!\|?\|@\|#\|\*\|_\|\\\|\/\|\[\|]\|\[\|\([0-9]\{1,3\}\.\)\{3\}[0-9]\{1,3\}/d' -e '/\.$/d' -e '/^\s*$/d' | awk '{$1=$1};1' | dos2unix | idn2 --no-alabelroundtrip --no-tr46 | LC_ALL=C sort -u > whitelist-temp.txt

comm -23 category-ads-all-temp.txt whitelist-temp.txt > category-ads-all.txt
comm -23 category-ads-all-temp.txt whitelist-temp.txt > category-ads-all-temp-temp.txt
sed -e 's/^/\./' -e 's/\./\\./g' -e 's/\-/\\-/g' -e 's/$/\$/' category-ads-all-temp-temp.txt > category-ads-all-sub.txt
cat category-ads-all-temp-temp.txt | LC_ALL=C grep -f category-ads-all-sub.txt | LC_ALL=C sort -u > category-ads-all-redundant-sub.txt
comm -23 category-ads-all-temp-temp.txt category-ads-all-redundant-sub.txt > category-ads-all.txt
echo "TOTAL_ADS=$(wc -l < category-ads-all.txt)" >> $GITHUB_ENV
rm -f category-ads-all-raw.txt whitelist-raw.txt category-ads-all-temp.txt whitelist-temp.txt
mv ads.txt category-ads-all.txt release

- name: Generate malware list
run: |
curl -sSL https://malware-filter.gitlab.io/malware-filter/urlhaus-filter-dnscrypt-blocked-names-online.txt | sed -e 's/^\(|\|@\|\*\|\.\|\-\|0\.0\.0\.0\|127\.0\.0\.1\)*//g' -e 's/\^.*$//g' -e '/!\|?\|@\|#\|\*\|_\|\\\|\/\|\[\|]\|\[\|\([0-9]\{1,3\}\.\)\{3\}[0-9]\{1,3\}/d' -e '/\.$/d' -e '/^\s*$/d' | awk '{$1=$1};1' | dos2unix | idn2 --no-alabelroundtrip --no-tr46 | sort -u > malware.txt
curl -sSL https://malware-filter.gitlab.io/malware-filter/urlhaus-filter-dnscrypt-blocked-names-online.txt | sed -e 's/^\(|\|@\|\*\|\.\|\-\|0\.0\.0\.0\|127\.0\.0\.1\)*//g' -e 's/\^.*$//g' -e '/!\|?\|@\|#\|\*\|_\|\\\|\/\|\[\|]\|\[\|\([0-9]\{1,3\}\.\)\{3\}[0-9]\{1,3\}/d' -e '/\.$/d' -e '/^\s*$/d' | awk '{$1=$1};1' | dos2unix | idn2 --no-alabelroundtrip --no-tr46 | LC_ALL=C sort -u > malware-temp.txt
curl -sSL https://malware-filter.gitlab.io/malware-filter/urlhaus-filter-dnscrypt-blocked-ips-online.txt | sed '/#/d' > malware-ip.txt
sed -e 's/^/\./' -e 's/\./\\./g' -e 's/\-/\\-/g' -e 's/$/\$/' malware-temp.txt > malware-sub.txt
cat malware-temp.txt | LC_ALL=C grep -f malware-sub.txt | LC_ALL=C sort -u > malware-redundant-sub.txt
comm -23 malware-temp.txt malware-redundant-sub.txt > malware.txt
echo "TOTAL_MALWARE=$(wc -l < malware.txt)" >> $GITHUB_ENV
mv malware.txt release

- name: Generate phishing list
run: |
curl -sSL https://malware-filter.gitlab.io/malware-filter/phishing-filter-dnscrypt-blocked-names.txt | sed -e 's/^\(|\|@\|\*\|\.\|\-\|0\.0\.0\.0\|127\.0\.0\.1\)*//g' -e 's/\^.*$//g' -e '/!\|?\|@\|#\|\*\|_\|\\\|\/\|\[\|]\|\[\|\([0-9]\{1,3\}\.\)\{3\}[0-9]\{1,3\}/d' -e '/\.$/d' -e '/^\s*$/d' | awk '{$1=$1};1' | dos2unix | idn2 --no-alabelroundtrip --no-tr46 | sort -u > phishing.txt
curl -sSL https://malware-filter.gitlab.io/malware-filter/phishing-filter-dnscrypt-blocked-names.txt | sed -e 's/^\(|\|@\|\*\|\.\|\-\|0\.0\.0\.0\|127\.0\.0\.1\)*//g' -e 's/\^.*$//g' -e '/!\|?\|@\|#\|\*\|_\|\\\|\/\|\[\|]\|\[\|\([0-9]\{1,3\}\.\)\{3\}[0-9]\{1,3\}/d' -e '/\.$/d' -e '/^\s*$/d' | awk '{$1=$1};1' | dos2unix | idn2 --no-alabelroundtrip --no-tr46 | LC_ALL=C sort -u > phishing-temp.txt
curl -sSL https://malware-filter.gitlab.io/malware-filter/phishing-filter-dnscrypt-blocked-ips.txt | sed '/#/d' > phishing-ip.txt
sed -e 's/^/\./' -e 's/\./\\./g' -e 's/\-/\\-/g' -e 's/$/\$/' phishing-temp.txt > phishing-sub.txt
cat phishing-temp.txt | LC_ALL=C grep -f phishing-sub.txt | LC_ALL=C sort -u > phishing-redundant-sub.txt
comm -23 phishing-temp.txt phishing-redundant-sub.txt > phishing.txt
echo "TOTAL_PHISHING=$(wc -l < phishing.txt)" >> $GITHUB_ENV
mv phishing.txt release

- name: Generate cryptominers domains list
run: |
curl -sSL https://raw.githubusercontent.com/hoshsadiq/adblock-nocoin-list/master/hosts.txt | sed -e 's/^\(|\|\*\|\.\|\-\|0\.0\.0\.0\|127\.0\.0\.1\)*//g' -e 's/\^.*$//g' -e '/!\|?\|@\|#\|\*\|_\|\\\|\/\|\[\|]\|\[\|\([0-9]\{1,3\}\.\)\{3\}[0-9]\{1,3\}/d' -e '/\.$/d' -e '/^\s*$/d' | awk '{$1=$1};1' | dos2unix | idn2 --no-alabelroundtrip --no-tr46 | sort -u > cryptominers.txt
curl -sSL https://raw.githubusercontent.com/hoshsadiq/adblock-nocoin-list/master/hosts.txt | sed -e 's/^\(|\|\*\|\.\|\-\|0\.0\.0\.0\|127\.0\.0\.1\)*//g' -e 's/\^.*$//g' -e '/!\|?\|@\|#\|\*\|_\|\\\|\/\|\[\|]\|\[\|\([0-9]\{1,3\}\.\)\{3\}[0-9]\{1,3\}/d' -e '/\.$/d' -e '/^\s*$/d' | awk '{$1=$1};1' | dos2unix | idn2 --no-alabelroundtrip --no-tr46 | LC_ALL=C sort -u > cryptominers-temp.txt
sed -e 's/^/\./' -e 's/\./\\./g' -e 's/\-/\\-/g' -e 's/$/\$/' cryptominers-temp.txt > cryptominers-sub.txt
cat cryptominers-temp.txt | LC_ALL=C grep -f cryptominers-sub.txt | LC_ALL=C sort -u > cryptominers-redundant-sub.txt
comm -23 cryptominers-temp.txt cryptominers-redundant-sub.txt > cryptominers.txt
echo "TOTAL_CRYPTO=$(wc -l < cryptominers.txt)" >> $GITHUB_ENV
mv cryptominers.txt release

- name: Generate Social Media domains list
run: |
curl -sSL https://raw.githubusercontent.com/StevenBlack/hosts/master/alternates/social-only/hosts | sed -e 's/^\(|\|\*\|\.\|\-\|0\.0\.0\.0 \|127\.0\.0\.1 \)*//g' -e 's/\^.*$//g' -e 's/^\(www\.\)*//g' -e '/!\|?\|@\|#\|\*\|_\|\\\|\/\|\[\|]\|\[\|\([0-9]\{1,3\}\.\)\{3\}[0-9]\{1,3\}/d' -e '/\.$/d' -e '/^\s*$/d' | awk '{$1=$1};1' | dos2unix | idn2 --no-alabelroundtrip --no-tr46 | sort -u > social.txt
curl -sSL https://raw.githubusercontent.com/StevenBlack/hosts/master/alternates/social-only/hosts | sed -e 's/^\(|\|\*\|\.\|\-\|0\.0\.0\.0 \|127\.0\.0\.1 \)*//g' -e 's/\^.*$//g' -e 's/^\(www\.\)*//g' -e '/!\|?\|@\|#\|\*\|_\|\\\|\/\|\[\|]\|\[\|\([0-9]\{1,3\}\.\)\{3\}[0-9]\{1,3\}/d' -e '/\.$/d' -e '/^\s*$/d' | awk '{$1=$1};1' | dos2unix | idn2 --no-alabelroundtrip --no-tr46 | LC_ALL=C sort -u > social-temp.txt
sed -e 's/^/\./' -e 's/\./\\./g' -e 's/\-/\\-/g' -e 's/$/\$/' social-temp.txt > social-sub.txt
cat social-temp.txt | LC_ALL=C grep -f social-sub.txt | LC_ALL=C sort -u > social-redundant-sub.txt
comm -23 social-temp.txt social-redundant-sub.txt > social.txt
echo "TOTAL_SOCIAL=$(wc -l < social.txt)" >> $GITHUB_ENV
mv social.txt release

- name: Generate nsfw domains list
run: |
curl -sSL https://raw.githubusercontent.com/StevenBlack/hosts/master/alternates/gambling-porn-only/hosts | sed -e 's/^\(|\|\*\|\.\|\-\|0\.0\.0\.0 \|127\.0\.0\.1 \)*//g' -e 's/\^.*$//g' -e 's/^\(www\.\)*//g' -e '/!\|?\|@\|#\|\*\|_\|\\\|\/\|\[\|]\|\[\|\([0-9]\{1,3\}\.\)\{3\}[0-9]\{1,3\}/d' -e '/\.$/d' -e '/^\s*$/d' | awk '{$1=$1};1' | dos2unix | idn2 --no-alabelroundtrip --no-tr46 | sort -u > nsfw.txt
curl -sSL https://raw.githubusercontent.com/StevenBlack/hosts/master/alternates/gambling-porn-only/hosts | sed -e 's/^\(|\|\*\|\.\|\-\|0\.0\.0\.0 \|127\.0\.0\.1 \)*//g' -e 's/\^.*$//g' -e 's/^\(www\.\)*//g' -e '/!\|?\|@\|#\|\*\|_\|\\\|\/\|\[\|]\|\[\|\([0-9]\{1,3\}\.\)\{3\}[0-9]\{1,3\}/d' -e '/\.$/d' -e '/^\s*$/d' | awk '{$1=$1};1' | dos2unix | idn2 --no-alabelroundtrip --no-tr46 | LC_ALL=C sort -u > nsfw-temp.txt
sed -e 's/^/\./' -e 's/\./\\./g' -e 's/\-/\\-/g' -e 's/$/\$/' nsfw-temp.txt > nsfw-sub.txt
cat nsfw-temp.txt | LC_ALL=C grep -f nsfw-sub.txt | LC_ALL=C sort -u > nsfw-redundant-sub.txt
comm -23 nsfw-temp.txt nsfw-redundant-sub.txt > nsfw.txt
echo "TOTAL_NSFW=$(wc -l < nsfw.txt)" >> $GITHUB_ENV
mv nsfw.txt release

- name: Generate geoip.dat, geoip-lite.dat, Country.mmdb and Country-lite.mmdb files
- name: Generate geoip.dat, geoip-lite.dat, security-ip.dat, Country.mmdb, Country-lite.mmdb and Security-ip.mmdb files
run: |
go install -v github.com/Loyalsoldier/geoip@latest
$(go env GOPATH)/bin/geoip -c config.json
cp output/dat/geoip.dat output/dat/geoip-lite.dat release
cp output/maxmind/Country.mmdb output/maxmind/Country-lite.mmdb release
cp output/dat/geoip.dat output/dat/geoip-lite.dat output/dat/security-ip.dat release
cp output/maxmind/Country.mmdb output/maxmind/Country-lite.mmdb output/maxmind/Security-ip.mmdb release
cp -fpPR output/text release

- name: Verify Country.mmdb and Country-lite.mmdb files
- name: Verify Country.mmdb, Country-lite.mmdb and Security-ip.mmdb files
run: |
go install -v github.com/maxmind/mmdbverify@latest
$(go env GOPATH)/bin/mmdbverify -file release/Country.mmdb
$(go env GOPATH)/bin/mmdbverify -file release/Country-lite.mmdb
$(go env GOPATH)/bin/mmdbverify -file release/Security-ip.mmdb

- name: Generate geosite.dat and geosite-lite.dat file
- name: Generate geosite.dat, geosite-lite.dat and security.dat files
run: |
cd v2ray-geosite
mkdir data datalite
cp ../release/ir.txt data/ir
cp ../release/ir-lite.txt datalite/ir
cp ../release/ads.txt data/ads
cp ../release/ads.txt datalite/ads
cp ../release/cn.txt data/cn
cp ../release/category-ads-all.txt data/category-ads-all
cp ../release/malware.txt data/malware
cp ../release/phishing.txt data/phishing
cp ../release/cryptominers.txt data/cryptominers
cp ../release/social.txt data/social
cp ../release/nsfw.txt data/nsfw


go mod edit -go=1.21
go get -u
go mod tidy
go run ./ --datapath=data --outputdir=../release --outputname=geosite.dat

mkdir datalite
cp ../release/ir-lite.txt datalite/ir
cp ../release/ads.txt datalite/ads
go run ./ --datapath=datalite --outputdir=../release --outputname=geosite-lite.dat

mkdir security
cp ../release/category-ads-all.txt security/category-ads-all
cp ../release/malware.txt security/malware
cp ../release/phishing.txt security/phishing
cp ../release/cryptominers.txt security/cryptominers
go run ./ --datapath=security --outputdir=../release --outputname=security.dat

- name: Generate sha256sum
run: |
sha256sum release/geoip.dat > release/geoip.dat.sha256sum
sha256sum release/geoip-lite.dat > release/geoip-lite.dat.sha256sum
sha256sum release/security-ip.dat > release/security-ip.dat.sha256sum
sha256sum release/Country.mmdb > release/Country.mmdb.sha256sum
sha256sum release/Country-lite.mmdb > release/Country-lite.mmdb.sha256sum
sha256sum release/Security-ip.mmdb > release/Security-ip.mmdb.sha256sum
sha256sum release/geosite.dat > release/geosite.dat.sha256sum
sha256sum release/geosite-lite.dat > release/geosite-lite.dat.sha256sum

sha256sum release/security.dat > release/security.dat.sha256sum

- name: Generate Release Notes
run: |
run: |
echo "* Updated on ${{ env.RELEASE_DATE }}" > RELEASE_NOTES
echo "* IR Domains (Excluding .ir Domains) -> ${{ env.TOTAL_IR }}" >> RELEASE_NOTES
echo "* Active IR Domains (Excluding .ir Domains, Used in geosite-lite) -> ${{ env.TOTAL_IR_LITE }}" >> RELEASE_NOTES
Expand All @@ -188,6 +219,7 @@ jobs:
echo "* NSFW Domains -> ${{ env.TOTAL_NSFW }}" >> RELEASE_NOTES

- name: Push assets to release branch
if: ${{ inputs.PRE_RELEASE == 'false' }}
run: |
cd release || exit 1
git init
Expand All @@ -200,6 +232,7 @@ jobs:
git push -f origin release

- name: Purge jsDelivr CDN cache
if: ${{ inputs.PRE_RELEASE == 'false' }}
run: |
cd release || exit 1
for file in $(ls); do
Expand All @@ -213,15 +246,16 @@ jobs:
tag_name: ${{ env.TAG_NAME }}
body_path: RELEASE_NOTES
draft: false
prerelease: false
prerelease: ${{ inputs.PRE_RELEASE }}
files: |
release/*.dat
release/*.mmdb
release/*.sha256sum
env:
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}

- name: Clean older releases
if: ${{ inputs.PRE_RELEASE == 'false' }}
uses: dev-drprasad/delete-older-releases@v0.3.2
with:
keep_latest: 7
Expand Down
Loading