custom_cn_mark/custom_mod/Enhanced data processing

Dockerfile

@@ -1,4 +1,6 @@
 FROM alpine:edge AS builder
+RUN apk update && \
+    apk upgrade --no-cache
 #actions COPY build_test_ok /
 COPY --from=sliamb/prebuild-paopaodns /src/ /src/
 COPY src/ /src/
@@ -37,12 +39,14 @@ RUN if /src/redis-server -v|grep build;then echo redis_check > /redis_check;else
 
 FROM alpine:edge
 COPY --from=builder /src/ /usr/sbin/
-RUN apk add --no-cache ca-certificates dcron tzdata hiredis libevent dnscrypt-proxy inotify-tools bind-tools libgcc xz && \
-    apk upgrade --no-cache &&\
+RUN apk update && \
+    apk upgrade --no-cache && \
+    apk add --no-cache ca-certificates dcron tzdata hiredis libevent dnscrypt-proxy inotify-tools bind-tools libgcc xz && \
     mkdir -p /etc/unbound && \
-    mv /usr/sbin/named.cache /etc/unbound/named.cache &&           \
-    adduser -D -H unbound &&\
-    mv /usr/sbin/repositories /etc/apk/repositories
+    mv /usr/sbin/named.cache /etc/unbound/named.cache && \
+    adduser -D -H unbound && \
+    mv /usr/sbin/repositories /etc/apk/repositories && \
+    rm -rf /var/cache/apk/*
 ARG DEVLOG_SW
 ENV TZ=Asia/Shanghai \
     DEVLOG=$DEVLOG_SW \
@@ -60,7 +64,7 @@ ENV TZ=Asia/Shanghai \
     CUSTOM_FORWARD_TTL=0 \
     AUTO_FORWARD=no \
     AUTO_FORWARD_CHECK=yes \
-    USE_MARK_DATA=no \
+    USE_MARK_DATA=yes \
     RULES_TTL=0 \
     HTTP_FILE=no \
     QUERY_TIME=2000ms \

src/data_update.sh

@@ -10,8 +10,7 @@ comp_trackerslist() {
         update_file_wait="/data/trackerslist.txt"
         wait_apply
     fi
-    sort -u /tmp/trackerslist/trackerslist.txt | grep "." >/tmp/trackerslist/trackerslist.txt.gen
-    grep -Eo "^[a-z]+://.+" /tmp/trackerslist/trackerslist.txt.gen >/data/trackerslist.txt
+    sort -u /tmp/trackerslist/trackerslist.txt | grep -Eo "^[a-z]+://.+" > /data/trackerslist.txt
     rm -rf /tmp/trackerslist/
     return 0
 }

src/debug.sh

@@ -28,6 +28,7 @@ echo "[INFO]" NETWORK
 blank
 ip a | grep -E "UP|inet"
 ip r
+traceroute -m4 -w1 120.53.53.53
 ping 223.5.5.5 -c1
 ping 119.29.29.29 -c1
 nslookup www.taobao.com 223.5.5.5

src/force_forward_list.txt

@@ -5,9 +5,6 @@
 
 # Global bing
 domain:bing.com
-domain:bing.net
-domain:msn.com
-domain:msn.cn
 
 # Google play download issues
 domain:googleapis.cn

src/init.sh

@@ -249,7 +249,10 @@ redis-server /tmp/redis.conf
 if ! ps -ef | grep -v grep | grep -q redis-server; then
     redis-server /tmp/redis.conf --ignore-warnings ARM64-COW-BUG
 fi
-sleep 3
+while ! redis-cli -s /tmp/redis.sock info | grep -q human; do
+    echo "Waiting for redis..."
+    sleep 1
+done
 sed "s/{CORES}/$CORES/g" /data/unbound.conf | sed "s/{POWCORES}/$POWCORES/g" | sed "s/{FDLIM}/$FDLIM/g" | sed "s/{MEM1}/$MEM1/g" | sed "s/{MEM2}/$MEM2/g" | sed "s/{MEM3}/$MEM3/g" | sed "s/{ETHIP}/$ETHIP/g" | sed "s/{DNS_SERVERNAME}/$DNS_SERVERNAME/g" >/tmp/unbound.conf
 # if [ "$DEVLOG" = "yes" ]; then
 #     sed -i "s/verbosity: 0/verbosity: 2/g" /tmp/unbound.conf
@@ -263,6 +266,23 @@ if echo "$SERVER_IP" | grep -Eoq "[.0-9]+"; then
     sed -i "s/{SERVER_IP}/$SERVER_IP/g" /tmp/unbound.conf
     sed -i "s/#serverip-enable//g" /tmp/unbound.conf
 fi
+if [ "$FDLIM" -gt 1 ] && [ "$SAFEMODE" != "yes" ]; then
+    calc_r=$(mosdns eat calc "$lim" "$REALCORES" "r")
+    calc_f=$(mosdns eat calc "$lim" "$REALCORES" "f")
+    r_outgoing=$(echo "$calc_r" | cut -d':' -f2)
+    f_outgoing=$(echo "$calc_f" | cut -d':' -f2)
+    r_outgoing_half=$(echo "$calc_r" | cut -d':' -f4)
+    f_outgoing_half=$(echo "$calc_f" | cut -d':' -f4)
+    r_numQueriesPerThread=$(echo "$calc_r" | cut -d':' -f6)
+    f_numQueriesPerThread=$(echo "$calc_f" | cut -d':' -f6)
+    sed -i "s/{r_outgoing}/$r_outgoing/g" /tmp/unbound.conf
+    sed -i "s/{f_outgoing}/$f_outgoing/g" /tmp/unbound.conf
+    sed -i "s/{r_outgoing_half}/$r_outgoing_half/g" /tmp/unbound.conf
+    sed -i "s/{f_outgoing_half}/$f_outgoing_half/g" /tmp/unbound.conf
+    sed -i "s/{r_numQueriesPerThread}/$r_numQueriesPerThread/g" /tmp/unbound.conf
+    sed -i "s/{f_numQueriesPerThread}/$f_numQueriesPerThread/g" /tmp/unbound.conf
+    sed -i "s/#safeoff//g" /tmp/unbound.conf
+fi
 if [ "$CNAUTO" != "no" ]; then
     DNSPORT="5301"
     if [ ! -f /data/mosdns.yaml ]; then
@@ -296,7 +316,6 @@ if [ "$CNAUTO" != "no" ]; then
         sed "s/#socksok//g" /data/dnscrypt.toml | sed "s/{SOCKS5}/$SOCKS5/g" | sed -r "s/listen_addresses.+/listen_addresses = ['0.0.0.0:5303']/g" | sed -r "s/^force_tcp.+/force_tcp = true/g" >/data/dnscrypt-resolvers/dnscrypt_socks.toml
         sed "s/{DNSPORT}/5304/g" /tmp/unbound.conf | sed "s/#CNAUTO//g" | sed "s/#socksok//g" >/tmp/unbound_forward.conf
         sed "s/#socksok//g" /data/mosdns.yaml >/tmp/mosdns.yaml
-        sleep 5
     else
         sed "s/{DNSPORT}/5304/g" /tmp/unbound.conf | sed "s/#CNAUTO//g" | sed "s/#nosocks//g" >/tmp/unbound_forward.conf
         sed "s/#nosocks//g" /data/mosdns.yaml >/tmp/mosdns.yaml
@@ -379,16 +398,7 @@ if [ "$CNAUTO" != "no" ]; then
     fi
     #convert hosts
     if [ "$USE_HOSTS" = "yes" ]; then
-        grep -vE "^#" /etc/hosts | grep . | sort -u >/tmp/hosts.cp.gen
-        echo "" >>/tmp/hosts.cp.gen
-        echo "" >>/tmp/hosts.cp.gen
-        echo "" >/tmp/hosts.txt
-        while read line; do
-            record=$(echo "$line" | grep -Eo "[.:a-f0-9]+" | head -1)
-            domain=$(echo "$line" | grep -Eo "[-_.a-zA-Z0-9]+" | tail -1)
-            echo "$domain" "$record" >>/tmp/hosts.txt
-        done </tmp/hosts.cp.gen
-        rm /tmp/hosts.cp.gen
+        mosdns eat hosts
         sed -i "s/#usehosts-yes//g" /tmp/mosdns.yaml
         sed -i "s/#usehosts-enable//g" /tmp/mosdns.yaml
     fi
@@ -445,6 +455,7 @@ if [ "$CNAUTO" != "no" ]; then
     unbound -c /tmp/unbound_forward.conf -p
     # Add Mods
     touch /data/custom_mod.yaml
+    cp /tmp/mosdns.yaml /tmp/mosdns_base.yaml
     mosdns AddMod
     if [ -f /tmp/mosdns_mod.yaml ]; then
         cat /tmp/mosdns_mod.yaml >/tmp/mosdns.yaml

src/mosdns.yaml

@@ -18,9 +18,6 @@ plugins:
 #usehosts-yes#usehosts-enable      files:
 #usehosts-yes#usehosts-enable        - "/tmp/hosts.txt" 
 
-##swaps_ipset_start##
-##swaps_ipset_end##
-
   - tag: "force_recurse_list"
     type: "domain_set"
     args:
@@ -51,13 +48,14 @@ plugins:
 #global_mark_yes    args:
 #global_mark_yes      files:
 #global_mark_yes        - "/tmp/cn_mark.dat"
+#global_mark_yes        - "/tmp/custom_cn_mark.txt"
 
 #global_mark_yes  - tag: "global_mark_cn"
 #global_mark_yes    type: "domain_set"
 #global_mark_yes    args:
 #global_mark_yes      files:
 #global_mark_yes        - "/tmp/global_mark_cn.dat"
-#global_mark_yes        - "/tmp/cn_mark.dat"
+#global_mark_yes        - "/tmp/custom_cn_mark.txt"
 
 #customforward-seted  - tag: "force_forward_list"
 #customforward-seted    type: "domain_set"

src/reload.sh

@@ -0,0 +1,18 @@
+#!/bin/sh
+. /etc/profile
+if [ -f /data/custom_env.ini ]; then
+    grep -Eo "^[_a-zA-Z0-9]+=\".+\"" /data/custom_env.ini >/tmp/custom_env.ini
+    if [ -f "/tmp/custom_env.ini" ]; then
+        while IFS= read -r line; do
+            line=$(echo "$line" | sed 's/"//g' | sed "s/'//g")
+            export "$line"
+        done <"/tmp/custom_env.ini"
+    fi
+fi
+/usr/sbin/mosdns version
+/usr/sbin/mosdns AddMod
+if [ -f /tmp/mosdns_mod.yaml ]; then
+    cat /tmp/mosdns_mod.yaml >/tmp/mosdns.yaml
+    sed -i '/^#/d' /tmp/mosdns.yaml
+fi
+/usr/sbin/watch_list.sh reload_dns

src/test.sh

@@ -85,6 +85,7 @@ if ps -ef | grep -v grep | grep -q mosdns.yaml; then
 else
     if [ "$CNAUTO" != "no" ]; then
         echo "DNS NOT READY.❌"
+        echo "Please wait until the DNS server has fully started before attempting to execute test.sh."
     else
         echo "UNBOUND MODE TEST."
         result=$t1$t2$t3$t4$t5$tredis

src/unbound.conf

@@ -90,7 +90,8 @@ server:
 	# number of ports to allocate per thread, determines the size of the
 	# port range that can be open simultaneously.  About double the
 	# num-queries-per-thread, or, use as many as the OS will allow you.
-#safemem	outgoing-range: 8192
+#safemem#RAWDNS#safeoff	outgoing-range: {r_outgoing}
+#safemem#CNAUTO#safeoff	outgoing-range: {f_outgoing}
 #lowrmem	outgoing-range: 60
 
 	# permit Unbound to use this port number or port range for
@@ -107,10 +108,12 @@ server:
 	# outgoing-port-avoid: "3200-3208"
 
 	# number of outgoing simultaneous tcp buffers to hold per thread.
-#RAWDNS			outgoing-num-tcp: {FDLIM}
+#RAWDNS#safeoff			outgoing-num-tcp: {r_outgoing_half}
+#CNAUTO#safeoff			outgoing-num-tcp: {f_outgoing_half}
 
 	# number of incoming simultaneous tcp buffers to hold per thread.
-#RAWDNS			incoming-num-tcp: {FDLIM}
+#RAWDNS#safeoff			incoming-num-tcp: {r_outgoing_half}
+#CNAUTO#safeoff			incoming-num-tcp: {f_outgoing_half}
 
 	# buffer size for UDP port 53 incoming (SO_RCVBUF socket option).
 	# 0 is system default.  Use 4m to catch query spikes for busy servers.
@@ -164,7 +167,8 @@ server:
 #RAWDNS	msg-cache-slabs: {POWCORES}
 
 	# the number of queries that a thread gets to service.
-#safemem	num-queries-per-thread: 4096
+#safemem#RAWDNS#safeoff	num-queries-per-thread: {r_numQueriesPerThread}
+#safemem#CNAUTO#safeoff	num-queries-per-thread: {f_numQueriesPerThread}
 #lowrmem num-queries-per-thread: 30
 	# if very busy, 50% queries run to completion, 50% get timeout in msec
 	# jostle-timeout: 200