Merge branch 'bol-van:master' into master

common/base.sh

@@ -241,7 +241,7 @@ fix_sbin_path()
 # it can calculate floating point expr
 calc()
 {
-	awk "BEGIN { print $*}";
+	LC_ALL=C awk "BEGIN { print $*}";
 }
 
 fsleep_setup()

common/ipt.sh

@@ -1,5 +1,9 @@
 std_ports
 ipt_connbytes="-m connbytes --connbytes-dir=original --connbytes-mode=packets --connbytes"
+IPSET_EXCLUDE="-m set ! --match-set nozapret"
+IPSET_EXCLUDE6="-m set ! --match-set nozapret6"
+IPBAN_EXCLUDE="-m set ! --match-set ipban"
+IPBAN_EXCLUDE6="-m set ! --match-set ipban6"
 
 ipt()
 {
@@ -132,7 +136,7 @@ _fw_tpws4()
 
 		ipt_print_op $1 "$2" "tpws (port $3)"
 
-		rule="$2 $IPSET_EXCLUDE dst -j DNAT --to $TPWS_LOCALHOST4:$3"
+		rule="$2 $IPSET_EXCLUDE dst $IPBAN_EXCLUDE dst -j DNAT --to $TPWS_LOCALHOST4:$3"
 		for i in $4 ; do
 			ipt_add_del $1 PREROUTING -t nat -i $i $rule
 	 	done
@@ -160,7 +164,7 @@ _fw_tpws6()
 
 		ipt_print_op $1 "$2" "tpws (port $3)" 6
 
-		rule="$2 $IPSET_EXCLUDE6 dst"
+		rule="$2 $IPSET_EXCLUDE6 dst $IPBAN_EXCLUDE6 dst"
 		for i in $4 ; do
 			_dnat6_target $i DNAT6
 			[ -n "$DNAT6" -a "$DNAT6" != "-" ] && ipt6_add_del $1 PREROUTING -t nat -i $i $rule -j DNAT --to [$DNAT6]:$3

common/linux_iphelper.sh

@@ -4,6 +4,8 @@
 # PREROUTING - can't DNAT to ::1. can DNAT to link local of -i interface or to any global addr
 # not a good idea to expose tpws to the world (bind to ::)
 
+# max wait time for the link local ipv6 on the LAN interface
+LINKLOCAL_WAIT_SEC=${LINKLOCAL_WAIT_SEC:-5}
 
 get_ipv6_linklocal()
 {

common/nft.sh

@@ -387,8 +387,8 @@ _nft_fw_tpws4()
 	[ "$DISABLE_IPV4" = "1" -o -z "$1" ] || {
 		local filter="$1" port="$2"
 		nft_print_op "$filter" "tpws (port $2)" 4
-		nft_insert_rule dnat_output skuid != $WS_USER ${3:+oifname @wanif }$filter ip daddr != @nozapret $FW_EXTRA_POST dnat ip to $TPWS_LOCALHOST4:$port
-		nft_insert_rule dnat_pre iifname @lanif $filter ip daddr != @nozapret $FW_EXTRA_POST dnat ip to $TPWS_LOCALHOST4:$port
+		nft_insert_rule dnat_output skuid != $WS_USER ${3:+oifname @wanif }$filter ip daddr != @nozapret ip daddr != @ipban $FW_EXTRA_POST dnat ip to $TPWS_LOCALHOST4:$port
+		nft_insert_rule dnat_pre iifname @lanif $filter ip daddr != @nozapret ip daddr != @ipban $FW_EXTRA_POST dnat ip to $TPWS_LOCALHOST4:$port
 		prepare_route_localnet
 	}
 }
@@ -402,9 +402,9 @@ _nft_fw_tpws6()
 	[ "$DISABLE_IPV6" = "1" -o -z "$1" ] || {
 		local filter="$1" port="$2" DNAT6 i
 		nft_print_op "$filter" "tpws (port $port)" 6
-		nft_insert_rule dnat_output skuid != $WS_USER ${4:+oifname @wanif6 }$filter ip6 daddr != @nozapret6 $FW_EXTRA_POST dnat ip6 to [::1]:$port
+		nft_insert_rule dnat_output skuid != $WS_USER ${4:+oifname @wanif6 }$filter ip6 daddr != @nozapret6 ip6 daddr != @ipban6 $FW_EXTRA_POST dnat ip6 to [::1]:$port
 		[ -n "$3" ] && {
-			nft_insert_rule dnat_pre $filter ip6 daddr != @nozapret6 $FW_EXTRA_POST dnat ip6 to iifname map @link_local:$port
+			nft_insert_rule dnat_pre $filter ip6 daddr != @nozapret6 ip6 daddr != @ipban6 $FW_EXTRA_POST dnat ip6 to iifname map @link_local:$port
 			for i in $3; do
 				_dnat6_target $i DNAT6
 				# can be multiple tpws processes on different ports

docs/readme.md

@@ -1439,6 +1439,7 @@ PKTWS_EXTRA_1 .. PKTWS_EXTRA_9, TPWS_EXTRA_1 .. TPWS_EXTRA_9 - отдельно
 SECURE_DNS=0|1 - принудительно выключить или включить DoH
 DOH_SERVERS - список URL DoH через пробел для автоматического выбора работающего сервера
 DOH_SERVER - конкретный DoH URL, отказ от поиска
+UNBLOCKED_DOM - незаблокированный домен, который используется для тестов IP block
 ```
 
 Пример запуска с переменными:\

init.d/openwrt/functions

@@ -25,15 +25,8 @@ CUSTOM_DIR="$ZAPRET_RW/init.d/openwrt"
 
 TPWS_LOCALHOST4=127.0.0.127
 
-# max wait time for the link local ipv6 on the LAN interface
-LINKLOCAL_WAIT_SEC=5
-
 IPSET_CR="$ZAPRET_BASE/ipset/create_ipset.sh"
 
-IPSET_EXCLUDE="-m set ! --match-set nozapret"
-IPSET_EXCLUDE6="-m set ! --match-set nozapret6"
-
-
 # can be multiple ipv6 outgoing interfaces
 # uplink from isp, tunnelbroker, vpn, ...
 # want them all. who knows what's the real one that blocks sites

init.d/sysv/functions

@@ -90,13 +90,6 @@ TPWS_WAIT_SOCKS6="$TPWS_WAIT --bind-wait-ip-linklocal=30"
 # first wait for lan to ifup, then wait for bind-wait-ip-linklocal seconds for link local address and bind-wait-ip for any ipv6 as the worst case
 TPWS_OPT_BASE6_PRE="--bind-linklocal=prefer $TPWS_WAIT --bind-wait-ip-linklocal=3"
 
-# max wait time for the link local ipv6 on the LAN interface
-LINKLOCAL_WAIT_SEC=5
-
-IPSET_EXCLUDE="-m set ! --match-set nozapret"
-IPSET_EXCLUDE6="-m set ! --match-set nozapret6"
-
-
 dnat6_target()
 {
 	_dnat6_target "$@"

ipset/antifilter.helper

@@ -7,7 +7,7 @@ get_antifilter()
  [ "$DISABLE_IPV4" != "1" ] && {
   curl --fail --max-time 150 --connect-timeout 20 --max-filesize 41943040 -k -L "$1" | cut_local >"$ZIPLISTTMP" &&
   {
-   dlsize=$(LANG=C wc -c "$ZIPLISTTMP" | xargs | cut -f 1 -d ' ')
+   dlsize=$(LC_ALL=C LANG=C wc -c "$ZIPLISTTMP" | xargs | cut -f 1 -d ' ')
    if [ $dlsize -lt 102400 ]; then
     echo list file is too small. can be bad.
     exit 2

ipset/def.sh

@@ -5,7 +5,7 @@ ZAPRET_RW=${ZAPRET_RW:-"$ZAPRET_BASE"}
 ZAPRET_CONFIG=${ZAPRET_CONFIG:-"$ZAPRET_RW/config"}
 IPSET_RW_DIR="$ZAPRET_RW/ipset"
 
-. "$ZAPRET_CONFIG"
+[ -f "$ZAPRET_CONFIG" ] && . "$ZAPRET_CONFIG"
 . "$ZAPRET_BASE/common/base.sh"
 
 [ -z "$TMPDIR" ] && TMPDIR=/tmp
@@ -141,6 +141,18 @@ zzsize()
   printf 0
  fi
 }
+zzcopy()
+{
+ local is_gz=0
+ zztest "$1" && is_gz=1
+ if [ "$GZIP_LISTS" = 1 -a $is_gz = 1 ]; then
+  cp "$1" "${2}.gz"
+ elif [ "$GZIP_LISTS" != 1 -a $is_gz != 1 ]; then
+  cp "$1" "$2"
+ else
+  zzcat "$1" | zz "$2"
+ fi
+}
 
 digger()
 {

ipset/get_antizapret_domains.sh

@@ -21,7 +21,7 @@ curl -H "Accept-Encoding: gzip" -k --fail --max-time 600 --connect-timeout 5 --r
  exit 2
 }
 
-dlsize=$(LANG=C wc -c "$ZDOM" | xargs | cut -f 1 -d ' ')
+dlsize=$(LC_ALL=C LANG=C wc -c "$ZDOM" | xargs | cut -f 1 -d ' ')
 if test $dlsize -lt 102400; then
  echo list file is too small. can be bad.
  exit 2

ipset/get_config.sh

@@ -4,7 +4,7 @@
 IPSET_DIR="$(dirname "$0")"
 IPSET_DIR="$(cd "$IPSET_DIR"; pwd)"
 
-. "$IPSET_DIR/../config"
+[ -f "$IPSET_DIR/../config" ] && . "$IPSET_DIR/../config"
 
 [ -z "$GETLIST" ] && GETLIST=get_ipban.sh
 [ -x "$IPSET_DIR/$GETLIST" ] && exec "$IPSET_DIR/$GETLIST"

ipset/get_reestr_hostlist.sh

@@ -5,9 +5,9 @@ IPSET_DIR="$(cd "$IPSET_DIR"; pwd)"
 
 . "$IPSET_DIR/def.sh"
 
-ZREESTR="$TMPDIR/zapret.txt"
+ZREESTR="$TMPDIR/zapret.txt.gz"
 IPB="$TMPDIR/ipb.txt"
-ZURL_REESTR=https://raw.githubusercontent.com/zapret-info/z-i/master/dump.csv
+ZURL_REESTR=https://raw.githubusercontent.com/zapret-info/z-i/master/dump.csv.gz
 
 dl_checked()
 {
@@ -21,7 +21,7 @@ dl_checked()
    echo list download failed : $1
    return 2
   }
-  dlsize=$(LANG=C wc -c "$2" | xargs | cut -f 1 -d ' ')
+  dlsize=$(LC_ALL=C LANG=C wc -c "$2" | xargs | cut -f 1 -d ' ')
   if test $dlsize -lt $3; then
    echo list is too small : $dlsize bytes. can be bad.
    return 2
@@ -31,11 +31,11 @@ dl_checked()
 
 reestr_list()
 {
- LANG=C cut -s -f2 -d';' "$ZREESTR" | LANG=C nice -n 5 sed -Ee 's/^\*\.(.+)$/\1/' -ne 's/^[a-z0-9A-Z._-]+$/&/p' | $AWK '{ print tolower($0) }'
+ LC_ALL=C LANG=C gunzip -c "$ZREESTR" | cut -s -f2 -d';' | LC_ALL=C  LANG=C nice -n 5 sed -Ee 's/^\*\.(.+)$/\1/' -ne 's/^[a-z0-9A-Z._-]+$/&/p' | $AWK '{ print tolower($0) }'
 }
 reestr_extract_ip()
 {
- LANG=C nice -n 5 $AWK -F ';' '($1 ~ /^([0-9]{1,3}\.){3}[0-9]{1,3}/) && (($2 == "" && $3 == "") || ($1 == $2)) {gsub(/ \| /, RS); print $1}' "$ZREESTR" | LANG=C $AWK '{split($1, a, /\|/); for (i in a) {print a[i]}}'
+ LC_ALL=C LANG=C gunzip -c | nice -n 5 $AWK -F ';' '($1 ~ /^([0-9]{1,3}\.){3}[0-9]{1,3}/) && (($2 == "" && $3 == "") || ($1 == $2)) {gsub(/ \| /, RS); print $1}' | LC_ALL=C  LANG=C $AWK '{split($1, a, /\|/); for (i in a) {print a[i]}}'
 }
 
 ipban_fin()

ipset/get_reestr_preresolved.sh

@@ -24,12 +24,12 @@ dl()
    echo list download failed : $1
    exit 2
   }
-  dlsize=$(LANG=C wc -c "$TMPLIST" | xargs | cut -f 1 -d ' ')
+  dlsize=$(LC_ALL=C LANG=C wc -c "$TMPLIST" | xargs | cut -f 1 -d ' ')
   if test $dlsize -lt $3; then
    echo list is too small : $dlsize bytes. can be bad.
    exit 2
   fi
-  zzcat "$TMPLIST" | zz "$2"
+  zzcopy "$TMPLIST" "$2"
   rm -f "$TMPLIST"
 }
 

ipset/get_reestr_preresolved_smart.sh

@@ -24,12 +24,12 @@ dl()
    echo list download failed : $1
    exit 2
   }
-  dlsize=$(LANG=C wc -c "$TMPLIST" | xargs | cut -f 1 -d ' ')
+  dlsize=$(LC_ALL=C LANG=C wc -c "$TMPLIST" | xargs | cut -f 1 -d ' ')
   if test $dlsize -lt $3; then
    echo list is too small : $dlsize bytes. can be bad.
    exit 2
   fi
-  zzcat "$TMPLIST" | zz "$2"
+  zzcopy "$TMPLIST" "$2"
   rm -f "$TMPLIST"
 }
 

ipset/get_reestr_resolvable_domains.sh

@@ -23,12 +23,12 @@ dl()
    echo list download failed : $1
    exit 2
   }
-  dlsize=$(LANG=C wc -c "$TMPLIST" | xargs | cut -f 1 -d ' ')
+  dlsize=$(LC_ALL=C LANG=C wc -c "$TMPLIST" | xargs | cut -f 1 -d ' ')
   if test $dlsize -lt $3; then
    echo list is too small : $dlsize bytes. can be bad.
    exit 2
   fi
-  zzcat "$TMPLIST" | zz "$2"
+  zzcopy "$TMPLIST" "$2"
   rm -f "$TMPLIST"
 }
 

ipset/get_reestr_resolve.sh

@@ -5,12 +5,12 @@ IPSET_DIR="$(cd "$IPSET_DIR"; pwd)"
 
 . "$IPSET_DIR/def.sh"
 
-ZREESTR="$TMPDIR/zapret.txt"
+ZREESTR="$TMPDIR/zapret.txt.gz"
 ZDIG="$TMPDIR/zapret-dig.txt"
 IPB="$TMPDIR/ipb.txt"
 ZIPLISTTMP="$TMPDIR/zapret-ip.txt"
 #ZURL=https://reestr.rublacklist.net/api/current
-ZURL_REESTR=https://raw.githubusercontent.com/zapret-info/z-i/master/dump.csv
+ZURL_REESTR=https://raw.githubusercontent.com/zapret-info/z-i/master/dump.csv.gz
 
 dl_checked()
 {
@@ -24,7 +24,7 @@ dl_checked()
    echo list download failed : $1
    return 2
   }
-  dlsize=$(LANG=C wc -c "$2" | xargs | cut -f 1 -d ' ')
+  dlsize=$(LC_ALL=C LANG=C wc -c "$2" | xargs | cut -f 1 -d ' ')
   if test $dlsize -lt $3; then
    echo list is too small : $dlsize bytes. can be bad.
    return 2
@@ -34,11 +34,11 @@ dl_checked()
 
 reestr_list()
 {
- LANG=C cut -s -f2 -d';' "$ZREESTR" | LANG=C nice -n 5 sed -Ee 's/^\*\.(.+)$/\1/' -ne 's/^[a-z0-9A-Z._-]+$/&/p'
+ LC_ALL=C LANG=C gunzip -c "$ZREESTR" | cut -s -f2 -d';' | LC_ALL=C LANG=C nice -n 5 sed -Ee 's/^\*\.(.+)$/\1/' -ne 's/^[a-z0-9A-Z._-]+$/&/p' | $AWK '{ print tolower($0) }'
 }
 reestr_extract_ip()
 {
- LANG=C nice -n 5 $AWK -F ';' '($1 ~ /^([0-9]{1,3}\.){3}[0-9]{1,3}/) && (($2 == "" && $3 == "") || ($1 == $2)) {gsub(/ \| /, RS); print $1}' "$ZREESTR" | LANG=C $AWK '{split($1, a, /\|/); for (i in a) {print a[i]}}'
+ LC_ALL=C LANG=C gunzip -c | nice -n 5 $AWK -F ';' '($1 ~ /^([0-9]{1,3}\.){3}[0-9]{1,3}/) && (($2 == "" && $3 == "") || ($1 == $2)) {gsub(/ \| /, RS); print $1}' | LC_ALL=C LANG=C $AWK '{split($1, a, /\|/); for (i in a) {print a[i]}}'
 }
 
 getuser && {

ipset/get_refilter_domains.sh

@@ -20,12 +20,12 @@ dl()
    echo list download failed : $1
    exit 2
   }
-  dlsize=$(LANG=C wc -c "$TMPLIST" | xargs | cut -f 1 -d ' ')
+  dlsize=$(LC_ALL=C LANG=C wc -c "$TMPLIST" | xargs | cut -f 1 -d ' ')
   if test $dlsize -lt $3; then
    echo list is too small : $dlsize bytes. can be bad.
    exit 2
   fi
-  zzcat "$TMPLIST" | tr -d '\015' | zz "$2"
+  zzcopy "$TMPLIST" "$2"
   rm -f "$TMPLIST"
 }
 

ipset/get_refilter_ipsum.sh

@@ -20,13 +20,12 @@ dl()
    echo list download failed : $1
    exit 2
   }
-  dlsize=$(LANG=C wc -c "$TMPLIST" | xargs | cut -f 1 -d ' ')
+  dlsize=$(LC_ALL=C LANG=C wc -c "$TMPLIST" | xargs | cut -f 1 -d ' ')
   if test $dlsize -lt $3; then
    echo list is too small : $dlsize bytes. can be bad.
    exit 2
   fi
-  # remove DOS EOL \r
-  zzcat "$TMPLIST" | tr -d '\015' | zz "$2"
+  zzcopy "$TMPLIST" "$2"
   rm -f "$TMPLIST"
 }