gsd
/
zapret
mirror of https://github.com/bol-van/zapret/
1
0
Fork 0
Code Issues Projects Releases Wiki Activity
Browse Source

ipset: check dns before processing lists

pull/80/head
bol-van 4 years ago
parent
7355b20544
commit
6c3cf5ffc3
8 changed files with 136 additions and 108 deletions
Whitespace
Show all changes Ignore whitespace when comparing lines Ignore changes in amount of whitespace Ignore changes in whitespace at EOL
Unified View
Diff Options
Show Stats Download Patch File Download Diff File
  1. 56
      ipset/def.sh
  2. 9
      ipset/get_antifilter_ip.sh
  3. 9
      ipset/get_antifilter_ipsmart.sh
  4. 9
      ipset/get_antifilter_ipsum.sh
  5. 41
      ipset/get_reestr_combined.sh
  6. 4
      ipset/get_reestr_hostlist.sh
  7. 43
      ipset/get_reestr_ip.sh
  8. 73
      ipset/get_reestr_resolve.sh

56
ipset/def.sh
View File

@ -1,3 +1,8 @@
[ -n "$IPSET_DIR" ] || {
IPSET_DIR="$(dirname "$0")"
IPSET_DIR="$(cd "$IPSET_DIR"; pwd)"
}
. "$IPSET_DIR/../config" . "$IPSET_DIR/../config"
[ -z "$TMPDIR" ] && TMPDIR=/tmp [ -z "$TMPDIR" ] && TMPDIR=/tmp
@ -47,12 +52,15 @@ exists()
# MacoS in cron does not include /usr/local/bin to PATH # MacoS in cron does not include /usr/local/bin to PATH
if [ -x /usr/local/bin/ggrep ] ; then if [ -x /usr/local/bin/ggrep ] ; then
GREP=/usr/local/bin/ggrep GREP=/usr/local/bin/ggrep
elif [ -x /usr/local/bin/grep ] ; then
GREP=/usr/local/bin/grep
elif exists ggrep; then elif exists ggrep; then
GREP=$(which ggrep) GREP=$(which ggrep)
else else
GREP=$(which grep) GREP=$(which grep)
fi fi
grep_supports_b() grep_supports_b()
{ {
# \b does not work with BSD grep # \b does not work with BSD grep
@ -119,18 +127,31 @@ zzsize()
digger() digger()
{ {
# $1 - hostlist # $1 - family (4|6)
# $2 - family (4|6) # $2 - s=enable mdig stats
>&2 echo digging $(wc -l <"$1" | xargs) ipv$2 domains : "$1"
if [ -x "$MDIG" ]; then if [ -x "$MDIG" ]; then
zzcat "$1" | "$MDIG" --family=$2 --threads=$MDIG_THREADS --stats=1000 local cmd
[ "$2" == "s" ] && cmd=--stats=1000
"$MDIG" --family=$1 --threads=$MDIG_THREADS $1
else else
local A=A local A=A
[ "$2" = "6" ] && A=AAAA [ "$1" = "6" ] && A=AAAA
zzcat "$1" | dig $A +short +time=8 +tries=2 -f - | $GREP -E '^[^;].*[^\.]$' dig $A +short +time=8 +tries=2 -f - | $GREP -E '^[^;].*[^\.]$'
fi fi
} }
filedigger()
{
# $1 - hostlist
# $2 - family (4|6)
>&2 echo digging $(wc -l <"$1" | xargs) ipv$2 domains : "$1"
zzcat "$1" | digger $2 s
}
dnstest()
{
local ip=$(echo w3.org | digger 46)
[ -n "$ip" ]
}
cut_local() cut_local()
{ {
@ -152,24 +173,29 @@ oom_adjust_high()
getexclude() getexclude()
{ {
oom_adjust_high oom_adjust_high
dnstest || {
echo "! DNS is not working. list processing aborted."
return 1
}
[ -f "$ZUSERLIST_EXCLUDE" ] && { [ -f "$ZUSERLIST_EXCLUDE" ] && {
[ "$DISABLE_IPV4" != "1" ] && digger "$ZUSERLIST_EXCLUDE" 4 | sort -u > "$ZIPLIST_EXCLUDE" [ "$DISABLE_IPV4" != "1" ] && filedigger "$ZUSERLIST_EXCLUDE" 4 | sort -u > "$ZIPLIST_EXCLUDE"
[ "$DISABLE_IPV6" != "1" ] && digger "$ZUSERLIST_EXCLUDE" 6 | sort -u > "$ZIPLIST_EXCLUDE6" [ "$DISABLE_IPV6" != "1" ] && filedigger "$ZUSERLIST_EXCLUDE" 6 | sort -u > "$ZIPLIST_EXCLUDE6"
} }
return 0
} }
getuser() getuser()
{ {
getexclude getexclude || return
[ -f "$ZUSERLIST" ] && { [ -f "$ZUSERLIST" ] && {
[ "$DISABLE_IPV4" != "1" ] && digger "$ZUSERLIST" 4 | cut_local | sort -u > "$ZIPLIST_USER" [ "$DISABLE_IPV4" != "1" ] && filedigger "$ZUSERLIST" 4 | cut_local | sort -u > "$ZIPLIST_USER"
[ "$DISABLE_IPV6" != "1" ] && digger "$ZUSERLIST" 6 | cut_local6 | sort -u > "$ZIPLIST_USER6" [ "$DISABLE_IPV6" != "1" ] && filedigger "$ZUSERLIST" 6 | cut_local6 | sort -u > "$ZIPLIST_USER6"
} }
[ -f "$ZUSERLIST_IPBAN" ] && { [ -f "$ZUSERLIST_IPBAN" ] && {
[ "$DISABLE_IPV4" != "1" ] && digger "$ZUSERLIST_IPBAN" 4 | cut_local | sort -u > "$ZIPLIST_USER_IPBAN" [ "$DISABLE_IPV4" != "1" ] && filedigger "$ZUSERLIST_IPBAN" 4 | cut_local | sort -u > "$ZIPLIST_USER_IPBAN"
[ "$DISABLE_IPV6" != "1" ] && digger "$ZUSERLIST_IPBAN" 6 | cut_local6 | sort -u > "$ZIPLIST_USER_IPBAN6" [ "$DISABLE_IPV6" != "1" ] && filedigger "$ZUSERLIST_IPBAN" 6 | cut_local6 | sort -u > "$ZIPLIST_USER_IPBAN6"
} }
return 0
} }
hup_zapret_daemons() hup_zapret_daemons()

9
ipset/get_antifilter_ip.sh
View File

@ -5,10 +5,9 @@ IPSET_DIR="$(cd "$IPSET_DIR"; pwd)"
. "$IPSET_DIR/def.sh" . "$IPSET_DIR/def.sh"
getuser getuser && {
. "$IPSET_DIR/antifilter.helper"
. "$IPSET_DIR/antifilter.helper" get_antifilter https://antifilter.network/download/ip.lst "$ZIPLIST"
}
get_antifilter https://antifilter.network/download/ip.lst "$ZIPLIST"
"$IPSET_DIR/create_ipset.sh" "$IPSET_DIR/create_ipset.sh"

9
ipset/get_antifilter_ipsmart.sh
View File

@ -5,10 +5,9 @@ IPSET_DIR="$(cd "$IPSET_DIR"; pwd)"
. "$IPSET_DIR/def.sh" . "$IPSET_DIR/def.sh"
getuser getuser && {
. "$IPSET_DIR/antifilter.helper"
. "$IPSET_DIR/antifilter.helper" get_antifilter https://antifilter.network/download/ipsmart.lst "$ZIPLIST"
}
get_antifilter https://antifilter.network/download/ipsmart.lst "$ZIPLIST"
"$IPSET_DIR/create_ipset.sh" "$IPSET_DIR/create_ipset.sh"

9
ipset/get_antifilter_ipsum.sh
View File

@ -5,10 +5,9 @@ IPSET_DIR="$(cd "$IPSET_DIR"; pwd)"
. "$IPSET_DIR/def.sh" . "$IPSET_DIR/def.sh"
getuser getuser && {
. "$IPSET_DIR/antifilter.helper"
. "$IPSET_DIR/antifilter.helper" get_antifilter https://antifilter.network/download/ipsum.lst "$ZIPLIST"
}
get_antifilter https://antifilter.network/download/ipsum.lst "$ZIPLIST"
"$IPSET_DIR/create_ipset.sh" "$IPSET_DIR/create_ipset.sh"

41
ipset/get_reestr_combined.sh
View File

@ -9,7 +9,6 @@ ZREESTR="$TMPDIR/reestr.txt"
#ZURL_REESTR=https://reestr.rublacklist.net/api/current #ZURL_REESTR=https://reestr.rublacklist.net/api/current
ZURL_REESTR=https://raw.githubusercontent.com/zapret-info/z-i/master/dump.csv ZURL_REESTR=https://raw.githubusercontent.com/zapret-info/z-i/master/dump.csv
getuser
dig_reestr() dig_reestr()
{ {
@ -36,29 +35,31 @@ dig_reestr()
rm -f "$TMP" rm -f "$TMP"
} }
getuser && {
curl -k --fail --max-time 600 --connect-timeout 5 --retry 3 --max-filesize 251658240 "$ZURL_REESTR" -o "$ZREESTR" || curl -k --fail --max-time 600 --connect-timeout 5 --retry 3 --max-filesize 251658240 "$ZURL_REESTR" -o "$ZREESTR" ||
{ {
echo reestr list download failed echo reestr list download failed
exit 2 exit 2
} }
dlsize=$(LANG=C wc -c "$ZREESTR" | xargs | cut -f 1 -d ' ') dlsize=$(LANG=C wc -c "$ZREESTR" | xargs | cut -f 1 -d ' ')
if test $dlsize -lt 1048576; then if test $dlsize -lt 1048576; then
echo reestr ip list is too small. can be bad. echo reestr ip list is too small. can be bad.
exit 2 exit 2
fi fi
#sed -i 's/\\n/\r\n/g' $ZREESTR #sed -i 's/\\n/\r\n/g' $ZREESTR
get_ip_regex get_ip_regex
[ "$DISABLE_IPV4" != "1" ] && { [ "$DISABLE_IPV4" != "1" ] && {
dig_reestr "$REG_IPV4" "$ZIPLIST" "$ZIPLIST_IPBAN" 4 dig_reestr "$REG_IPV4" "$ZIPLIST" "$ZIPLIST_IPBAN" 4
} }
[ "$DISABLE_IPV6" != "1" ] && { [ "$DISABLE_IPV6" != "1" ] && {
dig_reestr "$REG_IPV6" "$ZIPLIST6" "$ZIPLIST_IPBAN6" 6 dig_reestr "$REG_IPV6" "$ZIPLIST6" "$ZIPLIST_IPBAN6" 6
} }
rm -f "$ZREESTR" rm -f "$ZREESTR"
}
"$IPSET_DIR/create_ipset.sh" "$IPSET_DIR/create_ipset.sh"

4
ipset/get_reestr_hostlist.sh
View File

@ -6,8 +6,10 @@ IPSET_DIR="$(cd "$IPSET_DIR"; pwd)"
. "$IPSET_DIR/def.sh" . "$IPSET_DIR/def.sh"
# useful in case ipban set is used in custom scripts # useful in case ipban set is used in custom scripts
getuser FAIL=
getuser || FAIL=1
"$IPSET_DIR/create_ipset.sh" "$IPSET_DIR/create_ipset.sh"
[ -n "$FAIL" ] && exit
ZREESTR="$TMPDIR/zapret.txt" ZREESTR="$TMPDIR/zapret.txt"
#ZURL=https://reestr.rublacklist.net/api/current #ZURL=https://reestr.rublacklist.net/api/current

43
ipset/get_reestr_ip.sh
View File

@ -9,7 +9,6 @@ ZREESTR="$TMPDIR/reestr.txt"
#ZURL_REESTR=https://reestr.rublacklist.net/api/current #ZURL_REESTR=https://reestr.rublacklist.net/api/current
ZURL_REESTR=https://raw.githubusercontent.com/zapret-info/z-i/master/dump.csv ZURL_REESTR=https://raw.githubusercontent.com/zapret-info/z-i/master/dump.csv
getuser
dig_reestr() dig_reestr()
{ {
@ -23,29 +22,31 @@ dig_reestr()
} }
# assume all https banned by ip getuser && {
curl -k --fail --max-time 600 --connect-timeout 5 --retry 3 --max-filesize 251658240 "$ZURL_REESTR" -o "$ZREESTR" || # assume all https banned by ip
{ curl -k --fail --max-time 600 --connect-timeout 5 --retry 3 --max-filesize 251658240 "$ZURL_REESTR" -o "$ZREESTR" ||
echo reestr list download failed {
exit 2 echo reestr list download failed
} exit 2
dlsize=$(LANG=C wc -c "$ZREESTR" | xargs | cut -f 1 -d ' ') }
if test $dlsize -lt 1048576; then dlsize=$(LANG=C wc -c "$ZREESTR" | xargs | cut -f 1 -d ' ')
echo reestr ip list is too small. can be bad. if test $dlsize -lt 1048576; then
exit 2 echo reestr ip list is too small. can be bad.
fi exit 2
#sed -i 's/\\n/\r\n/g' $ZREESTR fi
#sed -i 's/\\n/\r\n/g' $ZREESTR
get_ip_regex get_ip_regex
[ "$DISABLE_IPV4" != "1" ] && { [ "$DISABLE_IPV4" != "1" ] && {
dig_reestr "$REG_IPV4" "$ZIPLIST" 4 dig_reestr "$REG_IPV4" "$ZIPLIST" 4
} }
[ "$DISABLE_IPV6" != "1" ] && { [ "$DISABLE_IPV6" != "1" ] && {
dig_reestr "$REG_IPV6" "$ZIPLIST6" 6 dig_reestr "$REG_IPV6" "$ZIPLIST6" 6
} }
rm -f "$ZREESTR" rm -f "$ZREESTR"
}
"$IPSET_DIR/create_ipset.sh" "$IPSET_DIR/create_ipset.sh"

73
ipset/get_reestr_resolve.sh
View File

@ -11,44 +11,45 @@ ZIPLISTTMP="$TMPDIR/zapret-ip.txt"
#ZURL=https://reestr.rublacklist.net/api/current #ZURL=https://reestr.rublacklist.net/api/current
ZURL=https://raw.githubusercontent.com/zapret-info/z-i/master/dump.csv ZURL=https://raw.githubusercontent.com/zapret-info/z-i/master/dump.csv
getuser getuser && {
# both disabled
# both disabled [ "$DISABLE_IPV4" = "1" ] && [ "$DISABLE_IPV6" = "1" ] && exit 0
[ "$DISABLE_IPV4" = "1" ] && [ "$DISABLE_IPV6" = "1" ] && exit 0
curl -k --fail --max-time 600 --connect-timeout 5 --retry 3 --max-filesize 251658240 "$ZURL" >"$ZREESTR" ||
curl -k --fail --max-time 600 --connect-timeout 5 --retry 3 --max-filesize 251658240 "$ZURL" >"$ZREESTR" || {
{ echo reestr list download failed
echo reestr list download failed exit 2
exit 2 }
}
dlsize=$(LANG=C wc -c "$ZREESTR" | xargs | cut -f 1 -d ' ')
if test $dlsize -lt 204800; then
echo list file is too small. can be bad.
exit 2
fi
echo preparing dig list ..
LANG=C cut -f2 -d ';' "$ZREESTR" | LANG=C sed -Ee 's/^\*\.(.+)$/\1/' -ne 's/^[a-z0-9A-Z._-]+$/&/p' >"$ZDIG"
rm -f "$ZREESTR"
echo digging started. this can take long ...
[ "$DISABLE_IPV4" != "1" ] && { dlsize=$(LANG=C wc -c "$ZREESTR" | xargs | cut -f 1 -d ' ')
digger "$ZDIG" 4 | cut_local >"$ZIPLISTTMP" || { if test $dlsize -lt 204800; then
rm -f "$ZDIG" echo list file is too small. can be bad.
exit 1 exit 2
fi
echo preparing dig list ..
LANG=C cut -f2 -d ';' "$ZREESTR" | LANG=C sed -Ee 's/^\*\.(.+)$/\1/' -ne 's/^[a-z0-9A-Z._-]+$/&/p' >"$ZDIG"
rm -f "$ZREESTR"
echo digging started. this can take long ...
[ "$DISABLE_IPV4" != "1" ] && {
filedigger "$ZDIG" 4 | cut_local >"$ZIPLISTTMP" || {
rm -f "$ZDIG"
exit 1
}
ip2net4 <"$ZIPLISTTMP" | zz "$ZIPLIST"
rm -f "$ZIPLISTTMP"
} }
ip2net4 <"$ZIPLISTTMP" | zz "$ZIPLIST" [ "$DISABLE_IPV6" != "1" ] && {
rm -f "$ZIPLISTTMP" filedigger "$ZDIG" 6 | cut_local6 >"$ZIPLISTTMP" || {
} rm -f "$ZDIG"
[ "$DISABLE_IPV6" != "1" ] && { exit 1
digger "$ZDIG" 6 | cut_local6 >"$ZIPLISTTMP" || { }
rm -f "$ZDIG" ip2net6 <"$ZIPLISTTMP" | zz "$ZIPLIST6"
exit 1 rm -f "$ZIPLISTTMP"
} }
ip2net6 <"$ZIPLISTTMP" | zz "$ZIPLIST6" rm -f "$ZDIG"
rm -f "$ZIPLISTTMP"
} }
rm -f "$ZDIG"
"$IPSET_DIR/create_ipset.sh" "$IPSET_DIR/create_ipset.sh"

Write Preview
Loading…
Cancel
Save