gsd
/
zapret
mirror of https://github.com/bol-van/zapret/
1
0
Fork 0
Code Issues Projects Releases Wiki Activity
Browse Source

reestr lists: support ipban

pull/172/head
bol-van 1 year ago
parent
55c2783aea
commit
e716971739
5 changed files with 120 additions and 41 deletions
Whitespace
Show all changes Ignore whitespace when comparing lines Ignore changes in amount of whitespace Ignore changes in whitespace at EOL
Split View
Diff Options
Show Stats Download Patch File Download Diff File
  1. 51
      ipset/get_reestr_hostlist.sh
  2. 21
      ipset/get_reestr_preresolved.sh
  3. 21
      ipset/get_reestr_preresolved_smart.sh
  4. 16
      ipset/get_reestr_resolvable_domains.sh
  5. 52
      ipset/get_reestr_resolve.sh

51
ipset/get_reestr_hostlist.sh
View File

@ -5,37 +5,60 @@ IPSET_DIR="$(cd "$IPSET_DIR"; pwd)"
. "$IPSET_DIR/def.sh"
# useful in case ipban set is used in custom scripts
FAIL=
getipban || FAIL=1
"$IPSET_DIR/create_ipset.sh"
[ -n "$FAIL" ] && exit
ZREESTR="$TMPDIR/zapret.txt"
#ZURL=https://reestr.rublacklist.net/api/current
IPB="$TMPDIR/ipb.txt"
ZURL_REESTR=https://raw.githubusercontent.com/zapret-info/z-i/master/dump.csv
curl -H "Accept-Encoding: gzip" -k --fail --max-time 600 --connect-timeout 5 --retry 3 --max-filesize 251658240 "$ZURL_REESTR" | gunzip - >"$ZREESTR" ||
dl_checked()
{
echo reestr list download failed
exit 2
# $1 - url
# $2 - file
# $3 - minsize
# $4 - maxsize
# $5 - maxtime
curl -H "Accept-Encoding: gzip" -k --fail --max-time $5 --connect-timeout 10 --retry 4 --max-filesize $4 "$1" | gunzip - >"$2" ||
{
echo list download failed : $1
return 2
}
dlsize=$(LANG=C wc -c "$2" | xargs | cut -f 1 -d ' ')
if test $dlsize -lt $3; then
echo list is too small : $dlsize bytes. can be bad.
return 2
fi
return 0
}
reestr_list()
{
LANG=C cut -s -f2 -d';' "$ZREESTR" | LANG=C nice -n 5 sed -Ee 's/^\*\.(.+)$/\1/' -ne 's/^[a-z0-9A-Z._-]+$/&/p' | $AWK '{ print tolower($0) }'
}
reestr_extract_ip()
{
LANG=C nice -n 5 $AWK -F ';' '($1 ~ /^([0-9]{1,3}\.){3}[0-9]{1,3}/) && (($2 == "" && $3 == "") || ($1 == $2)) {gsub(/ \| /, RS); print $1}' "$ZREESTR" | LANG=C $AWK '{split($1, a, /\|/); for (i in a) {print a[i]}}'
}
dlsize=$(LANG=C wc -c "$ZREESTR" | xargs | cut -f 1 -d ' ')
if test $dlsize -lt 204800; then
echo list file is too small. can be bad.
ipban_fin()
{
getipban
"$IPSET_DIR/create_ipset.sh"
}
dl_checked "$ZURL_REESTR" "$ZREESTR" 204800 251658240 600 || {
ipban_fin
exit 2
fi
}
reestr_list | sort -u | zz "$ZHOSTLIST"
reestr_extract_ip <"$ZREESTR" >"$IPB"
rm -f "$ZREESTR"
$AWK '/^([0-9]{1,3}\.){3}[0-9]{1,3}$/' "$IPB" | ip2net4 | zz "$ZIPLIST_IPBAN"
$AWK '/^([0-9a-fA-F]{0,4}:){1,7}[0-9a-fA-F]{0,4}$/' "$IPB" | ip2net6 | zz "$ZIPLIST_IPBAN6"
rm -f "$IPB"
hup_zapret_daemons
ipban_fin
exit 0

21
ipset/get_reestr_preresolved.sh
View File

@ -5,24 +5,27 @@ IPSET_DIR="$(cd "$IPSET_DIR"; pwd)"
. "$IPSET_DIR/def.sh"
TMPLIST="$TMPDIR/list_nethub.txt"
TMPLIST="$TMPDIR/list.txt"
BASEURL="https://raw.githubusercontent.com/bol-van/rulist/main"
URL4="$BASEURL/reestr_resolved4.txt"
URL6="$BASEURL/reestr_resolved6.txt"
IPB4="$BASEURL/reestr_ipban4.txt"
IPB6="$BASEURL/reestr_ipban6.txt"
dl()
{
# $1 - url
# $2 - file
curl -H "Accept-Encoding: gzip" -k --fail --max-time 180 --connect-timeout 10 --retry 4 --max-filesize 33554432 "$1" | gunzip - >"$TMPLIST" ||
# $3 - minsize
# $4 - maxsize
curl -H "Accept-Encoding: gzip" -k --fail --max-time 120 --connect-timeout 10 --retry 4 --max-filesize $4 "$1" | gunzip - >"$TMPLIST" ||
{
echo list download failed : $1
exit 2
}
dlsize=$(LANG=C wc -c "$TMPLIST" | xargs | cut -f 1 -d ' ')
if test $dlsize -lt 32768; then
if test $dlsize -lt $3; then
echo list is too small : $dlsize bytes. can be bad.
exit 2
fi
@ -31,8 +34,14 @@ dl()
}
getuser && {
[ "$DISABLE_IPV4" != "1" ] && dl "$URL4" "$ZIPLIST"
[ "$DISABLE_IPV6" != "1" ] && dl "$URL6" "$ZIPLIST6"
[ "$DISABLE_IPV4" != "1" ] && {
dl "$URL4" "$ZIPLIST" 32768 4194304
dl "$IPB4" "$ZIPLIST_IPBAN" 8192 1048576
}
[ "$DISABLE_IPV6" != "1" ] && {
dl "$URL6" "$ZIPLIST6" 8192 4194304
dl "$IPB6" "$ZIPLIST_IPBAN6" 128 1048576
}
}
"$IPSET_DIR/create_ipset.sh"

21
ipset/get_reestr_preresolved_smart.sh
View File

@ -5,24 +5,27 @@ IPSET_DIR="$(cd "$IPSET_DIR"; pwd)"
. "$IPSET_DIR/def.sh"
TMPLIST="$TMPDIR/list_nethub.txt"
TMPLIST="$TMPDIR/list.txt"
BASEURL="https://raw.githubusercontent.com/bol-van/rulist/main"
URL4="$BASEURL/reestr_smart4.txt"
URL6="$BASEURL/reestr_smart6.txt"
IPB4="$BASEURL/reestr_ipban4.txt"
IPB6="$BASEURL/reestr_ipban6.txt"
dl()
{
# $1 - url
# $2 - file
curl -H "Accept-Encoding: gzip" -k --fail --max-time 180 --connect-timeout 10 --retry 4 --max-filesize 33554432 "$1" | gunzip - >"$TMPLIST" ||
# $3 - minsize
# $4 - maxsize
curl -H "Accept-Encoding: gzip" -k --fail --max-time 120 --connect-timeout 10 --retry 4 --max-filesize $4 "$1" | gunzip - >"$TMPLIST" ||
{
echo list download failed : $1
exit 2
}
dlsize=$(LANG=C wc -c "$TMPLIST" | xargs | cut -f 1 -d ' ')
if test $dlsize -lt 32768; then
if test $dlsize -lt $3; then
echo list is too small : $dlsize bytes. can be bad.
exit 2
fi
@ -31,8 +34,14 @@ dl()
}
getuser && {
[ "$DISABLE_IPV4" != "1" ] && dl "$URL4" "$ZIPLIST"
[ "$DISABLE_IPV6" != "1" ] && dl "$URL6" "$ZIPLIST6"
[ "$DISABLE_IPV4" != "1" ] && {
dl "$URL4" "$ZIPLIST" 32768 4194304
dl "$IPB4" "$ZIPLIST_IPBAN" 8192 1048576
}
[ "$DISABLE_IPV6" != "1" ] && {
dl "$URL6" "$ZIPLIST6" 8192 4194304
dl "$IPB6" "$ZIPLIST_IPBAN6" 128 1048576
}
}
"$IPSET_DIR/create_ipset.sh"

16
ipset/get_reestr_resolvable_domains.sh
View File

@ -9,18 +9,22 @@ TMPLIST="$TMPDIR/list_nethub.txt"
BASEURL="https://raw.githubusercontent.com/bol-van/rulist/main"
URL="$BASEURL/reestr_hostname_resolvable.txt"
IPB4="$BASEURL/reestr_ipban4.txt"
IPB6="$BASEURL/reestr_ipban6.txt"
dl()
{
# $1 - url
# $2 - file
curl -H "Accept-Encoding: gzip" -k --fail --max-time 180 --connect-timeout 10 --retry 4 --max-filesize 33554432 "$1" | gunzip - >"$TMPLIST" ||
# $3 - minsize
# $4 - maxsize
curl -H "Accept-Encoding: gzip" -k --fail --max-time 120 --connect-timeout 10 --retry 4 --max-filesize $4 "$1" | gunzip - >"$TMPLIST" ||
{
echo list download failed : $1
exit 2
}
dlsize=$(LANG=C wc -c "$TMPLIST" | xargs | cut -f 1 -d ' ')
if test $dlsize -lt 65536; then
if test $dlsize -lt $3; then
echo list is too small : $dlsize bytes. can be bad.
exit 2
fi
@ -28,8 +32,14 @@ dl()
rm -f "$TMPLIST"
}
dl "$URL" "$ZHOSTLIST"
dl "$URL" "$ZHOSTLIST" 65536 67108864
hup_zapret_daemons
[ "$DISABLE_IPV4" != "1" ] && dl "$IPB4" "$ZIPLIST_IPBAN" 8192 1048576
[ "$DISABLE_IPV6" != "1" ] && dl "$IPB6" "$ZIPLIST_IPBAN6" 128 1048576
getipban
"$IPSET_DIR/create_ipset.sh"
exit 0

52
ipset/get_reestr_resolve.sh
View File

@ -7,28 +7,56 @@ IPSET_DIR="$(cd "$IPSET_DIR"; pwd)"
ZREESTR="$TMPDIR/zapret.txt"
ZDIG="$TMPDIR/zapret-dig.txt"
IPB="$TMPDIR/ipb.txt"
ZIPLISTTMP="$TMPDIR/zapret-ip.txt"
#ZURL=https://reestr.rublacklist.net/api/current
ZURL_REESTR=https://raw.githubusercontent.com/zapret-info/z-i/master/dump.csv
dl_checked()
{
# $1 - url
# $2 - file
# $3 - minsize
# $4 - maxsize
# $5 - maxtime
curl -H "Accept-Encoding: gzip" -k --fail --max-time $5 --connect-timeout 10 --retry 4 --max-filesize $4 "$1" | gunzip - >"$2" ||
{
echo list download failed : $1
return 2
}
dlsize=$(LANG=C wc -c "$2" | xargs | cut -f 1 -d ' ')
if test $dlsize -lt $3; then
echo list is too small : $dlsize bytes. can be bad.
return 2
fi
return 0
}
reestr_list()
{
LANG=C cut -s -f2 -d';' "$ZREESTR" | LANG=C nice -n 5 sed -Ee 's/^\*\.(.+)$/\1/' -ne 's/^[a-z0-9A-Z._-]+$/&/p'
}
reestr_extract_ip()
{
LANG=C nice -n 5 $AWK -F ';' '($1 ~ /^([0-9]{1,3}\.){3}[0-9]{1,3}/) && (($2 == "" && $3 == "") || ($1 == $2)) {gsub(/ \| /, RS); print $1}' "$ZREESTR" | LANG=C $AWK '{split($1, a, /\|/); for (i in a) {print a[i]}}'
}
getuser && {
# both disabled
[ "$DISABLE_IPV4" = "1" ] && [ "$DISABLE_IPV6" = "1" ] && exit 0
curl -H "Accept-Encoding: gzip" -k --fail --max-time 600 --connect-timeout 5 --retry 3 --max-filesize 251658240 "$ZURL_REESTR" | gunzip - >"$ZREESTR" ||
{
echo reestr list download failed
exit 2
}
dlsize=$(LANG=C wc -c "$ZREESTR" | xargs | cut -f 1 -d ' ')
if test $dlsize -lt 204800; then
echo list file is too small. can be bad.
exit 2
fi
dl_checked "$ZURL_REESTR" "$ZREESTR" 204800 251658240 600 || exit 2
echo preparing ipban list ..
reestr_extract_ip <"$ZREESTR" >"$IPB"
$AWK '/^([0-9]{1,3}\.){3}[0-9]{1,3}$/' "$IPB" | ip2net4 | zz "$ZIPLIST_IPBAN"
$AWK '/^([0-9a-fA-F]{0,4}:){1,7}[0-9a-fA-F]{0,4}$/' "$IPB" | ip2net6 | zz "$ZIPLIST_IPBAN6"
rm -f "$IPB"
echo preparing dig list ..
LANG=C cut -f2 -d ';' "$ZREESTR" | LANG=C sed -Ee 's/^\*\.(.+)$/\1/' -ne 's/^[a-z0-9A-Z._-]+$/&/p' >"$ZDIG"
reestr_list | sort -u >"$ZDIG"
rm -f "$ZREESTR"
echo digging started. this can take long ...

Write Preview
Loading…
Cancel
Save