gsd
/
zapret
mirror of https://github.com/bol-van/zapret/
1
0
Fork 0
Code Issues Projects Releases Wiki Activity
Browse Source

ipset: add ip lists compression

pull/33/head
bolvan 6 years ago
parent
4013b75e1c
commit
51810df337
8 changed files with 72 additions and 41 deletions
Whitespace
Show all changes Ignore whitespace when comparing lines Ignore changes in amount of whitespace Ignore changes in whitespace at EOL
Split View
Diff Options
Show Stats Download Patch File Download Diff File
  1. 7
      docs/readme.txt
  2. 5
      install_easy.sh
  3. 17
      ipset/create_ipset.sh
  4. 33
      ipset/def.sh
  5. 5
      ipset/get_antizapret.sh
  6. 21
      ipset/get_combined.sh
  7. 6
      ipset/get_hostlist.sh
  8. 17
      ipset/get_reestr.sh

7
docs/readme.txt
View File

@ -161,6 +161,9 @@ get_reestr.sh может использовать мультипоточный
5) ipset/get_config.sh. этот скрипт вызывает то, что прописано в переменной GETLIST из файла config
Если переменная не определена, то никаких действий не выполняется.
Листы zapret-ip.txt и zapret-ipban.txt сохраняются в сжатом виде в файлы .gz.
Это позволяет снизить их размер во много раз и сэкономить место на роутере.
На роутерах не рекомендуется вызывать эти скрипты чаще раза за 2 суток, поскольку сохранение идет
либо во внутреннюю флэш память роутера, либо в случае extroot - на флэшку.
В обоих случаях слишком частая запись может убить флэшку, но если это произойдет с внутренней
@ -270,7 +273,7 @@ TPWS_OPT_HTTPS="--split-pos=3"
systemctl enable zapret
Удалить старые листы, если они были созданы ранее :
rm /opt/zapret/ipset/zapret-ip.txt /opt/zapret/ipset/zapret-ip-user.txt /opt/zapret/ipset/zapret-ip-ipban.txt /opt/zapret/ipset/zapret-ip-user-ipban.txt /opt/zapret/ipset/zapret-hosts.txt
rm /opt/zapret/ipset/zapret-ip.txt* /opt/zapret/ipset/zapret-ip-user.txt* /opt/zapret/ipset/zapret-ip-ipban.txt* /opt/zapret/ipset/zapret-ip-user-ipban.txt* /opt/zapret/ipset/zapret-hosts.txt
По желанию прописать в /opt/zapret/ipset/zapret-hosts-user.txt свои домены.
Выполнить скрипт обновления листа :
/opt/zapret/ipset/get_config.sh
@ -418,7 +421,7 @@ ipset можно выкинуть, если не будем пользовать
Настроить параметры согласно разделу "Выбор параметров".
Удалить старые листы, если они были созданы ранее :
rm /opt/zapret/ipset/zapret-ip.txt /opt/zapret/ipset/zapret-ip-user.txt /opt/zapret/ipset/zapret-ip-ipban.txt /opt/zapret/ipset/zapret-ip-user-ipban.txt /opt/zapret/ipset/zapret-hosts.txt
rm /opt/zapret/ipset/zapret-ip.txt* /opt/zapret/ipset/zapret-ip-user.txt* /opt/zapret/ipset/zapret-ip-ipban.txt* /opt/zapret/ipset/zapret-ip-user-ipban.txt* /opt/zapret/ipset/zapret-hosts.txt
По желанию прописать в /opt/zapret/ipset/zapret-hosts-user.txt свои домены.
Выполнить скрипт обновления листа :
/opt/zapret/ipset/get_config.sh

5
install_easy.sh
View File

@ -348,8 +348,9 @@ download_list()
[ -x "$GET_LIST" ] && {
echo \* downloading blocked ip/host list ...
rm -f "$EXEDIR/ipset/zapret-ip.txt" "$EXEDIR/ipset/zapret-ip-user.txt" \
"$EXEDIR/ipset/zapret-ip-ipban.txt" "$EXEDIR/ipset/zapret-ip-user-ipban.txt"
# can be txt or txt.gz
rm -f "$EXEDIR/ipset/zapret-ip.txt*" "$EXEDIR/ipset/zapret-ip-user.txt*" \
"$EXEDIR/ipset/zapret-ip-ipban.txt*" "$EXEDIR/ipset/zapret-ip-user-ipban.txt*"
"$GET_LIST" || {
echo could not download ip list
exitp 25

17
ipset/create_ipset.sh
View File

@ -8,6 +8,7 @@ IP2NET=$EXEDIR/../ip2net/ip2net
. "$EXEDIR/def.sh"
create_ipset()
{
local IPSTYPE
@ -19,20 +20,20 @@ fi
ipset flush $2 2>/dev/null || ipset create $2 $IPSTYPE $IPSET_OPT
for f in "$3" "$4"
do
[ -f "$f" ] && {
if [ -x $IP2NET ]; then
zzexist "$f" && {
if [ -x "$IP2NET" ]; then
echo Adding to ipset $2 \($IPSTYPE , ip2net\) : $f
if [ -f "$ZIPLIST_EXCLUDE" ] ; then
grep -vxFf $ZIPLIST_EXCLUDE "$f" | $IP2NET | sed -nre "s/^.+$/add $2 &/p" | ipset -! restore
zzcat "$f" | grep -vxFf "$ZIPLIST_EXCLUDE" | "$IP2NET" | sed -nre "s/^.+$/add $2 &/p" | ipset -! restore
else
$IP2NET <"$f" | sed -nre "s/^.+$/add $2 &/p" | ipset -! restore
zzcat "$f" | "$IP2NET" | sed -nre "s/^.+$/add $2 &/p" | ipset -! restore
fi
else
echo Adding to ipset $2 \($IPSTYPE\) : $f
if [ -f "$ZIPLIST_EXCLUDE" ] ; then
grep -vxFf $ZIPLIST_EXCLUDE "$f" | sort -u | sed -nre "s/^.+$/add $2 &/p" | ipset -! restore
zzcat "$f" | grep -vxFf "$ZIPLIST_EXCLUDE" | sort -u | sed -nre "s/^.+$/add $2 &/p" | ipset -! restore
else
sort -u "$f" | sed -nre "s/^.+$/add $2 &/p" | ipset -! restore
zzcat "$f" | sort -u | sed -nre "s/^.+$/add $2 &/p" | ipset -! restore
fi
fi
}
@ -40,5 +41,5 @@ done
return 0
}
create_ipset hash:ip $ZIPSET $ZIPLIST $ZIPLIST_USER
create_ipset hash:ip $ZIPSET_IPBAN $ZIPLIST_IPBAN $ZIPLIST_USER_IPBAN
create_ipset hash:ip $ZIPSET "$ZIPLIST" "$ZIPLIST_USER"
create_ipset hash:ip $ZIPSET_IPBAN "$ZIPLIST_IPBAN" "$ZIPLIST_USER_IPBAN"

33
ipset/def.sh
View File

@ -14,12 +14,29 @@ ZUSERLIST_IPBAN=$EXEDIR/zapret-hosts-user-ipban.txt
MDIG=$EXEDIR/../mdig/mdig
MDIG_THREADS=30
zzexist()
{
[ -f "$1.gz" ] || [ -f "$1" ]
}
zzcat()
{
if [ -f "$1.gz" ]; then
gunzip -c "$1"
else
cat "$1"
fi
}
zz()
{
gzip -c >"$1.gz"
}
digger()
{
if [ -x $MDIG ]; then
$MDIG --family=4 --threads=$MDIG_THREADS <$1
if [ -x "$MDIG" ]; then
zzcat "$1" | "$MDIG" --family=4 --threads=$MDIG_THREADS
else
dig A +short +time=8 +tries=2 -f $1 | grep -E '^[^;].*[^\.]$'
zzcat "$1" | dig A +short +time=8 +tries=2 -f - | grep -E '^[^;].*[^\.]$'
fi
}
@ -30,12 +47,14 @@ cut_local()
grep -vE '^10\.[0-9]+\.[0-9]+\.[0-9]+$'
}
getuser()
{
[ -f $ZUSERLIST ] && {
digger $ZUSERLIST | cut_local | sort -u >$ZIPLIST_USER
[ -f "$ZUSERLIST" ] && {
digger "$ZUSERLIST" | cut_local | sort -u > "$ZIPLIST_USER"
}
[ -f $ZUSERLIST_IPBAN ] && {
digger $ZUSERLIST_IPBAN | cut_local | sort -u >$ZIPLIST_USER_IPBAN
[ -f "$ZUSERLIST_IPBAN" ] && {
digger "$ZUSERLIST_IPBAN" | cut_local | sort -u > "$ZIPLIST_USER_IPBAN"
}
}

5
ipset/get_antizapret.sh
View File

@ -12,13 +12,14 @@ ZIPLISTTMP=$TMPDIR/zapret-ip.txt
getuser
curl --fail --max-time 150 --max-filesize 20971520 -k -L "$ZURL" | cut_local >$ZIPLISTTMP &&
curl --fail --max-time 150 --max-filesize 20971520 -k -L "$ZURL" | cut_local >"$ZIPLISTTMP" &&
{
dlsize=$(wc -c "$ZIPLISTTMP" | cut -f 1 -d ' ')
if test $dlsize -lt 204800; then
echo list file is too small. can be bad.
exit 2
fi
mv -f $ZIPLISTTMP $ZIPLIST
cat "$ZIPLISTTMP" | zz "$ZIPLIST"
rm -f "$ZIPLISTTMP" "$ZIPLIST"
"$EXEDIR/create_ipset.sh"
}

21
ipset/get_combined.sh
View File

@ -11,11 +11,12 @@ ZREESTR=$TMPDIR/reestr.txt
ZURL_REESTR=https://raw.githubusercontent.com/zapret-info/z-i/master/dump.csv
ZAZ=$TMPDIR/zapret-ip.txt
ZURL_AZ=http://antizapret.prostovpn.org/iplist.txt
ZIPLIST_IPBAN_TMP=/tmp/zapret-ipban.txt
getuser
# assume all https banned by ip
curl -k --fail --max-time 300 --max-filesize 62914560 "$ZURL_REESTR" -o $ZREESTR ||
curl -k --fail --max-time 300 --max-filesize 62914560 "$ZURL_REESTR" -o "$ZREESTR" ||
{
echo reestr list download failed
exit 2
@ -27,24 +28,30 @@ if test $dlsize -lt 1048576; then
fi
#sed -i 's/\\n/\r\n/g' $ZREESTR
# find entries with https or without domain name - they should be banned by IP
(grep -a "https://" $ZREESTR ; grep -avE "^.*;[^;:/]+\.[^;:/]+;" $ZREESTR ) |
(grep -a "https://" "$ZREESTR" ; grep -avE "^.*;[^;:/]+\.[^;:/]+;" "$ZREESTR" ) |
grep -oE '[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}(/[0-9]+)?' |
cut_local |
sort -u >$ZIPLIST_IPBAN
sort -u >"$ZIPLIST_IPBAN_TMP"
rm -f $ZREESTR
rm -f "$ZREESTR"
curl --fail --max-time 150 --max-filesize 20971520 -k -L "$ZURL_AZ" | cut_local >$ZAZ ||
curl --fail --max-time 150 --max-filesize 20971520 -k -L "$ZURL_AZ" | cut_local >"$ZAZ" ||
{
rm -f "$ZIPLIST_IPBAN_TMP"
echo antizapret list download failed
exit 2
}
dlsize=$(wc -c "$ZAZ" | cut -f 1 -d ' ')
if test $dlsize -lt 204800; then
rm -f "$ZIPLIST_IPBAN_TMP"
echo antizapret list file is too small. can be bad.
exit 2
fi
# do not include hosts banned by ip
grep -xvFf $ZIPLIST_IPBAN $ZAZ >$ZIPLIST
rm -f $ZAZ
grep -xvFf "$ZIPLIST_IPBAN_TMP" "$ZAZ" | zz "$ZIPLIST"
rm -f "$ZAZ"
cat "$ZIPLIST_IPBAN_TMP" | zz "$ZIPLIST_IPBAN"
rm -f "$ZIPLIST_IPBAN_TMP"
"$EXEDIR/create_ipset.sh"

6
ipset/get_hostlist.sh
View File

@ -10,7 +10,7 @@ ZREESTR=$TMPDIR/zapret.txt
#ZURL=https://reestr.rublacklist.net/api/current
ZURL=https://raw.githubusercontent.com/zapret-info/z-i/master/dump.csv
curl -k --fail --max-time 300 --max-filesize 62914560 "$ZURL" >$ZREESTR ||
curl -k --fail --max-time 300 --max-filesize 62914560 "$ZURL" >"$ZREESTR" ||
{
echo reestr list download failed
exit 2
@ -20,8 +20,8 @@ if test $dlsize -lt 204800; then
echo list file is too small. can be bad.
exit 2
fi
(cut -s -f2 -d';' $ZREESTR | grep -a . | sed -re 's/^\*\.(.+)$/\1/' | awk '{ print tolower($0) }' ; cat $ZUSERLIST ) | sort -u >$ZHOSTLIST
rm -f $ZREESTR
(cut -s -f2 -d';' "$ZREESTR" | grep -a . | sed -re 's/^\*\.(.+)$/\1/' | awk '{ print tolower($0) }' ; cat "$ZUSERLIST" ) | sort -u >"$ZHOSTLIST"
rm -f "$ZREESTR"
# force tpws to reload if its running
killall -HUP tpws 2>/dev/null

17
ipset/get_reestr.sh
View File

@ -14,7 +14,7 @@ ZURL=https://raw.githubusercontent.com/zapret-info/z-i/master/dump.csv
getuser
curl -k --fail --max-time 300 --max-filesize 62914560 "$ZURL" >$ZREESTR ||
curl -k --fail --max-time 300 --max-filesize 62914560 "$ZURL" >"$ZREESTR" ||
{
echo reestr list download failed
exit 2
@ -27,15 +27,14 @@ fi
echo preparing dig list ..
#sed -i 's/\\n/\r\n/g' $ZREESTR
#sed -nre 's/^[^;]*;([^;|\\]{4,250})\;.*$/\1/p' $ZREESTR | sort | uniq >$ZDIG
cut -f2 -d';' $ZREESTR | grep -avE '^$|\*|:' >$ZDIG
rm -f $ZREESTR
cut -f2 -d ';' "$ZREESTR" | grep -avE '^$|\*|:' >"$ZDIG"
rm -f "$ZREESTR"
echo digging started ...
digger $ZDIG | cut_local >$ZIPLISTTMP || {
rm -f $ZDIG
digger "$ZDIG" | cut_local >"$ZIPLISTTMP" || {
rm -f "$ZDIG"
exit 1
}
rm -f $ZDIG $ZIPLIST
sort -u $ZIPLISTTMP >$ZIPLIST
rm -f $ZIPLISTTMP
rm -f "$ZDIG"
sort -u "$ZIPLISTTMP" | zz "$ZIPLIST"
rm -f "$ZIPLISTTMP"
"$EXEDIR/create_ipset.sh"

Write Preview
Loading…
Cancel
Save