switched reestr list source to github (rublacklist enable cloudflare ddos)

ipset/get_combined.sh

@@ -7,37 +7,44 @@ EXEDIR=$(dirname $SCRIPT)
 . "$EXEDIR/def.sh"
 
 ZREESTR=$TMPDIR/reestr.txt
-ZURL_REESTR=https://reestr.rublacklist.net/api/current
+#ZURL_REESTR=https://reestr.rublacklist.net/api/current
+ZURL_REESTR=https://raw.githubusercontent.com/zapret-info/z-i/master/dump.csv
 ZAZ=$TMPDIR/zapret-ip.txt
 ZURL_AZ=http://antizapret.prostovpn.org/iplist.txt
 
 getuser
 
 # assume all https banned by ip
-curl -k --fail --max-time 300 --max-filesize 41943040 "$ZURL_REESTR" -o $ZREESTR
+curl -k --fail --max-time 300 --max-filesize 41943040 "$ZURL_REESTR" -o $ZREESTR ||
+{
+ echo reestr list download failed
+ exit 2
+}
 dlsize=$(wc -c "$ZREESTR" | cut -f 1 -d ' ')
 if test $dlsize -lt 1048576; then
  echo reestr ip list is too small. can be bad.
  exit 2
 fi
-sed -i 's/\\n/\r\n/g' $ZREESTR
+#sed -i 's/\\n/\r\n/g' $ZREESTR
 # find entries with https or without domain name - they should be banned by IP
-(grep "https://" $ZREESTR ; grep -vE ";[a-z|*|_|0-9|\-]+\.[a-z|*|_|0-9|\-]+" $ZREESTR ) |
+(grep -a "https://" $ZREESTR ; grep -avE "^.*;[^;:/]+\.[^;:/]+;" $ZREESTR ) |
  grep -oE '[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}(/[0-9]+)?' |
  cut_local |
  sort -u >$ZIPLIST_IPBAN
-
+ 
 rm -f $ZREESTR
 
-curl --fail --max-time 120 --max-filesize 10485760 -k -L "$ZURL_AZ" | cut_local >$ZAZ &&
+curl --fail --max-time 120 --max-filesize 10485760 -k -L "$ZURL_AZ" | cut_local >$ZAZ ||
 {
- dlsize=$(wc -c "$ZAZ" | cut -f 1 -d ' ')
+ echo antizapret list download failed
- if test $dlsize -lt 204800; then
+ exit 2
-  echo antizapret list file is too small. can be bad.
-  exit 2
- fi
- # do not include hosts banned by ip
- grep -xvFf $ZIPLIST_IPBAN $ZAZ >$ZIPLIST
- rm -f $ZAZ
- "$EXEDIR/create_ipset.sh"
 }
+dlsize=$(wc -c "$ZAZ" | cut -f 1 -d ' ')
+if test $dlsize -lt 204800; then
+ echo antizapret list file is too small. can be bad.
+ exit 2
+fi
+# do not include hosts banned by ip
+grep -xvFf $ZIPLIST_IPBAN $ZAZ >$ZIPLIST
+rm -f $ZAZ
+"$EXEDIR/create_ipset.sh"

ipset/get_reestr.sh

@@ -9,26 +9,33 @@ EXEDIR=$(dirname $SCRIPT)
 ZREESTR=$TMPDIR/zapret.txt
 ZDIG=$TMPDIR/zapret-dig.txt
 ZIPLISTTMP=$TMPDIR/zapret-ip.txt
-ZURL=https://reestr.rublacklist.net/api/current
+#ZURL=https://reestr.rublacklist.net/api/current
+ZURL=https://raw.githubusercontent.com/zapret-info/z-i/master/dump.csv
 
 getuser
 
-curl -k --fail --max-time 300 --max-filesize 41943040 "$ZURL" >$ZREESTR && {
+curl -k --fail --max-time 300 --max-filesize 41943040 "$ZURL" >$ZREESTR ||
- dlsize=$(wc -c "$ZREESTR" | cut -f 1 -d ' ')
+{
- if test $dlsize -lt 204800; then
+ echo reestr list download failed   
-  echo list file is too small. can be bad.
+ exit 2
-  exit 2
- fi
- sed -i 's/\\n/\r\n/g' $ZREESTR
- sed -nre 's/^[^;]*;([^;|\\]{4,250})\;.*/\1/p' $ZREESTR | sort | uniq >$ZDIG
- rm -f $ZREESTR
- echo digging started ...
- dig A +short +time=8 +tries=2 -f $ZDIG | grep -E '^[^;].*[^\.]$' | grep -vE '^192\.168\.[0-9]+\.[0-9]+$' | grep -vE '^127\.[0-9]+\.[0-9]+\.[0-9]+$' | grep -vE '^10\.[0-9]+\.[0-9]+\.[0-9]+$' >$ZIPLISTTMP || {
-  rm -f $ZDIG
-  exit 1
- }
- rm -f $ZDIG $ZIPLIST
- sort $ZIPLISTTMP | uniq >$ZIPLIST
- rm -f $ZIPLISTTMP
- "$EXEDIR/create_ipset.sh"
 }
+dlsize=$(wc -c "$ZREESTR" | cut -f 1 -d ' ')
+if test $dlsize -lt 204800; then
+ echo list file is too small. can be bad.
+ exit 2
+fi
+echo preparing dig list ..
+#sed -i 's/\\n/\r\n/g' $ZREESTR
+#sed -nre 's/^[^;]*;([^;|\\]{4,250})\;.*$/\1/p' $ZREESTR | sort | uniq >$ZDIG
+cut -f2 -d';' $ZREESTR  | grep -avE '^$|\*|:' >$ZDIG
+rm -f $ZREESTR
+echo digging started ...
+dig A +short +time=8 +tries=2 -f $ZDIG | grep -E '^[^;].*[^\.]$' | grep -vE '^192\.168\.[0-9]+\.[0-9]+$' | grep -vE '^127\.[0-9]+\.[0-9]+\.[0-9]+$' | grep -vE '^10\.[0-9]+\.[0-9]+\.[0-9]+$' >$ZIPLISTTMP || {
+ rm -f $ZDIG
+ exit 1
+}
+rm -f $ZDIG $ZIPLIST
+sort $ZIPLISTTMP | uniq >$ZIPLIST
+rm -f $ZIPLISTTMP
+"$EXEDIR/create_ipset.sh"
+