ipset: scripts maintenance

ipset/get_reestr_hostlist.sh

@@ -5,9 +5,9 @@ IPSET_DIR="$(cd "$IPSET_DIR"; pwd)"
 
 . "$IPSET_DIR/def.sh"
 
-ZREESTR="$TMPDIR/zapret.txt"
+ZREESTR="$TMPDIR/zapret.txt.gz"
 IPB="$TMPDIR/ipb.txt"
-ZURL_REESTR=https://raw.githubusercontent.com/zapret-info/z-i/master/dump.csv
+ZURL_REESTR=https://raw.githubusercontent.com/zapret-info/z-i/master/dump.csv.gz
 
 dl_checked()
 {
@@ -31,11 +31,11 @@ dl_checked()
 
 reestr_list()
 {
- LANG=C cut -s -f2 -d';' "$ZREESTR" | LANG=C nice -n 5 sed -Ee 's/^\*\.(.+)$/\1/' -ne 's/^[a-z0-9A-Z._-]+$/&/p' | $AWK '{ print tolower($0) }'
+ LANG=C gunzip -c "$ZREESTR" | cut -s -f2 -d';' | LANG=C nice -n 5 sed -Ee 's/^\*\.(.+)$/\1/' -ne 's/^[a-z0-9A-Z._-]+$/&/p' | $AWK '{ print tolower($0) }'
 }
 reestr_extract_ip()
 {
- LANG=C nice -n 5 $AWK -F ';' '($1 ~ /^([0-9]{1,3}\.){3}[0-9]{1,3}/) && (($2 == "" && $3 == "") || ($1 == $2)) {gsub(/ \| /, RS); print $1}' "$ZREESTR" | LANG=C $AWK '{split($1, a, /\|/); for (i in a) {print a[i]}}'
+ LANG=C gunzip -c | nice -n 5 $AWK -F ';' '($1 ~ /^([0-9]{1,3}\.){3}[0-9]{1,3}/) && (($2 == "" && $3 == "") || ($1 == $2)) {gsub(/ \| /, RS); print $1}' | LANG=C $AWK '{split($1, a, /\|/); for (i in a) {print a[i]}}'
 }
 
 ipban_fin()

ipset/get_reestr_resolve.sh

@@ -5,12 +5,12 @@ IPSET_DIR="$(cd "$IPSET_DIR"; pwd)"
 
 . "$IPSET_DIR/def.sh"
 
-ZREESTR="$TMPDIR/zapret.txt"
+ZREESTR="$TMPDIR/zapret.txt.gz"
 ZDIG="$TMPDIR/zapret-dig.txt"
 IPB="$TMPDIR/ipb.txt"
 ZIPLISTTMP="$TMPDIR/zapret-ip.txt"
 #ZURL=https://reestr.rublacklist.net/api/current
-ZURL_REESTR=https://raw.githubusercontent.com/zapret-info/z-i/master/dump.csv
+ZURL_REESTR=https://raw.githubusercontent.com/zapret-info/z-i/master/dump.csv.gz
 
 dl_checked()
 {
@@ -34,11 +34,11 @@ dl_checked()
 
 reestr_list()
 {
- LANG=C cut -s -f2 -d';' "$ZREESTR" | LANG=C nice -n 5 sed -Ee 's/^\*\.(.+)$/\1/' -ne 's/^[a-z0-9A-Z._-]+$/&/p'
+ LANG=C gunzip -c "$ZREESTR" | cut -s -f2 -d';' | LANG=C nice -n 5 sed -Ee 's/^\*\.(.+)$/\1/' -ne 's/^[a-z0-9A-Z._-]+$/&/p' | $AWK '{ print tolower($0) }'
 }
 reestr_extract_ip()
 {
- LANG=C nice -n 5 $AWK -F ';' '($1 ~ /^([0-9]{1,3}\.){3}[0-9]{1,3}/) && (($2 == "" && $3 == "") || ($1 == $2)) {gsub(/ \| /, RS); print $1}' "$ZREESTR" | LANG=C $AWK '{split($1, a, /\|/); for (i in a) {print a[i]}}'
+ LANG=C gunzip -c | nice -n 5 $AWK -F ';' '($1 ~ /^([0-9]{1,3}\.){3}[0-9]{1,3}/) && (($2 == "" && $3 == "") || ($1 == $2)) {gsub(/ \| /, RS); print $1}' | LANG=C $AWK '{split($1, a, /\|/); for (i in a) {print a[i]}}'
 }
 
 getuser && {

ipset/get_refilter_domains.sh

@@ -25,7 +25,7 @@ dl()
    echo list is too small : $dlsize bytes. can be bad.
    exit 2
   fi
-  zzcat "$TMPLIST" | tr -d '\015' | zz "$2"
+  zzcat "$TMPLIST" | zz "$2"
   rm -f "$TMPLIST"
 }
 

ipset/get_refilter_ipsum.sh

@@ -26,7 +26,7 @@ dl()
    exit 2
   fi
   # remove DOS EOL \r
-  zzcat "$TMPLIST" | tr -d '\015' | zz "$2"
+  zzcat "$TMPLIST" | zz "$2"
   rm -f "$TMPLIST"
 }