autohostlist debug

common/list.sh

@@ -15,6 +15,7 @@ find_hostlists()
 	[ -f "$HOSTLIST_EXCLUDE" ] || HOSTLIST_EXCLUDE=
 
 	HOSTLIST_AUTO="$HOSTLIST_BASE/zapret-hosts-auto.txt"
+	HOSTLIST_AUTO_DEBUGLOG="$HOSTLIST_BASE/zapret-hosts-auto-debug.log"
 }
 
 filter_apply_autohostlist_target()
@@ -23,9 +24,10 @@ filter_apply_autohostlist_target()
 	
 	local parm1="${AUTOHOSTLIST_FAIL_THRESHOLD:+--hostlist-auto-fail-threshold=$AUTOHOSTLIST_FAIL_THRESHOLD}"
 	local parm2="${AUTOHOSTLIST_FAIL_TIME:+--hostlist-auto-fail-time=$AUTOHOSTLIST_FAIL_TIME}"
-	local parm3
+	local parm3 parm4
 	[ "$MODE" = "tpws" -o "$MODE" = "tpws-socks" ] || parm3="${AUTOHOSTLIST_RETRANS_THRESHOLD:+--hostlist-auto-retrans-threshold=$AUTOHOSTLIST_RETRANS_THRESHOLD}"
-	eval $1="\"\$$1 --hostlist-auto=$HOSTLIST_AUTO $parm1 $parm2 $parm3\""
+	[ "$AUTOHOSTLIST_DEBUGLOG" = 1 ] && parm4="--hostlist-auto-debug=$HOSTLIST_AUTO_DEBUGLOG"
+	eval $1="\"\$$1 --hostlist-auto=$HOSTLIST_AUTO $parm1 $parm2 $parm3 $parm4\""
 }
 
 filter_apply_hostlist_target()

config

@@ -23,6 +23,8 @@ IP2NET_OPT6="--prefix-length=56-64 --v6-threshold=5"
 AUTOHOSTLIST_RETRANS_THRESHOLD=3
 AUTOHOSTLIST_FAIL_THRESHOLD=2
 AUTOHOSTLIST_FAIL_TIME=60
+# 1 = debug autohostlist positives to ipset/zapret-hosts-auto-debug.log
+AUTOHOSTLIST_DEBUGLOG=0
 
 # number of parallel threads for domain list resolves
 MDIG_THREADS=30

docs/readme.eng.md

@@ -184,6 +184,7 @@ nfqws takes the following parameters:
  --hostlist-auto-fail-threshold=<int>           ; how many failed attempts cause hostname to be added to auto hostlist (default : 2)
  --hostlist-auto-fail-time=<int>                ; all failed attemps must be within these seconds (default : 60)
  --hostlist-auto-retrans-threshold=<int>        ; how many request retransmissions cause attempt to fail (default : 3)
+ --hostlist-auto-debug=<logfile>        	; debug auto hostlist positives
 ```
 
 The manipulation parameters can be combined in any way.
@@ -539,6 +540,8 @@ tpws is transparent proxy.
  --hostlist-auto=<filename>            ; detect DPI blocks and build hostlist automatically
  --hostlist-auto-fail-threshold=<int>  ; how many failed attempts cause hostname to be added to auto hostlist (default : 2)
  --hostlist-auto-fail-time=<int>       ; all failed attemps must be within these seconds (default : 60)
+ --hostlist-auto-debug=<logfile>       	; debug auto hostlist positives
+
  --split-http-req=method|host	; split http request at specified logical position.
  --split-pos=<numeric_offset>   ; split at specified pos. split-http-req takes precedence over split-pos for http reqs.
  --split-any-protocol		; split not only http and https
@@ -911,6 +914,7 @@ autohostlist mode tuning.
 AUTOHOSTLIST_RETRANS_THRESHOLD=3
 AUTOHOSTLIST_FAIL_THRESHOLD=2
 AUTOHOSTLIST_FAIL_TIME=60
+AUTOHOSTLIST_DEBUG=0
 ```
 
 Enable gzip compression for large lists. Used by ipset/*.sh scripts.

docs/readme.txt

@@ -234,6 +234,7 @@ nfqws
  --hostlist-auto-fail-threshold=<int>           ; сколько раз нужно обнаружить ситуацию, похожую на блокировку, чтобы добавить хост в лист (по умолчанию: 2)
  --hostlist-auto-fail-time=<int>                ; все эти ситуации должны быть в пределах указанного количества секунд (по умолчанию: 60)
  --hostlist-auto-retrans-threshold=<int>        ; сколько ретрансмиссий запроса считать блокировкой (по умолчанию: 3)
+ --hostlist-auto-debug=<logfile>        	; лог положительных решений по autohostlist. позволяет разобраться почему там появляются хосты.
 
 Параметры манипуляции могут сочетаться в любых комбинациях.
 
@@ -609,6 +610,7 @@ tpws - это transparent proxy.
  --hostlist-auto=<filename>             ; обнаруживать автоматически блокировки и заполнять автоматический hostlist (требует перенаправления входящего трафика)
  --hostlist-auto-fail-threshold=<int>   ; сколько раз нужно обнаружить ситуацию, похожую на блокировку, чтобы добавить хост в лист (по умолчанию: 2)
  --hostlist-auto-fail-time=<int>        ; все эти ситуации должны быть в пределах указанного количества секунд (по умолчанию: 60)
+ --hostlist-auto-debug=<logfile>       	; лог положительных решений по autohostlist. позволяет разобраться почему там появляются хосты.
 
 
 Параметры манипуляции могут сочетаться в любых комбинациях.
@@ -1123,6 +1125,7 @@ IP2NET_OPT6="--prefix-length=56-64 --v6-threshold=5"
 AUTOHOSTLIST_RETRANS_THRESHOLD=3
 AUTOHOSTLIST_FAIL_THRESHOLD=2
 AUTOHOSTLIST_FAIL_TIME=60
+AUTOHOSTLIST_DEBUG=0
 
 Включить или выключить сжатие больших листов в скриптах ipset/*.sh. По умолчанию включено.
 GZIP_LISTS=1

nfq/desync.c

@@ -175,8 +175,11 @@ static void wssize_cutoff(t_ctrack *ctrack)
 
 static void ctrack_stop_req_counter(t_ctrack *ctrack)
 {
-	ctrack->req_retrans_counter = RETRANS_COUNTER_STOP;
-	maybe_cutoff(ctrack, IPPROTO_TCP);
+	if (ctrack && *params.hostlist_auto_filename)
+	{
+		ctrack->req_retrans_counter = RETRANS_COUNTER_STOP;
+		maybe_cutoff(ctrack, IPPROTO_TCP);
+	}
 }
 
 // return true if retrans trigger fires
@@ -221,6 +224,7 @@ static void auto_hostlist_failed(const char *hostname)
 	}
 	fail_counter->counter++;
 	DLOG("auto hostlist : %s : fail counter %d/%d\n", hostname, fail_counter->counter, params.hostlist_auto_fail_threshold);
+	HOSTLIST_DEBUGLOG_APPEND("%s : fail counter %d/%d", hostname, fail_counter->counter, params.hostlist_auto_fail_threshold);
 	if (fail_counter->counter >= params.hostlist_auto_fail_threshold)
 	{
 		DLOG("auto hostlist : fail threshold reached. adding %s to auto hostlist\n", hostname);
@@ -231,6 +235,7 @@ static void auto_hostlist_failed(const char *hostname)
 		if (!HostlistCheck(params.hostlist, params.hostlist_exclude, hostname, &bExcluded) && !bExcluded)
 		{
 			DLOG("auto hostlist : adding %s\n", hostname);
+			HOSTLIST_DEBUGLOG_APPEND("%s : adding", hostname);
 			if (!StrPoolAddStr(&params.hostlist, hostname))
 			{
 				fprintf(stderr, "StrPoolAddStr out of memory\n");
@@ -243,7 +248,10 @@ static void auto_hostlist_failed(const char *hostname)
 			}
 		}
 		else
+		{
 			DLOG("auto hostlist : NOT adding %s\n", hostname);
+			HOSTLIST_DEBUGLOG_APPEND("%s : NOT adding, duplicate detected", hostname);
+		}
 	}
 }
 
@@ -289,6 +297,7 @@ packet_process_result dpi_desync_tcp_packet(uint32_t fwmark, const char *ifout,
 			if (tcphdr->th_flags & TH_RST)
 			{
 				DLOG("incoming RST detected for hostname %s\n", ctrack->hostname);
+				HOSTLIST_DEBUGLOG_APPEND("%s : incoming RST", ctrack->hostname);
 				bFail = bStop = true;
 			}
 			else if (len_payload && ctrack->l7proto==HTTP)
@@ -298,7 +307,10 @@ packet_process_result dpi_desync_tcp_packet(uint32_t fwmark, const char *ifout,
 					DLOG("incoming HTTP reply detected for hostname %s\n", ctrack->hostname);
 					bFail = HttpReplyLooksLikeDPIRedirect(data_payload, len_payload, ctrack->hostname);
 					if (bFail)
+					{
 						DLOG("redirect to another domain detected. possibly DPI redirect.\n")
+						HOSTLIST_DEBUGLOG_APPEND("%s : redirect to another domain", ctrack->hostname);
+					}
 					else
 						DLOG("local or in-domain redirect detected. it's not a DPI redirect.\n")
 				}
@@ -439,9 +451,8 @@ packet_process_result dpi_desync_tcp_packet(uint32_t fwmark, const char *ifout,
 		}
 		else
 		{
-			if (ctrack && *params.hostlist_auto_filename)
-				// received unknown payload. it means we are out of the request retransmission phase. stop counter
-				ctrack_stop_req_counter(ctrack);
+			// received unknown payload. it means we are out of the request retransmission phase. stop counter
+			ctrack_stop_req_counter(ctrack);
 			
 			if (!params.desync_any_proto) return res;
 			DLOG("applying tampering to unknown protocol\n")
@@ -456,12 +467,14 @@ packet_process_result dpi_desync_tcp_packet(uint32_t fwmark, const char *ifout,
 			if ((params.hostlist || params.hostlist_exclude) && !HostlistCheck(params.hostlist, params.hostlist_exclude, host, &bExcluded))
 			{
 				DLOG("not applying tampering to this request\n")
-				if (!bExcluded)
+				if (!bExcluded && *params.hostlist_auto_filename && ctrack)
 				{
-					if (*params.hostlist_auto_filename && ctrack && !ctrack->hostname)
-						ctrack->hostname=strdup(host);
+					if (!ctrack->hostname) ctrack->hostname=strdup(host);
 					if (auto_hostlist_retrans(ctrack, IPPROTO_TCP, params.hostlist_auto_retrans_threshold))
+					{
+						HOSTLIST_DEBUGLOG_APPEND("%s : tcp retrans threshold reached", ctrack->hostname);
 						auto_hostlist_failed(host);
+					}
 				}
 				return res;
 			}
@@ -867,32 +880,34 @@ packet_process_result dpi_desync_udp_packet(uint32_t fwmark, const char *ifout,
 			}
 			bKnownProtocol = true;
 		}
-		else if (IsWireguardHandshakeInitiation(data_payload,len_payload))
-		{
-			DLOG("packet contains wireguard handshake initiation\n")
-			if (ctrack && !ctrack->l7proto) ctrack->l7proto = WIREGUARD;
-			fake = params.fake_wg;
-			fake_size = params.fake_wg_size;
-			bKnownProtocol = true;
-		}
-		else if (IsDhtD1(data_payload,len_payload))
-		{
-			DLOG("packet contains DHT d1...e\n")
-			if (ctrack && !ctrack->l7proto) ctrack->l7proto = DHT;
-			fake = params.fake_dht;
-			fake_size = params.fake_dht_size;
-			bKnownProtocol = true;
-		}
 		else
 		{
-			if (ctrack && *params.hostlist_auto_filename)
-				// received unknown payload. it means we are out of the request retransmission phase. stop counter
-				ctrack_stop_req_counter(ctrack);
-			
-			if (!params.desync_any_proto) return res;
-			DLOG("applying tampering to unknown protocol\n")
-			fake = params.fake_unknown_udp;
-			fake_size = params.fake_unknown_udp_size;
+			// received payload without host. it means we are out of the request retransmission phase. stop counter
+			ctrack_stop_req_counter(ctrack);
+
+			if (IsWireguardHandshakeInitiation(data_payload,len_payload))
+			{
+				DLOG("packet contains wireguard handshake initiation\n")
+				if (ctrack && !ctrack->l7proto) ctrack->l7proto = WIREGUARD;
+				fake = params.fake_wg;
+				fake_size = params.fake_wg_size;
+				bKnownProtocol = true;
+			}
+			else if (IsDhtD1(data_payload,len_payload))
+			{
+				DLOG("packet contains DHT d1...e\n")
+				if (ctrack && !ctrack->l7proto) ctrack->l7proto = DHT;
+				fake = params.fake_dht;
+				fake_size = params.fake_dht_size;
+				bKnownProtocol = true;
+			}
+			else
+			{
+				if (!params.desync_any_proto) return res;
+				DLOG("applying tampering to unknown protocol\n")
+				fake = params.fake_unknown_udp;
+				fake_size = params.fake_unknown_udp_size;
+			}
 		}
 
 		if (bHaveHost)
@@ -902,12 +917,14 @@ packet_process_result dpi_desync_udp_packet(uint32_t fwmark, const char *ifout,
 			if ((params.hostlist || params.hostlist_exclude) && !HostlistCheck(params.hostlist, params.hostlist_exclude, host, &bExcluded))
 			{
 				DLOG("not applying tampering to this request\n")
-				if (!bExcluded)
+				if (!bExcluded && *params.hostlist_auto_filename && ctrack)
 				{
-					if (*params.hostlist_auto_filename && ctrack && !ctrack->hostname)
-						ctrack->hostname=strdup(host);
+					if (!ctrack->hostname) ctrack->hostname=strdup(host);
 					if (auto_hostlist_retrans(ctrack, IPPROTO_UDP, params.hostlist_auto_retrans_threshold))
+					{
+						HOSTLIST_DEBUGLOG_APPEND("%s : udp retrans threshold reached", ctrack->hostname);
 						auto_hostlist_failed(host);
+					}
 				}
 				return res;
 			}

nfq/helpers.c

@@ -188,6 +188,12 @@ void phton64(uint8_t *p, uint64_t v)
 	p[7] = (uint8_t)(v >> 0);
 }
 
+bool ipv6_addr_is_zero(const struct in6_addr *a)
+{
+    return !memcmp(a,"\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00",16);
+}
+
+
 #define INVALID_HEX_DIGIT ((uint8_t)-1)
 static inline uint8_t parse_hex_digit(char c)
 {
@@ -234,3 +240,13 @@ void fill_pattern(uint8_t *buf,size_t bufsize,const void *pattern,size_t patsize
 		bufsize -= size;
 	}
 }
+
+int fprint_localtime(FILE *F)
+{
+	struct tm t;
+	time_t now;
+
+	time(&now);
+	localtime_r(&now,&t);
+	return fprintf(F, "%02d.%02d.%04d %02d:%02d:%02d", t.tm_mday, t.tm_mon + 1, t.tm_year + 1900, t.tm_hour, t.tm_min, t.tm_sec);
+}

nfq/helpers.h

@@ -25,6 +25,8 @@ bool set_socket_buffers(int fd, int rcvbuf, int sndbuf);
 uint64_t pntoh64(const void *p);
 void phton64(uint8_t *p, uint64_t v);
 
+bool ipv6_addr_is_zero(const struct in6_addr *a);
+
 static inline uint16_t pntoh16(const uint8_t *p) {
 	return ((uint16_t)p[0] << 8) | (uint16_t)p[1];
 }
@@ -38,3 +40,5 @@ static inline uint32_t pntoh32(const uint8_t *p) {
 
 bool parse_hex_str(const char *s, uint8_t *pbuf, size_t *size);
 void fill_pattern(uint8_t *buf,size_t bufsize,const void *pattern,size_t patsize);
+
+int fprint_localtime(FILE *F);

nfq/nfqws.c

@@ -567,7 +567,8 @@ static void exithelp(void)
 		" --hostlist-auto=<filename>\t\t\t; detect DPI blocks and build hostlist automatically\n"
 		" --hostlist-auto-fail-threshold=<int>\t\t; how many failed attempts cause hostname to be added to auto hostlist (default : %d)\n"
 		" --hostlist-auto-fail-time=<int>\t\t; all failed attemps must be within these seconds (default : %d)\n"
-		" --hostlist-auto-retrans-threshold=<int>\t; how many request retransmissions cause attempt to fail (default : %d)\n",
+		" --hostlist-auto-retrans-threshold=<int>\t; how many request retransmissions cause attempt to fail (default : %d)\n"
+		" --hostlist-auto-debug=<logfile>\t\t; debug auto hostlist positives\n",
 		CTRACK_T_SYN, CTRACK_T_EST, CTRACK_T_FIN, CTRACK_T_UDP,
 #if defined(__linux__) || defined(SO_USER_COOKIE)
 		DPI_DESYNC_FWMARK_DEFAULT,DPI_DESYNC_FWMARK_DEFAULT,
@@ -752,10 +753,11 @@ int main(int argc, char **argv)
 		{"hostlist-auto-fail-threshold",required_argument,0,0},	// optidx=41
 		{"hostlist-auto-fail-time",required_argument,0,0},	// optidx=42
 		{"hostlist-auto-retrans-threshold",required_argument,0,0},	// optidx=43
+		{"hostlist-auto-debug",required_argument,0,0},	// optidx=44
 	
 #ifdef __linux__
-		{"bind-fix4",no_argument,0,0},		// optidx=44
-		{"bind-fix6",no_argument,0,0},		// optidx=45
+		{"bind-fix4",no_argument,0,0},		// optidx=45
+		{"bind-fix6",no_argument,0,0},		// optidx=46
 #endif
 		{NULL,0,NULL,0}
 	};
@@ -1140,11 +1142,26 @@ int main(int argc, char **argv)
 				exit_clean(1);
 			}
 			break;
+		case 44: /* hostlist-auto-debug */
+			{
+				FILE *F = fopen(optarg,"a+t");
+				if (!F)
+				{
+					fprintf(stderr, "cannot create %s\n", optarg);
+					exit_clean(1);
+				}
+				fclose(F);
+				if (params.droproot && chown(optarg, params.uid, -1))
+					fprintf(stderr, "could not chown %s. auto hostlist debug log may not be writable after privilege drop\n", optarg);
+				strncpy(params.hostlist_auto_debuglog, optarg, sizeof(params.hostlist_auto_debuglog));
+				params.hostlist_auto_debuglog[sizeof(params.hostlist_auto_debuglog) - 1] = '\0';
+			}
+			break;
 #ifdef __linux__
-		case 44: /* bind-fix4 */
+		case 45: /* bind-fix4 */
 			params.bind_fix4 = true;
 			break;
-		case 45: /* bind-fix6 */
+		case 46: /* bind-fix6 */
 			params.bind_fix6 = true;
 			break;
 #endif

nfq/params.h

@@ -64,7 +64,7 @@ struct params_s
 
 	strpool *hostlist, *hostlist_exclude;
 	struct str_list_head hostlist_files, hostlist_exclude_files;
-	char hostlist_auto_filename[PATH_MAX];
+	char hostlist_auto_filename[PATH_MAX], hostlist_auto_debuglog[PATH_MAX];
 	int hostlist_auto_fail_threshold, hostlist_auto_fail_time, hostlist_auto_retrans_threshold;
 	hostfail_pool *hostlist_auto_fail_counters;
 
@@ -75,3 +75,15 @@ struct params_s
 extern struct params_s params;
 
 #define DLOG(format, ...) {if (params.debug) printf(format, ##__VA_ARGS__);}
+
+#define LOG_APPEND(filename, format, ...) \
+{ \
+	FILE *F = fopen(filename,"at"); \
+	if (F) \
+	{ \
+		fprint_localtime(F); \
+		fprintf(F, " : " format "\n", ##__VA_ARGS__); \
+		fclose(F); \
+	} \
+}
+#define HOSTLIST_DEBUGLOG_APPEND(format, ...) if (*params.hostlist_auto_debuglog) LOG_APPEND(params.hostlist_auto_debuglog, format, ##__VA_ARGS__)

tpws/helpers.c

@@ -8,6 +8,7 @@
 #include <sys/socket.h>
 #include <arpa/inet.h>
 #include <ifaddrs.h>
+#include <time.h>
 
 char *strncasestr(const char *s,const char *find, size_t slen)
 {
@@ -211,3 +212,13 @@ int get_so_error(int fd)
 		errn=errno;
 	return errn;
 }
+
+int fprint_localtime(FILE *F)
+{
+	struct tm t;
+	time_t now;
+
+	time(&now);
+	localtime_r(&now,&t);
+	return fprintf(F, "%02d.%02d.%04d %02d:%02d:%02d", t.tm_mday, t.tm_mon + 1, t.tm_year + 1900, t.tm_hour, t.tm_min, t.tm_sec);
+}

tpws/helpers.h

@@ -5,6 +5,7 @@
 #include <netinet/in.h>
 #include <sys/socket.h>
 #include <netdb.h>
+#include <stdio.h>
 
 char *strncasestr(const char *s,const char *find, size_t slen);
 
@@ -41,3 +42,5 @@ static inline void phton16(uint8_t *p, uint16_t v) {
 	p[0] = (uint8_t)(v>>8);
 	p[1] = (uint8_t)v;
 }
+
+int fprint_localtime(FILE *F);

tpws/params.h

@@ -56,7 +56,7 @@ struct params_s
 
 	strpool *hostlist, *hostlist_exclude;
 	struct str_list_head hostlist_files, hostlist_exclude_files;
-	char hostlist_auto_filename[PATH_MAX];
+	char hostlist_auto_filename[PATH_MAX], hostlist_auto_debuglog[PATH_MAX];
 	int hostlist_auto_fail_threshold, hostlist_auto_fail_time;
 	hostfail_pool *hostlist_auto_fail_counters;
 
@@ -72,3 +72,15 @@ extern struct params_s params;
 #define _DBGPRINT(format, level, ...) { if (params.debug>=level) printf(format "\n", ##__VA_ARGS__); }
 #define VPRINT(format, ...) _DBGPRINT(format,1,##__VA_ARGS__)
 #define DBGPRINT(format, ...) _DBGPRINT(format,2,##__VA_ARGS__)
+
+#define LOG_APPEND(filename, format, ...) \
+{ \
+	FILE *F = fopen(filename,"at"); \
+	if (F) \
+	{ \
+		fprint_localtime(F); \
+		fprintf(F, " : " format "\n", ##__VA_ARGS__); \
+		fclose(F); \
+	} \
+}
+#define HOSTLIST_DEBUGLOG_APPEND(format, ...) if (*params.hostlist_auto_debuglog) LOG_APPEND(params.hostlist_auto_debuglog, format, ##__VA_ARGS__)

tpws/tamper.c

@@ -299,6 +299,7 @@ static void auto_hostlist_failed(const char *hostname)
 	}
 	fail_counter->counter++;
 	VPRINT("auto hostlist : %s : fail counter %d/%d", hostname, fail_counter->counter, params.hostlist_auto_fail_threshold);
+	HOSTLIST_DEBUGLOG_APPEND("%s : fail counter %d/%d", hostname, fail_counter->counter, params.hostlist_auto_fail_threshold);
 	if (fail_counter->counter >= params.hostlist_auto_fail_threshold)
 	{
 		VPRINT("auto hostlist : fail threshold reached. adding %s to auto hostlist", hostname);
@@ -309,6 +310,7 @@ static void auto_hostlist_failed(const char *hostname)
 		if (!HostlistCheck(params.hostlist, params.hostlist_exclude, hostname, &bExcluded) && !bExcluded)
 		{
 			VPRINT("auto hostlist : adding %s", hostname);
+			HOSTLIST_DEBUGLOG_APPEND("%s : adding", hostname);
 			if (!StrPoolAddStr(&params.hostlist, hostname))
 			{
 				fprintf(stderr, "StrPoolAddStr out of memory\n");
@@ -321,7 +323,10 @@ static void auto_hostlist_failed(const char *hostname)
 			}
 		}
 		else
+		{
 			VPRINT("auto hostlist: NOT adding %s", hostname);
+			HOSTLIST_DEBUGLOG_APPEND("%s : NOT adding, duplicate detected", hostname);
+		}
 	}
 }
 
@@ -340,7 +345,10 @@ void tamper_in(t_ctrack *ctrack, uint8_t *segment,size_t segment_buffer_size,siz
 			VPRINT("incoming HTTP reply detected for hostname %s", ctrack->hostname);
 			bFail = HttpReplyLooksLikeDPIRedirect(segment, *size, ctrack->hostname);
 			if (bFail)
+			{
 				VPRINT("redirect to another domain detected. possibly DPI redirect.")
+				HOSTLIST_DEBUGLOG_APPEND("%s : redirect to another domain", ctrack->hostname);
+			}
 			else
 				VPRINT("local or in-domain redirect detected. it's not a DPI redirect.")
 		}
@@ -365,6 +373,7 @@ void rst_in(t_ctrack *ctrack)
 	if (!ctrack->bTamperInCutoff && ctrack->hostname)
 	{
 		VPRINT("incoming RST detected for hostname %s", ctrack->hostname);
+		HOSTLIST_DEBUGLOG_APPEND("%s : incoming RST", ctrack->hostname);
 		auto_hostlist_failed(ctrack->hostname);
 	}
 }
@@ -378,6 +387,7 @@ void hup_out(t_ctrack *ctrack)
 	{
 		// local leg dropped connection after first request. probably due to timeout.
 		VPRINT("local leg closed connection after first request (timeout ?). hostname: %s", ctrack->hostname);
+		HOSTLIST_DEBUGLOG_APPEND("%s : client closed connection without server reply", ctrack->hostname);
 		auto_hostlist_failed(ctrack->hostname);
 	}
 }

tpws/tpws.c

@@ -166,6 +166,7 @@ static void exithelp(void)
 		" --hostlist-auto=<filename>\t\t; detect DPI blocks and build hostlist automatically\n"
 		" --hostlist-auto-fail-threshold=<int>\t; how many failed attempts cause hostname to be added to auto hostlist (default : %d)\n"
 		" --hostlist-auto-fail-time=<int>\t; all failed attemps must be within these seconds (default : %d)\n"
+		" --hostlist-auto-debug=<logfile>\t; debug auto hostlist positives\n"
 		"\nTAMPER:\n"
 		" --split-http-req=method|host\t\t; split at specified logical part of plain http request\n"
 		" --split-pos=<numeric_offset>\t\t; split at specified pos. split-http-req takes precedence for http.\n"
@@ -304,17 +305,18 @@ void parse_params(int argc, char *argv[])
 		{ "hostlist-auto",required_argument,0,0}, // optidx=35
 		{ "hostlist-auto-fail-threshold",required_argument,0,0}, // optidx=36
 		{ "hostlist-auto-fail-time",required_argument,0,0},	// optidx=37
-		{ "pidfile",required_argument,0,0 },// optidx=38
-		{ "debug",optional_argument,0,0 },// optidx=39
-		{ "local-rcvbuf",required_argument,0,0 },// optidx=40
-		{ "local-sndbuf",required_argument,0,0 },// optidx=41
-		{ "remote-rcvbuf",required_argument,0,0 },// optidx=42
-		{ "remote-sndbuf",required_argument,0,0 },// optidx=43
-		{ "socks",no_argument,0,0 },// optidx=44
-		{ "no-resolve",no_argument,0,0 },// optidx=45
-		{ "skip-nodelay",no_argument,0,0 },// optidx=46
+		{ "hostlist-auto-debug",required_argument,0,0}, // optidx=38
+		{ "pidfile",required_argument,0,0 },// optidx=39
+		{ "debug",optional_argument,0,0 },// optidx=40
+		{ "local-rcvbuf",required_argument,0,0 },// optidx=41
+		{ "local-sndbuf",required_argument,0,0 },// optidx=42
+		{ "remote-rcvbuf",required_argument,0,0 },// optidx=43
+		{ "remote-sndbuf",required_argument,0,0 },// optidx=44
+		{ "socks",no_argument,0,0 },// optidx=45
+		{ "no-resolve",no_argument,0,0 },// optidx=46
+		{ "skip-nodelay",no_argument,0,0 },// optidx=47
 #if defined(BSD) && !defined(__OpenBSD__) && !defined(__APPLE__)
-		{ "enable-pf",no_argument,0,0 },// optidx=47
+		{ "enable-pf",no_argument,0,0 },// optidx=48
 #endif
 		{ "hostlist-auto-retrans-threshold",optional_argument,0,0}, // ignored. for nfqws command line compatibility
 		{ NULL,0,NULL,0 }
@@ -603,36 +605,51 @@ void parse_params(int argc, char *argv[])
 				exit_clean(1);
 			}
 			break;
-		case 38: /* pidfile */
+		case 38: /* hostlist-auto-debug */
+			{
+				FILE *F = fopen(optarg,"a+t");
+				if (!F)
+				{
+					fprintf(stderr, "cannot create %s\n", optarg);
+					exit_clean(1);
+				}
+				fclose(F);
+				if (params.droproot && chown(optarg, params.uid, -1))
+					fprintf(stderr, "could not chown %s. auto hostlist debug log may not be writable after privilege drop\n", optarg);
+				strncpy(params.hostlist_auto_debuglog, optarg, sizeof(params.hostlist_auto_debuglog));
+				params.hostlist_auto_debuglog[sizeof(params.hostlist_auto_debuglog) - 1] = '\0';
+			}
+			break;
+		case 39: /* pidfile */
 			strncpy(params.pidfile,optarg,sizeof(params.pidfile));
 			params.pidfile[sizeof(params.pidfile)-1]='\0';
 			break;
-		case 39:
+		case 40:
 			params.debug = optarg ? atoi(optarg) : 1;
 			break;
-		case 40: /* local-rcvbuf */
+		case 41: /* local-rcvbuf */
 			params.local_rcvbuf = atoi(optarg)/2;
 			break;
-		case 41: /* local-sndbuf */
+		case 42: /* local-sndbuf */
 			params.local_sndbuf = atoi(optarg)/2;
 			break;
-		case 42: /* remote-rcvbuf */
+		case 43: /* remote-rcvbuf */
 			params.remote_rcvbuf = atoi(optarg)/2;
 			break;
-		case 43: /* remote-sndbuf */
+		case 44: /* remote-sndbuf */
 			params.remote_sndbuf = atoi(optarg)/2;
 			break;
-		case 44: /* socks */
+		case 45: /* socks */
 			params.proxy_type = CONN_TYPE_SOCKS;
 			break;
-		case 45: /* no-resolve */
+		case 46: /* no-resolve */
 			params.no_resolve = true;
 			break;
-		case 46: /* skip-nodelay */
+		case 47: /* skip-nodelay */
 			params.skip_nodelay = true;
 			break;
 #if defined(BSD) && !defined(__OpenBSD__) && !defined(__APPLE__)
-		case 47: /* enable-pf */
+		case 48: /* enable-pf */
 			params.pf_enable = true;
 			break;
 #endif