nfqws: tls client hello reassemble

common/ipt.sh

@@ -328,9 +328,9 @@ zapret_do_firewall_rules_ipt()
 	# autohostlist mode requires incoming traffic sample
 	# always use conntrack packet limiter or nfqws will deal with gigabytes
 	if [ "$MODE_FILTER" = "autohostlist" ]; then
-		n=$((4+${AUTOHOSTLIST_RETRANS_THRESHOLD:-3}))
+		n=$((6+${AUTOHOSTLIST_RETRANS_THRESHOLD:-3}))
 	else
-		n=4
+		n=6
 	fi
 	first_packet_only="${first_packet_only}$n"
 

common/nft.sh

@@ -576,9 +576,9 @@ zapret_apply_firewall_rules_nft()
 	# autohostlist mode requires incoming traffic sample
 	# always use conntrack packet limiter or nfqws will deal with gigabytes
 	if [ "$MODE_FILTER" = "autohostlist" ]; then
-		first_packet_only=$((4+${AUTOHOSTLIST_RETRANS_THRESHOLD:-3}))
+		first_packet_only=$((6+${AUTOHOSTLIST_RETRANS_THRESHOLD:-3}))
 	else
-		first_packet_only=4
+		first_packet_only=6
 	fi
 	first_packet_only="ct original packets 1-$first_packet_only"
 

docs/changes.txt

@@ -251,3 +251,7 @@ tpws --tlsrec attack.
 v52
 
 autohostlist mode
+
+v53
+
+nfqws: tcp session reassemble for TLS ClientHello

docs/iptables.txt

@@ -17,7 +17,7 @@ iptables -t mangle -I POSTROUTING -p udp --dport 443 -m mark ! --mark 0x40000000
 # auto hostlist with avoiding wrong ACK numbers in RST,ACK packets sent by russian DPI
 sysctl net.netfilter.nf_conntrack_tcp_be_liberal=1 
 iptables -t mangle -I POSTROUTING -p tcp -m multiport --dports 80,443 -m connbytes --connbytes-dir=original --connbytes-mode=packets --connbytes 1:12 -m mark ! --mark 0x40000000/0x40000000 -j NFQUEUE --queue-num 200 --queue-bypass
-iptables -t mangle -I PREROUTING -p tcp -m multiport --sports 80,443 -m connbytes --connbytes-dir=reply --connbytes-mode=packets --connbytes 1:4 -m mark ! --mark 0x40000000/0x40000000 -j NFQUEUE --queue-num 200 --queue-bypass
+iptables -t mangle -I PREROUTING -p tcp -m multiport --sports 80,443 -m connbytes --connbytes-dir=reply --connbytes-mode=packets --connbytes 1:6 -m mark ! --mark 0x40000000/0x40000000 -j NFQUEUE --queue-num 200 --queue-bypass
 
 
 For TPROXY :

docs/readme.eng.md

@@ -92,10 +92,10 @@ into IP addresses and put them to ipset 'zapret', then add a filter to the comma
 Some DPIs catch only the first http request, ignoring subsequent requests in a keep-alive session.
 Then we can reduce CPU load, refusing to process unnecessary packets.
 
-`iptables -t mangle -I POSTROUTING -o <external_interface> -p tcp --dport 80 -m connbytes --connbytes-dir=original --connbytes-mode=packets --connbytes 1:4 -m mark ! --mark 0x40000000/0x40000000 -m set --match-set zapret dst -j NFQUEUE --queue-num 200 --queue-bypass`
+`iptables -t mangle -I POSTROUTING -o <external_interface> -p tcp --dport 80 -m connbytes --connbytes-dir=original --connbytes-mode=packets --connbytes 1:6 -m mark ! --mark 0x40000000/0x40000000 -m set --match-set zapret dst -j NFQUEUE --queue-num 200 --queue-bypass`
 
 Mark filter does not allow nfqws-generated packets to enter the queue again.
-Its necessary to use this filter when also using `connbytes 1:4`. Without it packet ordering can be changed breaking the whole idea.
+Its necessary to use this filter when also using `connbytes 1:6`. Without it packet ordering can be changed breaking the whole idea.
 
 ## ip6tables
 
@@ -295,13 +295,13 @@ Subdomains are applied automatically. gzip lists are supported.
 
 iptables for performing the attack on the first packet :
 
-`iptables -t mangle -I POSTROUTING -o <external_interface> -p tcp -m multiport --dports 80,443 -m connbytes --connbytes-dir=original --connbytes-mode=packets --connbytes 1:4 -m mark ! --mark 0x40000000/0x40000000 -j NFQUEUE --queue-num 200 --queue-bypass`
+`iptables -t mangle -I POSTROUTING -o <external_interface> -p tcp -m multiport --dports 80,443 -m connbytes --connbytes-dir=original --connbytes-mode=packets --connbytes 1:6 -m mark ! --mark 0x40000000/0x40000000 -j NFQUEUE --queue-num 200 --queue-bypass`
 
 This is good if DPI does not track all requests in http keep-alive session.
 If it does, then pass all outgoing packets for http and only first data packet for https :
 
 ```
-iptables -t mangle -I POSTROUTING -o <external_interface> -p tcp --dport 443 -m connbytes --connbytes-dir=original --connbytes-mode=packets --connbytes 1:4 -m mark ! --mark 0x40000000/0x40000000 -j NFQUEUE --queue-num 200 --queue-bypass
+iptables -t mangle -I POSTROUTING -o <external_interface> -p tcp --dport 443 -m connbytes --connbytes-dir=original --connbytes-mode=packets --connbytes 1:6 -m mark ! --mark 0x40000000/0x40000000 -j NFQUEUE --queue-num 200 --queue-bypass
 iptables -t mangle -I POSTROUTING -o <external_interface> -p tcp --dport 80 -m mark ! --mark 0x40000000/0x40000000 -j NFQUEUE --queue-num 200 --queue-bypass
 ```
 

docs/readme.txt

@@ -1,4 +1,4 @@
-zapret v.52
+zapret v.53
 
 English
 -------
@@ -116,10 +116,10 @@ iptables -t mangle -I POSTROUTING -o <внешний_интерфейс> -p tcp
 DPI может ловить только первый http запрос, игнорируя последующие запросы в keep-alive сессии.
 Тогда можем уменьшить нагрузку на проц, отказавшись от процессинга ненужных пакетов.
 
-iptables -t mangle -I POSTROUTING -o <внешний_интерфейс> -p tcp --dport 80 -m connbytes --connbytes-dir=original --connbytes-mode=packets --connbytes 1:4 -m mark ! --mark 0x40000000/0x40000000 -m set --match-set zapret dst -j NFQUEUE --queue-num 200 --queue-bypass
+iptables -t mangle -I POSTROUTING -o <внешний_интерфейс> -p tcp --dport 80 -m connbytes --connbytes-dir=original --connbytes-mode=packets --connbytes 1:6 -m mark ! --mark 0x40000000/0x40000000 -m set --match-set zapret dst -j NFQUEUE --queue-num 200 --queue-bypass
 
 Фильтр по mark нужен для отсечения от очереди пакетов, сгенерированных внутри nfqws.
-Если применяется фильтр по connbytes 1:4, то обязательно добавлять в iptables и фильтр по mark. Иначе возможно
+Если применяется фильтр по connbytes 1:6, то обязательно добавлять в iptables и фильтр по mark. Иначе возможно
 перепутывание порядка следования пакетов, что приведет к неработоспособности метода.
 
 
@@ -356,12 +356,12 @@ DPI может отстать от потока, если ClientHello его у
 
 iptables для задействования атаки на первый пакет данных :
 
-iptables -t mangle -I POSTROUTING -o <внешний_интерфейс> -p tcp -m multiport --dports 80,443 -m connbytes --connbytes-dir=original --connbytes-mode=packets --connbytes 1:4 -m mark ! --mark 0x40000000/0x40000000 -j NFQUEUE --queue-num 200 --queue-bypass
+iptables -t mangle -I POSTROUTING -o <внешний_интерфейс> -p tcp -m multiport --dports 80,443 -m connbytes --connbytes-dir=original --connbytes-mode=packets --connbytes 1:6 -m mark ! --mark 0x40000000/0x40000000 -j NFQUEUE --queue-num 200 --queue-bypass
 
 Этот вариант применяем, когда DPI не следит за всеми запросами http внутри keep-alive сессии.
 Если следит, направляем только первый пакет от https и все пакеты от http :
 
-iptables -t mangle -I POSTROUTING -o <внешний_интерфейс> -p tcp --dport 443 -m connbytes --connbytes-dir=original --connbytes-mode=packets --connbytes 1:4 -m mark ! --mark 0x40000000/0x40000000 -j NFQUEUE --queue-num 200 --queue-bypass
+iptables -t mangle -I POSTROUTING -o <внешний_интерфейс> -p tcp --dport 443 -m connbytes --connbytes-dir=original --connbytes-mode=packets --connbytes 1:6 -m mark ! --mark 0x40000000/0x40000000 -j NFQUEUE --queue-num 200 --queue-bypass
 iptables -t mangle -I POSTROUTING -o <внешний_интерфейс> -p tcp --dport 80 -m mark ! --mark 0x40000000/0x40000000 -j NFQUEUE --queue-num 200 --queue-bypass
 
 mark нужен, чтобы сгенерированный поддельный пакет не попал опять к нам на обработку. nfqws выставляет fwmark при его отсылке.
@@ -372,11 +372,11 @@ mark нужен, чтобы сгенерированный поддельный
 При отсутствии ограничения на connbytes, атака будет работать и без фильтра по mark.
 Но лучше его все же оставить для увеличения скорости.
 
-Почему --connbytes 1:4 :
+Почему --connbytes 1:6 :
 1 - для работы методов десинхронизации 0-й фазы и wssize
 2 - иногда данные идут в 3-м пакете 3-way handshake
-3 - стандартная ситуация
-4 - для надежности. на случай, если выполнялась одна ретрансмиссия
+3 - стандартная ситуация приема одного пакета запроса
+4-6 - на случай ретрансмиссии или запроса длиной в несколько пакетов (TLSClientHello с kyber, например)
 
 КОМБИНИРОВАНИЕ МЕТОДОВ ДЕСИНХРОНИЗАЦИИ
 В параметре dpi-desync можно указать до 3 режимов через запятую.
@@ -416,7 +416,6 @@ ip6tables -D zone_wan_output -m comment --comment '!fw3' -j zone_wan_dest_ACCEPT
 CONNTRACK
 nfqws оснащен ограниченной реализацией слежения за состоянием tcp соединений (conntrack).
 Он включается для реализации некоторых методов противодействия DPI.
-На текущий момент это параметры --wssize и --dpi-desync-cutoff.
 conntrack способен следить за фазой соединения : SYN,ESTABLISHED,FIN , количеством пакетов в каждую сторону,
 sequence numbers. conntrack способен "кормиться" пакетами в обе или только в одну сторону.
 Соединение попадает в таблицу при обнаружении пакетов с выставленными флагами SYN или SYN,ACK.
@@ -468,6 +467,17 @@ window size итоговый размер окна стал максимальн
 На склонных к бездействию соединениях следует изменить таймауты conntrack.
 Если соединение выпало из conntrack и задана опция --dpi-desync-cutoff, dpi desync применяться не будет.
 
+РЕАССЕМБЛИНГ TCP
+nfqws поддерживает реассемблинг некоторых видов tcp запросов.
+На текущий момент это TLS ClientHello. Он бывает длинным, если в chrome включить пост-квантовую
+криптографию tls-kyber, и занимает как правило 2 пакета.
+chrome рандомизирует фингерпринт TLS. SNI может оказаться как в начале, так и в конце, то есть
+попасть в 1 или 2 пакет. stateful DPI обычно реассемблирует запрос целиком, и только потом
+принимает решение о блокировке.
+nfqws реагирует десинхронизацией на каждый пакет из TLSClientHello, если задана опция
+--dpi-desync-skip-nosni=0. В противном случае десинхронизация идет на сам пакет,
+включающий SNI, и все последующие.
+
 ПОДДЕРЖКА UDP
 Атаки на udp более ограничены в возможностях. udp нельзя фрагментировать иначе, чем на уровне ip.
 Для UDP действуют только режимы десинхронизации fake,hopbyhop,destopt,ipfrag1,ipfrag2,udplen,tamper.
@@ -487,6 +497,11 @@ udplen увеличивает размер udp пакета на указанн
 Атака fake полезна только для stateful DPI, она бесполезна для анализа на уровне отдельных пакетов.
 По умолчанию fake наполнение - 64 нуля. Можно указать файл в --dpi-desync-fake-unknown-udp.
 
+РЕАССЕМБЛИНГ QUIC
+tls-kyber может так же размазываться по 2 пакетам QUIC Initial.
+Пока их реассемблинг не реализован, поскольку русский DPI не регирует на такие пакеты.
+Идет десинхронизация полных hello в одном пакете и частичных hello, где SNI попал в 1-й пакет.
+
 IP ФРАГМЕНТАЦИЯ
 В современной сети с этом все очень плохо. Фрагментированные пакеты застревают по пути, часто отбрасываются.
 Иногда доходят. Иногда то доходят, то не доходят. Может зависеть от версии ipv4/ipv6.
@@ -964,7 +979,7 @@ nfqws и tpws могут сечь варианты 1-3, 4 они не распо
 Заносите такие домены в ipset/zapret-hosts-user-exclude.txt, чтобы избежать повторения.
 Чтобы впоследствии разобраться почему домен был занесен в лист, можно включить autohostlist debug log.
 Он полезен тем, что работает без постоянного просмотра вывода nfqws в режиме debug.
-В лог заносятся только основные события, ведушие к занесению хоста в лист.
+В лог заносятся только основные события, ведущие к занесению хоста в лист.
 По логу можно понять как избежать ложных срабатываний и подходит ли вообще вам этот режим.
 
 Скрипты zapret ведут autohostlist в ipset/zapret-hosts-auto.txt.

nfq/conntrack.c

@@ -25,9 +25,23 @@ static void connswap(const t_conn *c, t_conn *c2)
 	c2->dport = c->sport;
 }
 
+void ConntrackClearHostname(t_ctrack *track)
+{
+	if (track->hostname)
+	{
+		free(track->hostname);
+		track->hostname = NULL;
+	}
+}
+static void ConntrackClearTrack(t_ctrack *track)
+{
+	ConntrackClearHostname(track);
+	ReasmClear(&track->reasm_orig);
+}
+
 static void ConntrackFreeElem(t_conntrack_pool *elem)
 {
-	if (elem->track.hostname) free(elem->track.hostname);
+	ConntrackClearTrack(&elem->track);
 	free(elem);
 }
 
@@ -309,3 +323,36 @@ void ConntrackPoolDump(const t_conntrack *p)
 			t->track.req_retrans_counter, t->track.b_cutoff, t->track.b_wssize_cutoff, t->track.b_desync_cutoff, t->track.hostname, ConntrackProtoName(t->track.l7proto));
 	};
 }
+
+
+void ReasmClear(t_reassemble *reasm)
+{
+	if (reasm->packet)
+	{
+		free(reasm->packet);
+		reasm->packet = NULL;
+	}
+	reasm->size = reasm->size_present = 0;
+}
+bool ReasmInit(t_reassemble *reasm, size_t size_requested, uint32_t seq_start)
+{
+	reasm->packet = malloc(size_requested);
+	if (!reasm->packet) return false;
+	reasm->size = size_requested;
+	reasm->size_present = 0;
+	reasm->seq = seq_start;
+	return true;
+}
+bool ReasmFeed(t_reassemble *reasm, uint32_t seq, const void *payload, size_t len)
+{
+	if (reasm->seq!=seq) return false; // fail session if out of sequence
+	
+	size_t szcopy;
+	szcopy = reasm->size - reasm->size_present;
+	if (len<szcopy) szcopy = len;
+	memcpy(reasm->packet + reasm->size_present, payload, szcopy);
+	reasm->size_present += szcopy;
+	reasm->seq += (uint32_t)szcopy;
+
+	return true;
+}

nfq/conntrack.h

@@ -34,6 +34,14 @@ typedef struct
 	uint8_t l4proto; // IPPROTO_TCP, IPPROTO_UDP
 } t_conn;
 
+// this structure helps to reassemble continuous packets streams. it does not support out-of-orders
+typedef struct {
+	uint8_t *packet;		// allocated for size during reassemble request. requestor must know the message size.
+	uint32_t seq;			// current seq number. if a packet comes with an unexpected seq - it fails reassemble session.
+	size_t size;			// expected message size. success means that we have received exactly 'size' bytes and have them in 'packet'
+	size_t size_present;		// how many bytes already stored in 'packet'
+} t_reassemble;
+
 // SYN - SYN or SYN/ACK received
 // ESTABLISHED - any except SYN or SYN/ACK received
 // FIN - FIN or RST received
@@ -55,11 +63,14 @@ typedef struct
 	uint8_t scale_orig, scale_reply;	// last seen window scale factor. SCALE_NONE if none
 	
 	uint8_t req_retrans_counter;		// number of request retransmissions
-	uint32_t req_seq;			// sequence number of the request (to track retransmissions)
+	bool req_seq_start_present, req_seq_present;
+	uint32_t req_seq_start,req_seq_end;	// sequence interval of the request (to track retransmissions)
 
 	bool b_cutoff;				// mark for deletion
 	bool b_wssize_cutoff, b_desync_cutoff;
 
+	t_reassemble reasm_orig;
+
 	t_l7proto l7proto;
 	char *hostname;
 } t_ctrack;
@@ -85,3 +96,11 @@ bool ConntrackPoolDrop(t_conntrack *p, const struct ip *ip, const struct ip6_hdr
 void CaonntrackExtractConn(t_conn *c, bool bReverse, const struct ip *ip, const struct ip6_hdr *ip6, const struct tcphdr *tcphdr, const struct udphdr *udphdr);
 void ConntrackPoolDump(const t_conntrack *p);
 void ConntrackPoolPurge(t_conntrack *p);
+void ConntrackClearHostname(t_ctrack *track);
+
+bool ReasmInit(t_reassemble *reasm, size_t size_requested, uint32_t seq_start);
+void ReasmClear(t_reassemble *reasm);
+// false means reassemble session has failed and we should ReasmClear() it
+bool ReasmFeed(t_reassemble *reasm, uint32_t seq, const void *payload, size_t len);
+inline static bool ReasmIsEmpty(t_reassemble *reasm) {return !reasm->size;}
+inline static bool ReasmIsFull(t_reassemble *reasm) {return !ReasmIsEmpty(reasm) && (reasm->size==reasm->size_present);}

nfq/desync.c

@@ -155,13 +155,14 @@ static void maybe_cutoff(t_ctrack *ctrack, uint8_t proto)
 			ctrack->b_wssize_cutoff |= cutoff_test(ctrack, params.wssize_cutoff, params.wssize_cutoff_mode);
 		ctrack->b_desync_cutoff |= cutoff_test(ctrack, params.desync_cutoff, params.desync_cutoff_mode);
 
+		// we do not need conntrack entry anymore if all cutoff conditions are either not defined or reached
+		// do not drop udp entry because it will be recreated when next packet arrives
 		if (proto==IPPROTO_TCP)
-			// we do not need conntrack entry anymore if all cutoff conditions are either not defined or reached
-			// do not drop udp entry because it will be recreated when next packet arrives
 			ctrack->b_cutoff |= \
 			    (!params.wssize || ctrack->b_wssize_cutoff) &&
 			    (!params.desync_cutoff || ctrack->b_desync_cutoff) &&
-			    (!*params.hostlist_auto_filename || ctrack->req_retrans_counter==RETRANS_COUNTER_STOP);
+			    (!*params.hostlist_auto_filename || ctrack->req_retrans_counter==RETRANS_COUNTER_STOP) &&
+			    ReasmIsEmpty(&ctrack->reasm_orig);
 	}
 }
 static void wssize_cutoff(t_ctrack *ctrack)
@@ -172,8 +173,16 @@ static void wssize_cutoff(t_ctrack *ctrack)
 		maybe_cutoff(ctrack, IPPROTO_TCP);
 	}
 }
+static void forced_wssize_cutoff(t_ctrack *ctrack)
+{
+ 	if (ctrack && params.wssize && !ctrack->b_wssize_cutoff)
+	{
+		DLOG("forced wssize-cutoff\n");
+		wssize_cutoff(ctrack);
+	}
+}
 
-static void ctrack_stop_req_counter(t_ctrack *ctrack)
+static void ctrack_stop_retrans_counter(t_ctrack *ctrack)
 {
 	if (ctrack && *params.hostlist_auto_filename)
 	{
@@ -187,24 +196,26 @@ static bool auto_hostlist_retrans(t_ctrack *ctrack, uint8_t l4proto, int thresho
 {
 	if (*params.hostlist_auto_filename && ctrack && ctrack->req_retrans_counter!=RETRANS_COUNTER_STOP)
 	{
+		if (l4proto==IPPROTO_TCP)
+		{
+			if (!ctrack->req_seq_present)
+				return false;
+			if (!seq_within(ctrack->seq_last, ctrack->req_seq_start, ctrack->req_seq_end))
+			{
+				DLOG("req retrans : tcp seq %u not within the req range %u-%u. stop tracking.\n", ctrack->seq_last, ctrack->req_seq_start, ctrack->req_seq_end);
+				ctrack_stop_retrans_counter(ctrack);
+				ctrack->req_seq_present = false;
+				return false;
+			}
+		}
 		ctrack->req_retrans_counter++;
-		DLOG("req retrans counter : %u/%u\n",ctrack->req_retrans_counter, threshold);
 		if (ctrack->req_retrans_counter >= threshold)
 		{
-			DLOG("req retrans threshold reached\n");
-			ctrack_stop_req_counter(ctrack);
+			DLOG("req retrans threshold reached : %u/%u\n",ctrack->req_retrans_counter, threshold);
+			ctrack_stop_retrans_counter(ctrack);
 			return true;
 		}
-		if (l4proto==IPPROTO_TCP)
-		{
-		    if (!ctrack->req_seq) ctrack->req_seq = ctrack->seq_last;
-		    if (ctrack->seq_last != ctrack->req_seq)
-		    {
-			    DLOG("another request, not retransmission. stop tracking.\n");
-			    ctrack_stop_req_counter(ctrack);
-			    return false;
-		    }
-		}
+		DLOG("req retrans counter : %u/%u\n",ctrack->req_retrans_counter, threshold);
 	}
 	return false;
 }
@@ -227,7 +238,7 @@ static void auto_hostlist_failed(const char *hostname)
 	HOSTLIST_DEBUGLOG_APPEND("%s : fail counter %d/%d", hostname, fail_counter->counter, params.hostlist_auto_fail_threshold);
 	if (fail_counter->counter >= params.hostlist_auto_fail_threshold)
 	{
-		DLOG("auto hostlist : fail threshold reached. adding %s to auto hostlist\n", hostname);
+		DLOG("auto hostlist : fail threshold reached. about to add %s to auto hostlist\n", hostname);
 		HostFailPoolDel(&params.hostlist_auto_fail_counters, fail_counter);
 		
 		DLOG("auto hostlist : rechecking %s to avoid duplicates\n", hostname);
@@ -255,7 +266,69 @@ static void auto_hostlist_failed(const char *hostname)
 	}
 }
 
-#define CONNTRACK_REQUIRED (params.wssize || params.desync_cutoff || *params.hostlist_auto_filename)
+static void process_retrans_fail(t_ctrack *ctrack, uint8_t proto)
+{
+	if (ctrack && ctrack->hostname && auto_hostlist_retrans(ctrack, proto, params.hostlist_auto_retrans_threshold))
+	{
+		HOSTLIST_DEBUGLOG_APPEND("%s : tcp retrans threshold reached", ctrack->hostname);
+		auto_hostlist_failed(ctrack->hostname);
+	}
+}
+
+static bool reasm_start(t_ctrack *ctrack, t_reassemble *reasm, size_t sz, size_t szMax, const uint8_t *data_payload, size_t len_payload)
+{
+	ReasmClear(reasm);
+	if (sz<=szMax)
+	{
+		if (ReasmInit(reasm,sz,ctrack->seq_last))
+		{
+			ReasmFeed(reasm,ctrack->seq_last,data_payload,len_payload);
+			DLOG("starting reassemble. now we have %zu/%zu\n",reasm->size_present,reasm->size);
+			return true;
+		}
+		else
+			DLOG("reassemble init failed. out of memory\n");
+	}
+	else
+		DLOG("unexpected large payload for reassemble: size=%zu\n",sz);
+	return false;
+}
+static bool reasm_orig_start(t_ctrack *ctrack, size_t sz, size_t szMax, const uint8_t *data_payload, size_t len_payload)
+{
+	return reasm_start(ctrack,&ctrack->reasm_orig,sz,szMax,data_payload,len_payload);
+}
+static bool reasm_feed(t_ctrack *ctrack, t_reassemble *reasm, const uint8_t *data_payload, size_t len_payload)
+{
+	if (ctrack && !ReasmIsEmpty(reasm))
+	{
+		if (ReasmFeed(reasm,ctrack->seq_last,data_payload,len_payload))
+		{
+			DLOG("reassemble : feeding data payload size=%zu. now we have %zu/%zu\n",len_payload,reasm->size_present,reasm->size)
+			return true;
+		}
+		else
+		{
+			ReasmClear(reasm);
+			DLOG("reassemble session failed\n")
+		}
+	}
+	return false;
+}
+static bool reasm_orig_feed(t_ctrack *ctrack, const uint8_t *data_payload, size_t len_payload)
+{
+	return reasm_feed(ctrack, &ctrack->reasm_orig, data_payload, len_payload);
+}
+static void reasm_orig_fin(t_ctrack *ctrack)
+{
+	if (ctrack && ReasmIsFull(&ctrack->reasm_orig))
+	{
+		DLOG("reassemble session finished\n");
+		ReasmClear(&ctrack->reasm_orig);
+	}
+}
+
+
+
 // result : true - drop original packet, false = dont drop
 packet_process_result dpi_desync_tcp_packet(uint32_t fwmark, const char *ifout, uint8_t *data_pkt, size_t len_pkt, struct ip *ip, struct ip6_hdr *ip6hdr, struct tcphdr *tcphdr, size_t len_tcp, uint8_t *data_payload, size_t len_payload)
 {
@@ -271,13 +344,10 @@ packet_process_result dpi_desync_tcp_packet(uint32_t fwmark, const char *ifout,
 
 	if (!!ip == !!ip6hdr) return res; // one and only one must be present
 
-	if (CONNTRACK_REQUIRED)
-	{
-		ConntrackPoolPurge(&params.conntrack);
-		if (ConntrackPoolFeed(&params.conntrack, ip, ip6hdr, tcphdr, NULL, len_payload, &ctrack, &bReverse))
-			maybe_cutoff(ctrack, IPPROTO_TCP);
-		HostFailPoolPurgeRateLimited(&params.hostlist_auto_fail_counters);
-	}
+	ConntrackPoolPurge(&params.conntrack);
+	if (ConntrackPoolFeed(&params.conntrack, ip, ip6hdr, tcphdr, NULL, len_payload, &ctrack, &bReverse))
+		maybe_cutoff(ctrack, IPPROTO_TCP);
+	HostFailPoolPurgeRateLimited(&params.hostlist_auto_fail_counters);
 
 	//ConntrackPoolDump(&params.conntrack);
 
@@ -290,15 +360,16 @@ packet_process_result dpi_desync_tcp_packet(uint32_t fwmark, const char *ifout,
 	if (bReverse)
 	{
 		// process reply packets for auto hostlist mode
-		// by looking at RSTs or HTTP replies we decide whether original request looks to be blocked by DPI
-		if (*params.hostlist_auto_filename && ctrack && ctrack->hostname && ctrack->req_retrans_counter != RETRANS_COUNTER_STOP)
+		// by looking at RSTs or HTTP replies we decide whether original request looks like DPI blocked
+		// we only process first-sequence replies. do not react to subsequent redirects or RSTs
+		if (*params.hostlist_auto_filename && ctrack && ctrack->hostname && (ctrack->ack_last-ctrack->ack0)==1)
 		{
-			bool bFail=false, bStop=false;
+			bool bFail=false;
 			if (tcphdr->th_flags & TH_RST)
 			{
 				DLOG("incoming RST detected for hostname %s\n", ctrack->hostname);
 				HOSTLIST_DEBUGLOG_APPEND("%s : incoming RST", ctrack->hostname);
-				bFail = bStop = true;
+				bFail = true;
 			}
 			else if (len_payload && ctrack->l7proto==HTTP)
 			{
@@ -319,12 +390,11 @@ packet_process_result dpi_desync_tcp_packet(uint32_t fwmark, const char *ifout,
 					// received not http reply. do not monitor this connection anymore
 					DLOG("incoming unknown HTTP data detected for hostname %s\n", ctrack->hostname);
 				}
-				bStop = true;
 			}
 			if (bFail)
 				auto_hostlist_failed(ctrack->hostname);
-			if (bStop)
-				ctrack_stop_req_counter(ctrack);
+			if (tcphdr->th_flags & TH_RST)
+				ConntrackClearHostname(ctrack); // do not react to further dup RSTs
 		}
 	
 		return res; // nothing to do. do not waste cpu
@@ -408,76 +478,117 @@ packet_process_result dpi_desync_tcp_packet(uint32_t fwmark, const char *ifout,
 		bool bIsHttp;
 		bool bKnownProtocol = false;
 		uint8_t *p, *phost;
-		if ((bIsHttp = IsHttp(data_payload,len_payload)))
+		const uint8_t *rdata_payload = data_payload;
+		size_t rlen_payload = len_payload;
+		
+		if (reasm_orig_feed(ctrack,data_payload,len_payload))
+		{
+			rdata_payload = ctrack->reasm_orig.packet;
+			rlen_payload = ctrack->reasm_orig.size_present;
+		}
+
+		if ((bIsHttp = IsHttp(rdata_payload,rlen_payload)))
 		{
 			DLOG("packet contains HTTP request\n")
 			if (ctrack && !ctrack->l7proto) ctrack->l7proto = HTTP;
-			if (params.wssize)
-			{
-				DLOG("forced wssize-cutoff\n");
-				wssize_cutoff(ctrack);
-			}
+			forced_wssize_cutoff(ctrack);
 			fake = params.fake_http;
 			fake_size = params.fake_http_size;
-			if (params.hostlist || params.debug) bHaveHost=HttpExtractHost(data_payload,len_payload,host,sizeof(host));
-			if (params.hostlist && !bHaveHost)
+			if (params.hostlist || params.hostlist_exclude)
 			{
-				DLOG("not applying tampering to HTTP without Host:\n")
-				return res;
+				bHaveHost=HttpExtractHost(rdata_payload,rlen_payload,host,sizeof(host));
+				if (!bHaveHost)
+				{
+					DLOG("not applying tampering to HTTP without Host:\n")
+					process_retrans_fail(ctrack, IPPROTO_TCP);
+					reasm_orig_fin(ctrack);
+					return res;
+				}
+			}
+			if (ctrack)
+			{
+				// we do not reassemble http
+				if (!ctrack->req_seq_present)
+				{
+					ctrack->req_seq_start=ctrack->seq_last;
+					ctrack->req_seq_end=ctrack->pos_orig-1;
+					ctrack->req_seq_start_present=ctrack->req_seq_present=true;
+					DLOG("req retrans : tcp seq interval %u-%u\n",ctrack->req_seq_start,ctrack->req_seq_end);
+				}
 			}
 			bKnownProtocol = true;
 		}
-		else if (IsTLSClientHello(data_payload,len_payload))
+		else if (IsTLSClientHello(rdata_payload,rlen_payload,TLS_PARTIALS_ENABLE))
 		{
-			DLOG("packet contains TLS ClientHello\n")
-			if (ctrack && !ctrack->l7proto) ctrack->l7proto = TLS;
-			if (params.wssize)
-			{
-				DLOG("forced wssize-cutoff\n");
-				wssize_cutoff(ctrack);
-			}
+			bool bReqFull = IsTLSRecordFull(rdata_payload,rlen_payload);
+			DLOG(bReqFull ? "packet contains full TLS ClientHello\n" : "packet contains partial TLS ClientHello\n")
 			fake = params.fake_tls;
 			fake_size = params.fake_tls_size;
-			if (params.hostlist || params.desync_skip_nosni || params.debug)
+			bHaveHost=TLSHelloExtractHost(rdata_payload,rlen_payload,host,sizeof(host),TLS_PARTIALS_ENABLE);
+			if (ctrack)
 			{
-				bHaveHost=TLSHelloExtractHost(data_payload,len_payload,host,sizeof(host));
-				if (params.desync_skip_nosni && !bHaveHost)
+				if (!ctrack->l7proto) ctrack->l7proto = TLS;
+				if (!bReqFull && ReasmIsEmpty(&ctrack->reasm_orig))
+					// do not reconstruct unexpected large payload (they are feeding garbage ?)
+					reasm_orig_start(ctrack,TLSRecordLen(data_payload),4096,data_payload,len_payload);
+				if (!ctrack->req_seq_start_present)
 				{
-					DLOG("not applying tampering to TLS ClientHello without hostname in the SNI\n")
-					return res;
+					// lower bound of request seq interval
+					ctrack->req_seq_start=ctrack->seq_last;
+					ctrack->req_seq_start_present=true;
 				}
+				if (!ctrack->req_seq_present && bReqFull)
+				{
+					// upper bound of request seq interval
+					ctrack->req_seq_end=ctrack->pos_orig-1;
+					ctrack->req_seq_present=ctrack->req_seq_start_present;
+					DLOG("req retrans : seq interval %u-%u\n",ctrack->req_seq_start,ctrack->req_seq_end);
+				}
+			}
+			if (bReqFull || !ctrack || ReasmIsEmpty(&ctrack->reasm_orig)) forced_wssize_cutoff(ctrack);
+
+			if (params.desync_skip_nosni && !bHaveHost)
+			{
+				DLOG("not applying tampering to TLS ClientHello without hostname in the SNI\n")
+				process_retrans_fail(ctrack, IPPROTO_TCP);
+				reasm_orig_fin(ctrack);
+				return res;
 			}
 			bKnownProtocol = true;
 		}
-		else
-		{
-			// received unknown payload. it means we are out of the request retransmission phase. stop counter
-			ctrack_stop_req_counter(ctrack);
-			
-			if (!params.desync_any_proto) return res;
-			DLOG("applying tampering to unknown protocol\n")
-			fake = params.fake_unknown;
-			fake_size = params.fake_unknown_size;
-		}
+
+		reasm_orig_fin(ctrack);
+		rdata_payload=NULL;
 
 		if (bHaveHost)
 		{
-			DLOG("hostname: %s\n",host)
 			bool bExcluded;
+			DLOG("hostname: %s\n",host)
 			if ((params.hostlist || params.hostlist_exclude) && !HostlistCheck(params.hostlist, params.hostlist_exclude, host, &bExcluded))
 			{
 				DLOG("not applying tampering to this request\n")
-				if (!bExcluded && *params.hostlist_auto_filename && ctrack)
+				if (ctrack)
 				{
-					if (!ctrack->hostname) ctrack->hostname=strdup(host);
-					if (auto_hostlist_retrans(ctrack, IPPROTO_TCP, params.hostlist_auto_retrans_threshold))
+					if (!bExcluded && *params.hostlist_auto_filename)
 					{
-						HOSTLIST_DEBUGLOG_APPEND("%s : tcp retrans threshold reached", ctrack->hostname);
-						auto_hostlist_failed(host);
+						if (!ctrack->hostname) ctrack->hostname=strdup(host);
+						process_retrans_fail(ctrack, IPPROTO_TCP);
 					}
+					else
+						ctrack_stop_retrans_counter(ctrack);
 				}
 				return res;
 			}
+			ctrack_stop_retrans_counter(ctrack);
+		}
+		process_retrans_fail(ctrack, IPPROTO_TCP);
+		
+		if (!bKnownProtocol)
+		{
+			if (!params.desync_any_proto) return res;
+			DLOG("applying tampering to unknown protocol\n")
+			fake = params.fake_unknown;
+			fake_size = params.fake_unknown_size;
 		}
 
 		if (bIsHttp && (params.hostcase || params.hostnospace || params.domcase) && (phost = (uint8_t*)memmem(data_payload, len_payload, "\r\nHost: ", 8)))
@@ -773,6 +884,7 @@ packet_process_result dpi_desync_tcp_packet(uint32_t fwmark, const char *ifout,
 					return frag;
 				}
 		}
+	
 	}
 
 	return res;
@@ -793,13 +905,10 @@ packet_process_result dpi_desync_udp_packet(uint32_t fwmark, const char *ifout,
 
 	if (!!ip == !!ip6hdr) return res; // one and only one must be present
 
-	if (CONNTRACK_REQUIRED)
-	{
-		ConntrackPoolPurge(&params.conntrack);
-		if (ConntrackPoolFeed(&params.conntrack, ip, ip6hdr, NULL, udphdr, len_payload, &ctrack, &bReverse))
-			maybe_cutoff(ctrack, IPPROTO_UDP);
-		HostFailPoolPurgeRateLimited(&params.hostlist_auto_fail_counters);
-	}
+	ConntrackPoolPurge(&params.conntrack);
+	if (ConntrackPoolFeed(&params.conntrack, ip, ip6hdr, NULL, udphdr, len_payload, &ctrack, &bReverse))
+		maybe_cutoff(ctrack, IPPROTO_UDP);
+	HostFailPoolPurgeRateLimited(&params.hostlist_auto_fail_counters);
 
 	//ConntrackPoolDump(&params.conntrack);
 
@@ -883,7 +992,7 @@ packet_process_result dpi_desync_udp_packet(uint32_t fwmark, const char *ifout,
 		else
 		{
 			// received payload without host. it means we are out of the request retransmission phase. stop counter
-			ctrack_stop_req_counter(ctrack);
+			ctrack_stop_retrans_counter(ctrack);
 
 			if (IsWireguardHandshakeInitiation(data_payload,len_payload))
 			{
@@ -920,11 +1029,7 @@ packet_process_result dpi_desync_udp_packet(uint32_t fwmark, const char *ifout,
 				if (!bExcluded && *params.hostlist_auto_filename && ctrack)
 				{
 					if (!ctrack->hostname) ctrack->hostname=strdup(host);
-					if (auto_hostlist_retrans(ctrack, IPPROTO_UDP, params.hostlist_auto_retrans_threshold))
-					{
-						HOSTLIST_DEBUGLOG_APPEND("%s : udp retrans threshold reached", ctrack->hostname);
-						auto_hostlist_failed(host);
-					}
+					process_retrans_fail(ctrack, IPPROTO_UDP);
 				}
 				return res;
 			}

nfq/helpers.c

@@ -5,6 +5,7 @@
 #include <string.h>
 #include <unistd.h>
 #include <ctype.h>
+#include <time.h>
 
 void hexdump_limited_dlog(const uint8_t *data, size_t size, size_t limit)
 {
@@ -149,12 +150,12 @@ void dbgprint_socket_buffers(int fd)
 bool set_socket_buffers(int fd, int rcvbuf, int sndbuf)
 {
 	DLOG("set_socket_buffers fd=%d rcvbuf=%d sndbuf=%d\n", fd, rcvbuf, sndbuf)
-		if (rcvbuf && setsockopt(fd, SOL_SOCKET, SO_RCVBUF, &rcvbuf, sizeof(int)) < 0)
-		{
-			perror("setsockopt (SO_RCVBUF)");
-			close(fd);
-			return false;
-		}
+	if (rcvbuf && setsockopt(fd, SOL_SOCKET, SO_RCVBUF, &rcvbuf, sizeof(int)) < 0)
+	{
+		perror("setsockopt (SO_RCVBUF)");
+		close(fd);
+		return false;
+	}
 	if (sndbuf && setsockopt(fd, SOL_SOCKET, SO_SNDBUF, &sndbuf, sizeof(int)) < 0)
 	{
 		perror("setsockopt (SO_SNDBUF)");
@@ -188,6 +189,11 @@ void phton64(uint8_t *p, uint64_t v)
 	p[7] = (uint8_t)(v >> 0);
 }
 
+bool seq_within(uint32_t s, uint32_t s1, uint32_t s2)
+{
+	return s2>=s1 && s>=s1 && s<=s2 || s2<s1 && (s<=s2 || s>=s1);
+}
+
 bool ipv6_addr_is_zero(const struct in6_addr *a)
 {
     return !memcmp(a,"\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00",16);

nfq/helpers.h

@@ -5,6 +5,7 @@
 #include <sys/socket.h>
 #include <stddef.h>
 #include <stdbool.h>
+#include <stdio.h>
 
 #include "params.h"
 
@@ -19,6 +20,8 @@ void print_sockaddr(const struct sockaddr *sa);
 void ntop46(const struct sockaddr *sa, char *str, size_t len);
 void ntop46_port(const struct sockaddr *sa, char *str, size_t len);
 
+bool seq_within(uint32_t s, uint32_t s1, uint32_t s2);
+
 void dbgprint_socket_buffers(int fd);
 bool set_socket_buffers(int fd, int rcvbuf, int sndbuf);
 

nfq/params.h

@@ -1,6 +1,5 @@
 #pragma once
 
-#include "params.h"
 #include "pools.h"
 #include "conntrack.h"
 #include "desync.h"
@@ -12,6 +11,8 @@
 #include <stdbool.h>
 #include <stdio.h>
 
+#define TLS_PARTIALS_ENABLE	true
+
 #define Q_RCVBUF	(128*1024)	// in bytes
 #define Q_SNDBUF	(64*1024)	// in bytes
 #define RAW_SNDBUF	(64*1024)	// in bytes

nfq/protocol.c

@@ -108,12 +108,25 @@ bool HttpReplyLooksLikeDPIRedirect(const uint8_t *data, size_t len, const char *
 }
 
 
-
-bool IsTLSClientHello(const uint8_t *data, size_t len)
+uint16_t TLSRecordDataLen(const uint8_t *data)
+{
+	return pntoh16(data + 3);
+}
+size_t TLSRecordLen(const uint8_t *data)
+{
+	return TLSRecordDataLen(data) + 5;
+}
+bool IsTLSRecordFull(const uint8_t *data, size_t len)
+{
+	return TLSRecordLen(data)<=len;
+}
+bool IsTLSClientHello(const uint8_t *data, size_t len, bool bPartialIsOK)
 {
-	return len >= 6 && data[0] == 0x16 && data[1] == 0x03 && data[2] >= 0x01 && data[2] <= 0x03 && data[5] == 0x01 && (pntoh16(data + 3) + 5) <= len;
+	return len >= 6 && data[0] == 0x16 && data[1] == 0x03 && data[2] >= 0x01 && data[2] <= 0x03 && data[5] == 0x01 && (bPartialIsOK || TLSRecordLen(data) <= len);
 }
-bool TLSFindExtInHandshake(const uint8_t *data, size_t len, uint16_t type, const uint8_t **ext, size_t *len_ext)
+
+// bPartialIsOK=true - accept partial packets not containing the whole TLS message
+bool TLSFindExtInHandshake(const uint8_t *data, size_t len, uint16_t type, const uint8_t **ext, size_t *len_ext, bool bPartialIsOK)
 {
 	// +0
 	// u8	HandshakeType: ClientHello
@@ -133,8 +146,11 @@ bool TLSFindExtInHandshake(const uint8_t *data, size_t len, uint16_t type, const
 	l = 1 + 3 + 2 + 32;
 	// SessionIDLength
 	if (len < (l + 1)) return false;
-	ll = data[1] << 16 | data[2] << 8 | data[3]; // HandshakeProtocol length
-	if (len < (ll + 4)) return false;
+	if (!bPartialIsOK)
+	{
+	    ll = data[1] << 16 | data[2] << 8 | data[3]; // HandshakeProtocol length
+	    if (len < (ll + 4)) return false;
+	}
 	l += data[l] + 1;
 	// CipherSuitesLength
 	if (len < (l + 2)) return false;
@@ -148,7 +164,15 @@ bool TLSFindExtInHandshake(const uint8_t *data, size_t len, uint16_t type, const
 	data += l; len -= l;
 	l = pntoh16(data);
 	data += 2; len -= 2;
-	if (len < l) return false;
+	
+	if (bPartialIsOK)
+	{
+		if (len < l) l = len;
+	}
+	else
+	{
+		if (len < l) return false;
+	}
 
 	while (l >= 4)
 	{
@@ -170,14 +194,14 @@ bool TLSFindExtInHandshake(const uint8_t *data, size_t len, uint16_t type, const
 
 	return false;
 }
-bool TLSFindExt(const uint8_t *data, size_t len, uint16_t type, const uint8_t **ext, size_t *len_ext)
+bool TLSFindExt(const uint8_t *data, size_t len, uint16_t type, const uint8_t **ext, size_t *len_ext, bool bPartialIsOK)
 {
 	// +0
 	// u8	ContentType: Handshake
 	// u16	Version: TLS1.0
 	// u16	Length
-	if (!IsTLSClientHello(data, len)) return false;
-	return TLSFindExtInHandshake(data + 5, len - 5, type, ext, len_ext);
+	if (!IsTLSClientHello(data, len, bPartialIsOK)) return false;
+	return TLSFindExtInHandshake(data + 5, len - 5, type, ext, len_ext, bPartialIsOK);
 }
 static bool TLSExtractHostFromExt(const uint8_t *ext, size_t elen, char *host, size_t len_host)
 {
@@ -196,20 +220,20 @@ static bool TLSExtractHostFromExt(const uint8_t *ext, size_t elen, char *host, s
 	}
 	return true;
 }
-bool TLSHelloExtractHost(const uint8_t *data, size_t len, char *host, size_t len_host)
+bool TLSHelloExtractHost(const uint8_t *data, size_t len, char *host, size_t len_host, bool bPartialIsOK)
 {
 	const uint8_t *ext;
 	size_t elen;
 
-	if (!TLSFindExt(data, len, 0, &ext, &elen)) return false;
+	if (!TLSFindExt(data, len, 0, &ext, &elen, bPartialIsOK)) return false;
 	return TLSExtractHostFromExt(ext, elen, host, len_host);
 }
-bool TLSHelloExtractHostFromHandshake(const uint8_t *data, size_t len, char *host, size_t len_host)
+bool TLSHelloExtractHostFromHandshake(const uint8_t *data, size_t len, char *host, size_t len_host, bool bPartialIsOK)
 {
 	const uint8_t *ext;
 	size_t elen;
 
-	if (!TLSFindExtInHandshake(data, len, 0, &ext, &elen)) return false;
+	if (!TLSFindExtInHandshake(data, len, 0, &ext, &elen, bPartialIsOK)) return false;
 	return TLSExtractHostFromExt(ext, elen, host, len_host);
 }
 
@@ -580,7 +604,7 @@ bool QUICExtractHostFromInitial(const uint8_t *data, size_t data_len, char *host
 	if (!IsQUICCryptoHello(defrag, defrag_len, &hello_offset, &hello_len)) return false;
 	if (bIsCryptoHello) *bIsCryptoHello=true;
 
-	return TLSHelloExtractHostFromHandshake(defrag + hello_offset, hello_len, host, len_host);
+	return TLSHelloExtractHostFromHandshake(defrag + hello_offset, hello_len, host, len_host, true);
 }
 
 bool IsQUICInitial(const uint8_t *data, size_t len)

nfq/protocol.h

@@ -5,6 +5,7 @@
 #include <stdbool.h>
 #include "crypto/sha.h"
 #include "crypto/aes-gcm.h"
+#include "helpers.h"
 
 bool IsHttp(const uint8_t *data, size_t len);
 // header must be passed like this : "\nHost:"
@@ -17,11 +18,14 @@ int HttpReplyCode(const uint8_t *data, size_t len);
 // must be pre-checked by IsHttpReply
 bool HttpReplyLooksLikeDPIRedirect(const uint8_t *data, size_t len, const char *host);
 
-bool IsTLSClientHello(const uint8_t *data, size_t len);
-bool TLSFindExt(const uint8_t *data, size_t len, uint16_t type, const uint8_t **ext, size_t *len_ext);
-bool TLSFindExtInHandshake(const uint8_t *data, size_t len, uint16_t type, const uint8_t **ext, size_t *len_ext);
-bool TLSHelloExtractHost(const uint8_t *data, size_t len, char *host, size_t len_host);
-bool TLSHelloExtractHostFromHandshake(const uint8_t *data, size_t len, char *host, size_t len_host);
+uint16_t TLSRecordDataLen(const uint8_t *data);
+size_t TLSRecordLen(const uint8_t *data);
+bool IsTLSRecordFull(const uint8_t *data, size_t len);
+bool IsTLSClientHello(const uint8_t *data, size_t len, bool bPartialIsOK);
+bool TLSFindExt(const uint8_t *data, size_t len, uint16_t type, const uint8_t **ext, size_t *len_ext, bool bPartialIsOK);
+bool TLSFindExtInHandshake(const uint8_t *data, size_t len, uint16_t type, const uint8_t **ext, size_t *len_ext, bool bPartialIsOK);
+bool TLSHelloExtractHost(const uint8_t *data, size_t len, char *host, size_t len_host, bool bPartialIsOK);
+bool TLSHelloExtractHostFromHandshake(const uint8_t *data, size_t len, char *host, size_t len_host, bool bPartialIsOK);
 
 bool IsWireguardHandshakeInitiation(const uint8_t *data, size_t len);
 bool IsDhtD1(const uint8_t *data, size_t len);