161 tcp_dsack_block_exists(struct tcpcb *tp)  in tcp_dsack_block_exists()  argument
164 	if (tp->rcv_numsacks == 0)  in tcp_dsack_block_exists()
166 	if (SEQ_LEQ(tp->sackblks[0].end, tp->rcv_nxt))  in tcp_dsack_block_exists()
176 tcp_update_dsack_list(struct tcpcb *tp, tcp_seq rcv_start, tcp_seq rcv_end)  in tcp_update_dsack_list()  argument
182 	INP_WLOCK_ASSERT(tptoinpcb(tp));  in tcp_update_dsack_list()
186 	if (SEQ_LT(rcv_end, tp->rcv_nxt) ||  in tcp_update_dsack_list()
187 	    ((rcv_end == tp->rcv_nxt) &&  in tcp_update_dsack_list()
188 	     (tp->rcv_numsacks > 0 ) &&  in tcp_update_dsack_list()
189 	     (tp->sackblks[0].end == tp->rcv_nxt))) {  in tcp_update_dsack_list()
201 	for (i = 0; i < tp->rcv_numsacks; i++) {  in tcp_update_dsack_list()
202 		start = tp->sackblks[i].start;  in tcp_update_dsack_list()
203 		end = tp->sackblks[i].end;  in tcp_update_dsack_list()
212 		if (SEQ_GT(tp->rcv_nxt, end)) {  in tcp_update_dsack_list()
244 	if ((SEQ_LT(tp->rcv_nxt, mid_blk.end) &&  in tcp_update_dsack_list()
251 	for (j = 0; (j < tp->rcv_numsacks) && (n < MAX_SACK_BLKS); j++) {  in tcp_update_dsack_list()
252 		if (((SEQ_LT(tp->sackblks[j].end, mid_blk.start) ||  in tcp_update_dsack_list()
253 		      SEQ_GT(tp->sackblks[j].start, mid_blk.end)) &&  in tcp_update_dsack_list()
254 		    (SEQ_GT(tp->sackblks[j].start, tp->rcv_nxt))))  in tcp_update_dsack_list()
255 		saved_blks[n++] = tp->sackblks[j];  in tcp_update_dsack_list()
261 			tp->sackblks[j++] = saved_blks[i];  in tcp_update_dsack_list()
264 	tp->rcv_numsacks = j;  in tcp_update_dsack_list()
272 tcp_update_sack_list(struct tcpcb *tp, tcp_seq rcv_start, tcp_seq rcv_end)  in tcp_update_sack_list()  argument
283 	INP_WLOCK_ASSERT(tptoinpcb(tp));  in tcp_update_sack_list()
289 	    (tp->rcv_numsacks >= 1) &&  in tcp_update_sack_list()
290 	    (rcv_end == tp->sackblks[0].end)) {  in tcp_update_sack_list()
292 		head_blk = tp->sackblks[0];  in tcp_update_sack_list()
305 	for (i = 0; i < tp->rcv_numsacks; i++) {  in tcp_update_sack_list()
306 		tcp_seq start = tp->sackblks[i].start;  in tcp_update_sack_list()
307 		tcp_seq end = tp->sackblks[i].end;  in tcp_update_sack_list()
308 		if (SEQ_GEQ(start, end) || SEQ_LEQ(start, tp->rcv_nxt)) {  in tcp_update_sack_list()
383 		tp->sackblks[0] = head_blk;  in tcp_update_sack_list()
393 	    (rcv_start == tp->sackblks[0].end)) {  in tcp_update_sack_list()
400 		bcopy(saved_blks, &tp->sackblks[num_head],  in tcp_update_sack_list()
405 	tp->rcv_numsacks = num_head + num_saved;  in tcp_update_sack_list()
409 tcp_clean_dsack_blocks(struct tcpcb *tp)  in tcp_clean_dsack_blocks()  argument
414 	INP_WLOCK_ASSERT(tptoinpcb(tp));  in tcp_clean_dsack_blocks()
421 	for (i = 0; i < tp->rcv_numsacks; i++) {  in tcp_clean_dsack_blocks()
422 		tcp_seq start = tp->sackblks[i].start;  in tcp_clean_dsack_blocks()
423 		tcp_seq end = tp->sackblks[i].end;  in tcp_clean_dsack_blocks()
424 		if (SEQ_GEQ(start, end) || SEQ_LEQ(start, tp->rcv_nxt)) {  in tcp_clean_dsack_blocks()
441 		bcopy(saved_blks, &tp->sackblks[0],  in tcp_clean_dsack_blocks()
444 	tp->rcv_numsacks = num_saved;  in tcp_clean_dsack_blocks()
451 tcp_clean_sackreport(struct tcpcb *tp)  in tcp_clean_sackreport()  argument
455 	INP_WLOCK_ASSERT(tptoinpcb(tp));  in tcp_clean_sackreport()
456 	tp->rcv_numsacks = 0;  in tcp_clean_sackreport()
458 		tp->sackblks[i].start = tp->sackblks[i].end=0;  in tcp_clean_sackreport()
465 tcp_sackhole_alloc(struct tcpcb *tp, tcp_seq start, tcp_seq end)  in tcp_sackhole_alloc()  argument
469 	if (tp->snd_numholes >= V_tcp_sack_maxholes ||  in tcp_sackhole_alloc()
483 	tp->snd_numholes++;  in tcp_sackhole_alloc()
493 tcp_sackhole_free(struct tcpcb *tp, struct sackhole *hole)  in tcp_sackhole_free()  argument
498 	tp->snd_numholes--;  in tcp_sackhole_free()
501 	KASSERT(tp->snd_numholes >= 0, ("tp->snd_numholes >= 0"));  in tcp_sackhole_free()
509 tcp_sackhole_insert(struct tcpcb *tp, tcp_seq start, tcp_seq end,  in tcp_sackhole_insert()  argument
515 	hole = tcp_sackhole_alloc(tp, start, end);  in tcp_sackhole_insert()
521 		TAILQ_INSERT_AFTER(&tp->snd_holes, after, hole, scblink);  in tcp_sackhole_insert()
523 		TAILQ_INSERT_TAIL(&tp->snd_holes, hole, scblink);  in tcp_sackhole_insert()
526 	if (tp->sackhint.nexthole == NULL)  in tcp_sackhole_insert()
527 		tp->sackhint.nexthole = hole;  in tcp_sackhole_insert()
536 tcp_sackhole_remove(struct tcpcb *tp, struct sackhole *hole)  in tcp_sackhole_remove()  argument
540 	if (tp->sackhint.nexthole == hole)  in tcp_sackhole_remove()
541 		tp->sackhint.nexthole = TAILQ_NEXT(hole, scblink);  in tcp_sackhole_remove()
544 	TAILQ_REMOVE(&tp->snd_holes, hole, scblink);  in tcp_sackhole_remove()
547 	tcp_sackhole_free(tp, hole);  in tcp_sackhole_remove()
559 tcp_sack_doack(struct tcpcb *tp, struct tcpopt *to, tcp_seq th_ack)  in tcp_sack_doack()  argument
566 	int maxseg = tp->t_maxseg - MAX_TCPOPTLEN;  in tcp_sack_doack()
573 	INP_WLOCK_ASSERT(tptoinpcb(tp));  in tcp_sack_doack()
584 	if (SEQ_LT(tp->snd_una, th_ack) && !TAILQ_EMPTY(&tp->snd_holes)) {  in tcp_sack_doack()
585 		left_edge_delta = th_ack - tp->snd_una;  in tcp_sack_doack()
586 		sack_blocks[num_sack_blks].start = tp->snd_una;  in tcp_sack_doack()
592 		if (SEQ_LT(tp->snd_fack, th_ack)) {  in tcp_sack_doack()
593 			delivered_data += th_ack - tp->snd_una;  in tcp_sack_doack()
594 			tp->snd_fack = th_ack;  in tcp_sack_doack()
609 			    SEQ_GT(sack.start, tp->snd_una) &&  in tcp_sack_doack()
611 			    SEQ_LT(sack.start, tp->snd_max) &&  in tcp_sack_doack()
612 			    SEQ_GT(sack.end, tp->snd_una) &&  in tcp_sack_doack()
613 			    SEQ_LEQ(sack.end, tp->snd_max) &&  in tcp_sack_doack()
615 			     SEQ_GEQ(sack.end, tp->snd_max))) {  in tcp_sack_doack()
622 				tcp_record_dsack(tp, sack.start, sack.end, 0);  in tcp_sack_doack()
647 	if (TAILQ_EMPTY(&tp->snd_holes)) {  in tcp_sack_doack()
655 		tp->snd_fack = SEQ_MAX(tp->snd_una, th_ack);  in tcp_sack_doack()
656 		tp->sackhint.sacked_bytes = 0;	/* reset */  in tcp_sack_doack()
657 		tp->sackhint.hole_bytes = 0;  in tcp_sack_doack()
672 	tp->sackhint.last_sack_ack = sblkp->end;  in tcp_sack_doack()
673 	if (SEQ_LT(tp->snd_fack, sblkp->start)) {  in tcp_sack_doack()
680 		if (((temp = TAILQ_LAST(&tp->snd_holes, sackhole_head)) != NULL) &&  in tcp_sack_doack()
681 		    SEQ_LEQ(tp->snd_fack,temp->end)) {  in tcp_sack_doack()
682 			tp->sackhint.hole_bytes -= temp->end - temp->start;  in tcp_sack_doack()
683 			temp->start = SEQ_MAX(tp->snd_fack, SEQ_MAX(tp->snd_una, th_ack));  in tcp_sack_doack()
687 			tp->sackhint.hole_bytes += temp->end - temp->start;  in tcp_sack_doack()
688 			KASSERT(tp->sackhint.hole_bytes >= 0,  in tcp_sack_doack()
690 			tp->snd_fack = sblkp->end;  in tcp_sack_doack()
700 			temp = tcp_sackhole_insert(tp, tp->snd_fack,sblkp->start,NULL);  in tcp_sack_doack()
703 				tp->sackhint.hole_bytes += temp->end - temp->start;  in tcp_sack_doack()
704 				tp->snd_fack = sblkp->end;  in tcp_sack_doack()
718 				       SEQ_LT(tp->snd_fack, sblkp->start))  in tcp_sack_doack()
721 				    SEQ_LT(tp->snd_fack, sblkp->end)) {  in tcp_sack_doack()
722 					delivered_data += sblkp->end - tp->snd_fack;  in tcp_sack_doack()
723 					tp->snd_fack = sblkp->end;  in tcp_sack_doack()
733 	} else if (SEQ_LT(tp->snd_fack, sblkp->end)) {  in tcp_sack_doack()
735 		delivered_data += sblkp->end - tp->snd_fack;  in tcp_sack_doack()
736 		tp->snd_fack = sblkp->end;  in tcp_sack_doack()
739 	cur = TAILQ_LAST(&tp->snd_holes, sackhole_head); /* Last SACK hole. */  in tcp_sack_doack()
740 	loss_hiack = tp->snd_fack;  in tcp_sack_doack()
749 			    (loss_thresh > (tcprexmtthresh-1)*tp->t_maxseg)))   in tcp_sack_doack()
755 			    (loss_thresh > (tcprexmtthresh-1)*tp->t_maxseg))) {  in tcp_sack_doack()
780 			    (loss_thresh > (tcprexmtthresh-1)*tp->t_maxseg)))  in tcp_sack_doack()
785 		tp->sackhint.sack_bytes_rexmit -=  in tcp_sack_doack()
787 		KASSERT(tp->sackhint.sack_bytes_rexmit >= 0,  in tcp_sack_doack()
797 				tp->sackhint.hole_bytes -= temp->end - temp->start;  in tcp_sack_doack()
798 				tcp_sackhole_remove(tp, temp);  in tcp_sack_doack()
808 				tp->sackhint.hole_bytes -= sblkp->end - cur->start;  in tcp_sack_doack()
817 				tp->sackhint.hole_bytes -= cur->end - sblkp->start;  in tcp_sack_doack()
820 				if ((tp->t_flags & TF_LRD) && SEQ_GEQ(cur->rxmit, cur->end))  in tcp_sack_doack()
821 					cur->rxmit = tp->snd_recover;  in tcp_sack_doack()
827 				temp = tcp_sackhole_insert(tp, sblkp->end,  in tcp_sack_doack()
833 						tp->sackhint.sack_bytes_rexmit +=  in tcp_sack_doack()
837 					tp->sackhint.hole_bytes -= sblkp->end - sblkp->start;  in tcp_sack_doack()
842 					    (loss_thresh > (tcprexmtthresh-1)*tp->t_maxseg)))  in tcp_sack_doack()
847 					if ((tp->t_flags & TF_LRD) && SEQ_GEQ(cur->rxmit, cur->end))  in tcp_sack_doack()
848 						cur->rxmit = tp->snd_recover;  in tcp_sack_doack()
853 		tp->sackhint.sack_bytes_rexmit +=  in tcp_sack_doack()
865 			    (loss_thresh > (tcprexmtthresh-1)*tp->t_maxseg)))  in tcp_sack_doack()
873 	KASSERT(!(TAILQ_EMPTY(&tp->snd_holes) && (tp->sackhint.hole_bytes != 0)),  in tcp_sack_doack()
876 	KASSERT(notlost_bytes <= tp->sackhint.hole_bytes,  in tcp_sack_doack()
889 	tp->sackhint.delivered_data = delivered_data;  in tcp_sack_doack()
890 	tp->sackhint.sacked_bytes += delivered_data - left_edge_delta;  in tcp_sack_doack()
891 	tp->sackhint.lost_bytes = tp->sackhint.hole_bytes - notlost_bytes;  in tcp_sack_doack()
893 	KASSERT((tp->sackhint.sacked_bytes >= 0), ("sacked_bytes < 0"));  in tcp_sack_doack()
901 tcp_free_sackholes(struct tcpcb *tp)  in tcp_free_sackholes()  argument
905 	INP_WLOCK_ASSERT(tptoinpcb(tp));  in tcp_free_sackholes()
906 	while ((q = TAILQ_FIRST(&tp->snd_holes)) != NULL)  in tcp_free_sackholes()
907 		tcp_sackhole_remove(tp, q);  in tcp_free_sackholes()
908 	tp->sackhint.sack_bytes_rexmit = 0;  in tcp_free_sackholes()
909 	tp->sackhint.delivered_data = 0;  in tcp_free_sackholes()
910 	tp->sackhint.sacked_bytes = 0;  in tcp_free_sackholes()
911 	tp->sackhint.hole_bytes = 0;  in tcp_free_sackholes()
912 	tp->sackhint.lost_bytes = 0;  in tcp_free_sackholes()
914 	KASSERT(tp->snd_numholes == 0, ("tp->snd_numholes == 0"));  in tcp_free_sackholes()
915 	KASSERT(tp->sackhint.nexthole == NULL,  in tcp_free_sackholes()
926 tcp_resend_sackholes(struct tcpcb *tp)  in tcp_resend_sackholes()  argument
930 	INP_WLOCK_ASSERT(tptoinpcb(tp));  in tcp_resend_sackholes()
931 	TAILQ_FOREACH(p, &tp->snd_holes, scblink) {  in tcp_resend_sackholes()
934 	tp->sackhint.nexthole = TAILQ_FIRST(&tp->snd_holes);  in tcp_resend_sackholes()
935 	tp->sackhint.sack_bytes_rexmit = 0;  in tcp_resend_sackholes()
947 tcp_sack_partialack(struct tcpcb *tp, struct tcphdr *th, u_int *maxsegp)  in tcp_sack_partialack()  argument
953 	INP_WLOCK_ASSERT(tptoinpcb(tp));  in tcp_sack_partialack()
956 		*maxsegp = tcp_maxseg(tp);  in tcp_sack_partialack()
959 	tcp_timer_activate(tp, TT_REXMT, 0);  in tcp_sack_partialack()
960 	tp->t_rtttime = 0;  in tcp_sack_partialack()
962 	if ((BYTES_THIS_ACK(tp, th) / maxseg) >= 2)  in tcp_sack_partialack()
965 		tp->snd_cwnd = imax(tp->snd_nxt - th->th_ack +  in tcp_sack_partialack()
966 				tp->sackhint.sack_bytes_rexmit -  in tcp_sack_partialack()
967 				tp->sackhint.sacked_bytes -  in tcp_sack_partialack()
968 				tp->sackhint.lost_bytes, maxseg) +  in tcp_sack_partialack()
971 		tp->snd_cwnd = (tp->sackhint.sack_bytes_rexmit +  in tcp_sack_partialack()
972 		    imax(0, tp->snd_nxt - tp->snd_recover) +  in tcp_sack_partialack()
975 	if (tp->snd_cwnd > tp->snd_ssthresh)  in tcp_sack_partialack()
976 		tp->snd_cwnd = tp->snd_ssthresh;  in tcp_sack_partialack()
977 	tp->t_flags |= TF_ACKNOW;  in tcp_sack_partialack()
995 	    SEQ_LT(th->th_ack, tp->snd_recover) &&  in tcp_sack_partialack()
996 	    TAILQ_EMPTY(&tp->snd_holes) &&  in tcp_sack_partialack()
997 	    (tp->sackhint.delivered_data > 0)) {  in tcp_sack_partialack()
1004 		tcp_seq highdata = tp->snd_max;  in tcp_sack_partialack()
1005 		if (tp->t_flags & TF_SENTFIN)  in tcp_sack_partialack()
1007 		highdata = SEQ_MIN(highdata, tp->snd_recover);  in tcp_sack_partialack()
1009 			tp->snd_fack = th->th_ack;  in tcp_sack_partialack()
1010 			if ((temp = tcp_sackhole_insert(tp, SEQ_MAX(th->th_ack,  in tcp_sack_partialack()
1012 				tp->sackhint.hole_bytes +=  in tcp_sack_partialack()
1017 	(void) tcp_output(tp);  in tcp_sack_partialack()
1038 tcp_sack_output(struct tcpcb *tp, int *sack_bytes_rexmt)  in tcp_sack_output()  argument
1042 	INP_WLOCK_ASSERT(tptoinpcb(tp));  in tcp_sack_output()
1043 	*sack_bytes_rexmt = tp->sackhint.sack_bytes_rexmit;  in tcp_sack_output()
1044 	hole = tp->sackhint.nexthole;  in tcp_sack_output()
1053 				tp->sackhint.nexthole = hole;  in tcp_sack_output()
1060 		KASSERT(SEQ_LT(hole->start, tp->snd_fack), ("%s: hole.start >= snd.fack", __func__));  in tcp_sack_output()
1061 		KASSERT(SEQ_LT(hole->end, tp->snd_fack), ("%s: hole.end >= snd.fack", __func__));  in tcp_sack_output()
1062 		KASSERT(SEQ_LT(hole->rxmit, tp->snd_fack), ("%s: hole.rxmit >= snd.fack", __func__));  in tcp_sack_output()
1064 		    SEQ_GEQ(hole->start, tp->snd_fack) ||  in tcp_sack_output()
1065 		    SEQ_GEQ(hole->end, tp->snd_fack) ||  in tcp_sack_output()
1066 		    SEQ_GEQ(hole->rxmit, tp->snd_fack)) {  in tcp_sack_output()
1068 					hole->start, hole->end, hole->rxmit, tp->snd_fack);  in tcp_sack_output()
1081 tcp_sack_adjust(struct tcpcb *tp)  in tcp_sack_adjust()  argument
1083 	struct sackhole *p, *cur = TAILQ_FIRST(&tp->snd_holes);  in tcp_sack_adjust()
1085 	INP_WLOCK_ASSERT(tptoinpcb(tp));  in tcp_sack_adjust()
1090 	if (SEQ_GEQ(tp->snd_nxt, tp->snd_fack)) {  in tcp_sack_adjust()
1100 		if (SEQ_LT(tp->snd_nxt, cur->end)) {  in tcp_sack_adjust()
1103 		if (SEQ_GEQ(tp->snd_nxt, p->start)) {  in tcp_sack_adjust()
1106 			tp->snd_nxt = p->start;  in tcp_sack_adjust()
1110 	if (SEQ_LT(tp->snd_nxt, cur->end)) {  in tcp_sack_adjust()
1113 	tp->snd_nxt = tp->snd_fack;  in tcp_sack_adjust()
1123 tcp_sack_lost_retransmission(struct tcpcb *tp, struct tcphdr *th)  in tcp_sack_lost_retransmission()  argument
1127 	if (IN_RECOVERY(tp->t_flags) &&  in tcp_sack_lost_retransmission()
1128 	    SEQ_GT(tp->snd_fack, tp->snd_recover) &&  in tcp_sack_lost_retransmission()
1129 	    ((temp = TAILQ_FIRST(&tp->snd_holes)) != NULL) &&  in tcp_sack_lost_retransmission()
1131 	    SEQ_GEQ(tp->snd_fack, temp->rxmit)) {  in tcp_sack_lost_retransmission()
1138 		tp->sackhint.nexthole = temp;  in tcp_sack_lost_retransmission()
1139 		TAILQ_FOREACH(temp, &tp->snd_holes, scblink) {  in tcp_sack_lost_retransmission()
1140 			if (SEQ_GEQ(tp->snd_fack, temp->rxmit) &&  in tcp_sack_lost_retransmission()
1150 		tp->snd_cwnd = tp->snd_ssthresh;  in tcp_sack_lost_retransmission()
1155 		EXIT_RECOVERY(tp->t_flags);  in tcp_sack_lost_retransmission()
1156 		cc_cong_signal(tp, th, CC_NDUPACK);  in tcp_sack_lost_retransmission()
1163 		tp->snd_cwnd = tcp_maxseg(tp);  in tcp_sack_lost_retransmission()
1164 		tp->sackhint.recover_fs = (tp->snd_max - tp->snd_una) -  in tcp_sack_lost_retransmission()
1165 					    tp->sackhint.recover_fs;  in tcp_sack_lost_retransmission()