1f870fa0bSMat Martineau // SPDX-License-Identifier: GPL-2.0 2f870fa0bSMat Martineau /* Multipath TCP 3f870fa0bSMat Martineau * 4f870fa0bSMat Martineau * Copyright (c) 2017 - 2019, Intel Corporation. 5f870fa0bSMat Martineau */ 6f870fa0bSMat Martineau 7f870fa0bSMat Martineau #define pr_fmt(fmt) "MPTCP: " fmt 8f870fa0bSMat Martineau 9f870fa0bSMat Martineau #include <linux/kernel.h> 10f870fa0bSMat Martineau #include <linux/module.h> 11f870fa0bSMat Martineau #include <linux/netdevice.h> 127a6a6cbcSPaolo Abeni #include <linux/sched/signal.h> 137a6a6cbcSPaolo Abeni #include <linux/atomic.h> 14f870fa0bSMat Martineau #include <net/sock.h> 15f870fa0bSMat Martineau #include <net/inet_common.h> 16f870fa0bSMat Martineau #include <net/inet_hashtables.h> 17f870fa0bSMat Martineau #include <net/protocol.h> 18f870fa0bSMat Martineau #include <net/tcp.h> 193721b9b6SMat Martineau #include <net/tcp_states.h> 20cf7da0d6SPeter Krystad #if IS_ENABLED(CONFIG_MPTCP_IPV6) 21cf7da0d6SPeter Krystad #include <net/transp_v6.h> 22cf7da0d6SPeter Krystad #endif 23f870fa0bSMat Martineau #include <net/mptcp.h> 24e16163b6SPaolo Abeni #include <net/xfrm.h> 25644807e3SFlorian Westphal #include <asm/ioctls.h> 26f870fa0bSMat Martineau #include "protocol.h" 27fc518953SFlorian Westphal #include "mib.h" 28f870fa0bSMat Martineau 29e10a9892SGeliang Tang #define CREATE_TRACE_POINTS 30e10a9892SGeliang Tang #include <trace/events/mptcp.h> 31e10a9892SGeliang Tang 32b0519de8SFlorian Westphal #if IS_ENABLED(CONFIG_MPTCP_IPV6) 33b0519de8SFlorian Westphal struct mptcp6_sock { 34b0519de8SFlorian Westphal struct mptcp_sock msk; 35b0519de8SFlorian Westphal struct ipv6_pinfo np; 36b0519de8SFlorian Westphal }; 37b0519de8SFlorian Westphal #endif 38b0519de8SFlorian Westphal 39b7f653b2SFlorian Westphal enum { 40b7f653b2SFlorian Westphal MPTCP_CMSG_TS = BIT(0), 412c9e7765SFlorian Westphal MPTCP_CMSG_INQ = BIT(1), 42b7f653b2SFlorian Westphal }; 43b7f653b2SFlorian Westphal 4491b6d325SEric Dumazet static struct percpu_counter mptcp_sockets_allocated ____cacheline_aligned_in_smp; 45d027236cSPaolo Abeni 46e16163b6SPaolo Abeni static void __mptcp_destroy_sock(struct sock *sk); 4781c1d029SPaolo Abeni static void mptcp_check_send_data_fin(struct sock *sk); 48e16163b6SPaolo Abeni 49b19bc294SPaolo Abeni DEFINE_PER_CPU(struct mptcp_delegated_action, mptcp_delegated_actions); 50b19bc294SPaolo Abeni static struct net_device mptcp_napi_dev; 51b19bc294SPaolo Abeni 526f8a612aSFlorian Westphal /* Returns end sequence number of the receiver's advertised window */ 536f8a612aSFlorian Westphal static u64 mptcp_wnd_end(const struct mptcp_sock *msk) 546f8a612aSFlorian Westphal { 557439d687SPaolo Abeni return READ_ONCE(msk->wnd_end); 566f8a612aSFlorian Westphal } 576f8a612aSFlorian Westphal 58d2f77c53SPaolo Abeni static bool mptcp_is_tcpsk(struct sock *sk) 590b4f33deSFlorian Westphal { 600b4f33deSFlorian Westphal struct socket *sock = sk->sk_socket; 610b4f33deSFlorian Westphal 620b4f33deSFlorian Westphal if (unlikely(sk->sk_prot == &tcp_prot)) { 630b4f33deSFlorian Westphal /* we are being invoked after mptcp_accept() has 640b4f33deSFlorian Westphal * accepted a non-mp-capable flow: sk is a tcp_sk, 650b4f33deSFlorian Westphal * not an mptcp one. 660b4f33deSFlorian Westphal * 670b4f33deSFlorian Westphal * Hand the socket over to tcp so all further socket ops 680b4f33deSFlorian Westphal * bypass mptcp. 690b4f33deSFlorian Westphal */ 701ded5e5aSEric Dumazet WRITE_ONCE(sock->ops, &inet_stream_ops); 71d2f77c53SPaolo Abeni return true; 720b4f33deSFlorian Westphal #if IS_ENABLED(CONFIG_MPTCP_IPV6) 730b4f33deSFlorian Westphal } else if (unlikely(sk->sk_prot == &tcpv6_prot)) { 741ded5e5aSEric Dumazet WRITE_ONCE(sock->ops, &inet6_stream_ops); 75d2f77c53SPaolo Abeni return true; 760b4f33deSFlorian Westphal #endif 770b4f33deSFlorian Westphal } 780b4f33deSFlorian Westphal 79d2f77c53SPaolo Abeni return false; 800b4f33deSFlorian Westphal } 810b4f33deSFlorian Westphal 82fa68018dSPaolo Abeni static int __mptcp_socket_create(struct mptcp_sock *msk) 832303f994SPeter Krystad { 842303f994SPeter Krystad struct mptcp_subflow_context *subflow; 852303f994SPeter Krystad struct sock *sk = (struct sock *)msk; 862303f994SPeter Krystad struct socket *ssock; 872303f994SPeter Krystad int err; 882303f994SPeter Krystad 896bc1fe7dSPaolo Abeni err = mptcp_subflow_create_socket(sk, sk->sk_family, &ssock); 902303f994SPeter Krystad if (err) 91fa68018dSPaolo Abeni return err; 922303f994SPeter Krystad 93b8dc6d6cSPaolo Abeni msk->scaling_ratio = tcp_sk(ssock->sk)->scaling_ratio; 941b1b43eeSPaolo Abeni WRITE_ONCE(msk->first, ssock->sk); 952303f994SPeter Krystad subflow = mptcp_subflow_ctx(ssock->sk); 96cec37a6eSPeter Krystad list_add(&subflow->node, &msk->conn_list); 97e16163b6SPaolo Abeni sock_hold(ssock->sk); 982303f994SPeter Krystad subflow->request_mptcp = 1; 996f06b4d4SPaolo Abeni subflow->subflow_id = msk->subflow_id++; 1004cf86ae8SPaolo Abeni 1014cf86ae8SPaolo Abeni /* This is the first subflow, always with id 0 */ 1024cf86ae8SPaolo Abeni subflow->local_id_valid = 1; 103866f26f2SPaolo Abeni mptcp_sock_graft(msk->first, sk->sk_socket); 10439880bd8SPaolo Abeni iput(SOCK_INODE(ssock)); 105e1ff9e82SDavide Caratti 106fa68018dSPaolo Abeni return 0; 1072303f994SPeter Krystad } 1082303f994SPeter Krystad 109ddb1a072SPaolo Abeni /* If the MPC handshake is not started, returns the first subflow, 110ddb1a072SPaolo Abeni * eventually allocating it. 111ddb1a072SPaolo Abeni */ 1123f326a82SPaolo Abeni struct sock *__mptcp_nmpc_sk(struct mptcp_sock *msk) 113ddb1a072SPaolo Abeni { 114ddb1a072SPaolo Abeni struct sock *sk = (struct sock *)msk; 115ddb1a072SPaolo Abeni int ret; 116ddb1a072SPaolo Abeni 117ddb1a072SPaolo Abeni if (!((1 << sk->sk_state) & (TCPF_CLOSE | TCPF_LISTEN))) 118ddb1a072SPaolo Abeni return ERR_PTR(-EINVAL); 119ddb1a072SPaolo Abeni 1203f326a82SPaolo Abeni if (!msk->first) { 121ddb1a072SPaolo Abeni ret = __mptcp_socket_create(msk); 122ddb1a072SPaolo Abeni if (ret) 123ddb1a072SPaolo Abeni return ERR_PTR(ret); 124ddb1a072SPaolo Abeni } 125ddb1a072SPaolo Abeni 1263f326a82SPaolo Abeni return msk->first; 127ddb1a072SPaolo Abeni } 128ddb1a072SPaolo Abeni 129ab174ad8SPaolo Abeni static void mptcp_drop(struct sock *sk, struct sk_buff *skb) 130ab174ad8SPaolo Abeni { 131ab174ad8SPaolo Abeni sk_drops_add(sk, skb); 132ab174ad8SPaolo Abeni __kfree_skb(skb); 133ab174ad8SPaolo Abeni } 134ab174ad8SPaolo Abeni 1359531e4a8SEric Dumazet static void mptcp_rmem_fwd_alloc_add(struct sock *sk, int size) 1369531e4a8SEric Dumazet { 1379531e4a8SEric Dumazet WRITE_ONCE(mptcp_sk(sk)->rmem_fwd_alloc, 1389531e4a8SEric Dumazet mptcp_sk(sk)->rmem_fwd_alloc + size); 1399531e4a8SEric Dumazet } 1409531e4a8SEric Dumazet 1416511882cSPaolo Abeni static void mptcp_rmem_charge(struct sock *sk, int size) 1426511882cSPaolo Abeni { 1439531e4a8SEric Dumazet mptcp_rmem_fwd_alloc_add(sk, -size); 1446511882cSPaolo Abeni } 1456511882cSPaolo Abeni 1468268ed4cSPaolo Abeni static bool mptcp_try_coalesce(struct sock *sk, struct sk_buff *to, 1478268ed4cSPaolo Abeni struct sk_buff *from) 1488268ed4cSPaolo Abeni { 1498268ed4cSPaolo Abeni bool fragstolen; 1508268ed4cSPaolo Abeni int delta; 1518268ed4cSPaolo Abeni 1528268ed4cSPaolo Abeni if (MPTCP_SKB_CB(from)->offset || 1538268ed4cSPaolo Abeni !skb_try_coalesce(to, from, &fragstolen, &delta)) 1548268ed4cSPaolo Abeni return false; 1558268ed4cSPaolo Abeni 15606242e44SPaolo Abeni pr_debug("colesced seq %llx into %llx new len %d new end seq %llx", 15706242e44SPaolo Abeni MPTCP_SKB_CB(from)->map_seq, MPTCP_SKB_CB(to)->map_seq, 15806242e44SPaolo Abeni to->len, MPTCP_SKB_CB(from)->end_seq); 159ab174ad8SPaolo Abeni MPTCP_SKB_CB(to)->end_seq = MPTCP_SKB_CB(from)->end_seq; 1607288ff6eSPaolo Abeni 1617288ff6eSPaolo Abeni /* note the fwd memory can reach a negative value after accounting 1627288ff6eSPaolo Abeni * for the delta, but the later skb free will restore a non 1637288ff6eSPaolo Abeni * negative one 1647288ff6eSPaolo Abeni */ 1658268ed4cSPaolo Abeni atomic_add(delta, &sk->sk_rmem_alloc); 1666511882cSPaolo Abeni mptcp_rmem_charge(sk, delta); 1677288ff6eSPaolo Abeni kfree_skb_partial(from, fragstolen); 1687288ff6eSPaolo Abeni 1698268ed4cSPaolo Abeni return true; 1708268ed4cSPaolo Abeni } 1718268ed4cSPaolo Abeni 172ab174ad8SPaolo Abeni static bool mptcp_ooo_try_coalesce(struct mptcp_sock *msk, struct sk_buff *to, 173ab174ad8SPaolo Abeni struct sk_buff *from) 1746771bfd9SFlorian Westphal { 175ab174ad8SPaolo Abeni if (MPTCP_SKB_CB(from)->map_seq != MPTCP_SKB_CB(to)->end_seq) 176ab174ad8SPaolo Abeni return false; 177ab174ad8SPaolo Abeni 178ab174ad8SPaolo Abeni return mptcp_try_coalesce((struct sock *)msk, to, from); 179ab174ad8SPaolo Abeni } 180ab174ad8SPaolo Abeni 1816511882cSPaolo Abeni static void __mptcp_rmem_reclaim(struct sock *sk, int amount) 1826511882cSPaolo Abeni { 183100fdd1fSEric Dumazet amount >>= PAGE_SHIFT; 1849531e4a8SEric Dumazet mptcp_rmem_charge(sk, amount << PAGE_SHIFT); 1856511882cSPaolo Abeni __sk_mem_reduce_allocated(sk, amount); 1866511882cSPaolo Abeni } 1876511882cSPaolo Abeni 1886511882cSPaolo Abeni static void mptcp_rmem_uncharge(struct sock *sk, int size) 1896511882cSPaolo Abeni { 1906511882cSPaolo Abeni struct mptcp_sock *msk = mptcp_sk(sk); 1916511882cSPaolo Abeni int reclaimable; 1926511882cSPaolo Abeni 1939531e4a8SEric Dumazet mptcp_rmem_fwd_alloc_add(sk, size); 1946511882cSPaolo Abeni reclaimable = msk->rmem_fwd_alloc - sk_unused_reserved_mem(sk); 1956511882cSPaolo Abeni 1966511882cSPaolo Abeni /* see sk_mem_uncharge() for the rationale behind the following schema */ 197d24141feSPaolo Abeni if (unlikely(reclaimable >= PAGE_SIZE)) 198d24141feSPaolo Abeni __mptcp_rmem_reclaim(sk, reclaimable); 1996511882cSPaolo Abeni } 2006511882cSPaolo Abeni 2016511882cSPaolo Abeni static void mptcp_rfree(struct sk_buff *skb) 2026511882cSPaolo Abeni { 2036511882cSPaolo Abeni unsigned int len = skb->truesize; 2046511882cSPaolo Abeni struct sock *sk = skb->sk; 2056511882cSPaolo Abeni 2066511882cSPaolo Abeni atomic_sub(len, &sk->sk_rmem_alloc); 2076511882cSPaolo Abeni mptcp_rmem_uncharge(sk, len); 2086511882cSPaolo Abeni } 2096511882cSPaolo Abeni 21036b122baSDmytro Shytyi void mptcp_set_owner_r(struct sk_buff *skb, struct sock *sk) 2116511882cSPaolo Abeni { 2126511882cSPaolo Abeni skb_orphan(skb); 2136511882cSPaolo Abeni skb->sk = sk; 2146511882cSPaolo Abeni skb->destructor = mptcp_rfree; 2156511882cSPaolo Abeni atomic_add(skb->truesize, &sk->sk_rmem_alloc); 2166511882cSPaolo Abeni mptcp_rmem_charge(sk, skb->truesize); 2176511882cSPaolo Abeni } 2186511882cSPaolo Abeni 219ab174ad8SPaolo Abeni /* "inspired" by tcp_data_queue_ofo(), main differences: 220ab174ad8SPaolo Abeni * - use mptcp seqs 221ab174ad8SPaolo Abeni * - don't cope with sacks 222ab174ad8SPaolo Abeni */ 223ab174ad8SPaolo Abeni static void mptcp_data_queue_ofo(struct mptcp_sock *msk, struct sk_buff *skb) 224ab174ad8SPaolo Abeni { 225ab174ad8SPaolo Abeni struct sock *sk = (struct sock *)msk; 226ab174ad8SPaolo Abeni struct rb_node **p, *parent; 227ab174ad8SPaolo Abeni u64 seq, end_seq, max_seq; 228ab174ad8SPaolo Abeni struct sk_buff *skb1; 229ab174ad8SPaolo Abeni 230ab174ad8SPaolo Abeni seq = MPTCP_SKB_CB(skb)->map_seq; 231ab174ad8SPaolo Abeni end_seq = MPTCP_SKB_CB(skb)->end_seq; 232f3589be0SPaolo Abeni max_seq = atomic64_read(&msk->rcv_wnd_sent); 233ab174ad8SPaolo Abeni 23406242e44SPaolo Abeni pr_debug("msk=%p seq=%llx limit=%llx empty=%d", msk, seq, max_seq, 23506242e44SPaolo Abeni RB_EMPTY_ROOT(&msk->out_of_order_queue)); 236fa3fe2b1SFlorian Westphal if (after64(end_seq, max_seq)) { 237ab174ad8SPaolo Abeni /* out of window */ 238ab174ad8SPaolo Abeni mptcp_drop(sk, skb); 239fa3fe2b1SFlorian Westphal pr_debug("oow by %lld, rcv_wnd_sent %llu\n", 240fa3fe2b1SFlorian Westphal (unsigned long long)end_seq - (unsigned long)max_seq, 241f3589be0SPaolo Abeni (unsigned long long)atomic64_read(&msk->rcv_wnd_sent)); 24206242e44SPaolo Abeni MPTCP_INC_STATS(sock_net(sk), MPTCP_MIB_NODSSWINDOW); 243ab174ad8SPaolo Abeni return; 244ab174ad8SPaolo Abeni } 245ab174ad8SPaolo Abeni 246ab174ad8SPaolo Abeni p = &msk->out_of_order_queue.rb_node; 24706242e44SPaolo Abeni MPTCP_INC_STATS(sock_net(sk), MPTCP_MIB_OFOQUEUE); 248ab174ad8SPaolo Abeni if (RB_EMPTY_ROOT(&msk->out_of_order_queue)) { 249ab174ad8SPaolo Abeni rb_link_node(&skb->rbnode, NULL, p); 250ab174ad8SPaolo Abeni rb_insert_color(&skb->rbnode, &msk->out_of_order_queue); 251ab174ad8SPaolo Abeni msk->ooo_last_skb = skb; 252ab174ad8SPaolo Abeni goto end; 253ab174ad8SPaolo Abeni } 254ab174ad8SPaolo Abeni 255ab174ad8SPaolo Abeni /* with 2 subflows, adding at end of ooo queue is quite likely 256ab174ad8SPaolo Abeni * Use of ooo_last_skb avoids the O(Log(N)) rbtree lookup. 257ab174ad8SPaolo Abeni */ 25806242e44SPaolo Abeni if (mptcp_ooo_try_coalesce(msk, msk->ooo_last_skb, skb)) { 25906242e44SPaolo Abeni MPTCP_INC_STATS(sock_net(sk), MPTCP_MIB_OFOMERGE); 26006242e44SPaolo Abeni MPTCP_INC_STATS(sock_net(sk), MPTCP_MIB_OFOQUEUETAIL); 261ab174ad8SPaolo Abeni return; 26206242e44SPaolo Abeni } 263ab174ad8SPaolo Abeni 264ab174ad8SPaolo Abeni /* Can avoid an rbtree lookup if we are adding skb after ooo_last_skb */ 265ab174ad8SPaolo Abeni if (!before64(seq, MPTCP_SKB_CB(msk->ooo_last_skb)->end_seq)) { 26606242e44SPaolo Abeni MPTCP_INC_STATS(sock_net(sk), MPTCP_MIB_OFOQUEUETAIL); 267ab174ad8SPaolo Abeni parent = &msk->ooo_last_skb->rbnode; 268ab174ad8SPaolo Abeni p = &parent->rb_right; 269ab174ad8SPaolo Abeni goto insert; 270ab174ad8SPaolo Abeni } 271ab174ad8SPaolo Abeni 272ab174ad8SPaolo Abeni /* Find place to insert this segment. Handle overlaps on the way. */ 273ab174ad8SPaolo Abeni parent = NULL; 274ab174ad8SPaolo Abeni while (*p) { 275ab174ad8SPaolo Abeni parent = *p; 276ab174ad8SPaolo Abeni skb1 = rb_to_skb(parent); 277ab174ad8SPaolo Abeni if (before64(seq, MPTCP_SKB_CB(skb1)->map_seq)) { 278ab174ad8SPaolo Abeni p = &parent->rb_left; 279ab174ad8SPaolo Abeni continue; 280ab174ad8SPaolo Abeni } 281ab174ad8SPaolo Abeni if (before64(seq, MPTCP_SKB_CB(skb1)->end_seq)) { 282ab174ad8SPaolo Abeni if (!after64(end_seq, MPTCP_SKB_CB(skb1)->end_seq)) { 283ab174ad8SPaolo Abeni /* All the bits are present. Drop. */ 284ab174ad8SPaolo Abeni mptcp_drop(sk, skb); 28506242e44SPaolo Abeni MPTCP_INC_STATS(sock_net(sk), MPTCP_MIB_DUPDATA); 286ab174ad8SPaolo Abeni return; 287ab174ad8SPaolo Abeni } 288ab174ad8SPaolo Abeni if (after64(seq, MPTCP_SKB_CB(skb1)->map_seq)) { 289ab174ad8SPaolo Abeni /* partial overlap: 290ab174ad8SPaolo Abeni * | skb | 291ab174ad8SPaolo Abeni * | skb1 | 292ab174ad8SPaolo Abeni * continue traversing 293ab174ad8SPaolo Abeni */ 294ab174ad8SPaolo Abeni } else { 295ab174ad8SPaolo Abeni /* skb's seq == skb1's seq and skb covers skb1. 296ab174ad8SPaolo Abeni * Replace skb1 with skb. 297ab174ad8SPaolo Abeni */ 298ab174ad8SPaolo Abeni rb_replace_node(&skb1->rbnode, &skb->rbnode, 299ab174ad8SPaolo Abeni &msk->out_of_order_queue); 300ab174ad8SPaolo Abeni mptcp_drop(sk, skb1); 30106242e44SPaolo Abeni MPTCP_INC_STATS(sock_net(sk), MPTCP_MIB_DUPDATA); 302ab174ad8SPaolo Abeni goto merge_right; 303ab174ad8SPaolo Abeni } 304ab174ad8SPaolo Abeni } else if (mptcp_ooo_try_coalesce(msk, skb1, skb)) { 30506242e44SPaolo Abeni MPTCP_INC_STATS(sock_net(sk), MPTCP_MIB_OFOMERGE); 306ab174ad8SPaolo Abeni return; 307ab174ad8SPaolo Abeni } 308ab174ad8SPaolo Abeni p = &parent->rb_right; 309ab174ad8SPaolo Abeni } 31006242e44SPaolo Abeni 311ab174ad8SPaolo Abeni insert: 312ab174ad8SPaolo Abeni /* Insert segment into RB tree. */ 313ab174ad8SPaolo Abeni rb_link_node(&skb->rbnode, parent, p); 314ab174ad8SPaolo Abeni rb_insert_color(&skb->rbnode, &msk->out_of_order_queue); 315ab174ad8SPaolo Abeni 316ab174ad8SPaolo Abeni merge_right: 317ab174ad8SPaolo Abeni /* Remove other segments covered by skb. */ 318ab174ad8SPaolo Abeni while ((skb1 = skb_rb_next(skb)) != NULL) { 319ab174ad8SPaolo Abeni if (before64(end_seq, MPTCP_SKB_CB(skb1)->end_seq)) 320ab174ad8SPaolo Abeni break; 321ab174ad8SPaolo Abeni rb_erase(&skb1->rbnode, &msk->out_of_order_queue); 322ab174ad8SPaolo Abeni mptcp_drop(sk, skb1); 32306242e44SPaolo Abeni MPTCP_INC_STATS(sock_net(sk), MPTCP_MIB_DUPDATA); 324ab174ad8SPaolo Abeni } 325ab174ad8SPaolo Abeni /* If there is no skb after us, we are the last_skb ! */ 326ab174ad8SPaolo Abeni if (!skb1) 327ab174ad8SPaolo Abeni msk->ooo_last_skb = skb; 328ab174ad8SPaolo Abeni 329ab174ad8SPaolo Abeni end: 330ab174ad8SPaolo Abeni skb_condense(skb); 3316511882cSPaolo Abeni mptcp_set_owner_r(skb, sk); 3326511882cSPaolo Abeni } 3336511882cSPaolo Abeni 3346511882cSPaolo Abeni static bool mptcp_rmem_schedule(struct sock *sk, struct sock *ssk, int size) 3356511882cSPaolo Abeni { 3366511882cSPaolo Abeni struct mptcp_sock *msk = mptcp_sk(sk); 3376511882cSPaolo Abeni int amt, amount; 3386511882cSPaolo Abeni 33969d93daeSPaolo Abeni if (size <= msk->rmem_fwd_alloc) 3406511882cSPaolo Abeni return true; 3416511882cSPaolo Abeni 34269d93daeSPaolo Abeni size -= msk->rmem_fwd_alloc; 3436511882cSPaolo Abeni amt = sk_mem_pages(size); 344100fdd1fSEric Dumazet amount = amt << PAGE_SHIFT; 3454aaa1685SPaolo Abeni if (!__sk_mem_raise_allocated(sk, size, amt, SK_MEM_RECV)) 3466511882cSPaolo Abeni return false; 3476511882cSPaolo Abeni 3489531e4a8SEric Dumazet mptcp_rmem_fwd_alloc_add(sk, amount); 3496511882cSPaolo Abeni return true; 350ab174ad8SPaolo Abeni } 351ab174ad8SPaolo Abeni 352ab174ad8SPaolo Abeni static bool __mptcp_move_skb(struct mptcp_sock *msk, struct sock *ssk, 353ab174ad8SPaolo Abeni struct sk_buff *skb, unsigned int offset, 354ab174ad8SPaolo Abeni size_t copy_len) 355ab174ad8SPaolo Abeni { 356ab174ad8SPaolo Abeni struct mptcp_subflow_context *subflow = mptcp_subflow_ctx(ssk); 3576771bfd9SFlorian Westphal struct sock *sk = (struct sock *)msk; 3584e637c70SFlorian Westphal struct sk_buff *tail; 359b7f653b2SFlorian Westphal bool has_rxtstamp; 3606771bfd9SFlorian Westphal 3616771bfd9SFlorian Westphal __skb_unlink(skb, &ssk->sk_receive_queue); 3624e637c70SFlorian Westphal 3634e637c70SFlorian Westphal skb_ext_reset(skb); 3644e637c70SFlorian Westphal skb_orphan(skb); 3654e637c70SFlorian Westphal 3669c3f94e1SPaolo Abeni /* try to fetch required memory from subflow */ 3676511882cSPaolo Abeni if (!mptcp_rmem_schedule(sk, ssk, skb->truesize)) 3689c3f94e1SPaolo Abeni goto drop; 36972f96132SPaolo Abeni 370b7f653b2SFlorian Westphal has_rxtstamp = TCP_SKB_CB(skb)->has_rxtstamp; 371b7f653b2SFlorian Westphal 372ab174ad8SPaolo Abeni /* the skb map_seq accounts for the skb offset: 373ab174ad8SPaolo Abeni * mptcp_subflow_get_mapped_dsn() is based on the current tp->copied_seq 374ab174ad8SPaolo Abeni * value 375ab174ad8SPaolo Abeni */ 376ab174ad8SPaolo Abeni MPTCP_SKB_CB(skb)->map_seq = mptcp_subflow_get_mapped_dsn(subflow); 377ab174ad8SPaolo Abeni MPTCP_SKB_CB(skb)->end_seq = MPTCP_SKB_CB(skb)->map_seq + copy_len; 378ab174ad8SPaolo Abeni MPTCP_SKB_CB(skb)->offset = offset; 379b7f653b2SFlorian Westphal MPTCP_SKB_CB(skb)->has_rxtstamp = has_rxtstamp; 380ab174ad8SPaolo Abeni 381ab174ad8SPaolo Abeni if (MPTCP_SKB_CB(skb)->map_seq == msk->ack_seq) { 382ab174ad8SPaolo Abeni /* in sequence */ 38338967f42SPaolo Abeni msk->bytes_received += copy_len; 384917944daSMat Martineau WRITE_ONCE(msk->ack_seq, msk->ack_seq + copy_len); 3854e637c70SFlorian Westphal tail = skb_peek_tail(&sk->sk_receive_queue); 3868268ed4cSPaolo Abeni if (tail && mptcp_try_coalesce(sk, tail, skb)) 387ab174ad8SPaolo Abeni return true; 3884e637c70SFlorian Westphal 3896511882cSPaolo Abeni mptcp_set_owner_r(skb, sk); 3906771bfd9SFlorian Westphal __skb_queue_tail(&sk->sk_receive_queue, skb); 391ab174ad8SPaolo Abeni return true; 392ab174ad8SPaolo Abeni } else if (after64(MPTCP_SKB_CB(skb)->map_seq, msk->ack_seq)) { 393ab174ad8SPaolo Abeni mptcp_data_queue_ofo(msk, skb); 394ab174ad8SPaolo Abeni return false; 395ab174ad8SPaolo Abeni } 396ab174ad8SPaolo Abeni 397ab174ad8SPaolo Abeni /* old data, keep it simple and drop the whole pkt, sender 398ab174ad8SPaolo Abeni * will retransmit as needed, if needed. 399ab174ad8SPaolo Abeni */ 40006242e44SPaolo Abeni MPTCP_INC_STATS(sock_net(sk), MPTCP_MIB_DUPDATA); 4019c3f94e1SPaolo Abeni drop: 402ab174ad8SPaolo Abeni mptcp_drop(sk, skb); 403ab174ad8SPaolo Abeni return false; 4046771bfd9SFlorian Westphal } 4056771bfd9SFlorian Westphal 406f6909dc1SPaolo Abeni static void mptcp_stop_rtx_timer(struct sock *sk) 40716a9a9daSMat Martineau { 40816a9a9daSMat Martineau struct inet_connection_sock *icsk = inet_csk(sk); 40916a9a9daSMat Martineau 41016a9a9daSMat Martineau sk_stop_timer(sk, &icsk->icsk_retransmit_timer); 41116a9a9daSMat Martineau mptcp_sk(sk)->timer_ival = 0; 41216a9a9daSMat Martineau } 41316a9a9daSMat Martineau 414e16163b6SPaolo Abeni static void mptcp_close_wake_up(struct sock *sk) 415e16163b6SPaolo Abeni { 416e16163b6SPaolo Abeni if (sock_flag(sk, SOCK_DEAD)) 417e16163b6SPaolo Abeni return; 418e16163b6SPaolo Abeni 419e16163b6SPaolo Abeni sk->sk_state_change(sk); 420e16163b6SPaolo Abeni if (sk->sk_shutdown == SHUTDOWN_MASK || 421e16163b6SPaolo Abeni sk->sk_state == TCP_CLOSE) 422e16163b6SPaolo Abeni sk_wake_async(sk, SOCK_WAKE_WAITD, POLL_HUP); 423e16163b6SPaolo Abeni else 424e16163b6SPaolo Abeni sk_wake_async(sk, SOCK_WAKE_WAITD, POLL_IN); 425e16163b6SPaolo Abeni } 426e16163b6SPaolo Abeni 4276e628cd3SPaolo Abeni static bool mptcp_pending_data_fin_ack(struct sock *sk) 4286e628cd3SPaolo Abeni { 4296e628cd3SPaolo Abeni struct mptcp_sock *msk = mptcp_sk(sk); 4306e628cd3SPaolo Abeni 43181c1d029SPaolo Abeni return ((1 << sk->sk_state) & 4326e628cd3SPaolo Abeni (TCPF_FIN_WAIT1 | TCPF_CLOSING | TCPF_LAST_ACK)) && 4336e628cd3SPaolo Abeni msk->write_seq == READ_ONCE(msk->snd_una); 4346e628cd3SPaolo Abeni } 4356e628cd3SPaolo Abeni 43616a9a9daSMat Martineau static void mptcp_check_data_fin_ack(struct sock *sk) 43716a9a9daSMat Martineau { 43816a9a9daSMat Martineau struct mptcp_sock *msk = mptcp_sk(sk); 43916a9a9daSMat Martineau 44016a9a9daSMat Martineau /* Look for an acknowledged DATA_FIN */ 4416e628cd3SPaolo Abeni if (mptcp_pending_data_fin_ack(sk)) { 44216a9a9daSMat Martineau WRITE_ONCE(msk->snd_data_fin_enable, 0); 44316a9a9daSMat Martineau 44416a9a9daSMat Martineau switch (sk->sk_state) { 44516a9a9daSMat Martineau case TCP_FIN_WAIT1: 44616a9a9daSMat Martineau inet_sk_state_store(sk, TCP_FIN_WAIT2); 44716a9a9daSMat Martineau break; 44816a9a9daSMat Martineau case TCP_CLOSING: 44916a9a9daSMat Martineau case TCP_LAST_ACK: 45016a9a9daSMat Martineau inet_sk_state_store(sk, TCP_CLOSE); 45116a9a9daSMat Martineau break; 45216a9a9daSMat Martineau } 45316a9a9daSMat Martineau 454e16163b6SPaolo Abeni mptcp_close_wake_up(sk); 45516a9a9daSMat Martineau } 45616a9a9daSMat Martineau } 45716a9a9daSMat Martineau 4583721b9b6SMat Martineau static bool mptcp_pending_data_fin(struct sock *sk, u64 *seq) 4593721b9b6SMat Martineau { 4603721b9b6SMat Martineau struct mptcp_sock *msk = mptcp_sk(sk); 4613721b9b6SMat Martineau 4623721b9b6SMat Martineau if (READ_ONCE(msk->rcv_data_fin) && 4633721b9b6SMat Martineau ((1 << sk->sk_state) & 4643721b9b6SMat Martineau (TCPF_ESTABLISHED | TCPF_FIN_WAIT1 | TCPF_FIN_WAIT2))) { 4653721b9b6SMat Martineau u64 rcv_data_fin_seq = READ_ONCE(msk->rcv_data_fin_seq); 4663721b9b6SMat Martineau 4673721b9b6SMat Martineau if (msk->ack_seq == rcv_data_fin_seq) { 4683721b9b6SMat Martineau if (seq) 4693721b9b6SMat Martineau *seq = rcv_data_fin_seq; 4703721b9b6SMat Martineau 4713721b9b6SMat Martineau return true; 4723721b9b6SMat Martineau } 4733721b9b6SMat Martineau } 4743721b9b6SMat Martineau 4753721b9b6SMat Martineau return false; 4763721b9b6SMat Martineau } 4773721b9b6SMat Martineau 478403a40f2SEric Dumazet static void mptcp_set_datafin_timeout(struct sock *sk) 4796477dd39SMat Martineau { 4806477dd39SMat Martineau struct inet_connection_sock *icsk = inet_csk(sk); 481877d11f0SMat Martineau u32 retransmits; 4826477dd39SMat Martineau 483877d11f0SMat Martineau retransmits = min_t(u32, icsk->icsk_retransmits, 484877d11f0SMat Martineau ilog2(TCP_RTO_MAX / TCP_RTO_MIN)); 485877d11f0SMat Martineau 486877d11f0SMat Martineau mptcp_sk(sk)->timer_ival = TCP_RTO_MIN << retransmits; 4876477dd39SMat Martineau } 4886477dd39SMat Martineau 48933d41c9cSPaolo Abeni static void __mptcp_set_timeout(struct sock *sk, long tout) 4903721b9b6SMat Martineau { 4913721b9b6SMat Martineau mptcp_sk(sk)->timer_ival = tout > 0 ? tout : TCP_RTO_MIN; 4923721b9b6SMat Martineau } 4933721b9b6SMat Martineau 49433d41c9cSPaolo Abeni static long mptcp_timeout_from_subflow(const struct mptcp_subflow_context *subflow) 49533d41c9cSPaolo Abeni { 49633d41c9cSPaolo Abeni const struct sock *ssk = mptcp_subflow_tcp_sock(subflow); 49733d41c9cSPaolo Abeni 49871b7dec2SPaolo Abeni return inet_csk(ssk)->icsk_pending && !subflow->stale_count ? 49971b7dec2SPaolo Abeni inet_csk(ssk)->icsk_timeout - jiffies : 0; 50033d41c9cSPaolo Abeni } 50133d41c9cSPaolo Abeni 50233d41c9cSPaolo Abeni static void mptcp_set_timeout(struct sock *sk) 50333d41c9cSPaolo Abeni { 50433d41c9cSPaolo Abeni struct mptcp_subflow_context *subflow; 50533d41c9cSPaolo Abeni long tout = 0; 50633d41c9cSPaolo Abeni 50733d41c9cSPaolo Abeni mptcp_for_each_subflow(mptcp_sk(sk), subflow) 50833d41c9cSPaolo Abeni tout = max(tout, mptcp_timeout_from_subflow(subflow)); 50933d41c9cSPaolo Abeni __mptcp_set_timeout(sk, tout); 51033d41c9cSPaolo Abeni } 51133d41c9cSPaolo Abeni 51276a13b31SGeliang Tang static inline bool tcp_can_send_ack(const struct sock *ssk) 513fd897679SPaolo Abeni { 514fd897679SPaolo Abeni return !((1 << inet_sk_state_load(ssk)) & 51520bc80b6SPaolo Abeni (TCPF_SYN_SENT | TCPF_SYN_RECV | TCPF_TIME_WAIT | TCPF_CLOSE | TCPF_LISTEN)); 516fd897679SPaolo Abeni } 517fd897679SPaolo Abeni 518a6574302SMat Martineau void __mptcp_subflow_send_ack(struct sock *ssk) 519a6574302SMat Martineau { 520a6574302SMat Martineau if (tcp_can_send_ack(ssk)) 521a6574302SMat Martineau tcp_send_ack(ssk); 522a6574302SMat Martineau } 523a6574302SMat Martineau 524f5360e9bSPaolo Abeni static void mptcp_subflow_send_ack(struct sock *ssk) 5257ed90803SPaolo Abeni { 52675e908c3SPaolo Abeni bool slow; 5277ed90803SPaolo Abeni 52875e908c3SPaolo Abeni slow = lock_sock_fast(ssk); 529a6574302SMat Martineau __mptcp_subflow_send_ack(ssk); 53075e908c3SPaolo Abeni unlock_sock_fast(ssk, slow); 531fd897679SPaolo Abeni } 532340fa666SMat Martineau 533340fa666SMat Martineau static void mptcp_send_ack(struct mptcp_sock *msk) 534340fa666SMat Martineau { 535340fa666SMat Martineau struct mptcp_subflow_context *subflow; 536340fa666SMat Martineau 537340fa666SMat Martineau mptcp_for_each_subflow(msk, subflow) 538340fa666SMat Martineau mptcp_subflow_send_ack(mptcp_subflow_tcp_sock(subflow)); 539ea4ca586SPaolo Abeni } 540ea4ca586SPaolo Abeni 541fde56eeaSPaolo Abeni static void mptcp_subflow_cleanup_rbuf(struct sock *ssk) 542fd897679SPaolo Abeni { 54375e908c3SPaolo Abeni bool slow; 544fd897679SPaolo Abeni 54575e908c3SPaolo Abeni slow = lock_sock_fast(ssk); 546fde56eeaSPaolo Abeni if (tcp_can_send_ack(ssk)) 547fd897679SPaolo Abeni tcp_cleanup_rbuf(ssk, 1); 54875e908c3SPaolo Abeni unlock_sock_fast(ssk, slow); 549fde56eeaSPaolo Abeni } 550fde56eeaSPaolo Abeni 551fde56eeaSPaolo Abeni static bool mptcp_subflow_could_cleanup(const struct sock *ssk, bool rx_empty) 552fde56eeaSPaolo Abeni { 553fde56eeaSPaolo Abeni const struct inet_connection_sock *icsk = inet_csk(ssk); 554c4512c63SMatthieu Baerts u8 ack_pending = READ_ONCE(icsk->icsk_ack.pending); 555fde56eeaSPaolo Abeni const struct tcp_sock *tp = tcp_sk(ssk); 556fde56eeaSPaolo Abeni 557fde56eeaSPaolo Abeni return (ack_pending & ICSK_ACK_SCHED) && 558fde56eeaSPaolo Abeni ((READ_ONCE(tp->rcv_nxt) - READ_ONCE(tp->rcv_wup) > 559fde56eeaSPaolo Abeni READ_ONCE(icsk->icsk_ack.rcv_mss)) || 560fde56eeaSPaolo Abeni (rx_empty && ack_pending & 561fde56eeaSPaolo Abeni (ICSK_ACK_PUSHED2 | ICSK_ACK_PUSHED))); 562ea4ca586SPaolo Abeni } 563fd897679SPaolo Abeni 564fd897679SPaolo Abeni static void mptcp_cleanup_rbuf(struct mptcp_sock *msk) 565fd897679SPaolo Abeni { 566e3859603SPaolo Abeni int old_space = READ_ONCE(msk->old_wspace); 567fd897679SPaolo Abeni struct mptcp_subflow_context *subflow; 568e3859603SPaolo Abeni struct sock *sk = (struct sock *)msk; 569fde56eeaSPaolo Abeni int space = __mptcp_space(sk); 570fde56eeaSPaolo Abeni bool cleanup, rx_empty; 571e3859603SPaolo Abeni 572fde56eeaSPaolo Abeni cleanup = (space > 0) && (space >= (old_space << 1)); 573ce599c51SPaolo Abeni rx_empty = !__mptcp_rmem(sk); 574fd897679SPaolo Abeni 575fd897679SPaolo Abeni mptcp_for_each_subflow(msk, subflow) { 576fd897679SPaolo Abeni struct sock *ssk = mptcp_subflow_tcp_sock(subflow); 577fd897679SPaolo Abeni 578fde56eeaSPaolo Abeni if (cleanup || mptcp_subflow_could_cleanup(ssk, rx_empty)) 579fde56eeaSPaolo Abeni mptcp_subflow_cleanup_rbuf(ssk); 5807ed90803SPaolo Abeni } 5817ed90803SPaolo Abeni } 5827ed90803SPaolo Abeni 5837ed90803SPaolo Abeni static bool mptcp_check_data_fin(struct sock *sk) 5843721b9b6SMat Martineau { 5853721b9b6SMat Martineau struct mptcp_sock *msk = mptcp_sk(sk); 5863721b9b6SMat Martineau u64 rcv_data_fin_seq; 5877ed90803SPaolo Abeni bool ret = false; 5883721b9b6SMat Martineau 5893721b9b6SMat Martineau /* Need to ack a DATA_FIN received from a peer while this side 5903721b9b6SMat Martineau * of the connection is in ESTABLISHED, FIN_WAIT1, or FIN_WAIT2. 5913721b9b6SMat Martineau * msk->rcv_data_fin was set when parsing the incoming options 5923721b9b6SMat Martineau * at the subflow level and the msk lock was not held, so this 5933721b9b6SMat Martineau * is the first opportunity to act on the DATA_FIN and change 5943721b9b6SMat Martineau * the msk state. 5953721b9b6SMat Martineau * 5963721b9b6SMat Martineau * If we are caught up to the sequence number of the incoming 5973721b9b6SMat Martineau * DATA_FIN, send the DATA_ACK now and do state transition. If 5983721b9b6SMat Martineau * not caught up, do nothing and let the recv code send DATA_ACK 5993721b9b6SMat Martineau * when catching up. 6003721b9b6SMat Martineau */ 6013721b9b6SMat Martineau 6023721b9b6SMat Martineau if (mptcp_pending_data_fin(sk, &rcv_data_fin_seq)) { 603917944daSMat Martineau WRITE_ONCE(msk->ack_seq, msk->ack_seq + 1); 6043721b9b6SMat Martineau WRITE_ONCE(msk->rcv_data_fin, 0); 6053721b9b6SMat Martineau 6066b9831bfSPaolo Abeni WRITE_ONCE(sk->sk_shutdown, sk->sk_shutdown | RCV_SHUTDOWN); 60716a9a9daSMat Martineau smp_mb__before_atomic(); /* SHUTDOWN must be visible first */ 6083721b9b6SMat Martineau 6093721b9b6SMat Martineau switch (sk->sk_state) { 6103721b9b6SMat Martineau case TCP_ESTABLISHED: 6113721b9b6SMat Martineau inet_sk_state_store(sk, TCP_CLOSE_WAIT); 6123721b9b6SMat Martineau break; 6133721b9b6SMat Martineau case TCP_FIN_WAIT1: 6143721b9b6SMat Martineau inet_sk_state_store(sk, TCP_CLOSING); 6153721b9b6SMat Martineau break; 6163721b9b6SMat Martineau case TCP_FIN_WAIT2: 6173721b9b6SMat Martineau inet_sk_state_store(sk, TCP_CLOSE); 6183721b9b6SMat Martineau break; 6193721b9b6SMat Martineau default: 6203721b9b6SMat Martineau /* Other states not expected */ 6213721b9b6SMat Martineau WARN_ON_ONCE(1); 6223721b9b6SMat Martineau break; 6233721b9b6SMat Martineau } 6243721b9b6SMat Martineau 6257ed90803SPaolo Abeni ret = true; 62681c1d029SPaolo Abeni if (!__mptcp_check_fallback(msk)) 627fd897679SPaolo Abeni mptcp_send_ack(msk); 628e16163b6SPaolo Abeni mptcp_close_wake_up(sk); 6293721b9b6SMat Martineau } 6307ed90803SPaolo Abeni return ret; 6313721b9b6SMat Martineau } 6323721b9b6SMat Martineau 6336771bfd9SFlorian Westphal static bool __mptcp_move_skbs_from_subflow(struct mptcp_sock *msk, 6346771bfd9SFlorian Westphal struct sock *ssk, 6356771bfd9SFlorian Westphal unsigned int *bytes) 6366771bfd9SFlorian Westphal { 6376771bfd9SFlorian Westphal struct mptcp_subflow_context *subflow = mptcp_subflow_ctx(ssk); 638600911ffSFlorian Westphal struct sock *sk = (struct sock *)msk; 6396771bfd9SFlorian Westphal unsigned int moved = 0; 6406771bfd9SFlorian Westphal bool more_data_avail; 6416771bfd9SFlorian Westphal struct tcp_sock *tp; 6426771bfd9SFlorian Westphal bool done = false; 64313c7ba0cSFlorian Westphal int sk_rbuf; 64413c7ba0cSFlorian Westphal 64513c7ba0cSFlorian Westphal sk_rbuf = READ_ONCE(sk->sk_rcvbuf); 64613c7ba0cSFlorian Westphal 64713c7ba0cSFlorian Westphal if (!(sk->sk_userlocks & SOCK_RCVBUF_LOCK)) { 64813c7ba0cSFlorian Westphal int ssk_rbuf = READ_ONCE(ssk->sk_rcvbuf); 64913c7ba0cSFlorian Westphal 65013c7ba0cSFlorian Westphal if (unlikely(ssk_rbuf > sk_rbuf)) { 65113c7ba0cSFlorian Westphal WRITE_ONCE(sk->sk_rcvbuf, ssk_rbuf); 65213c7ba0cSFlorian Westphal sk_rbuf = ssk_rbuf; 65313c7ba0cSFlorian Westphal } 65413c7ba0cSFlorian Westphal } 655600911ffSFlorian Westphal 656ab174ad8SPaolo Abeni pr_debug("msk=%p ssk=%p", msk, ssk); 6576771bfd9SFlorian Westphal tp = tcp_sk(ssk); 6586771bfd9SFlorian Westphal do { 6596771bfd9SFlorian Westphal u32 map_remaining, offset; 6606771bfd9SFlorian Westphal u32 seq = tp->copied_seq; 6616771bfd9SFlorian Westphal struct sk_buff *skb; 6626771bfd9SFlorian Westphal bool fin; 6636771bfd9SFlorian Westphal 6646771bfd9SFlorian Westphal /* try to move as much data as available */ 6656771bfd9SFlorian Westphal map_remaining = subflow->map_data_len - 6666771bfd9SFlorian Westphal mptcp_subflow_get_map_offset(subflow); 6676771bfd9SFlorian Westphal 6686771bfd9SFlorian Westphal skb = skb_peek(&ssk->sk_receive_queue); 669d9fb8c50SPaolo Abeni if (!skb) { 670d89e3ed7SPaolo Abeni /* With racing move_skbs_to_msk() and __mptcp_move_skbs(), 671d89e3ed7SPaolo Abeni * a different CPU can have already processed the pending 672d89e3ed7SPaolo Abeni * data, stop here or we can enter an infinite loop 673d9fb8c50SPaolo Abeni */ 674d9fb8c50SPaolo Abeni if (!moved) 675d9fb8c50SPaolo Abeni done = true; 6766771bfd9SFlorian Westphal break; 677d9fb8c50SPaolo Abeni } 6786771bfd9SFlorian Westphal 679e1ff9e82SDavide Caratti if (__mptcp_check_fallback(msk)) { 680d89e3ed7SPaolo Abeni /* Under fallback skbs have no MPTCP extension and TCP could 681d89e3ed7SPaolo Abeni * collapse them between the dummy map creation and the 682d89e3ed7SPaolo Abeni * current dequeue. Be sure to adjust the map size. 683e1ff9e82SDavide Caratti */ 684e1ff9e82SDavide Caratti map_remaining = skb->len; 685e1ff9e82SDavide Caratti subflow->map_data_len = skb->len; 686e1ff9e82SDavide Caratti } 687e1ff9e82SDavide Caratti 6886771bfd9SFlorian Westphal offset = seq - TCP_SKB_CB(skb)->seq; 6896771bfd9SFlorian Westphal fin = TCP_SKB_CB(skb)->tcp_flags & TCPHDR_FIN; 6906771bfd9SFlorian Westphal if (fin) { 6916771bfd9SFlorian Westphal done = true; 6926771bfd9SFlorian Westphal seq++; 6936771bfd9SFlorian Westphal } 6946771bfd9SFlorian Westphal 6956771bfd9SFlorian Westphal if (offset < skb->len) { 6966771bfd9SFlorian Westphal size_t len = skb->len - offset; 6976771bfd9SFlorian Westphal 6986771bfd9SFlorian Westphal if (tp->urg_data) 6996771bfd9SFlorian Westphal done = true; 7006771bfd9SFlorian Westphal 701ab174ad8SPaolo Abeni if (__mptcp_move_skb(msk, ssk, skb, offset, len)) 7026771bfd9SFlorian Westphal moved += len; 703ab174ad8SPaolo Abeni seq += len; 7046771bfd9SFlorian Westphal 7056771bfd9SFlorian Westphal if (WARN_ON_ONCE(map_remaining < len)) 7066771bfd9SFlorian Westphal break; 7076771bfd9SFlorian Westphal } else { 7086771bfd9SFlorian Westphal WARN_ON_ONCE(!fin); 7096771bfd9SFlorian Westphal sk_eat_skb(ssk, skb); 7106771bfd9SFlorian Westphal done = true; 7116771bfd9SFlorian Westphal } 7126771bfd9SFlorian Westphal 7136771bfd9SFlorian Westphal WRITE_ONCE(tp->copied_seq, seq); 7146771bfd9SFlorian Westphal more_data_avail = mptcp_subflow_data_available(ssk); 715600911ffSFlorian Westphal 71613c7ba0cSFlorian Westphal if (atomic_read(&sk->sk_rmem_alloc) > sk_rbuf) { 717600911ffSFlorian Westphal done = true; 718600911ffSFlorian Westphal break; 719600911ffSFlorian Westphal } 7206771bfd9SFlorian Westphal } while (more_data_avail); 7216771bfd9SFlorian Westphal 7226719331cSPaolo Abeni *bytes += moved; 7236771bfd9SFlorian Westphal return done; 7246771bfd9SFlorian Westphal } 7256771bfd9SFlorian Westphal 72687952603SPaolo Abeni static bool __mptcp_ofo_queue(struct mptcp_sock *msk) 727ab174ad8SPaolo Abeni { 728ab174ad8SPaolo Abeni struct sock *sk = (struct sock *)msk; 729ab174ad8SPaolo Abeni struct sk_buff *skb, *tail; 730ab174ad8SPaolo Abeni bool moved = false; 731ab174ad8SPaolo Abeni struct rb_node *p; 732ab174ad8SPaolo Abeni u64 end_seq; 733ab174ad8SPaolo Abeni 734ab174ad8SPaolo Abeni p = rb_first(&msk->out_of_order_queue); 73506242e44SPaolo Abeni pr_debug("msk=%p empty=%d", msk, RB_EMPTY_ROOT(&msk->out_of_order_queue)); 736ab174ad8SPaolo Abeni while (p) { 737ab174ad8SPaolo Abeni skb = rb_to_skb(p); 738ab174ad8SPaolo Abeni if (after64(MPTCP_SKB_CB(skb)->map_seq, msk->ack_seq)) 739ab174ad8SPaolo Abeni break; 740ab174ad8SPaolo Abeni 741ab174ad8SPaolo Abeni p = rb_next(p); 742ab174ad8SPaolo Abeni rb_erase(&skb->rbnode, &msk->out_of_order_queue); 743ab174ad8SPaolo Abeni 744ab174ad8SPaolo Abeni if (unlikely(!after64(MPTCP_SKB_CB(skb)->end_seq, 745ab174ad8SPaolo Abeni msk->ack_seq))) { 746ab174ad8SPaolo Abeni mptcp_drop(sk, skb); 74706242e44SPaolo Abeni MPTCP_INC_STATS(sock_net(sk), MPTCP_MIB_DUPDATA); 748ab174ad8SPaolo Abeni continue; 749ab174ad8SPaolo Abeni } 750ab174ad8SPaolo Abeni 751ab174ad8SPaolo Abeni end_seq = MPTCP_SKB_CB(skb)->end_seq; 752ab174ad8SPaolo Abeni tail = skb_peek_tail(&sk->sk_receive_queue); 753ab174ad8SPaolo Abeni if (!tail || !mptcp_ooo_try_coalesce(msk, tail, skb)) { 754ab174ad8SPaolo Abeni int delta = msk->ack_seq - MPTCP_SKB_CB(skb)->map_seq; 755ab174ad8SPaolo Abeni 756ab174ad8SPaolo Abeni /* skip overlapping data, if any */ 75706242e44SPaolo Abeni pr_debug("uncoalesced seq=%llx ack seq=%llx delta=%d", 75806242e44SPaolo Abeni MPTCP_SKB_CB(skb)->map_seq, msk->ack_seq, 75906242e44SPaolo Abeni delta); 760ab174ad8SPaolo Abeni MPTCP_SKB_CB(skb)->offset += delta; 7612c9e7765SFlorian Westphal MPTCP_SKB_CB(skb)->map_seq += delta; 762ab174ad8SPaolo Abeni __skb_queue_tail(&sk->sk_receive_queue, skb); 763ab174ad8SPaolo Abeni } 76438967f42SPaolo Abeni msk->bytes_received += end_seq - msk->ack_seq; 765ab174ad8SPaolo Abeni msk->ack_seq = end_seq; 766ab174ad8SPaolo Abeni moved = true; 767ab174ad8SPaolo Abeni } 768ab174ad8SPaolo Abeni return moved; 769ab174ad8SPaolo Abeni } 770ab174ad8SPaolo Abeni 7719f1a9881SPaolo Abeni static bool __mptcp_subflow_error_report(struct sock *sk, struct sock *ssk) 772d5fbeff1SPaolo Abeni { 773d5fbeff1SPaolo Abeni int err = sock_error(ssk); 774d5fbeff1SPaolo Abeni int ssk_state; 775d5fbeff1SPaolo Abeni 776d5fbeff1SPaolo Abeni if (!err) 7779f1a9881SPaolo Abeni return false; 778d5fbeff1SPaolo Abeni 779d5fbeff1SPaolo Abeni /* only propagate errors on fallen-back sockets or 780d5fbeff1SPaolo Abeni * on MPC connect 781d5fbeff1SPaolo Abeni */ 7829f1a9881SPaolo Abeni if (sk->sk_state != TCP_SYN_SENT && !__mptcp_check_fallback(mptcp_sk(sk))) 7839f1a9881SPaolo Abeni return false; 784d5fbeff1SPaolo Abeni 785d5fbeff1SPaolo Abeni /* We need to propagate only transition to CLOSE state. 786d5fbeff1SPaolo Abeni * Orphaned socket will see such state change via 787d5fbeff1SPaolo Abeni * subflow_sched_work_if_closed() and that path will properly 788d5fbeff1SPaolo Abeni * destroy the msk as needed. 789d5fbeff1SPaolo Abeni */ 790d5fbeff1SPaolo Abeni ssk_state = inet_sk_state_load(ssk); 791d5fbeff1SPaolo Abeni if (ssk_state == TCP_CLOSE && !sock_flag(sk, SOCK_DEAD)) 792d5fbeff1SPaolo Abeni inet_sk_state_store(sk, ssk_state); 793d5fbeff1SPaolo Abeni WRITE_ONCE(sk->sk_err, -err); 794d5fbeff1SPaolo Abeni 795d5fbeff1SPaolo Abeni /* This barrier is coupled with smp_rmb() in mptcp_poll() */ 796d5fbeff1SPaolo Abeni smp_wmb(); 797d5fbeff1SPaolo Abeni sk_error_report(sk); 7989f1a9881SPaolo Abeni return true; 799d5fbeff1SPaolo Abeni } 8009f1a9881SPaolo Abeni 8019f1a9881SPaolo Abeni void __mptcp_error_report(struct sock *sk) 8029f1a9881SPaolo Abeni { 8039f1a9881SPaolo Abeni struct mptcp_subflow_context *subflow; 8049f1a9881SPaolo Abeni struct mptcp_sock *msk = mptcp_sk(sk); 8059f1a9881SPaolo Abeni 8069f1a9881SPaolo Abeni mptcp_for_each_subflow(msk, subflow) 8079f1a9881SPaolo Abeni if (__mptcp_subflow_error_report(sk, mptcp_subflow_tcp_sock(subflow))) 8089f1a9881SPaolo Abeni break; 809d5fbeff1SPaolo Abeni } 810d5fbeff1SPaolo Abeni 8112e52213cSFlorian Westphal /* In most cases we will be able to lock the mptcp socket. If its already 8122e52213cSFlorian Westphal * owned, we need to defer to the work queue to avoid ABBA deadlock. 8132e52213cSFlorian Westphal */ 81499d1055cSPaolo Abeni static bool move_skbs_to_msk(struct mptcp_sock *msk, struct sock *ssk) 8152e52213cSFlorian Westphal { 8162e52213cSFlorian Westphal struct sock *sk = (struct sock *)msk; 8172e52213cSFlorian Westphal unsigned int moved = 0; 8182e52213cSFlorian Westphal 8192e52213cSFlorian Westphal __mptcp_move_skbs_from_subflow(msk, ssk, &moved); 82087952603SPaolo Abeni __mptcp_ofo_queue(msk); 821499ada50SPaolo Abeni if (unlikely(ssk->sk_err)) { 822499ada50SPaolo Abeni if (!sock_owned_by_user(sk)) 823499ada50SPaolo Abeni __mptcp_error_report(sk); 824499ada50SPaolo Abeni else 825e9d09bacSPaolo Abeni __set_bit(MPTCP_ERROR_REPORT, &msk->cb_flags); 826499ada50SPaolo Abeni } 827ab174ad8SPaolo Abeni 828ab174ad8SPaolo Abeni /* If the moves have caught up with the DATA_FIN sequence number 829ab174ad8SPaolo Abeni * it's time to ack the DATA_FIN and change socket state, but 830ab174ad8SPaolo Abeni * this is not a good place to change state. Let the workqueue 831ab174ad8SPaolo Abeni * do it. 832ab174ad8SPaolo Abeni */ 833ba8f48f7SPaolo Abeni if (mptcp_pending_data_fin(sk, NULL)) 834ba8f48f7SPaolo Abeni mptcp_schedule_work(sk); 83599d1055cSPaolo Abeni return moved > 0; 8362e52213cSFlorian Westphal } 8372e52213cSFlorian Westphal 8382e52213cSFlorian Westphal void mptcp_data_ready(struct sock *sk, struct sock *ssk) 839101f6f85SFlorian Westphal { 8406719331cSPaolo Abeni struct mptcp_subflow_context *subflow = mptcp_subflow_ctx(ssk); 841101f6f85SFlorian Westphal struct mptcp_sock *msk = mptcp_sk(sk); 84213c7ba0cSFlorian Westphal int sk_rbuf, ssk_rbuf; 843101f6f85SFlorian Westphal 844d7b1bfd0SPaolo Abeni /* The peer can send data while we are shutting down this 845d7b1bfd0SPaolo Abeni * subflow at msk destruction time, but we must avoid enqueuing 846d7b1bfd0SPaolo Abeni * more data to the msk receive queue 847d7b1bfd0SPaolo Abeni */ 848d7b1bfd0SPaolo Abeni if (unlikely(subflow->disposable)) 849d7b1bfd0SPaolo Abeni return; 850d7b1bfd0SPaolo Abeni 85113c7ba0cSFlorian Westphal ssk_rbuf = READ_ONCE(ssk->sk_rcvbuf); 85213c7ba0cSFlorian Westphal sk_rbuf = READ_ONCE(sk->sk_rcvbuf); 85313c7ba0cSFlorian Westphal if (unlikely(ssk_rbuf > sk_rbuf)) 85413c7ba0cSFlorian Westphal sk_rbuf = ssk_rbuf; 85513c7ba0cSFlorian Westphal 85699d1055cSPaolo Abeni /* over limit? can't append more skbs to msk, Also, no need to wake-up*/ 857ce599c51SPaolo Abeni if (__mptcp_rmem(sk) > sk_rbuf) { 858ce599c51SPaolo Abeni MPTCP_INC_STATS(sock_net(sk), MPTCP_MIB_RCVPRUNED); 85999d1055cSPaolo Abeni return; 860ce599c51SPaolo Abeni } 8612e52213cSFlorian Westphal 86299d1055cSPaolo Abeni /* Wake-up the reader only for in-sequence data */ 86399d1055cSPaolo Abeni mptcp_data_lock(sk); 8645684ab1aSPaolo Abeni if (move_skbs_to_msk(msk, ssk) && mptcp_epollin_ready(sk)) 865101f6f85SFlorian Westphal sk->sk_data_ready(sk); 86699d1055cSPaolo Abeni mptcp_data_unlock(sk); 86799d1055cSPaolo Abeni } 868101f6f85SFlorian Westphal 86955b47ca7SPaolo Abeni static void mptcp_subflow_joined(struct mptcp_sock *msk, struct sock *ssk) 87055b47ca7SPaolo Abeni { 87155b47ca7SPaolo Abeni mptcp_subflow_ctx(ssk)->map_seq = READ_ONCE(msk->ack_seq); 87255b47ca7SPaolo Abeni WRITE_ONCE(msk->allow_infinite_fallback, false); 87355b47ca7SPaolo Abeni mptcp_event(MPTCP_EVENT_SUB_ESTABLISHED, msk, ssk, GFP_ATOMIC); 87455b47ca7SPaolo Abeni } 87555b47ca7SPaolo Abeni 8763e501490SPaolo Abeni static bool __mptcp_finish_join(struct mptcp_sock *msk, struct sock *ssk) 877ec3edaa7SPeter Krystad { 8783e501490SPaolo Abeni struct sock *sk = (struct sock *)msk; 8795cf92bbaSPaolo Abeni 8803e501490SPaolo Abeni if (sk->sk_state != TCP_ESTABLISHED) 88178962489SFlorian Westphal return false; 882ec3edaa7SPeter Krystad 8833e501490SPaolo Abeni /* attach to msk socket only after we are sure we will deal with it 8843e501490SPaolo Abeni * at close time 8853e501490SPaolo Abeni */ 8863e501490SPaolo Abeni if (sk->sk_socket && !ssk->sk_socket) 8873e501490SPaolo Abeni mptcp_sock_graft(ssk, sk->sk_socket); 88878962489SFlorian Westphal 8896f06b4d4SPaolo Abeni mptcp_subflow_ctx(ssk)->subflow_id = msk->subflow_id++; 8903e501490SPaolo Abeni mptcp_sockopt_sync_locked(msk, ssk); 89155b47ca7SPaolo Abeni mptcp_subflow_joined(msk, ssk); 89227e5ccc2SPaolo Abeni mptcp_stop_tout_timer(sk); 8938005184fSPaolo Abeni __mptcp_propagate_sndbuf(sk, ssk); 8943e501490SPaolo Abeni return true; 89578962489SFlorian Westphal } 89678962489SFlorian Westphal 89756a666c4SPaolo Abeni static void __mptcp_flush_join_list(struct sock *sk, struct list_head *join_list) 89878962489SFlorian Westphal { 8993e501490SPaolo Abeni struct mptcp_subflow_context *tmp, *subflow; 9003e501490SPaolo Abeni struct mptcp_sock *msk = mptcp_sk(sk); 90178962489SFlorian Westphal 90256a666c4SPaolo Abeni list_for_each_entry_safe(subflow, tmp, join_list, node) { 9033e501490SPaolo Abeni struct sock *ssk = mptcp_subflow_tcp_sock(subflow); 9043e501490SPaolo Abeni bool slow = lock_sock_fast(ssk); 9053e501490SPaolo Abeni 9063e501490SPaolo Abeni list_move_tail(&subflow->node, &msk->conn_list); 9073e501490SPaolo Abeni if (!__mptcp_finish_join(msk, ssk)) 9083e501490SPaolo Abeni mptcp_subflow_reset(ssk); 9093e501490SPaolo Abeni unlock_sock_fast(ssk, slow); 91078962489SFlorian Westphal } 911ec3edaa7SPeter Krystad } 912ec3edaa7SPeter Krystad 913f6909dc1SPaolo Abeni static bool mptcp_rtx_timer_pending(struct sock *sk) 914b51f9b80SPaolo Abeni { 915b51f9b80SPaolo Abeni return timer_pending(&inet_csk(sk)->icsk_retransmit_timer); 916b51f9b80SPaolo Abeni } 917b51f9b80SPaolo Abeni 918f6909dc1SPaolo Abeni static void mptcp_reset_rtx_timer(struct sock *sk) 919b51f9b80SPaolo Abeni { 920b51f9b80SPaolo Abeni struct inet_connection_sock *icsk = inet_csk(sk); 921b51f9b80SPaolo Abeni unsigned long tout; 922b51f9b80SPaolo Abeni 923e16163b6SPaolo Abeni /* prevent rescheduling on close */ 924e16163b6SPaolo Abeni if (unlikely(inet_sk_state_load(sk) == TCP_CLOSE)) 925e16163b6SPaolo Abeni return; 926e16163b6SPaolo Abeni 92733d41c9cSPaolo Abeni tout = mptcp_sk(sk)->timer_ival; 928b51f9b80SPaolo Abeni sk_reset_timer(sk, &icsk->icsk_retransmit_timer, jiffies + tout); 929b51f9b80SPaolo Abeni } 930b51f9b80SPaolo Abeni 931ba8f48f7SPaolo Abeni bool mptcp_schedule_work(struct sock *sk) 932ba8f48f7SPaolo Abeni { 933ba8f48f7SPaolo Abeni if (inet_sk_state_load(sk) != TCP_CLOSE && 934ba8f48f7SPaolo Abeni schedule_work(&mptcp_sk(sk)->work)) { 935ba8f48f7SPaolo Abeni /* each subflow already holds a reference to the sk, and the 936ba8f48f7SPaolo Abeni * workqueue is invoked by a subflow, so sk can't go away here. 937ba8f48f7SPaolo Abeni */ 938ba8f48f7SPaolo Abeni sock_hold(sk); 939ba8f48f7SPaolo Abeni return true; 940ba8f48f7SPaolo Abeni } 941ba8f48f7SPaolo Abeni return false; 942ba8f48f7SPaolo Abeni } 943ba8f48f7SPaolo Abeni 9447a6a6cbcSPaolo Abeni static struct sock *mptcp_subflow_recv_lookup(const struct mptcp_sock *msk) 9457a6a6cbcSPaolo Abeni { 9467a6a6cbcSPaolo Abeni struct mptcp_subflow_context *subflow; 9477a6a6cbcSPaolo Abeni 948109cdeb8SGeliang Tang msk_owned_by_me(msk); 9497a6a6cbcSPaolo Abeni 9507a6a6cbcSPaolo Abeni mptcp_for_each_subflow(msk, subflow) { 95199d1055cSPaolo Abeni if (READ_ONCE(subflow->data_avail)) 9527a6a6cbcSPaolo Abeni return mptcp_subflow_tcp_sock(subflow); 9537a6a6cbcSPaolo Abeni } 9547a6a6cbcSPaolo Abeni 9557a6a6cbcSPaolo Abeni return NULL; 9567a6a6cbcSPaolo Abeni } 9577a6a6cbcSPaolo Abeni 9583f8e0aaeSPaolo Abeni static bool mptcp_skb_can_collapse_to(u64 write_seq, 95957040755SPaolo Abeni const struct sk_buff *skb, 96057040755SPaolo Abeni const struct mptcp_ext *mpext) 9616d0060f6SMat Martineau { 96257040755SPaolo Abeni if (!tcp_skb_can_collapse_to(skb)) 96357040755SPaolo Abeni return false; 96457040755SPaolo Abeni 9655a369ca6SPaolo Abeni /* can collapse only if MPTCP level sequence is in order and this 9665a369ca6SPaolo Abeni * mapping has not been xmitted yet 9675a369ca6SPaolo Abeni */ 9685a369ca6SPaolo Abeni return mpext && mpext->data_seq + mpext->data_len == write_seq && 9695a369ca6SPaolo Abeni !mpext->frozen; 97057040755SPaolo Abeni } 97157040755SPaolo Abeni 97229249eacSPaolo Abeni /* we can append data to the given data frag if: 97329249eacSPaolo Abeni * - there is space available in the backing page_frag 97429249eacSPaolo Abeni * - the data frag tail matches the current page_frag free offset 97529249eacSPaolo Abeni * - the data frag end sequence number matches the current write seq 97629249eacSPaolo Abeni */ 97718b683bfSPaolo Abeni static bool mptcp_frag_can_collapse_to(const struct mptcp_sock *msk, 97818b683bfSPaolo Abeni const struct page_frag *pfrag, 97918b683bfSPaolo Abeni const struct mptcp_data_frag *df) 98018b683bfSPaolo Abeni { 98118b683bfSPaolo Abeni return df && pfrag->page == df->page && 982d9ca1de8SPaolo Abeni pfrag->size - pfrag->offset > 0 && 98329249eacSPaolo Abeni pfrag->offset == (df->offset + df->data_len) && 98418b683bfSPaolo Abeni df->data_seq + df->data_len == msk->write_seq; 98518b683bfSPaolo Abeni } 98618b683bfSPaolo Abeni 987d027236cSPaolo Abeni static void dfrag_uncharge(struct sock *sk, int len) 98818b683bfSPaolo Abeni { 989d027236cSPaolo Abeni sk_mem_uncharge(sk, len); 9907948f6ccSFlorian Westphal sk_wmem_queued_add(sk, -len); 991d027236cSPaolo Abeni } 992d027236cSPaolo Abeni 993d027236cSPaolo Abeni static void dfrag_clear(struct sock *sk, struct mptcp_data_frag *dfrag) 994d027236cSPaolo Abeni { 995d027236cSPaolo Abeni int len = dfrag->data_len + dfrag->overhead; 996d027236cSPaolo Abeni 99718b683bfSPaolo Abeni list_del(&dfrag->list); 998d027236cSPaolo Abeni dfrag_uncharge(sk, len); 99918b683bfSPaolo Abeni put_page(dfrag->page); 100018b683bfSPaolo Abeni } 100118b683bfSPaolo Abeni 10026e628cd3SPaolo Abeni static void __mptcp_clean_una(struct sock *sk) 100318b683bfSPaolo Abeni { 100418b683bfSPaolo Abeni struct mptcp_sock *msk = mptcp_sk(sk); 100518b683bfSPaolo Abeni struct mptcp_data_frag *dtmp, *dfrag; 1006e1ff9e82SDavide Caratti u64 snd_una; 1007e1ff9e82SDavide Caratti 10087439d687SPaolo Abeni snd_una = msk->snd_una; 100918b683bfSPaolo Abeni list_for_each_entry_safe(dfrag, dtmp, &msk->rtx_queue, list) { 101018b683bfSPaolo Abeni if (after64(dfrag->data_seq + dfrag->data_len, snd_una)) 101118b683bfSPaolo Abeni break; 101218b683bfSPaolo Abeni 10131e1d9d6fSPaolo Abeni if (unlikely(dfrag == msk->first_pending)) { 10141e1d9d6fSPaolo Abeni /* in recovery mode can see ack after the current snd head */ 10151e1d9d6fSPaolo Abeni if (WARN_ON_ONCE(!msk->recovery)) 1016d9ca1de8SPaolo Abeni break; 10171e1d9d6fSPaolo Abeni 10181e1d9d6fSPaolo Abeni WRITE_ONCE(msk->first_pending, mptcp_send_next(sk)); 10191e1d9d6fSPaolo Abeni } 10201e1d9d6fSPaolo Abeni 1021d027236cSPaolo Abeni dfrag_clear(sk, dfrag); 1022d027236cSPaolo Abeni } 1023d027236cSPaolo Abeni 10247948f6ccSFlorian Westphal dfrag = mptcp_rtx_head(sk); 10257948f6ccSFlorian Westphal if (dfrag && after64(snd_una, dfrag->data_seq)) { 102653eb4c38SPaolo Abeni u64 delta = snd_una - dfrag->data_seq; 102753eb4c38SPaolo Abeni 10281e1d9d6fSPaolo Abeni /* prevent wrap around in recovery mode */ 10291e1d9d6fSPaolo Abeni if (unlikely(delta > dfrag->already_sent)) { 10301e1d9d6fSPaolo Abeni if (WARN_ON_ONCE(!msk->recovery)) 103153eb4c38SPaolo Abeni goto out; 10321e1d9d6fSPaolo Abeni if (WARN_ON_ONCE(delta > dfrag->data_len)) 10331e1d9d6fSPaolo Abeni goto out; 10341e1d9d6fSPaolo Abeni dfrag->already_sent += delta - dfrag->already_sent; 10351e1d9d6fSPaolo Abeni } 10367948f6ccSFlorian Westphal 10377948f6ccSFlorian Westphal dfrag->data_seq += delta; 103853eb4c38SPaolo Abeni dfrag->offset += delta; 10397948f6ccSFlorian Westphal dfrag->data_len -= delta; 1040d9ca1de8SPaolo Abeni dfrag->already_sent -= delta; 10417948f6ccSFlorian Westphal 10427948f6ccSFlorian Westphal dfrag_uncharge(sk, delta); 10437948f6ccSFlorian Westphal } 10447948f6ccSFlorian Westphal 10451e1d9d6fSPaolo Abeni /* all retransmitted data acked, recovery completed */ 10461e1d9d6fSPaolo Abeni if (unlikely(msk->recovery) && after64(msk->snd_una, msk->recovery_snd_nxt)) 10471e1d9d6fSPaolo Abeni msk->recovery = false; 10481e1d9d6fSPaolo Abeni 104953eb4c38SPaolo Abeni out: 10503241a9c0SFlorian Westphal if (snd_una == READ_ONCE(msk->snd_nxt) && 10513241a9c0SFlorian Westphal snd_una == READ_ONCE(msk->write_seq)) { 1052f6909dc1SPaolo Abeni if (mptcp_rtx_timer_pending(sk) && !mptcp_data_fin_enabled(msk)) 1053f6909dc1SPaolo Abeni mptcp_stop_rtx_timer(sk); 10546e628cd3SPaolo Abeni } else { 1055f6909dc1SPaolo Abeni mptcp_reset_rtx_timer(sk); 10566e628cd3SPaolo Abeni } 10576e628cd3SPaolo Abeni } 10586e628cd3SPaolo Abeni 1059417789dfSPaolo Abeni static void __mptcp_clean_una_wakeup(struct sock *sk) 1060417789dfSPaolo Abeni { 1061765ff425SPaolo Abeni lockdep_assert_held_once(&sk->sk_lock.slock); 1062765ff425SPaolo Abeni 1063417789dfSPaolo Abeni __mptcp_clean_una(sk); 1064417789dfSPaolo Abeni mptcp_write_space(sk); 1065417789dfSPaolo Abeni } 1066417789dfSPaolo Abeni 1067b5941f06SPaolo Abeni static void mptcp_clean_una_wakeup(struct sock *sk) 1068b5941f06SPaolo Abeni { 1069b5941f06SPaolo Abeni mptcp_data_lock(sk); 1070b5941f06SPaolo Abeni __mptcp_clean_una_wakeup(sk); 1071b5941f06SPaolo Abeni mptcp_data_unlock(sk); 1072b5941f06SPaolo Abeni } 1073b5941f06SPaolo Abeni 1074724cfd2eSPaolo Abeni static void mptcp_enter_memory_pressure(struct sock *sk) 107518b683bfSPaolo Abeni { 1076d9ca1de8SPaolo Abeni struct mptcp_subflow_context *subflow; 1077d9ca1de8SPaolo Abeni struct mptcp_sock *msk = mptcp_sk(sk); 1078d9ca1de8SPaolo Abeni bool first = true; 1079d9ca1de8SPaolo Abeni 1080d9ca1de8SPaolo Abeni mptcp_for_each_subflow(msk, subflow) { 1081d9ca1de8SPaolo Abeni struct sock *ssk = mptcp_subflow_tcp_sock(subflow); 1082d9ca1de8SPaolo Abeni 1083d9ca1de8SPaolo Abeni if (first) 1084d9ca1de8SPaolo Abeni tcp_enter_memory_pressure(ssk); 1085d9ca1de8SPaolo Abeni sk_stream_moderate_sndbuf(ssk); 10868005184fSPaolo Abeni 1087d9ca1de8SPaolo Abeni first = false; 1088d9ca1de8SPaolo Abeni } 10898005184fSPaolo Abeni __mptcp_sync_sndbuf(sk); 1090724cfd2eSPaolo Abeni } 1091724cfd2eSPaolo Abeni 1092724cfd2eSPaolo Abeni /* ensure we get enough memory for the frag hdr, beyond some minimal amount of 1093724cfd2eSPaolo Abeni * data 1094724cfd2eSPaolo Abeni */ 1095724cfd2eSPaolo Abeni static bool mptcp_page_frag_refill(struct sock *sk, struct page_frag *pfrag) 1096724cfd2eSPaolo Abeni { 1097724cfd2eSPaolo Abeni if (likely(skb_page_frag_refill(32U + sizeof(struct mptcp_data_frag), 1098724cfd2eSPaolo Abeni pfrag, sk->sk_allocation))) 1099724cfd2eSPaolo Abeni return true; 1100724cfd2eSPaolo Abeni 1101724cfd2eSPaolo Abeni mptcp_enter_memory_pressure(sk); 110218b683bfSPaolo Abeni return false; 110318b683bfSPaolo Abeni } 110418b683bfSPaolo Abeni 110518b683bfSPaolo Abeni static struct mptcp_data_frag * 110618b683bfSPaolo Abeni mptcp_carve_data_frag(const struct mptcp_sock *msk, struct page_frag *pfrag, 110718b683bfSPaolo Abeni int orig_offset) 110818b683bfSPaolo Abeni { 110918b683bfSPaolo Abeni int offset = ALIGN(orig_offset, sizeof(long)); 111018b683bfSPaolo Abeni struct mptcp_data_frag *dfrag; 111118b683bfSPaolo Abeni 111218b683bfSPaolo Abeni dfrag = (struct mptcp_data_frag *)(page_to_virt(pfrag->page) + offset); 111318b683bfSPaolo Abeni dfrag->data_len = 0; 111418b683bfSPaolo Abeni dfrag->data_seq = msk->write_seq; 111518b683bfSPaolo Abeni dfrag->overhead = offset - orig_offset + sizeof(struct mptcp_data_frag); 111618b683bfSPaolo Abeni dfrag->offset = offset + sizeof(struct mptcp_data_frag); 1117d9ca1de8SPaolo Abeni dfrag->already_sent = 0; 111818b683bfSPaolo Abeni dfrag->page = pfrag->page; 111918b683bfSPaolo Abeni 112018b683bfSPaolo Abeni return dfrag; 112118b683bfSPaolo Abeni } 112218b683bfSPaolo Abeni 1123caf971dfSPaolo Abeni struct mptcp_sendmsg_info { 1124caf971dfSPaolo Abeni int mss_now; 1125caf971dfSPaolo Abeni int size_goal; 1126d9ca1de8SPaolo Abeni u16 limit; 1127d9ca1de8SPaolo Abeni u16 sent; 1128d9ca1de8SPaolo Abeni unsigned int flags; 11291094c6feSPaolo Abeni bool data_lock_held; 1130caf971dfSPaolo Abeni }; 1131caf971dfSPaolo Abeni 1132b713d006SPaolo Abeni static int mptcp_check_allowed_size(const struct mptcp_sock *msk, struct sock *ssk, 1133b713d006SPaolo Abeni u64 data_seq, int avail_size) 11346f8a612aSFlorian Westphal { 11356f8a612aSFlorian Westphal u64 window_end = mptcp_wnd_end(msk); 1136b713d006SPaolo Abeni u64 mptcp_snd_wnd; 11376f8a612aSFlorian Westphal 11386f8a612aSFlorian Westphal if (__mptcp_check_fallback(msk)) 11396f8a612aSFlorian Westphal return avail_size; 11406f8a612aSFlorian Westphal 1141b713d006SPaolo Abeni mptcp_snd_wnd = window_end - data_seq; 1142b713d006SPaolo Abeni avail_size = min_t(unsigned int, mptcp_snd_wnd, avail_size); 11436f8a612aSFlorian Westphal 114492be2f52SPaolo Abeni if (unlikely(tcp_sk(ssk)->snd_wnd < mptcp_snd_wnd)) { 1145b713d006SPaolo Abeni tcp_sk(ssk)->snd_wnd = min_t(u64, U32_MAX, mptcp_snd_wnd); 114692be2f52SPaolo Abeni MPTCP_INC_STATS(sock_net(ssk), MPTCP_MIB_SNDWNDSHARED); 114792be2f52SPaolo Abeni } 11486f8a612aSFlorian Westphal 11496f8a612aSFlorian Westphal return avail_size; 11506f8a612aSFlorian Westphal } 11516f8a612aSFlorian Westphal 1152724cfd2eSPaolo Abeni static bool __mptcp_add_ext(struct sk_buff *skb, gfp_t gfp) 1153724cfd2eSPaolo Abeni { 1154724cfd2eSPaolo Abeni struct skb_ext *mpext = __skb_ext_alloc(gfp); 1155724cfd2eSPaolo Abeni 1156724cfd2eSPaolo Abeni if (!mpext) 1157724cfd2eSPaolo Abeni return false; 1158724cfd2eSPaolo Abeni __skb_ext_set(skb, SKB_EXT_MPTCP, mpext); 1159724cfd2eSPaolo Abeni return true; 1160724cfd2eSPaolo Abeni } 1161724cfd2eSPaolo Abeni 11626e628cd3SPaolo Abeni static struct sk_buff *__mptcp_do_alloc_tx_skb(struct sock *sk, gfp_t gfp) 1163724cfd2eSPaolo Abeni { 1164724cfd2eSPaolo Abeni struct sk_buff *skb; 1165724cfd2eSPaolo Abeni 11666e628cd3SPaolo Abeni skb = alloc_skb_fclone(MAX_TCP_HEADER, gfp); 1167724cfd2eSPaolo Abeni if (likely(skb)) { 11686e628cd3SPaolo Abeni if (likely(__mptcp_add_ext(skb, gfp))) { 1169724cfd2eSPaolo Abeni skb_reserve(skb, MAX_TCP_HEADER); 1170a52fe46eSEric Dumazet skb->ip_summed = CHECKSUM_PARTIAL; 1171f70cad10SPaolo Abeni INIT_LIST_HEAD(&skb->tcp_tsorted_anchor); 1172724cfd2eSPaolo Abeni return skb; 1173724cfd2eSPaolo Abeni } 1174724cfd2eSPaolo Abeni __kfree_skb(skb); 1175724cfd2eSPaolo Abeni } else { 1176724cfd2eSPaolo Abeni mptcp_enter_memory_pressure(sk); 1177724cfd2eSPaolo Abeni } 1178724cfd2eSPaolo Abeni return NULL; 1179724cfd2eSPaolo Abeni } 1180724cfd2eSPaolo Abeni 1181f70cad10SPaolo Abeni static struct sk_buff *__mptcp_alloc_tx_skb(struct sock *sk, struct sock *ssk, gfp_t gfp) 1182724cfd2eSPaolo Abeni { 1183724cfd2eSPaolo Abeni struct sk_buff *skb; 1184724cfd2eSPaolo Abeni 11856e628cd3SPaolo Abeni skb = __mptcp_do_alloc_tx_skb(sk, gfp); 1186724cfd2eSPaolo Abeni if (!skb) 1187f70cad10SPaolo Abeni return NULL; 1188724cfd2eSPaolo Abeni 1189724cfd2eSPaolo Abeni if (likely(sk_wmem_schedule(ssk, skb->truesize))) { 1190f70cad10SPaolo Abeni tcp_skb_entail(ssk, skb); 1191f70cad10SPaolo Abeni return skb; 1192724cfd2eSPaolo Abeni } 11933ef3905aSYonglong Li tcp_skb_tsorted_anchor_cleanup(skb); 1194724cfd2eSPaolo Abeni kfree_skb(skb); 1195f70cad10SPaolo Abeni return NULL; 1196724cfd2eSPaolo Abeni } 1197724cfd2eSPaolo Abeni 1198f70cad10SPaolo Abeni static struct sk_buff *mptcp_alloc_tx_skb(struct sock *sk, struct sock *ssk, bool data_lock_held) 1199724cfd2eSPaolo Abeni { 12001094c6feSPaolo Abeni gfp_t gfp = data_lock_held ? GFP_ATOMIC : sk->sk_allocation; 1201724cfd2eSPaolo Abeni 12021094c6feSPaolo Abeni return __mptcp_alloc_tx_skb(sk, ssk, gfp); 1203724cfd2eSPaolo Abeni } 1204724cfd2eSPaolo Abeni 1205d0cc2987SGeliang Tang /* note: this always recompute the csum on the whole skb, even 1206d0cc2987SGeliang Tang * if we just appended a single frag. More status info needed 1207d0cc2987SGeliang Tang */ 1208d0cc2987SGeliang Tang static void mptcp_update_data_checksum(struct sk_buff *skb, int added) 1209d0cc2987SGeliang Tang { 1210d0cc2987SGeliang Tang struct mptcp_ext *mpext = mptcp_get_ext(skb); 1211d0cc2987SGeliang Tang __wsum csum = ~csum_unfold(mpext->csum); 1212d0cc2987SGeliang Tang int offset = skb->len - added; 1213d0cc2987SGeliang Tang 1214d0cc2987SGeliang Tang mpext->csum = csum_fold(csum_block_add(csum, skb_checksum(skb, offset, added, 0), offset)); 1215d0cc2987SGeliang Tang } 1216d0cc2987SGeliang Tang 12171e39e5a3SGeliang Tang static void mptcp_update_infinite_map(struct mptcp_sock *msk, 12181e39e5a3SGeliang Tang struct sock *ssk, 12191e39e5a3SGeliang Tang struct mptcp_ext *mpext) 12201e39e5a3SGeliang Tang { 12211e39e5a3SGeliang Tang if (!mpext) 12221e39e5a3SGeliang Tang return; 12231e39e5a3SGeliang Tang 12241e39e5a3SGeliang Tang mpext->infinite_map = 1; 12251e39e5a3SGeliang Tang mpext->data_len = 0; 12261e39e5a3SGeliang Tang 1227104125b8SGeliang Tang MPTCP_INC_STATS(sock_net(ssk), MPTCP_MIB_INFINITEMAPTX); 12281e39e5a3SGeliang Tang mptcp_subflow_ctx(ssk)->send_infinite_map = 0; 12291e39e5a3SGeliang Tang pr_fallback(msk); 1230d51991e2SPaolo Abeni mptcp_do_fallback(ssk); 12311e39e5a3SGeliang Tang } 12321e39e5a3SGeliang Tang 123357040755SPaolo Abeni static int mptcp_sendmsg_frag(struct sock *sk, struct sock *ssk, 1234d9ca1de8SPaolo Abeni struct mptcp_data_frag *dfrag, 1235caf971dfSPaolo Abeni struct mptcp_sendmsg_info *info) 123657040755SPaolo Abeni { 1237d9ca1de8SPaolo Abeni u64 data_seq = dfrag->data_seq + info->sent; 1238f70cad10SPaolo Abeni int offset = dfrag->offset + info->sent; 12396d0060f6SMat Martineau struct mptcp_sock *msk = mptcp_sk(sk); 12406f8a612aSFlorian Westphal bool zero_window_probe = false; 12416d0060f6SMat Martineau struct mptcp_ext *mpext = NULL; 1242f70cad10SPaolo Abeni bool can_coalesce = false; 1243f70cad10SPaolo Abeni bool reuse_skb = true; 1244f70cad10SPaolo Abeni struct sk_buff *skb; 1245f70cad10SPaolo Abeni size_t copy; 1246f70cad10SPaolo Abeni int i; 12476d0060f6SMat Martineau 1248e4b61351SGeliang Tang pr_debug("msk=%p ssk=%p sending dfrag at seq=%llu len=%u already sent=%u", 1249d9ca1de8SPaolo Abeni msk, ssk, dfrag->data_seq, dfrag->data_len, info->sent); 12506d0060f6SMat Martineau 1251f70cad10SPaolo Abeni if (WARN_ON_ONCE(info->sent > info->limit || 1252f70cad10SPaolo Abeni info->limit > dfrag->data_len)) 1253f70cad10SPaolo Abeni return 0; 1254f70cad10SPaolo Abeni 1255c886d702SPaolo Abeni if (unlikely(!__tcp_can_send(ssk))) 1256c886d702SPaolo Abeni return -EAGAIN; 1257c886d702SPaolo Abeni 1258d9ca1de8SPaolo Abeni /* compute send limit */ 1259d9ca1de8SPaolo Abeni info->mss_now = tcp_send_mss(ssk, &info->size_goal, info->flags); 1260f70cad10SPaolo Abeni copy = info->size_goal; 1261f70cad10SPaolo Abeni 126257040755SPaolo Abeni skb = tcp_write_queue_tail(ssk); 1263f70cad10SPaolo Abeni if (skb && copy > skb->len) { 126457040755SPaolo Abeni /* Limit the write to the size available in the 126557040755SPaolo Abeni * current skb, if any, so that we create at most a new skb. 126657040755SPaolo Abeni * Explicitly tells TCP internals to avoid collapsing on later 126757040755SPaolo Abeni * queue management operation, to avoid breaking the ext <-> 126857040755SPaolo Abeni * SSN association set here 126957040755SPaolo Abeni */ 1270*a16c054bSGeliang Tang mpext = mptcp_get_ext(skb); 12711094c6feSPaolo Abeni if (!mptcp_skb_can_collapse_to(data_seq, skb, mpext)) { 127257040755SPaolo Abeni TCP_SKB_CB(skb)->eor = 1; 12731094c6feSPaolo Abeni goto alloc_skb; 12741094c6feSPaolo Abeni } 12751094c6feSPaolo Abeni 1276f70cad10SPaolo Abeni i = skb_shinfo(skb)->nr_frags; 1277f70cad10SPaolo Abeni can_coalesce = skb_can_coalesce(skb, i, dfrag->page, offset); 1278657b991aSKuniyuki Iwashima if (!can_coalesce && i >= READ_ONCE(sysctl_max_skb_frags)) { 1279f70cad10SPaolo Abeni tcp_mark_push(tcp_sk(ssk), skb); 1280f70cad10SPaolo Abeni goto alloc_skb; 128115e6ca97SPaolo Abeni } 128218b683bfSPaolo Abeni 1283f70cad10SPaolo Abeni copy -= skb->len; 1284f70cad10SPaolo Abeni } else { 12851094c6feSPaolo Abeni alloc_skb: 1286f70cad10SPaolo Abeni skb = mptcp_alloc_tx_skb(sk, ssk, info->data_lock_held); 1287f70cad10SPaolo Abeni if (!skb) 1288f70cad10SPaolo Abeni return -ENOMEM; 1289f70cad10SPaolo Abeni 1290f70cad10SPaolo Abeni i = skb_shinfo(skb)->nr_frags; 1291f70cad10SPaolo Abeni reuse_skb = false; 1292*a16c054bSGeliang Tang mpext = mptcp_get_ext(skb); 1293f70cad10SPaolo Abeni } 12941094c6feSPaolo Abeni 12956f8a612aSFlorian Westphal /* Zero window and all data acked? Probe. */ 1296b713d006SPaolo Abeni copy = mptcp_check_allowed_size(msk, ssk, data_seq, copy); 1297f70cad10SPaolo Abeni if (copy == 0) { 12987439d687SPaolo Abeni u64 snd_una = READ_ONCE(msk->snd_una); 12997439d687SPaolo Abeni 130072377ab2SPaolo Abeni if (snd_una != msk->snd_nxt || tcp_write_queue_tail(ssk)) { 130127728ba8SEric Dumazet tcp_remove_empty_skb(ssk); 13026f8a612aSFlorian Westphal return 0; 13036f8a612aSFlorian Westphal } 13046f8a612aSFlorian Westphal 1305f70cad10SPaolo Abeni zero_window_probe = true; 1306f70cad10SPaolo Abeni data_seq = snd_una - 1; 1307f70cad10SPaolo Abeni copy = 1; 1308f70cad10SPaolo Abeni } 1309f70cad10SPaolo Abeni 1310f70cad10SPaolo Abeni copy = min_t(size_t, copy, info->limit - info->sent); 1311f70cad10SPaolo Abeni if (!sk_wmem_schedule(ssk, copy)) { 131227728ba8SEric Dumazet tcp_remove_empty_skb(ssk); 1313e2223995SPaolo Abeni return -ENOMEM; 131435759383SFlorian Westphal } 131518b683bfSPaolo Abeni 1316f70cad10SPaolo Abeni if (can_coalesce) { 1317f70cad10SPaolo Abeni skb_frag_size_add(&skb_shinfo(skb)->frags[i - 1], copy); 1318f70cad10SPaolo Abeni } else { 1319f70cad10SPaolo Abeni get_page(dfrag->page); 1320f70cad10SPaolo Abeni skb_fill_page_desc(skb, i, dfrag->page, offset, copy); 132157040755SPaolo Abeni } 132257040755SPaolo Abeni 1323f70cad10SPaolo Abeni skb->len += copy; 1324f70cad10SPaolo Abeni skb->data_len += copy; 1325f70cad10SPaolo Abeni skb->truesize += copy; 1326f70cad10SPaolo Abeni sk_wmem_queued_add(ssk, copy); 1327f70cad10SPaolo Abeni sk_mem_charge(ssk, copy); 1328f70cad10SPaolo Abeni WRITE_ONCE(tcp_sk(ssk)->write_seq, tcp_sk(ssk)->write_seq + copy); 1329f70cad10SPaolo Abeni TCP_SKB_CB(skb)->end_seq += copy; 1330f70cad10SPaolo Abeni tcp_skb_pcount_set(skb, 0); 1331f70cad10SPaolo Abeni 1332f70cad10SPaolo Abeni /* on skb reuse we just need to update the DSS len */ 1333f70cad10SPaolo Abeni if (reuse_skb) { 1334f70cad10SPaolo Abeni TCP_SKB_CB(skb)->tcp_flags &= ~TCPHDR_PSH; 1335f70cad10SPaolo Abeni mpext->data_len += copy; 1336f70cad10SPaolo Abeni goto out; 1337724cfd2eSPaolo Abeni } 13386d0060f6SMat Martineau 13396d0060f6SMat Martineau memset(mpext, 0, sizeof(*mpext)); 1340d9ca1de8SPaolo Abeni mpext->data_seq = data_seq; 13416d0060f6SMat Martineau mpext->subflow_seq = mptcp_subflow_ctx(ssk)->rel_write_seq; 1342f70cad10SPaolo Abeni mpext->data_len = copy; 13436d0060f6SMat Martineau mpext->use_map = 1; 13446d0060f6SMat Martineau mpext->dsn64 = 1; 13456d0060f6SMat Martineau 13466d0060f6SMat Martineau pr_debug("data_seq=%llu subflow_seq=%u data_len=%u dsn64=%d", 13476d0060f6SMat Martineau mpext->data_seq, mpext->subflow_seq, mpext->data_len, 13486d0060f6SMat Martineau mpext->dsn64); 13496d0060f6SMat Martineau 13506f8a612aSFlorian Westphal if (zero_window_probe) { 1351f70cad10SPaolo Abeni mptcp_subflow_ctx(ssk)->rel_write_seq += copy; 13526f8a612aSFlorian Westphal mpext->frozen = 1; 1353d0cc2987SGeliang Tang if (READ_ONCE(msk->csum_enabled)) 1354f70cad10SPaolo Abeni mptcp_update_data_checksum(skb, copy); 13556f8a612aSFlorian Westphal tcp_push_pending_frames(ssk); 1356d0cc2987SGeliang Tang return 0; 13576f8a612aSFlorian Westphal } 135857040755SPaolo Abeni out: 1359d0cc2987SGeliang Tang if (READ_ONCE(msk->csum_enabled)) 1360f70cad10SPaolo Abeni mptcp_update_data_checksum(skb, copy); 13611e39e5a3SGeliang Tang if (mptcp_subflow_ctx(ssk)->send_infinite_map) 13621e39e5a3SGeliang Tang mptcp_update_infinite_map(msk, ssk, mpext); 13630eb4e7eeSGeliang Tang trace_mptcp_sendmsg_frag(mpext); 1364f70cad10SPaolo Abeni mptcp_subflow_ctx(ssk)->rel_write_seq += copy; 1365f70cad10SPaolo Abeni return copy; 13666d0060f6SMat Martineau } 13676d0060f6SMat Martineau 1368d5f49190SPaolo Abeni #define MPTCP_SEND_BURST_SIZE ((1 << 16) - \ 1369d5f49190SPaolo Abeni sizeof(struct tcphdr) - \ 1370d5f49190SPaolo Abeni MAX_TCP_OPTION_SPACE - \ 1371d5f49190SPaolo Abeni sizeof(struct ipv6hdr) - \ 1372d5f49190SPaolo Abeni sizeof(struct frag_hdr)) 1373d5f49190SPaolo Abeni 1374d5f49190SPaolo Abeni struct subflow_send_info { 1375d5f49190SPaolo Abeni struct sock *ssk; 13763ce0852cSPaolo Abeni u64 linger_time; 1377d5f49190SPaolo Abeni }; 1378d5f49190SPaolo Abeni 1379ff5a0b42SPaolo Abeni void mptcp_subflow_set_active(struct mptcp_subflow_context *subflow) 1380ff5a0b42SPaolo Abeni { 1381ff5a0b42SPaolo Abeni if (!subflow->stale) 1382ff5a0b42SPaolo Abeni return; 1383ff5a0b42SPaolo Abeni 1384ff5a0b42SPaolo Abeni subflow->stale = 0; 1385fc1b4e3bSPaolo Abeni MPTCP_INC_STATS(sock_net(mptcp_subflow_tcp_sock(subflow)), MPTCP_MIB_SUBFLOWRECOVER); 1386ff5a0b42SPaolo Abeni } 1387ff5a0b42SPaolo Abeni 1388ff5a0b42SPaolo Abeni bool mptcp_subflow_active(struct mptcp_subflow_context *subflow) 1389ff5a0b42SPaolo Abeni { 1390ff5a0b42SPaolo Abeni if (unlikely(subflow->stale)) { 1391ff5a0b42SPaolo Abeni u32 rcv_tstamp = READ_ONCE(tcp_sk(mptcp_subflow_tcp_sock(subflow))->rcv_tstamp); 1392ff5a0b42SPaolo Abeni 1393ff5a0b42SPaolo Abeni if (subflow->stale_rcv_tstamp == rcv_tstamp) 1394ff5a0b42SPaolo Abeni return false; 1395ff5a0b42SPaolo Abeni 1396ff5a0b42SPaolo Abeni mptcp_subflow_set_active(subflow); 1397ff5a0b42SPaolo Abeni } 1398ff5a0b42SPaolo Abeni return __mptcp_subflow_active(subflow); 1399ff5a0b42SPaolo Abeni } 1400ff5a0b42SPaolo Abeni 14013ce0852cSPaolo Abeni #define SSK_MODE_ACTIVE 0 14023ce0852cSPaolo Abeni #define SSK_MODE_BACKUP 1 14033ce0852cSPaolo Abeni #define SSK_MODE_MAX 2 14043ce0852cSPaolo Abeni 140533d41c9cSPaolo Abeni /* implement the mptcp packet scheduler; 140633d41c9cSPaolo Abeni * returns the subflow that will transmit the next DSS 140733d41c9cSPaolo Abeni * additionally updates the rtx timeout 140833d41c9cSPaolo Abeni */ 140907336a87SGeliang Tang struct sock *mptcp_subflow_get_send(struct mptcp_sock *msk) 1410f296234cSPeter Krystad { 14113ce0852cSPaolo Abeni struct subflow_send_info send_info[SSK_MODE_MAX]; 1412f296234cSPeter Krystad struct mptcp_subflow_context *subflow; 141333d41c9cSPaolo Abeni struct sock *sk = (struct sock *)msk; 14143ce0852cSPaolo Abeni u32 pace, burst, wmem; 1415d5f49190SPaolo Abeni int i, nr_active = 0; 1416d5f49190SPaolo Abeni struct sock *ssk; 14173ce0852cSPaolo Abeni u64 linger_time; 141833d41c9cSPaolo Abeni long tout = 0; 1419f296234cSPeter Krystad 1420d5f49190SPaolo Abeni /* pick the subflow with the lower wmem/wspace ratio */ 14213ce0852cSPaolo Abeni for (i = 0; i < SSK_MODE_MAX; ++i) { 1422d5f49190SPaolo Abeni send_info[i].ssk = NULL; 14233ce0852cSPaolo Abeni send_info[i].linger_time = -1; 1424d5f49190SPaolo Abeni } 14253ce0852cSPaolo Abeni 1426d5f49190SPaolo Abeni mptcp_for_each_subflow(msk, subflow) { 1427e10a9892SGeliang Tang trace_mptcp_subflow_get_send(subflow); 1428d5f49190SPaolo Abeni ssk = mptcp_subflow_tcp_sock(subflow); 1429d5f49190SPaolo Abeni if (!mptcp_subflow_active(subflow)) 1430f296234cSPeter Krystad continue; 1431d5f49190SPaolo Abeni 143233d41c9cSPaolo Abeni tout = max(tout, mptcp_timeout_from_subflow(subflow)); 1433d5f49190SPaolo Abeni nr_active += !subflow->backup; 14343ce0852cSPaolo Abeni pace = subflow->avg_pacing_rate; 14353ce0852cSPaolo Abeni if (unlikely(!pace)) { 14363ce0852cSPaolo Abeni /* init pacing rate from socket */ 14373ce0852cSPaolo Abeni subflow->avg_pacing_rate = READ_ONCE(ssk->sk_pacing_rate); 14383ce0852cSPaolo Abeni pace = subflow->avg_pacing_rate; 1439d5f49190SPaolo Abeni if (!pace) 1440d5f49190SPaolo Abeni continue; 14413ce0852cSPaolo Abeni } 1442d5f49190SPaolo Abeni 14433ce0852cSPaolo Abeni linger_time = div_u64((u64)READ_ONCE(ssk->sk_wmem_queued) << 32, pace); 14443ce0852cSPaolo Abeni if (linger_time < send_info[subflow->backup].linger_time) { 1445d5f49190SPaolo Abeni send_info[subflow->backup].ssk = ssk; 14463ce0852cSPaolo Abeni send_info[subflow->backup].linger_time = linger_time; 1447d5f49190SPaolo Abeni } 1448f296234cSPeter Krystad } 144933d41c9cSPaolo Abeni __mptcp_set_timeout(sk, tout); 1450f296234cSPeter Krystad 1451d5f49190SPaolo Abeni /* pick the best backup if no other subflow is active */ 1452d5f49190SPaolo Abeni if (!nr_active) 14533ce0852cSPaolo Abeni send_info[SSK_MODE_ACTIVE].ssk = send_info[SSK_MODE_BACKUP].ssk; 1454d5f49190SPaolo Abeni 14553ce0852cSPaolo Abeni /* According to the blest algorithm, to avoid HoL blocking for the 14563ce0852cSPaolo Abeni * faster flow, we need to: 14573ce0852cSPaolo Abeni * - estimate the faster flow linger time 14583ce0852cSPaolo Abeni * - use the above to estimate the amount of byte transferred 14593ce0852cSPaolo Abeni * by the faster flow 14603ce0852cSPaolo Abeni * - check that the amount of queued data is greter than the above, 14613ce0852cSPaolo Abeni * otherwise do not use the picked, slower, subflow 14623ce0852cSPaolo Abeni * We select the subflow with the shorter estimated time to flush 14633ce0852cSPaolo Abeni * the queued mem, which basically ensure the above. We just need 14643ce0852cSPaolo Abeni * to check that subflow has a non empty cwin. 14653ce0852cSPaolo Abeni */ 14663ce0852cSPaolo Abeni ssk = send_info[SSK_MODE_ACTIVE].ssk; 1467b713d006SPaolo Abeni if (!ssk || !sk_stream_memory_free(ssk)) 1468d5f49190SPaolo Abeni return NULL; 14693ce0852cSPaolo Abeni 1470b713d006SPaolo Abeni burst = min_t(int, MPTCP_SEND_BURST_SIZE, mptcp_wnd_end(msk) - msk->snd_nxt); 14713ce0852cSPaolo Abeni wmem = READ_ONCE(ssk->sk_wmem_queued); 1472ebc1e08fSGeliang Tang if (!burst) 1473b713d006SPaolo Abeni return ssk; 1474b713d006SPaolo Abeni 14753ce0852cSPaolo Abeni subflow = mptcp_subflow_ctx(ssk); 14763ce0852cSPaolo Abeni subflow->avg_pacing_rate = div_u64((u64)subflow->avg_pacing_rate * wmem + 14773ce0852cSPaolo Abeni READ_ONCE(ssk->sk_pacing_rate) * burst, 14783ce0852cSPaolo Abeni burst + wmem); 14793ce0852cSPaolo Abeni msk->snd_burst = burst; 14803ce0852cSPaolo Abeni return ssk; 1481f296234cSPeter Krystad } 1482f296234cSPeter Krystad 1483b8e0def3SGeliang Tang static void mptcp_push_release(struct sock *ssk, struct mptcp_sendmsg_info *info) 1484d9ca1de8SPaolo Abeni { 1485d9ca1de8SPaolo Abeni tcp_push(ssk, 0, info->mss_now, tcp_sk(ssk)->nonagle, info->size_goal); 1486d9ca1de8SPaolo Abeni release_sock(ssk); 1487d9ca1de8SPaolo Abeni } 1488d9ca1de8SPaolo Abeni 14890d199e43SFlorian Westphal static void mptcp_update_post_push(struct mptcp_sock *msk, 14900d199e43SFlorian Westphal struct mptcp_data_frag *dfrag, 14910d199e43SFlorian Westphal u32 sent) 14920d199e43SFlorian Westphal { 14930d199e43SFlorian Westphal u64 snd_nxt_new = dfrag->data_seq; 14940d199e43SFlorian Westphal 14950d199e43SFlorian Westphal dfrag->already_sent += sent; 14960d199e43SFlorian Westphal 14970d199e43SFlorian Westphal msk->snd_burst -= sent; 14980d199e43SFlorian Westphal 14990d199e43SFlorian Westphal snd_nxt_new += dfrag->already_sent; 15000d199e43SFlorian Westphal 15010d199e43SFlorian Westphal /* snd_nxt_new can be smaller than snd_nxt in case mptcp 15020d199e43SFlorian Westphal * is recovering after a failover. In that event, this re-sends 15030d199e43SFlorian Westphal * old segments. 15040d199e43SFlorian Westphal * 15050d199e43SFlorian Westphal * Thus compute snd_nxt_new candidate based on 15060d199e43SFlorian Westphal * the dfrag->data_seq that was sent and the data 15070d199e43SFlorian Westphal * that has been handed to the subflow for transmission 15080d199e43SFlorian Westphal * and skip update in case it was old dfrag. 15090d199e43SFlorian Westphal */ 151038967f42SPaolo Abeni if (likely(after64(snd_nxt_new, msk->snd_nxt))) { 151138967f42SPaolo Abeni msk->bytes_sent += snd_nxt_new - msk->snd_nxt; 15120d199e43SFlorian Westphal msk->snd_nxt = snd_nxt_new; 15130d199e43SFlorian Westphal } 151438967f42SPaolo Abeni } 15150d199e43SFlorian Westphal 15168b38217aSMaxim Galaganov void mptcp_check_and_set_pending(struct sock *sk) 15173241a9c0SFlorian Westphal { 1518e9d09bacSPaolo Abeni if (mptcp_send_head(sk)) 1519e9d09bacSPaolo Abeni mptcp_sk(sk)->push_pending |= BIT(MPTCP_PUSH_PENDING); 15203241a9c0SFlorian Westphal } 15213241a9c0SFlorian Westphal 1522c5b4297dSGeliang Tang static int __subflow_push_pending(struct sock *sk, struct sock *ssk, 1523c5b4297dSGeliang Tang struct mptcp_sendmsg_info *info) 1524c5b4297dSGeliang Tang { 1525c5b4297dSGeliang Tang struct mptcp_sock *msk = mptcp_sk(sk); 1526c5b4297dSGeliang Tang struct mptcp_data_frag *dfrag; 1527c5b4297dSGeliang Tang int len, copied = 0, err = 0; 1528c5b4297dSGeliang Tang 1529c5b4297dSGeliang Tang while ((dfrag = mptcp_send_head(sk))) { 1530c5b4297dSGeliang Tang info->sent = dfrag->already_sent; 1531c5b4297dSGeliang Tang info->limit = dfrag->data_len; 1532c5b4297dSGeliang Tang len = dfrag->data_len - dfrag->already_sent; 1533c5b4297dSGeliang Tang while (len > 0) { 1534c5b4297dSGeliang Tang int ret = 0; 1535c5b4297dSGeliang Tang 1536c5b4297dSGeliang Tang ret = mptcp_sendmsg_frag(sk, ssk, dfrag, info); 1537c5b4297dSGeliang Tang if (ret <= 0) { 1538c5b4297dSGeliang Tang err = copied ? : ret; 1539c5b4297dSGeliang Tang goto out; 1540c5b4297dSGeliang Tang } 1541c5b4297dSGeliang Tang 1542c5b4297dSGeliang Tang info->sent += ret; 1543c5b4297dSGeliang Tang copied += ret; 1544c5b4297dSGeliang Tang len -= ret; 1545c5b4297dSGeliang Tang 1546c5b4297dSGeliang Tang mptcp_update_post_push(msk, dfrag, ret); 1547c5b4297dSGeliang Tang } 1548c5b4297dSGeliang Tang WRITE_ONCE(msk->first_pending, mptcp_send_next(sk)); 1549c5b4297dSGeliang Tang 1550c5b4297dSGeliang Tang if (msk->snd_burst <= 0 || 1551c5b4297dSGeliang Tang !sk_stream_memory_free(ssk) || 1552c5b4297dSGeliang Tang !mptcp_subflow_active(mptcp_subflow_ctx(ssk))) { 1553c5b4297dSGeliang Tang err = copied; 1554c5b4297dSGeliang Tang goto out; 1555c5b4297dSGeliang Tang } 1556c5b4297dSGeliang Tang mptcp_set_timeout(sk); 1557c5b4297dSGeliang Tang } 1558c5b4297dSGeliang Tang err = copied; 1559c5b4297dSGeliang Tang 1560c5b4297dSGeliang Tang out: 1561c5b4297dSGeliang Tang return err; 1562c5b4297dSGeliang Tang } 1563c5b4297dSGeliang Tang 1564ff5a0b42SPaolo Abeni void __mptcp_push_pending(struct sock *sk, unsigned int flags) 1565d9ca1de8SPaolo Abeni { 1566d9ca1de8SPaolo Abeni struct sock *prev_ssk = NULL, *ssk = NULL; 1567d9ca1de8SPaolo Abeni struct mptcp_sock *msk = mptcp_sk(sk); 1568d9ca1de8SPaolo Abeni struct mptcp_sendmsg_info info = { 1569d9ca1de8SPaolo Abeni .flags = flags, 1570d9ca1de8SPaolo Abeni }; 15710522b424SGeliang Tang bool do_check_data_fin = false; 15720fa1b378SGeliang Tang int push_count = 1; 1573d9ca1de8SPaolo Abeni 15740fa1b378SGeliang Tang while (mptcp_send_head(sk) && (push_count > 0)) { 15750fa1b378SGeliang Tang struct mptcp_subflow_context *subflow; 1576d9ca1de8SPaolo Abeni int ret = 0; 1577d9ca1de8SPaolo Abeni 15780fa1b378SGeliang Tang if (mptcp_sched_get_send(msk)) 15790fa1b378SGeliang Tang break; 1580d9ca1de8SPaolo Abeni 15810fa1b378SGeliang Tang push_count = 0; 15820fa1b378SGeliang Tang 15830fa1b378SGeliang Tang mptcp_for_each_subflow(msk, subflow) { 15840fa1b378SGeliang Tang if (READ_ONCE(subflow->scheduled)) { 15850fa1b378SGeliang Tang mptcp_subflow_set_scheduled(subflow, false); 15860fa1b378SGeliang Tang 15870fa1b378SGeliang Tang prev_ssk = ssk; 15880fa1b378SGeliang Tang ssk = mptcp_subflow_tcp_sock(subflow); 15890fa1b378SGeliang Tang if (ssk != prev_ssk) { 15909758f40eSPaolo Abeni /* First check. If the ssk has changed since 15919758f40eSPaolo Abeni * the last round, release prev_ssk 1592d9ca1de8SPaolo Abeni */ 15930fa1b378SGeliang Tang if (prev_ssk) 1594b8e0def3SGeliang Tang mptcp_push_release(prev_ssk, &info); 1595d9ca1de8SPaolo Abeni 15969758f40eSPaolo Abeni /* Need to lock the new subflow only if different 15979758f40eSPaolo Abeni * from the previous one, otherwise we are still 15989758f40eSPaolo Abeni * helding the relevant lock 15999758f40eSPaolo Abeni */ 1600d9ca1de8SPaolo Abeni lock_sock(ssk); 16010fa1b378SGeliang Tang } 16020fa1b378SGeliang Tang 16030fa1b378SGeliang Tang push_count++; 1604d9ca1de8SPaolo Abeni 1605c5b4297dSGeliang Tang ret = __subflow_push_pending(sk, ssk, &info); 1606d9ca1de8SPaolo Abeni if (ret <= 0) { 16070fa1b378SGeliang Tang if (ret != -EAGAIN || 16080fa1b378SGeliang Tang (1 << ssk->sk_state) & 16090fa1b378SGeliang Tang (TCPF_FIN_WAIT1 | TCPF_FIN_WAIT2 | TCPF_CLOSE)) 16100fa1b378SGeliang Tang push_count--; 1611c886d702SPaolo Abeni continue; 1612d9ca1de8SPaolo Abeni } 16130522b424SGeliang Tang do_check_data_fin = true; 1614d9ca1de8SPaolo Abeni } 16150fa1b378SGeliang Tang } 16160fa1b378SGeliang Tang } 1617d9ca1de8SPaolo Abeni 1618d9ca1de8SPaolo Abeni /* at this point we held the socket lock for the last subflow we used */ 1619d9ca1de8SPaolo Abeni if (ssk) 1620b8e0def3SGeliang Tang mptcp_push_release(ssk, &info); 1621d9ca1de8SPaolo Abeni 162233d41c9cSPaolo Abeni /* ensure the rtx timer is running */ 1623f6909dc1SPaolo Abeni if (!mptcp_rtx_timer_pending(sk)) 1624f6909dc1SPaolo Abeni mptcp_reset_rtx_timer(sk); 16250522b424SGeliang Tang if (do_check_data_fin) 162681c1d029SPaolo Abeni mptcp_check_send_data_fin(sk); 1627d9ca1de8SPaolo Abeni } 1628d9ca1de8SPaolo Abeni 162973a0052aSGeliang Tang static void __mptcp_subflow_push_pending(struct sock *sk, struct sock *ssk, bool first) 16306e628cd3SPaolo Abeni { 16316e628cd3SPaolo Abeni struct mptcp_sock *msk = mptcp_sk(sk); 16321094c6feSPaolo Abeni struct mptcp_sendmsg_info info = { 16331094c6feSPaolo Abeni .data_lock_held = true, 16341094c6feSPaolo Abeni }; 16350fa1b378SGeliang Tang bool keep_pushing = true; 1636b19bc294SPaolo Abeni struct sock *xmit_ssk; 1637c5b4297dSGeliang Tang int copied = 0; 16386e628cd3SPaolo Abeni 16396e628cd3SPaolo Abeni info.flags = 0; 16400fa1b378SGeliang Tang while (mptcp_send_head(sk) && keep_pushing) { 16410fa1b378SGeliang Tang struct mptcp_subflow_context *subflow = mptcp_subflow_ctx(ssk); 16426e628cd3SPaolo Abeni int ret = 0; 16436e628cd3SPaolo Abeni 164473a0052aSGeliang Tang /* check for a different subflow usage only after 1645b19bc294SPaolo Abeni * spooling the first chunk of data 1646b19bc294SPaolo Abeni */ 16470fa1b378SGeliang Tang if (first) { 16480fa1b378SGeliang Tang mptcp_subflow_set_scheduled(subflow, false); 1649c5b4297dSGeliang Tang ret = __subflow_push_pending(sk, ssk, &info); 1650b19bc294SPaolo Abeni first = false; 1651c5b4297dSGeliang Tang if (ret <= 0) 1652c5b4297dSGeliang Tang break; 1653c5b4297dSGeliang Tang copied += ret; 16540fa1b378SGeliang Tang continue; 16550fa1b378SGeliang Tang } 16560fa1b378SGeliang Tang 16570fa1b378SGeliang Tang if (mptcp_sched_get_send(msk)) 16580fa1b378SGeliang Tang goto out; 16590fa1b378SGeliang Tang 16600fa1b378SGeliang Tang if (READ_ONCE(subflow->scheduled)) { 16610fa1b378SGeliang Tang mptcp_subflow_set_scheduled(subflow, false); 16620fa1b378SGeliang Tang ret = __subflow_push_pending(sk, ssk, &info); 16630fa1b378SGeliang Tang if (ret <= 0) 16640fa1b378SGeliang Tang keep_pushing = false; 16650fa1b378SGeliang Tang copied += ret; 16660fa1b378SGeliang Tang } 16670fa1b378SGeliang Tang 16680fa1b378SGeliang Tang mptcp_for_each_subflow(msk, subflow) { 16690fa1b378SGeliang Tang if (READ_ONCE(subflow->scheduled)) { 16700fa1b378SGeliang Tang xmit_ssk = mptcp_subflow_tcp_sock(subflow); 16710fa1b378SGeliang Tang if (xmit_ssk != ssk) { 16720fa1b378SGeliang Tang mptcp_subflow_delegate(subflow, 16730fa1b378SGeliang Tang MPTCP_DELEGATE_SEND); 16740fa1b378SGeliang Tang keep_pushing = false; 16750fa1b378SGeliang Tang } 16760fa1b378SGeliang Tang } 16770fa1b378SGeliang Tang } 16786e628cd3SPaolo Abeni } 16796e628cd3SPaolo Abeni 16806e628cd3SPaolo Abeni out: 16816e628cd3SPaolo Abeni /* __mptcp_alloc_tx_skb could have released some wmem and we are 16826e628cd3SPaolo Abeni * not going to flush it via release_sock() 16836e628cd3SPaolo Abeni */ 16846e628cd3SPaolo Abeni if (copied) { 16856e628cd3SPaolo Abeni tcp_push(ssk, 0, info.mss_now, tcp_sk(ssk)->nonagle, 16866e628cd3SPaolo Abeni info.size_goal); 1687f6909dc1SPaolo Abeni if (!mptcp_rtx_timer_pending(sk)) 1688f6909dc1SPaolo Abeni mptcp_reset_rtx_timer(sk); 1689d09d818eSPaolo Abeni 16906e628cd3SPaolo Abeni if (msk->snd_data_fin_enable && 16916e628cd3SPaolo Abeni msk->snd_nxt + 1 == msk->write_seq) 16926e628cd3SPaolo Abeni mptcp_schedule_work(sk); 16936e628cd3SPaolo Abeni } 16946e628cd3SPaolo Abeni } 16956e628cd3SPaolo Abeni 16965cf92bbaSPaolo Abeni static void mptcp_set_nospace(struct sock *sk) 16975cf92bbaSPaolo Abeni { 16985cf92bbaSPaolo Abeni /* enable autotune */ 16995cf92bbaSPaolo Abeni set_bit(SOCK_NOSPACE, &sk->sk_socket->flags); 17005cf92bbaSPaolo Abeni 17015cf92bbaSPaolo Abeni /* will be cleared on avail space */ 17025cf92bbaSPaolo Abeni set_bit(MPTCP_NOSPACE, &mptcp_sk(sk)->flags); 17035cf92bbaSPaolo Abeni } 17045cf92bbaSPaolo Abeni 17057d803344SPaolo Abeni static int mptcp_disconnect(struct sock *sk, int flags); 17067d803344SPaolo Abeni 1707a2702a07SPaolo Abeni static int mptcp_sendmsg_fastopen(struct sock *sk, struct msghdr *msg, 1708fa9e5746SPaolo Abeni size_t len, int *copied_syn) 1709fa9e5746SPaolo Abeni { 1710fa9e5746SPaolo Abeni unsigned int saved_flags = msg->msg_flags; 1711fa9e5746SPaolo Abeni struct mptcp_sock *msk = mptcp_sk(sk); 1712a2702a07SPaolo Abeni struct sock *ssk; 1713fa9e5746SPaolo Abeni int ret; 1714fa9e5746SPaolo Abeni 1715a2702a07SPaolo Abeni /* on flags based fastopen the mptcp is supposed to create the 1716a2702a07SPaolo Abeni * first subflow right now. Otherwise we are in the defer_connect 1717a2702a07SPaolo Abeni * path, and the first subflow must be already present. 1718a2702a07SPaolo Abeni * Since the defer_connect flag is cleared after the first succsful 1719a2702a07SPaolo Abeni * fastopen attempt, no need to check for additional subflow status. 1720a2702a07SPaolo Abeni */ 1721ddb1a072SPaolo Abeni if (msg->msg_flags & MSG_FASTOPEN) { 17223f326a82SPaolo Abeni ssk = __mptcp_nmpc_sk(msk); 17233f326a82SPaolo Abeni if (IS_ERR(ssk)) 17243f326a82SPaolo Abeni return PTR_ERR(ssk); 1725ddb1a072SPaolo Abeni } 1726a2702a07SPaolo Abeni if (!msk->first) 1727a2702a07SPaolo Abeni return -EINVAL; 1728a2702a07SPaolo Abeni 1729a2702a07SPaolo Abeni ssk = msk->first; 1730a2702a07SPaolo Abeni 1731fa9e5746SPaolo Abeni lock_sock(ssk); 1732fa9e5746SPaolo Abeni msg->msg_flags |= MSG_DONTWAIT; 17337d803344SPaolo Abeni msk->fastopening = 1; 1734fa9e5746SPaolo Abeni ret = tcp_sendmsg_fastopen(ssk, msg, copied_syn, len, NULL); 17357d803344SPaolo Abeni msk->fastopening = 0; 1736fa9e5746SPaolo Abeni msg->msg_flags = saved_flags; 1737fa9e5746SPaolo Abeni release_sock(ssk); 1738fa9e5746SPaolo Abeni 1739fa9e5746SPaolo Abeni /* do the blocking bits of inet_stream_connect outside the ssk socket lock */ 1740fa9e5746SPaolo Abeni if (ret == -EINPROGRESS && !(msg->msg_flags & MSG_DONTWAIT)) { 1741fa9e5746SPaolo Abeni ret = __inet_stream_connect(sk->sk_socket, msg->msg_name, 1742fa9e5746SPaolo Abeni msg->msg_namelen, msg->msg_flags, 1); 1743fa9e5746SPaolo Abeni 1744fa9e5746SPaolo Abeni /* Keep the same behaviour of plain TCP: zero the copied bytes in 1745fa9e5746SPaolo Abeni * case of any error, except timeout or signal 1746fa9e5746SPaolo Abeni */ 1747fa9e5746SPaolo Abeni if (ret && ret != -EINPROGRESS && ret != -ERESTARTSYS && ret != -EINTR) 1748fa9e5746SPaolo Abeni *copied_syn = 0; 17497d803344SPaolo Abeni } else if (ret && ret != -EINPROGRESS) { 1750c2b2ae39SPaolo Abeni /* The disconnect() op called by tcp_sendmsg_fastopen()/ 1751c2b2ae39SPaolo Abeni * __inet_stream_connect() can fail, due to looking check, 1752c2b2ae39SPaolo Abeni * see mptcp_disconnect(). 1753c2b2ae39SPaolo Abeni * Attempt it again outside the problematic scope. 1754c2b2ae39SPaolo Abeni */ 1755c2b2ae39SPaolo Abeni if (!mptcp_disconnect(sk, 0)) 1756c2b2ae39SPaolo Abeni sk->sk_socket->state = SS_UNCONNECTED; 1757fa9e5746SPaolo Abeni } 175808e39c0dSEric Dumazet inet_clear_bit(DEFER_CONNECT, sk); 1759fa9e5746SPaolo Abeni 1760fa9e5746SPaolo Abeni return ret; 1761fa9e5746SPaolo Abeni } 1762fa9e5746SPaolo Abeni 17630ffe8e74SPaolo Abeni static int do_copy_data_nocache(struct sock *sk, int copy, 17640ffe8e74SPaolo Abeni struct iov_iter *from, char *to) 17650ffe8e74SPaolo Abeni { 17660ffe8e74SPaolo Abeni if (sk->sk_route_caps & NETIF_F_NOCACHE_COPY) { 17670ffe8e74SPaolo Abeni if (!copy_from_iter_full_nocache(to, copy, from)) 17680ffe8e74SPaolo Abeni return -EFAULT; 17690ffe8e74SPaolo Abeni } else if (!copy_from_iter_full(to, copy, from)) { 17700ffe8e74SPaolo Abeni return -EFAULT; 17710ffe8e74SPaolo Abeni } 17720ffe8e74SPaolo Abeni return 0; 17730ffe8e74SPaolo Abeni } 17740ffe8e74SPaolo Abeni 1775f870fa0bSMat Martineau static int mptcp_sendmsg(struct sock *sk, struct msghdr *msg, size_t len) 1776f870fa0bSMat Martineau { 1777f870fa0bSMat Martineau struct mptcp_sock *msk = mptcp_sk(sk); 177817091708SFlorian Westphal struct page_frag *pfrag; 17796d0060f6SMat Martineau size_t copied = 0; 1780caf971dfSPaolo Abeni int ret = 0; 17816d0060f6SMat Martineau long timeo; 1782f870fa0bSMat Martineau 1783987858e5SPaolo Abeni /* silently ignore everything else */ 17841e777f39SDmytro Shytyi msg->msg_flags &= MSG_MORE | MSG_DONTWAIT | MSG_NOSIGNAL | MSG_FASTOPEN; 1785987858e5SPaolo Abeni 17866511882cSPaolo Abeni lock_sock(sk); 17871954b860SMat Martineau 178808e39c0dSEric Dumazet if (unlikely(inet_test_bit(DEFER_CONNECT, sk) || 178908e39c0dSEric Dumazet msg->msg_flags & MSG_FASTOPEN)) { 1790d98a82a6SDmytro Shytyi int copied_syn = 0; 1791d98a82a6SDmytro Shytyi 1792a2702a07SPaolo Abeni ret = mptcp_sendmsg_fastopen(sk, msg, len, &copied_syn); 1793d98a82a6SDmytro Shytyi copied += copied_syn; 1794fa9e5746SPaolo Abeni if (ret == -EINPROGRESS && copied_syn > 0) 1795d98a82a6SDmytro Shytyi goto out; 1796fa9e5746SPaolo Abeni else if (ret) 179769800e51SPaolo Abeni goto do_error; 1798d98a82a6SDmytro Shytyi } 1799d98a82a6SDmytro Shytyi 18001954b860SMat Martineau timeo = sock_sndtimeo(sk, msg->msg_flags & MSG_DONTWAIT); 18011954b860SMat Martineau 18021954b860SMat Martineau if ((1 << sk->sk_state) & ~(TCPF_ESTABLISHED | TCPF_CLOSE_WAIT)) { 18031954b860SMat Martineau ret = sk_stream_wait_connect(sk, &timeo); 18041954b860SMat Martineau if (ret) 180569800e51SPaolo Abeni goto do_error; 18061954b860SMat Martineau } 18071954b860SMat Martineau 180869800e51SPaolo Abeni ret = -EPIPE; 180969800e51SPaolo Abeni if (unlikely(sk->sk_err || (sk->sk_shutdown & SEND_SHUTDOWN))) 181069800e51SPaolo Abeni goto do_error; 181169800e51SPaolo Abeni 181217091708SFlorian Westphal pfrag = sk_page_frag(sk); 181318b683bfSPaolo Abeni 1814d9ca1de8SPaolo Abeni while (msg_data_left(msg)) { 1815724cfd2eSPaolo Abeni int total_ts, frag_truesize = 0; 1816d9ca1de8SPaolo Abeni struct mptcp_data_frag *dfrag; 1817d9ca1de8SPaolo Abeni bool dfrag_collapsed; 1818d9ca1de8SPaolo Abeni size_t psize, offset; 1819d9ca1de8SPaolo Abeni 1820d9ca1de8SPaolo Abeni /* reuse tail pfrag, if possible, or carve a new one from the 1821d9ca1de8SPaolo Abeni * page allocator 1822fb529e62SFlorian Westphal */ 1823d9ca1de8SPaolo Abeni dfrag = mptcp_pending_tail(sk); 1824d9ca1de8SPaolo Abeni dfrag_collapsed = mptcp_frag_can_collapse_to(msk, pfrag, dfrag); 1825d9ca1de8SPaolo Abeni if (!dfrag_collapsed) { 1826d9ca1de8SPaolo Abeni if (!sk_stream_memory_free(sk)) 1827d9ca1de8SPaolo Abeni goto wait_for_memory; 18286e628cd3SPaolo Abeni 1829d9ca1de8SPaolo Abeni if (!mptcp_page_frag_refill(sk, pfrag)) 1830d9ca1de8SPaolo Abeni goto wait_for_memory; 1831d9ca1de8SPaolo Abeni 1832d9ca1de8SPaolo Abeni dfrag = mptcp_carve_data_frag(msk, pfrag, pfrag->offset); 1833d9ca1de8SPaolo Abeni frag_truesize = dfrag->overhead; 1834fb529e62SFlorian Westphal } 1835fb529e62SFlorian Westphal 1836d9ca1de8SPaolo Abeni /* we do not bound vs wspace, to allow a single packet. 1837d9ca1de8SPaolo Abeni * memory accounting will prevent execessive memory usage 1838d9ca1de8SPaolo Abeni * anyway 1839d9ca1de8SPaolo Abeni */ 1840d9ca1de8SPaolo Abeni offset = dfrag->offset + dfrag->data_len; 1841d9ca1de8SPaolo Abeni psize = pfrag->size - offset; 1842d9ca1de8SPaolo Abeni psize = min_t(size_t, psize, msg_data_left(msg)); 1843724cfd2eSPaolo Abeni total_ts = psize + frag_truesize; 18448ce568edSPaolo Abeni 18456511882cSPaolo Abeni if (!sk_wmem_schedule(sk, total_ts)) 1846d9ca1de8SPaolo Abeni goto wait_for_memory; 1847d9ca1de8SPaolo Abeni 18480ffe8e74SPaolo Abeni ret = do_copy_data_nocache(sk, psize, &msg->msg_iter, 18490ffe8e74SPaolo Abeni page_address(dfrag->page) + offset); 18500ffe8e74SPaolo Abeni if (ret) 185169800e51SPaolo Abeni goto do_error; 1852d9ca1de8SPaolo Abeni 1853d9ca1de8SPaolo Abeni /* data successfully copied into the write queue */ 18545e6300e7SEric Dumazet sk_forward_alloc_add(sk, -total_ts); 1855d9ca1de8SPaolo Abeni copied += psize; 1856d9ca1de8SPaolo Abeni dfrag->data_len += psize; 1857d9ca1de8SPaolo Abeni frag_truesize += psize; 1858d9ca1de8SPaolo Abeni pfrag->offset += frag_truesize; 1859d9ca1de8SPaolo Abeni WRITE_ONCE(msk->write_seq, msk->write_seq + psize); 1860d9ca1de8SPaolo Abeni 1861d9ca1de8SPaolo Abeni /* charge data on mptcp pending queue to the msk socket 1862d9ca1de8SPaolo Abeni * Note: we charge such data both to sk and ssk 1863d9ca1de8SPaolo Abeni */ 1864d9ca1de8SPaolo Abeni sk_wmem_queued_add(sk, frag_truesize); 1865d9ca1de8SPaolo Abeni if (!dfrag_collapsed) { 1866d9ca1de8SPaolo Abeni get_page(dfrag->page); 1867d9ca1de8SPaolo Abeni list_add_tail(&dfrag->list, &msk->rtx_queue); 1868d9ca1de8SPaolo Abeni if (!msk->first_pending) 1869d9ca1de8SPaolo Abeni WRITE_ONCE(msk->first_pending, dfrag); 1870d9ca1de8SPaolo Abeni } 1871e4b61351SGeliang Tang pr_debug("msk=%p dfrag at seq=%llu len=%u sent=%u new=%d", msk, 1872d9ca1de8SPaolo Abeni dfrag->data_seq, dfrag->data_len, dfrag->already_sent, 1873d9ca1de8SPaolo Abeni !dfrag_collapsed); 1874d9ca1de8SPaolo Abeni 1875d9ca1de8SPaolo Abeni continue; 1876d9ca1de8SPaolo Abeni 1877d9ca1de8SPaolo Abeni wait_for_memory: 18785cf92bbaSPaolo Abeni mptcp_set_nospace(sk); 1879c2e6048fSPaolo Abeni __mptcp_push_pending(sk, msg->msg_flags); 1880f296234cSPeter Krystad ret = sk_stream_wait_memory(sk, &timeo); 1881f296234cSPeter Krystad if (ret) 188269800e51SPaolo Abeni goto do_error; 1883cec37a6eSPeter Krystad } 1884cec37a6eSPeter Krystad 188513e16037SPaolo Abeni if (copied) 1886c2e6048fSPaolo Abeni __mptcp_push_pending(sk, msg->msg_flags); 1887da51aef5SPaolo Abeni 18881954b860SMat Martineau out: 1889cec37a6eSPeter Krystad release_sock(sk); 189069800e51SPaolo Abeni return copied; 189169800e51SPaolo Abeni 189269800e51SPaolo Abeni do_error: 189369800e51SPaolo Abeni if (copied) 189469800e51SPaolo Abeni goto out; 189569800e51SPaolo Abeni 189669800e51SPaolo Abeni copied = sk_stream_error(sk, msg->msg_flags, ret); 189769800e51SPaolo Abeni goto out; 1898f870fa0bSMat Martineau } 1899f870fa0bSMat Martineau 19006771bfd9SFlorian Westphal static int __mptcp_recvmsg_mskq(struct mptcp_sock *msk, 19016771bfd9SFlorian Westphal struct msghdr *msg, 1902b7f653b2SFlorian Westphal size_t len, int flags, 1903b7f653b2SFlorian Westphal struct scm_timestamping_internal *tss, 1904b7f653b2SFlorian Westphal int *cmsg_flags) 19056771bfd9SFlorian Westphal { 1906ca4fb892SYonglong Li struct sk_buff *skb, *tmp; 19076771bfd9SFlorian Westphal int copied = 0; 19086771bfd9SFlorian Westphal 1909ca4fb892SYonglong Li skb_queue_walk_safe(&msk->receive_queue, skb, tmp) { 19106771bfd9SFlorian Westphal u32 offset = MPTCP_SKB_CB(skb)->offset; 19116771bfd9SFlorian Westphal u32 data_len = skb->len - offset; 19126771bfd9SFlorian Westphal u32 count = min_t(size_t, len - copied, data_len); 19136771bfd9SFlorian Westphal int err; 19146771bfd9SFlorian Westphal 1915d976092cSPaolo Abeni if (!(flags & MSG_TRUNC)) { 19166771bfd9SFlorian Westphal err = skb_copy_datagram_msg(skb, offset, msg, count); 19176771bfd9SFlorian Westphal if (unlikely(err < 0)) { 19186771bfd9SFlorian Westphal if (!copied) 19196771bfd9SFlorian Westphal return err; 19206771bfd9SFlorian Westphal break; 19216771bfd9SFlorian Westphal } 1922d976092cSPaolo Abeni } 19236771bfd9SFlorian Westphal 1924b7f653b2SFlorian Westphal if (MPTCP_SKB_CB(skb)->has_rxtstamp) { 1925b7f653b2SFlorian Westphal tcp_update_recv_tstamps(skb, tss); 1926b7f653b2SFlorian Westphal *cmsg_flags |= MPTCP_CMSG_TS; 1927b7f653b2SFlorian Westphal } 1928b7f653b2SFlorian Westphal 19296771bfd9SFlorian Westphal copied += count; 19306771bfd9SFlorian Westphal 19316771bfd9SFlorian Westphal if (count < data_len) { 19322c9e7765SFlorian Westphal if (!(flags & MSG_PEEK)) { 19336771bfd9SFlorian Westphal MPTCP_SKB_CB(skb)->offset += count; 19342c9e7765SFlorian Westphal MPTCP_SKB_CB(skb)->map_seq += count; 19355684ab1aSPaolo Abeni msk->bytes_consumed += count; 19362c9e7765SFlorian Westphal } 19376771bfd9SFlorian Westphal break; 19386771bfd9SFlorian Westphal } 19396771bfd9SFlorian Westphal 1940ca4fb892SYonglong Li if (!(flags & MSG_PEEK)) { 194187952603SPaolo Abeni /* we will bulk release the skb memory later */ 194287952603SPaolo Abeni skb->destructor = NULL; 1943ce599c51SPaolo Abeni WRITE_ONCE(msk->rmem_released, msk->rmem_released + skb->truesize); 194487952603SPaolo Abeni __skb_unlink(skb, &msk->receive_queue); 19456771bfd9SFlorian Westphal __kfree_skb(skb); 19465684ab1aSPaolo Abeni msk->bytes_consumed += count; 1947ca4fb892SYonglong Li } 19486771bfd9SFlorian Westphal 19496771bfd9SFlorian Westphal if (copied >= len) 19506771bfd9SFlorian Westphal break; 19516771bfd9SFlorian Westphal } 19526771bfd9SFlorian Westphal 19536771bfd9SFlorian Westphal return copied; 19546771bfd9SFlorian Westphal } 19556771bfd9SFlorian Westphal 1956a6b118feSFlorian Westphal /* receive buffer autotuning. See tcp_rcv_space_adjust for more information. 1957a6b118feSFlorian Westphal * 1958a6b118feSFlorian Westphal * Only difference: Use highest rtt estimate of the subflows in use. 1959a6b118feSFlorian Westphal */ 1960a6b118feSFlorian Westphal static void mptcp_rcv_space_adjust(struct mptcp_sock *msk, int copied) 1961a6b118feSFlorian Westphal { 1962a6b118feSFlorian Westphal struct mptcp_subflow_context *subflow; 1963a6b118feSFlorian Westphal struct sock *sk = (struct sock *)msk; 1964b8dc6d6cSPaolo Abeni u8 scaling_ratio = U8_MAX; 1965a6b118feSFlorian Westphal u32 time, advmss = 1; 1966a6b118feSFlorian Westphal u64 rtt_us, mstamp; 1967a6b118feSFlorian Westphal 1968109cdeb8SGeliang Tang msk_owned_by_me(msk); 1969a6b118feSFlorian Westphal 1970a6b118feSFlorian Westphal if (copied <= 0) 1971a6b118feSFlorian Westphal return; 1972a6b118feSFlorian Westphal 1973a6b118feSFlorian Westphal msk->rcvq_space.copied += copied; 1974a6b118feSFlorian Westphal 1975a6b118feSFlorian Westphal mstamp = div_u64(tcp_clock_ns(), NSEC_PER_USEC); 1976a6b118feSFlorian Westphal time = tcp_stamp_us_delta(mstamp, msk->rcvq_space.time); 1977a6b118feSFlorian Westphal 1978a6b118feSFlorian Westphal rtt_us = msk->rcvq_space.rtt_us; 1979a6b118feSFlorian Westphal if (rtt_us && time < (rtt_us >> 3)) 1980a6b118feSFlorian Westphal return; 1981a6b118feSFlorian Westphal 1982a6b118feSFlorian Westphal rtt_us = 0; 1983a6b118feSFlorian Westphal mptcp_for_each_subflow(msk, subflow) { 1984a6b118feSFlorian Westphal const struct tcp_sock *tp; 1985a6b118feSFlorian Westphal u64 sf_rtt_us; 1986a6b118feSFlorian Westphal u32 sf_advmss; 1987a6b118feSFlorian Westphal 1988a6b118feSFlorian Westphal tp = tcp_sk(mptcp_subflow_tcp_sock(subflow)); 1989a6b118feSFlorian Westphal 1990a6b118feSFlorian Westphal sf_rtt_us = READ_ONCE(tp->rcv_rtt_est.rtt_us); 1991a6b118feSFlorian Westphal sf_advmss = READ_ONCE(tp->advmss); 1992a6b118feSFlorian Westphal 1993a6b118feSFlorian Westphal rtt_us = max(sf_rtt_us, rtt_us); 1994a6b118feSFlorian Westphal advmss = max(sf_advmss, advmss); 1995b8dc6d6cSPaolo Abeni scaling_ratio = min(tp->scaling_ratio, scaling_ratio); 1996a6b118feSFlorian Westphal } 1997a6b118feSFlorian Westphal 1998a6b118feSFlorian Westphal msk->rcvq_space.rtt_us = rtt_us; 1999b8dc6d6cSPaolo Abeni msk->scaling_ratio = scaling_ratio; 2000a6b118feSFlorian Westphal if (time < (rtt_us >> 3) || rtt_us == 0) 2001a6b118feSFlorian Westphal return; 2002a6b118feSFlorian Westphal 2003a6b118feSFlorian Westphal if (msk->rcvq_space.copied <= msk->rcvq_space.space) 2004a6b118feSFlorian Westphal goto new_measure; 2005a6b118feSFlorian Westphal 200678047648SKuniyuki Iwashima if (READ_ONCE(sock_net(sk)->ipv4.sysctl_tcp_moderate_rcvbuf) && 2007a6b118feSFlorian Westphal !(sk->sk_userlocks & SOCK_RCVBUF_LOCK)) { 2008a6b118feSFlorian Westphal u64 rcvwin, grow; 2009b8dc6d6cSPaolo Abeni int rcvbuf; 2010a6b118feSFlorian Westphal 2011a6b118feSFlorian Westphal rcvwin = ((u64)msk->rcvq_space.copied << 1) + 16 * advmss; 2012a6b118feSFlorian Westphal 2013a6b118feSFlorian Westphal grow = rcvwin * (msk->rcvq_space.copied - msk->rcvq_space.space); 2014a6b118feSFlorian Westphal 2015a6b118feSFlorian Westphal do_div(grow, msk->rcvq_space.space); 2016a6b118feSFlorian Westphal rcvwin += (grow << 1); 2017a6b118feSFlorian Westphal 2018b8dc6d6cSPaolo Abeni rcvbuf = min_t(u64, __tcp_space_from_win(scaling_ratio, rcvwin), 201902739545SKuniyuki Iwashima READ_ONCE(sock_net(sk)->ipv4.sysctl_tcp_rmem[2])); 2020a6b118feSFlorian Westphal 2021a6b118feSFlorian Westphal if (rcvbuf > sk->sk_rcvbuf) { 2022a6b118feSFlorian Westphal u32 window_clamp; 2023a6b118feSFlorian Westphal 2024b8dc6d6cSPaolo Abeni window_clamp = __tcp_win_from_space(scaling_ratio, rcvbuf); 2025a6b118feSFlorian Westphal WRITE_ONCE(sk->sk_rcvbuf, rcvbuf); 2026a6b118feSFlorian Westphal 2027a6b118feSFlorian Westphal /* Make subflows follow along. If we do not do this, we 2028a6b118feSFlorian Westphal * get drops at subflow level if skbs can't be moved to 2029a6b118feSFlorian Westphal * the mptcp rx queue fast enough (announced rcv_win can 2030a6b118feSFlorian Westphal * exceed ssk->sk_rcvbuf). 2031a6b118feSFlorian Westphal */ 2032a6b118feSFlorian Westphal mptcp_for_each_subflow(msk, subflow) { 2033a6b118feSFlorian Westphal struct sock *ssk; 2034c76c6956SPaolo Abeni bool slow; 2035a6b118feSFlorian Westphal 2036a6b118feSFlorian Westphal ssk = mptcp_subflow_tcp_sock(subflow); 2037c76c6956SPaolo Abeni slow = lock_sock_fast(ssk); 2038a6b118feSFlorian Westphal WRITE_ONCE(ssk->sk_rcvbuf, rcvbuf); 2039a6b118feSFlorian Westphal tcp_sk(ssk)->window_clamp = window_clamp; 2040c76c6956SPaolo Abeni tcp_cleanup_rbuf(ssk, 1); 2041c76c6956SPaolo Abeni unlock_sock_fast(ssk, slow); 2042a6b118feSFlorian Westphal } 2043a6b118feSFlorian Westphal } 2044a6b118feSFlorian Westphal } 2045a6b118feSFlorian Westphal 2046a6b118feSFlorian Westphal msk->rcvq_space.space = msk->rcvq_space.copied; 2047a6b118feSFlorian Westphal new_measure: 2048a6b118feSFlorian Westphal msk->rcvq_space.copied = 0; 2049a6b118feSFlorian Westphal msk->rcvq_space.time = mstamp; 2050a6b118feSFlorian Westphal } 2051a6b118feSFlorian Westphal 205287952603SPaolo Abeni static void __mptcp_update_rmem(struct sock *sk) 205387952603SPaolo Abeni { 205487952603SPaolo Abeni struct mptcp_sock *msk = mptcp_sk(sk); 205587952603SPaolo Abeni 205687952603SPaolo Abeni if (!msk->rmem_released) 205787952603SPaolo Abeni return; 205887952603SPaolo Abeni 205987952603SPaolo Abeni atomic_sub(msk->rmem_released, &sk->sk_rmem_alloc); 20606511882cSPaolo Abeni mptcp_rmem_uncharge(sk, msk->rmem_released); 2061ce599c51SPaolo Abeni WRITE_ONCE(msk->rmem_released, 0); 206287952603SPaolo Abeni } 206387952603SPaolo Abeni 206487952603SPaolo Abeni static void __mptcp_splice_receive_queue(struct sock *sk) 206587952603SPaolo Abeni { 206687952603SPaolo Abeni struct mptcp_sock *msk = mptcp_sk(sk); 206787952603SPaolo Abeni 206887952603SPaolo Abeni skb_queue_splice_tail_init(&sk->sk_receive_queue, &msk->receive_queue); 206987952603SPaolo Abeni } 207087952603SPaolo Abeni 2071e3859603SPaolo Abeni static bool __mptcp_move_skbs(struct mptcp_sock *msk) 20726771bfd9SFlorian Westphal { 207387952603SPaolo Abeni struct sock *sk = (struct sock *)msk; 20746771bfd9SFlorian Westphal unsigned int moved = 0; 207587952603SPaolo Abeni bool ret, done; 2076d5f49190SPaolo Abeni 20776771bfd9SFlorian Westphal do { 20786771bfd9SFlorian Westphal struct sock *ssk = mptcp_subflow_recv_lookup(msk); 207965f49fe7SFlorian Westphal bool slowpath; 20806771bfd9SFlorian Westphal 208187952603SPaolo Abeni /* we can have data pending in the subflows only if the msk 208287952603SPaolo Abeni * receive buffer was full at subflow_data_ready() time, 208387952603SPaolo Abeni * that is an unlikely slow path. 208487952603SPaolo Abeni */ 208587952603SPaolo Abeni if (likely(!ssk)) 20866771bfd9SFlorian Westphal break; 20876771bfd9SFlorian Westphal 208865f49fe7SFlorian Westphal slowpath = lock_sock_fast(ssk); 208987952603SPaolo Abeni mptcp_data_lock(sk); 2090e3859603SPaolo Abeni __mptcp_update_rmem(sk); 20916771bfd9SFlorian Westphal done = __mptcp_move_skbs_from_subflow(msk, ssk, &moved); 209287952603SPaolo Abeni mptcp_data_unlock(sk); 2093499ada50SPaolo Abeni 2094499ada50SPaolo Abeni if (unlikely(ssk->sk_err)) 2095499ada50SPaolo Abeni __mptcp_error_report(sk); 209665f49fe7SFlorian Westphal unlock_sock_fast(ssk, slowpath); 20976771bfd9SFlorian Westphal } while (!done); 20986771bfd9SFlorian Westphal 209987952603SPaolo Abeni /* acquire the data lock only if some input data is pending */ 210087952603SPaolo Abeni ret = moved > 0; 210187952603SPaolo Abeni if (!RB_EMPTY_ROOT(&msk->out_of_order_queue) || 210287952603SPaolo Abeni !skb_queue_empty_lockless(&sk->sk_receive_queue)) { 210387952603SPaolo Abeni mptcp_data_lock(sk); 210487952603SPaolo Abeni __mptcp_update_rmem(sk); 210587952603SPaolo Abeni ret |= __mptcp_ofo_queue(msk); 210687952603SPaolo Abeni __mptcp_splice_receive_queue(sk); 210787952603SPaolo Abeni mptcp_data_unlock(sk); 2108ab174ad8SPaolo Abeni } 210987952603SPaolo Abeni if (ret) 211087952603SPaolo Abeni mptcp_check_data_fin((struct sock *)msk); 211187952603SPaolo Abeni return !skb_queue_empty(&msk->receive_queue); 21126771bfd9SFlorian Westphal } 21136771bfd9SFlorian Westphal 21142c9e7765SFlorian Westphal static unsigned int mptcp_inq_hint(const struct sock *sk) 21152c9e7765SFlorian Westphal { 21162c9e7765SFlorian Westphal const struct mptcp_sock *msk = mptcp_sk(sk); 21172c9e7765SFlorian Westphal const struct sk_buff *skb; 21182c9e7765SFlorian Westphal 21192c9e7765SFlorian Westphal skb = skb_peek(&msk->receive_queue); 21202c9e7765SFlorian Westphal if (skb) { 21212c9e7765SFlorian Westphal u64 hint_val = msk->ack_seq - MPTCP_SKB_CB(skb)->map_seq; 21222c9e7765SFlorian Westphal 21232c9e7765SFlorian Westphal if (hint_val >= INT_MAX) 21242c9e7765SFlorian Westphal return INT_MAX; 21252c9e7765SFlorian Westphal 21262c9e7765SFlorian Westphal return (unsigned int)hint_val; 21272c9e7765SFlorian Westphal } 21282c9e7765SFlorian Westphal 21292c9e7765SFlorian Westphal if (sk->sk_state == TCP_CLOSE || (sk->sk_shutdown & RCV_SHUTDOWN)) 21302c9e7765SFlorian Westphal return 1; 21312c9e7765SFlorian Westphal 21322c9e7765SFlorian Westphal return 0; 21332c9e7765SFlorian Westphal } 21342c9e7765SFlorian Westphal 2135f870fa0bSMat Martineau static int mptcp_recvmsg(struct sock *sk, struct msghdr *msg, size_t len, 2136ec095263SOliver Hartkopp int flags, int *addr_len) 2137f870fa0bSMat Martineau { 2138f870fa0bSMat Martineau struct mptcp_sock *msk = mptcp_sk(sk); 2139b7f653b2SFlorian Westphal struct scm_timestamping_internal tss; 2140b7f653b2SFlorian Westphal int copied = 0, cmsg_flags = 0; 21417a6a6cbcSPaolo Abeni int target; 21427a6a6cbcSPaolo Abeni long timeo; 2143f870fa0bSMat Martineau 2144cb9d80f4SPaolo Abeni /* MSG_ERRQUEUE is really a no-op till we support IP_RECVERR */ 2145cb9d80f4SPaolo Abeni if (unlikely(flags & MSG_ERRQUEUE)) 2146cb9d80f4SPaolo Abeni return inet_recv_error(sk, msg, len, addr_len); 2147cb9d80f4SPaolo Abeni 21486511882cSPaolo Abeni lock_sock(sk); 2149fd897679SPaolo Abeni if (unlikely(sk->sk_state == TCP_LISTEN)) { 2150fd897679SPaolo Abeni copied = -ENOTCONN; 2151fd897679SPaolo Abeni goto out_err; 2152fd897679SPaolo Abeni } 2153fd897679SPaolo Abeni 2154ec095263SOliver Hartkopp timeo = sock_rcvtimeo(sk, flags & MSG_DONTWAIT); 21557a6a6cbcSPaolo Abeni 21567a6a6cbcSPaolo Abeni len = min_t(size_t, len, INT_MAX); 21577a6a6cbcSPaolo Abeni target = sock_rcvlowat(sk, flags & MSG_WAITALL, len); 21587a6a6cbcSPaolo Abeni 21592c9e7765SFlorian Westphal if (unlikely(msk->recvmsg_inq)) 21602c9e7765SFlorian Westphal cmsg_flags = MPTCP_CMSG_INQ; 21612c9e7765SFlorian Westphal 216205e3eceaSEric Dumazet while (copied < len) { 2163e3859603SPaolo Abeni int bytes_read; 21647a6a6cbcSPaolo Abeni 2165b7f653b2SFlorian Westphal bytes_read = __mptcp_recvmsg_mskq(msk, msg, len - copied, flags, &tss, &cmsg_flags); 21666771bfd9SFlorian Westphal if (unlikely(bytes_read < 0)) { 21677a6a6cbcSPaolo Abeni if (!copied) 21687a6a6cbcSPaolo Abeni copied = bytes_read; 21696771bfd9SFlorian Westphal goto out_err; 2170cec37a6eSPeter Krystad } 2171cec37a6eSPeter Krystad 21727a6a6cbcSPaolo Abeni copied += bytes_read; 21737a6a6cbcSPaolo Abeni 2174ea4ca586SPaolo Abeni /* be sure to advertise window change */ 2175fd897679SPaolo Abeni mptcp_cleanup_rbuf(msk); 2176ea4ca586SPaolo Abeni 2177e3859603SPaolo Abeni if (skb_queue_empty(&msk->receive_queue) && __mptcp_move_skbs(msk)) 2178e3859603SPaolo Abeni continue; 2179e3859603SPaolo Abeni 21807a6a6cbcSPaolo Abeni /* only the master socket status is relevant here. The exit 21817a6a6cbcSPaolo Abeni * conditions mirror closely tcp_recvmsg() 21827a6a6cbcSPaolo Abeni */ 21837a6a6cbcSPaolo Abeni if (copied >= target) 21847a6a6cbcSPaolo Abeni break; 21857a6a6cbcSPaolo Abeni 21867a6a6cbcSPaolo Abeni if (copied) { 21877a6a6cbcSPaolo Abeni if (sk->sk_err || 21887a6a6cbcSPaolo Abeni sk->sk_state == TCP_CLOSE || 21897a6a6cbcSPaolo Abeni (sk->sk_shutdown & RCV_SHUTDOWN) || 21907a6a6cbcSPaolo Abeni !timeo || 21917a6a6cbcSPaolo Abeni signal_pending(current)) 21927a6a6cbcSPaolo Abeni break; 21937a6a6cbcSPaolo Abeni } else { 21947a6a6cbcSPaolo Abeni if (sk->sk_err) { 21957a6a6cbcSPaolo Abeni copied = sock_error(sk); 21967a6a6cbcSPaolo Abeni break; 21977a6a6cbcSPaolo Abeni } 21987a6a6cbcSPaolo Abeni 219987952603SPaolo Abeni if (sk->sk_shutdown & RCV_SHUTDOWN) { 220087952603SPaolo Abeni /* race breaker: the shutdown could be after the 220187952603SPaolo Abeni * previous receive queue check 220287952603SPaolo Abeni */ 2203e3859603SPaolo Abeni if (__mptcp_move_skbs(msk)) 220487952603SPaolo Abeni continue; 22057a6a6cbcSPaolo Abeni break; 220687952603SPaolo Abeni } 22077a6a6cbcSPaolo Abeni 22087a6a6cbcSPaolo Abeni if (sk->sk_state == TCP_CLOSE) { 22097a6a6cbcSPaolo Abeni copied = -ENOTCONN; 22107a6a6cbcSPaolo Abeni break; 22117a6a6cbcSPaolo Abeni } 22127a6a6cbcSPaolo Abeni 22137a6a6cbcSPaolo Abeni if (!timeo) { 22147a6a6cbcSPaolo Abeni copied = -EAGAIN; 22157a6a6cbcSPaolo Abeni break; 22167a6a6cbcSPaolo Abeni } 22177a6a6cbcSPaolo Abeni 22187a6a6cbcSPaolo Abeni if (signal_pending(current)) { 22197a6a6cbcSPaolo Abeni copied = sock_intr_errno(timeo); 22207a6a6cbcSPaolo Abeni break; 22217a6a6cbcSPaolo Abeni } 22227a6a6cbcSPaolo Abeni } 22237a6a6cbcSPaolo Abeni 22247a6a6cbcSPaolo Abeni pr_debug("block timeout %ld", timeo); 2225612f71d7SPaolo Abeni sk_wait_data(sk, &timeo, NULL); 22267a6a6cbcSPaolo Abeni } 22273c90e377SPaolo Abeni 22286771bfd9SFlorian Westphal out_err: 2229b7f653b2SFlorian Westphal if (cmsg_flags && copied >= 0) { 2230b7f653b2SFlorian Westphal if (cmsg_flags & MPTCP_CMSG_TS) 2231b7f653b2SFlorian Westphal tcp_recv_timestamp(msg, sk, &tss); 22322c9e7765SFlorian Westphal 22332c9e7765SFlorian Westphal if (cmsg_flags & MPTCP_CMSG_INQ) { 22342c9e7765SFlorian Westphal unsigned int inq = mptcp_inq_hint(sk); 22352c9e7765SFlorian Westphal 22362c9e7765SFlorian Westphal put_cmsg(msg, SOL_TCP, TCP_CM_INQ, sizeof(inq), &inq); 22372c9e7765SFlorian Westphal } 2238b7f653b2SFlorian Westphal } 2239b7f653b2SFlorian Westphal 2240612f71d7SPaolo Abeni pr_debug("msk=%p rx queue empty=%d:%d copied=%d", 2241612f71d7SPaolo Abeni msk, skb_queue_empty_lockless(&sk->sk_receive_queue), 2242612f71d7SPaolo Abeni skb_queue_empty(&msk->receive_queue), copied); 2243ca4fb892SYonglong Li if (!(flags & MSG_PEEK)) 2244a6b118feSFlorian Westphal mptcp_rcv_space_adjust(msk, copied); 2245a6b118feSFlorian Westphal 2246cec37a6eSPeter Krystad release_sock(sk); 2247cec37a6eSPeter Krystad return copied; 2248cec37a6eSPeter Krystad } 2249cec37a6eSPeter Krystad 2250b51f9b80SPaolo Abeni static void mptcp_retransmit_timer(struct timer_list *t) 2251b51f9b80SPaolo Abeni { 2252b51f9b80SPaolo Abeni struct inet_connection_sock *icsk = from_timer(icsk, t, 2253b51f9b80SPaolo Abeni icsk_retransmit_timer); 2254b51f9b80SPaolo Abeni struct sock *sk = &icsk->icsk_inet.sk; 22552d6f5a2bSPaolo Abeni struct mptcp_sock *msk = mptcp_sk(sk); 2256b51f9b80SPaolo Abeni 2257b51f9b80SPaolo Abeni bh_lock_sock(sk); 2258b51f9b80SPaolo Abeni if (!sock_owned_by_user(sk)) { 22592d6f5a2bSPaolo Abeni /* we need a process context to retransmit */ 22602d6f5a2bSPaolo Abeni if (!test_and_set_bit(MPTCP_WORK_RTX, &msk->flags)) 22612d6f5a2bSPaolo Abeni mptcp_schedule_work(sk); 2262b51f9b80SPaolo Abeni } else { 2263b51f9b80SPaolo Abeni /* delegate our work to tcp_release_cb() */ 2264e9d09bacSPaolo Abeni __set_bit(MPTCP_RETRANSMIT, &msk->cb_flags); 2265b51f9b80SPaolo Abeni } 2266b51f9b80SPaolo Abeni bh_unlock_sock(sk); 2267b51f9b80SPaolo Abeni sock_put(sk); 2268b51f9b80SPaolo Abeni } 2269b51f9b80SPaolo Abeni 2270f6909dc1SPaolo Abeni static void mptcp_tout_timer(struct timer_list *t) 2271e16163b6SPaolo Abeni { 2272e16163b6SPaolo Abeni struct sock *sk = from_timer(sk, t, sk_timer); 2273e16163b6SPaolo Abeni 2274e16163b6SPaolo Abeni mptcp_schedule_work(sk); 2275b6d69fc8SFlorian Westphal sock_put(sk); 2276e16163b6SPaolo Abeni } 2277e16163b6SPaolo Abeni 22783b1d6210SPaolo Abeni /* Find an idle subflow. Return NULL if there is unacked data at tcp 22793b1d6210SPaolo Abeni * level. 22803b1d6210SPaolo Abeni * 22813b1d6210SPaolo Abeni * A backup subflow is returned only if that is the only kind available. 22823b1d6210SPaolo Abeni */ 228307336a87SGeliang Tang struct sock *mptcp_subflow_get_retrans(struct mptcp_sock *msk) 22843b1d6210SPaolo Abeni { 228571b7dec2SPaolo Abeni struct sock *backup = NULL, *pick = NULL; 22863b1d6210SPaolo Abeni struct mptcp_subflow_context *subflow; 228771b7dec2SPaolo Abeni int min_stale_count = INT_MAX; 22883b1d6210SPaolo Abeni 22893b1d6210SPaolo Abeni mptcp_for_each_subflow(msk, subflow) { 22903b1d6210SPaolo Abeni struct sock *ssk = mptcp_subflow_tcp_sock(subflow); 22913b1d6210SPaolo Abeni 2292ff5a0b42SPaolo Abeni if (!__mptcp_subflow_active(subflow)) 2293d5f49190SPaolo Abeni continue; 2294d5f49190SPaolo Abeni 229571b7dec2SPaolo Abeni /* still data outstanding at TCP level? skip this */ 229671b7dec2SPaolo Abeni if (!tcp_rtx_and_write_queues_empty(ssk)) { 229771b7dec2SPaolo Abeni mptcp_pm_subflow_chk_stale(msk, ssk); 229871b7dec2SPaolo Abeni min_stale_count = min_t(int, min_stale_count, subflow->stale_count); 2299860975c6SFlorian Westphal continue; 2300860975c6SFlorian Westphal } 23013b1d6210SPaolo Abeni 23023b1d6210SPaolo Abeni if (subflow->backup) { 23033b1d6210SPaolo Abeni if (!backup) 23043b1d6210SPaolo Abeni backup = ssk; 23053b1d6210SPaolo Abeni continue; 23063b1d6210SPaolo Abeni } 23073b1d6210SPaolo Abeni 230871b7dec2SPaolo Abeni if (!pick) 230971b7dec2SPaolo Abeni pick = ssk; 23103b1d6210SPaolo Abeni } 23113b1d6210SPaolo Abeni 231271b7dec2SPaolo Abeni if (pick) 231371b7dec2SPaolo Abeni return pick; 231471b7dec2SPaolo Abeni 231571b7dec2SPaolo Abeni /* use backup only if there are no progresses anywhere */ 231671b7dec2SPaolo Abeni return min_stale_count > 1 ? backup : NULL; 23173b1d6210SPaolo Abeni } 23183b1d6210SPaolo Abeni 23191e1d9d6fSPaolo Abeni bool __mptcp_retransmit_pending_data(struct sock *sk) 23201e1d9d6fSPaolo Abeni { 23211e1d9d6fSPaolo Abeni struct mptcp_data_frag *cur, *rtx_head; 23221e1d9d6fSPaolo Abeni struct mptcp_sock *msk = mptcp_sk(sk); 23231e1d9d6fSPaolo Abeni 232400df24f1SGeliang Tang if (__mptcp_check_fallback(msk)) 23251e1d9d6fSPaolo Abeni return false; 23261e1d9d6fSPaolo Abeni 23271e1d9d6fSPaolo Abeni if (tcp_rtx_and_write_queues_empty(sk)) 23281e1d9d6fSPaolo Abeni return false; 23291e1d9d6fSPaolo Abeni 23301e1d9d6fSPaolo Abeni /* the closing socket has some data untransmitted and/or unacked: 23311e1d9d6fSPaolo Abeni * some data in the mptcp rtx queue has not really xmitted yet. 23321e1d9d6fSPaolo Abeni * keep it simple and re-inject the whole mptcp level rtx queue 23331e1d9d6fSPaolo Abeni */ 23341e1d9d6fSPaolo Abeni mptcp_data_lock(sk); 23351e1d9d6fSPaolo Abeni __mptcp_clean_una_wakeup(sk); 23361e1d9d6fSPaolo Abeni rtx_head = mptcp_rtx_head(sk); 23371e1d9d6fSPaolo Abeni if (!rtx_head) { 23381e1d9d6fSPaolo Abeni mptcp_data_unlock(sk); 23391e1d9d6fSPaolo Abeni return false; 23401e1d9d6fSPaolo Abeni } 23411e1d9d6fSPaolo Abeni 23421e1d9d6fSPaolo Abeni msk->recovery_snd_nxt = msk->snd_nxt; 23431e1d9d6fSPaolo Abeni msk->recovery = true; 23441e1d9d6fSPaolo Abeni mptcp_data_unlock(sk); 23451e1d9d6fSPaolo Abeni 23461e1d9d6fSPaolo Abeni msk->first_pending = rtx_head; 23471e1d9d6fSPaolo Abeni msk->snd_burst = 0; 23481e1d9d6fSPaolo Abeni 23491e1d9d6fSPaolo Abeni /* be sure to clear the "sent status" on all re-injected fragments */ 23501e1d9d6fSPaolo Abeni list_for_each_entry(cur, &msk->rtx_queue, list) { 23511e1d9d6fSPaolo Abeni if (!cur->already_sent) 23521e1d9d6fSPaolo Abeni break; 23531e1d9d6fSPaolo Abeni cur->already_sent = 0; 23541e1d9d6fSPaolo Abeni } 23551e1d9d6fSPaolo Abeni 23561e1d9d6fSPaolo Abeni return true; 23571e1d9d6fSPaolo Abeni } 23581e1d9d6fSPaolo Abeni 2359b29fcfb5SPaolo Abeni /* flags for __mptcp_close_ssk() */ 2360b29fcfb5SPaolo Abeni #define MPTCP_CF_PUSH BIT(1) 2361b29fcfb5SPaolo Abeni #define MPTCP_CF_FASTCLOSE BIT(2) 2362b29fcfb5SPaolo Abeni 236314c56686SGeliang Tang /* be sure to send a reset only if the caller asked for it, also 236414c56686SGeliang Tang * clean completely the subflow status when the subflow reaches 236514c56686SGeliang Tang * TCP_CLOSE state 236614c56686SGeliang Tang */ 236714c56686SGeliang Tang static void __mptcp_subflow_disconnect(struct sock *ssk, 236814c56686SGeliang Tang struct mptcp_subflow_context *subflow, 236914c56686SGeliang Tang unsigned int flags) 237014c56686SGeliang Tang { 237114c56686SGeliang Tang if (((1 << ssk->sk_state) & (TCPF_CLOSE | TCPF_LISTEN)) || 237214c56686SGeliang Tang (flags & MPTCP_CF_FASTCLOSE)) { 237314c56686SGeliang Tang /* The MPTCP code never wait on the subflow sockets, TCP-level 237414c56686SGeliang Tang * disconnect should never fail 237514c56686SGeliang Tang */ 237614c56686SGeliang Tang WARN_ON_ONCE(tcp_disconnect(ssk, 0)); 237714c56686SGeliang Tang mptcp_subflow_ctx_reset(subflow); 237814c56686SGeliang Tang } else { 237914c56686SGeliang Tang tcp_shutdown(ssk, SEND_SHUTDOWN); 238014c56686SGeliang Tang } 238114c56686SGeliang Tang } 238214c56686SGeliang Tang 2383cec37a6eSPeter Krystad /* subflow sockets can be either outgoing (connect) or incoming 2384cec37a6eSPeter Krystad * (accept). 2385cec37a6eSPeter Krystad * 2386cec37a6eSPeter Krystad * Outgoing subflows use in-kernel sockets. 2387cec37a6eSPeter Krystad * Incoming subflows do not have their own 'struct socket' allocated, 2388cec37a6eSPeter Krystad * so we need to use tcp_close() after detaching them from the mptcp 2389cec37a6eSPeter Krystad * parent socket. 2390cec37a6eSPeter Krystad */ 2391a141e02eSFlorian Westphal static void __mptcp_close_ssk(struct sock *sk, struct sock *ssk, 2392b29fcfb5SPaolo Abeni struct mptcp_subflow_context *subflow, 2393b29fcfb5SPaolo Abeni unsigned int flags) 2394cec37a6eSPeter Krystad { 2395e0be4931SFlorian Westphal struct mptcp_sock *msk = mptcp_sk(sk); 239663740448SPaolo Abeni bool dispose_it, need_push = false; 239763740448SPaolo Abeni 239863740448SPaolo Abeni /* If the first subflow moved to a close state before accept, e.g. due 239927e5ccc2SPaolo Abeni * to an incoming reset or listener shutdown, the subflow socket is 240027e5ccc2SPaolo Abeni * already deleted by inet_child_forget() and the mptcp socket can't 240127e5ccc2SPaolo Abeni * survive too. 240263740448SPaolo Abeni */ 240327e5ccc2SPaolo Abeni if (msk->in_accept_queue && msk->first == ssk && 240427e5ccc2SPaolo Abeni (sock_flag(sk, SOCK_DEAD) || sock_flag(ssk, SOCK_DEAD))) { 240563740448SPaolo Abeni /* ensure later check in mptcp_worker() will dispose the msk */ 240663740448SPaolo Abeni sock_set_flag(sk, SOCK_DEAD); 2407d866ae9aSPaolo Abeni mptcp_set_close_tout(sk, tcp_jiffies32 - (mptcp_close_timeout(sk) + 1)); 240863740448SPaolo Abeni lock_sock_nested(ssk, SINGLE_DEPTH_NESTING); 240963740448SPaolo Abeni mptcp_subflow_drop_ctx(ssk); 241063740448SPaolo Abeni goto out_release; 241163740448SPaolo Abeni } 2412e0be4931SFlorian Westphal 241339880bd8SPaolo Abeni dispose_it = msk->free_first || ssk != msk->first; 2414b29fcfb5SPaolo Abeni if (dispose_it) 2415cec37a6eSPeter Krystad list_del(&subflow->node); 2416cec37a6eSPeter Krystad 24173f8b2667SPaolo Abeni lock_sock_nested(ssk, SINGLE_DEPTH_NESTING); 2418e16163b6SPaolo Abeni 2419ff18f9efSPaolo Abeni if ((flags & MPTCP_CF_FASTCLOSE) && !__mptcp_check_fallback(msk)) { 242014c56686SGeliang Tang /* be sure to force the tcp_close path 2421d21f8348SPaolo Abeni * to generate the egress reset 2422d21f8348SPaolo Abeni */ 2423d21f8348SPaolo Abeni ssk->sk_lingertime = 0; 2424d21f8348SPaolo Abeni sock_set_flag(ssk, SOCK_LINGER); 2425b29fcfb5SPaolo Abeni subflow->send_fastclose = 1; 2426d21f8348SPaolo Abeni } 2427b29fcfb5SPaolo Abeni 2428b29fcfb5SPaolo Abeni need_push = (flags & MPTCP_CF_PUSH) && __mptcp_retransmit_pending_data(sk); 2429b29fcfb5SPaolo Abeni if (!dispose_it) { 243014c56686SGeliang Tang __mptcp_subflow_disconnect(ssk, subflow, flags); 2431b29fcfb5SPaolo Abeni release_sock(ssk); 2432b29fcfb5SPaolo Abeni 2433b29fcfb5SPaolo Abeni goto out; 2434b29fcfb5SPaolo Abeni } 2435b29fcfb5SPaolo Abeni 2436d7b1bfd0SPaolo Abeni subflow->disposable = 1; 2437d7b1bfd0SPaolo Abeni 2438e16163b6SPaolo Abeni /* if ssk hit tcp_done(), tcp_cleanup_ulp() cleared the related ops 2439e16163b6SPaolo Abeni * the ssk has been already destroyed, we just need to release the 2440e16163b6SPaolo Abeni * reference owned by msk; 2441e16163b6SPaolo Abeni */ 2442e16163b6SPaolo Abeni if (!inet_csk(ssk)->icsk_ulp_ops) { 2443b6985b9bSPaolo Abeni WARN_ON_ONCE(!sock_flag(ssk, SOCK_DEAD)); 2444e16163b6SPaolo Abeni kfree_rcu(subflow, rcu); 2445e16163b6SPaolo Abeni } else { 2446d7b1bfd0SPaolo Abeni /* otherwise tcp will dispose of the ssk and subflow ctx */ 2447e16163b6SPaolo Abeni __tcp_close(ssk, 0); 2448e16163b6SPaolo Abeni 2449e16163b6SPaolo Abeni /* close acquired an extra ref */ 2450e16163b6SPaolo Abeni __sock_put(ssk); 2451e16163b6SPaolo Abeni } 245263740448SPaolo Abeni 245363740448SPaolo Abeni out_release: 24549f1a9881SPaolo Abeni __mptcp_subflow_error_report(sk, ssk); 2455e16163b6SPaolo Abeni release_sock(ssk); 2456e16163b6SPaolo Abeni 2457e16163b6SPaolo Abeni sock_put(ssk); 2458e0be4931SFlorian Westphal 2459c8fe62f0SFlorian Westphal if (ssk == msk->first) 24601b1b43eeSPaolo Abeni WRITE_ONCE(msk->first, NULL); 2461c8fe62f0SFlorian Westphal 2462b29fcfb5SPaolo Abeni out: 24638005184fSPaolo Abeni __mptcp_sync_sndbuf(sk); 24641e1d9d6fSPaolo Abeni if (need_push) 24651e1d9d6fSPaolo Abeni __mptcp_push_pending(sk, 0); 246627e5ccc2SPaolo Abeni 246727e5ccc2SPaolo Abeni /* Catch every 'all subflows closed' scenario, including peers silently 246827e5ccc2SPaolo Abeni * closing them, e.g. due to timeout. 246927e5ccc2SPaolo Abeni * For established sockets, allow an additional timeout before closing, 247027e5ccc2SPaolo Abeni * as the protocol can still create more subflows. 247127e5ccc2SPaolo Abeni */ 247227e5ccc2SPaolo Abeni if (list_is_singular(&msk->conn_list) && msk->first && 247327e5ccc2SPaolo Abeni inet_sk_state_load(msk->first) == TCP_CLOSE) { 247427e5ccc2SPaolo Abeni if (sk->sk_state != TCP_ESTABLISHED || 247527e5ccc2SPaolo Abeni msk->in_accept_queue || sock_flag(sk, SOCK_DEAD)) { 247627e5ccc2SPaolo Abeni inet_sk_state_store(sk, TCP_CLOSE); 247727e5ccc2SPaolo Abeni mptcp_close_wake_up(sk); 247827e5ccc2SPaolo Abeni } else { 247927e5ccc2SPaolo Abeni mptcp_start_tout_timer(sk); 248027e5ccc2SPaolo Abeni } 248127e5ccc2SPaolo Abeni } 2482f870fa0bSMat Martineau } 2483f870fa0bSMat Martineau 2484a141e02eSFlorian Westphal void mptcp_close_ssk(struct sock *sk, struct sock *ssk, 2485a141e02eSFlorian Westphal struct mptcp_subflow_context *subflow) 2486a141e02eSFlorian Westphal { 2487b911c97cSFlorian Westphal if (sk->sk_state == TCP_ESTABLISHED) 2488b911c97cSFlorian Westphal mptcp_event(MPTCP_EVENT_SUB_CLOSED, mptcp_sk(sk), ssk, GFP_KERNEL); 2489a88c9e49SPaolo Abeni 2490a88c9e49SPaolo Abeni /* subflow aborted before reaching the fully_established status 2491a88c9e49SPaolo Abeni * attempt the creation of the next subflow 2492a88c9e49SPaolo Abeni */ 249374cbb0c6SGeliang Tang mptcp_pm_subflow_check_next(mptcp_sk(sk), subflow); 2494a88c9e49SPaolo Abeni 2495b29fcfb5SPaolo Abeni __mptcp_close_ssk(sk, ssk, subflow, MPTCP_CF_PUSH); 2496a141e02eSFlorian Westphal } 2497a141e02eSFlorian Westphal 2498dc24f8b4SPaolo Abeni static unsigned int mptcp_sync_mss(struct sock *sk, u32 pmtu) 2499dc24f8b4SPaolo Abeni { 2500dc24f8b4SPaolo Abeni return 0; 2501dc24f8b4SPaolo Abeni } 2502dc24f8b4SPaolo Abeni 2503b6985b9bSPaolo Abeni static void __mptcp_close_subflow(struct sock *sk) 25040e4f35d7SPaolo Abeni { 25050e4f35d7SPaolo Abeni struct mptcp_subflow_context *subflow, *tmp; 2506b6985b9bSPaolo Abeni struct mptcp_sock *msk = mptcp_sk(sk); 25070e4f35d7SPaolo Abeni 25083abc05d9SFlorian Westphal might_sleep(); 25093abc05d9SFlorian Westphal 25105efbf6f7SMatthieu Baerts mptcp_for_each_subflow_safe(msk, subflow, tmp) { 25110e4f35d7SPaolo Abeni struct sock *ssk = mptcp_subflow_tcp_sock(subflow); 25120e4f35d7SPaolo Abeni 25130e4f35d7SPaolo Abeni if (inet_sk_state_load(ssk) != TCP_CLOSE) 25140e4f35d7SPaolo Abeni continue; 25150e4f35d7SPaolo Abeni 251640947e13SFlorian Westphal /* 'subflow_data_ready' will re-sched once rx queue is empty */ 251740947e13SFlorian Westphal if (!skb_queue_empty_lockless(&ssk->sk_receive_queue)) 251840947e13SFlorian Westphal continue; 251940947e13SFlorian Westphal 2520b6985b9bSPaolo Abeni mptcp_close_ssk(sk, ssk, subflow); 2521b6985b9bSPaolo Abeni } 2522b6985b9bSPaolo Abeni 25230e4f35d7SPaolo Abeni } 25240e4f35d7SPaolo Abeni 252527e5ccc2SPaolo Abeni static bool mptcp_close_tout_expired(const struct sock *sk) 2526e16163b6SPaolo Abeni { 252727e5ccc2SPaolo Abeni if (!inet_csk(sk)->icsk_mtup.probe_timestamp || 252827e5ccc2SPaolo Abeni sk->sk_state == TCP_CLOSE) 2529e16163b6SPaolo Abeni return false; 253027e5ccc2SPaolo Abeni 253127e5ccc2SPaolo Abeni return time_after32(tcp_jiffies32, 2532d866ae9aSPaolo Abeni inet_csk(sk)->icsk_mtup.probe_timestamp + mptcp_close_timeout(sk)); 2533e16163b6SPaolo Abeni } 2534e16163b6SPaolo Abeni 253550c504a2SFlorian Westphal static void mptcp_check_fastclose(struct mptcp_sock *msk) 253650c504a2SFlorian Westphal { 253750c504a2SFlorian Westphal struct mptcp_subflow_context *subflow, *tmp; 253880638684SGeliang Tang struct sock *sk = (struct sock *)msk; 253950c504a2SFlorian Westphal 254050c504a2SFlorian Westphal if (likely(!READ_ONCE(msk->rcv_fastclose))) 254150c504a2SFlorian Westphal return; 254250c504a2SFlorian Westphal 254350c504a2SFlorian Westphal mptcp_token_destroy(msk); 254450c504a2SFlorian Westphal 25455efbf6f7SMatthieu Baerts mptcp_for_each_subflow_safe(msk, subflow, tmp) { 254650c504a2SFlorian Westphal struct sock *tcp_sk = mptcp_subflow_tcp_sock(subflow); 254775e908c3SPaolo Abeni bool slow; 254850c504a2SFlorian Westphal 254975e908c3SPaolo Abeni slow = lock_sock_fast(tcp_sk); 255050c504a2SFlorian Westphal if (tcp_sk->sk_state != TCP_CLOSE) { 255150c504a2SFlorian Westphal tcp_send_active_reset(tcp_sk, GFP_ATOMIC); 255250c504a2SFlorian Westphal tcp_set_state(tcp_sk, TCP_CLOSE); 255350c504a2SFlorian Westphal } 255475e908c3SPaolo Abeni unlock_sock_fast(tcp_sk, slow); 255550c504a2SFlorian Westphal } 255650c504a2SFlorian Westphal 255769800e51SPaolo Abeni /* Mirror the tcp_reset() error propagation */ 255869800e51SPaolo Abeni switch (sk->sk_state) { 255969800e51SPaolo Abeni case TCP_SYN_SENT: 25609ae8e5adSEric Dumazet WRITE_ONCE(sk->sk_err, ECONNREFUSED); 256169800e51SPaolo Abeni break; 256269800e51SPaolo Abeni case TCP_CLOSE_WAIT: 25639ae8e5adSEric Dumazet WRITE_ONCE(sk->sk_err, EPIPE); 256469800e51SPaolo Abeni break; 256569800e51SPaolo Abeni case TCP_CLOSE: 256669800e51SPaolo Abeni return; 256769800e51SPaolo Abeni default: 25689ae8e5adSEric Dumazet WRITE_ONCE(sk->sk_err, ECONNRESET); 256969800e51SPaolo Abeni } 257069800e51SPaolo Abeni 257150c504a2SFlorian Westphal inet_sk_state_store(sk, TCP_CLOSE); 25726b9831bfSPaolo Abeni WRITE_ONCE(sk->sk_shutdown, SHUTDOWN_MASK); 257350c504a2SFlorian Westphal smp_mb__before_atomic(); /* SHUTDOWN must be visible first */ 257450c504a2SFlorian Westphal set_bit(MPTCP_WORK_CLOSE_SUBFLOW, &msk->flags); 257550c504a2SFlorian Westphal 257669800e51SPaolo Abeni /* the calling mptcp_worker will properly destroy the socket */ 257769800e51SPaolo Abeni if (sock_flag(sk, SOCK_DEAD)) 257869800e51SPaolo Abeni return; 257969800e51SPaolo Abeni 258069800e51SPaolo Abeni sk->sk_state_change(sk); 258169800e51SPaolo Abeni sk_error_report(sk); 258250c504a2SFlorian Westphal } 258350c504a2SFlorian Westphal 25842948d0a1SPaolo Abeni static void __mptcp_retrans(struct sock *sk) 258580992017SPaolo Abeni { 25862948d0a1SPaolo Abeni struct mptcp_sock *msk = mptcp_sk(sk); 2587ee2708aeSGeliang Tang struct mptcp_subflow_context *subflow; 2588caf971dfSPaolo Abeni struct mptcp_sendmsg_info info = {}; 25893b1d6210SPaolo Abeni struct mptcp_data_frag *dfrag; 25902948d0a1SPaolo Abeni struct sock *ssk; 2591ee2708aeSGeliang Tang int ret, err; 2592ee2708aeSGeliang Tang u16 len = 0; 25932948d0a1SPaolo Abeni 2594b5941f06SPaolo Abeni mptcp_clean_una_wakeup(sk); 25953241a9c0SFlorian Westphal 25963241a9c0SFlorian Westphal /* first check ssk: need to kick "stale" logic */ 2597ee2708aeSGeliang Tang err = mptcp_sched_get_retrans(msk); 25982948d0a1SPaolo Abeni dfrag = mptcp_rtx_head(sk); 25996477dd39SMat Martineau if (!dfrag) { 26006477dd39SMat Martineau if (mptcp_data_fin_enabled(msk)) { 26016477dd39SMat Martineau struct inet_connection_sock *icsk = inet_csk(sk); 26026477dd39SMat Martineau 26036477dd39SMat Martineau icsk->icsk_retransmits++; 26046477dd39SMat Martineau mptcp_set_datafin_timeout(sk); 26056477dd39SMat Martineau mptcp_send_ack(msk); 26066477dd39SMat Martineau 26076477dd39SMat Martineau goto reset_timer; 26086477dd39SMat Martineau } 26096477dd39SMat Martineau 26103241a9c0SFlorian Westphal if (!mptcp_send_head(sk)) 26112948d0a1SPaolo Abeni return; 26123241a9c0SFlorian Westphal 26133241a9c0SFlorian Westphal goto reset_timer; 26146477dd39SMat Martineau } 26152948d0a1SPaolo Abeni 2616ee2708aeSGeliang Tang if (err) 26172948d0a1SPaolo Abeni goto reset_timer; 26182948d0a1SPaolo Abeni 2619ee2708aeSGeliang Tang mptcp_for_each_subflow(msk, subflow) { 2620ee2708aeSGeliang Tang if (READ_ONCE(subflow->scheduled)) { 2621ee2708aeSGeliang Tang u16 copied = 0; 2622ee2708aeSGeliang Tang 2623ee2708aeSGeliang Tang mptcp_subflow_set_scheduled(subflow, false); 2624ee2708aeSGeliang Tang 2625ee2708aeSGeliang Tang ssk = mptcp_subflow_tcp_sock(subflow); 2626ee2708aeSGeliang Tang 26272948d0a1SPaolo Abeni lock_sock(ssk); 26282948d0a1SPaolo Abeni 26292948d0a1SPaolo Abeni /* limit retransmission to the bytes already sent on some subflows */ 26302948d0a1SPaolo Abeni info.sent = 0; 2631ee2708aeSGeliang Tang info.limit = READ_ONCE(msk->csum_enabled) ? dfrag->data_len : 2632ee2708aeSGeliang Tang dfrag->already_sent; 26334e14867dSPaolo Abeni while (info.sent < info.limit) { 26342948d0a1SPaolo Abeni ret = mptcp_sendmsg_frag(sk, ssk, dfrag, &info); 26352948d0a1SPaolo Abeni if (ret <= 0) 26362948d0a1SPaolo Abeni break; 26372948d0a1SPaolo Abeni 26382948d0a1SPaolo Abeni MPTCP_INC_STATS(sock_net(sk), MPTCP_MIB_RETRANSSEGS); 26392948d0a1SPaolo Abeni copied += ret; 26402948d0a1SPaolo Abeni info.sent += ret; 26412948d0a1SPaolo Abeni } 26424e14867dSPaolo Abeni if (copied) { 2643ee2708aeSGeliang Tang len = max(copied, len); 26442948d0a1SPaolo Abeni tcp_push(ssk, 0, info.mss_now, tcp_sk(ssk)->nonagle, 26452948d0a1SPaolo Abeni info.size_goal); 26460530020aSGeliang Tang WRITE_ONCE(msk->allow_infinite_fallback, false); 26474e14867dSPaolo Abeni } 26482948d0a1SPaolo Abeni 26492948d0a1SPaolo Abeni release_sock(ssk); 2650ee2708aeSGeliang Tang } 2651ee2708aeSGeliang Tang } 2652ee2708aeSGeliang Tang 2653ee2708aeSGeliang Tang msk->bytes_retrans += len; 2654ee2708aeSGeliang Tang dfrag->already_sent = max(dfrag->already_sent, len); 26552948d0a1SPaolo Abeni 26562948d0a1SPaolo Abeni reset_timer: 26573241a9c0SFlorian Westphal mptcp_check_and_set_pending(sk); 26583241a9c0SFlorian Westphal 2659f6909dc1SPaolo Abeni if (!mptcp_rtx_timer_pending(sk)) 2660f6909dc1SPaolo Abeni mptcp_reset_rtx_timer(sk); 26612948d0a1SPaolo Abeni } 26622948d0a1SPaolo Abeni 266376a13b31SGeliang Tang /* schedule the timeout timer for the relevant event: either close timeout 266476a13b31SGeliang Tang * or mp_fail timeout. The close timeout takes precedence on the mp_fail one 266576a13b31SGeliang Tang */ 2666f6909dc1SPaolo Abeni void mptcp_reset_tout_timer(struct mptcp_sock *msk, unsigned long fail_tout) 266776a13b31SGeliang Tang { 266876a13b31SGeliang Tang struct sock *sk = (struct sock *)msk; 266976a13b31SGeliang Tang unsigned long timeout, close_timeout; 267076a13b31SGeliang Tang 267127e5ccc2SPaolo Abeni if (!fail_tout && !inet_csk(sk)->icsk_mtup.probe_timestamp) 267276a13b31SGeliang Tang return; 267376a13b31SGeliang Tang 267427e5ccc2SPaolo Abeni close_timeout = inet_csk(sk)->icsk_mtup.probe_timestamp - tcp_jiffies32 + jiffies + 2675d866ae9aSPaolo Abeni mptcp_close_timeout(sk); 267676a13b31SGeliang Tang 267776a13b31SGeliang Tang /* the close timeout takes precedence on the fail one, and here at least one of 267876a13b31SGeliang Tang * them is active 267976a13b31SGeliang Tang */ 268027e5ccc2SPaolo Abeni timeout = inet_csk(sk)->icsk_mtup.probe_timestamp ? close_timeout : fail_tout; 268176a13b31SGeliang Tang 268276a13b31SGeliang Tang sk_reset_timer(sk, &sk->sk_timer, timeout); 268376a13b31SGeliang Tang } 268476a13b31SGeliang Tang 268549fa1919SGeliang Tang static void mptcp_mp_fail_no_response(struct mptcp_sock *msk) 268649fa1919SGeliang Tang { 268776a13b31SGeliang Tang struct sock *ssk = msk->first; 268849fa1919SGeliang Tang bool slow; 268949fa1919SGeliang Tang 269076a13b31SGeliang Tang if (!ssk) 269176a13b31SGeliang Tang return; 269276a13b31SGeliang Tang 269349fa1919SGeliang Tang pr_debug("MP_FAIL doesn't respond, reset the subflow"); 269449fa1919SGeliang Tang 269549fa1919SGeliang Tang slow = lock_sock_fast(ssk); 269649fa1919SGeliang Tang mptcp_subflow_reset(ssk); 269776a13b31SGeliang Tang WRITE_ONCE(mptcp_subflow_ctx(ssk)->fail_tout, 0); 269849fa1919SGeliang Tang unlock_sock_fast(ssk, slow); 269949fa1919SGeliang Tang } 270049fa1919SGeliang Tang 2701d21f8348SPaolo Abeni static void mptcp_do_fastclose(struct sock *sk) 2702d21f8348SPaolo Abeni { 2703d21f8348SPaolo Abeni struct mptcp_subflow_context *subflow, *tmp; 2704d21f8348SPaolo Abeni struct mptcp_sock *msk = mptcp_sk(sk); 2705d21f8348SPaolo Abeni 2706bbd49d11SPaolo Abeni inet_sk_state_store(sk, TCP_CLOSE); 2707d21f8348SPaolo Abeni mptcp_for_each_subflow_safe(msk, subflow, tmp) 2708d21f8348SPaolo Abeni __mptcp_close_ssk(sk, mptcp_subflow_tcp_sock(subflow), 2709d21f8348SPaolo Abeni subflow, MPTCP_CF_FASTCLOSE); 2710d21f8348SPaolo Abeni } 2711d21f8348SPaolo Abeni 27122948d0a1SPaolo Abeni static void mptcp_worker(struct work_struct *work) 27132948d0a1SPaolo Abeni { 27142948d0a1SPaolo Abeni struct mptcp_sock *msk = container_of(work, struct mptcp_sock, work); 271580638684SGeliang Tang struct sock *sk = (struct sock *)msk; 271676a13b31SGeliang Tang unsigned long fail_tout; 27172948d0a1SPaolo Abeni int state; 271880992017SPaolo Abeni 271980992017SPaolo Abeni lock_sock(sk); 2720e16163b6SPaolo Abeni state = sk->sk_state; 2721d6a04437SPaolo Abeni if (unlikely((1 << state) & (TCPF_CLOSE | TCPF_LISTEN))) 2722e16163b6SPaolo Abeni goto unlock; 2723e16163b6SPaolo Abeni 272450c504a2SFlorian Westphal mptcp_check_fastclose(msk); 272550c504a2SFlorian Westphal 2726e9801430SFlorian Westphal mptcp_pm_nl_work(msk); 2727b416268bSFlorian Westphal 272881c1d029SPaolo Abeni mptcp_check_send_data_fin(sk); 272981c1d029SPaolo Abeni mptcp_check_data_fin_ack(sk); 273043b54c6eSMat Martineau mptcp_check_data_fin(sk); 273143b54c6eSMat Martineau 2732b6985b9bSPaolo Abeni if (test_and_clear_bit(MPTCP_WORK_CLOSE_SUBFLOW, &msk->flags)) 2733b6985b9bSPaolo Abeni __mptcp_close_subflow(sk); 2734b6985b9bSPaolo Abeni 273527e5ccc2SPaolo Abeni if (mptcp_close_tout_expired(sk)) { 2736d21f8348SPaolo Abeni mptcp_do_fastclose(sk); 273727e5ccc2SPaolo Abeni mptcp_close_wake_up(sk); 273827e5ccc2SPaolo Abeni } 2739bbd49d11SPaolo Abeni 274027e5ccc2SPaolo Abeni if (sock_flag(sk, SOCK_DEAD) && sk->sk_state == TCP_CLOSE) { 2741e16163b6SPaolo Abeni __mptcp_destroy_sock(sk); 2742e16163b6SPaolo Abeni goto unlock; 2743e16163b6SPaolo Abeni } 2744e16163b6SPaolo Abeni 27452948d0a1SPaolo Abeni if (test_and_clear_bit(MPTCP_WORK_RTX, &msk->flags)) 27462948d0a1SPaolo Abeni __mptcp_retrans(sk); 27473b1d6210SPaolo Abeni 274876a13b31SGeliang Tang fail_tout = msk->first ? READ_ONCE(mptcp_subflow_ctx(msk->first)->fail_tout) : 0; 274976a13b31SGeliang Tang if (fail_tout && time_after(jiffies, fail_tout)) 275049fa1919SGeliang Tang mptcp_mp_fail_no_response(msk); 275149fa1919SGeliang Tang 27523b1d6210SPaolo Abeni unlock: 275380992017SPaolo Abeni release_sock(sk); 275480992017SPaolo Abeni sock_put(sk); 275580992017SPaolo Abeni } 275680992017SPaolo Abeni 2757e2636917SKuniyuki Iwashima static void __mptcp_init_sock(struct sock *sk) 2758f870fa0bSMat Martineau { 2759cec37a6eSPeter Krystad struct mptcp_sock *msk = mptcp_sk(sk); 2760cec37a6eSPeter Krystad 2761cec37a6eSPeter Krystad INIT_LIST_HEAD(&msk->conn_list); 2762ec3edaa7SPeter Krystad INIT_LIST_HEAD(&msk->join_list); 276318b683bfSPaolo Abeni INIT_LIST_HEAD(&msk->rtx_queue); 276480992017SPaolo Abeni INIT_WORK(&msk->work, mptcp_worker); 276587952603SPaolo Abeni __skb_queue_head_init(&msk->receive_queue); 2766ab174ad8SPaolo Abeni msk->out_of_order_queue = RB_ROOT; 2767f0e6a4cfSPaolo Abeni msk->first_pending = NULL; 27686511882cSPaolo Abeni msk->rmem_fwd_alloc = 0; 2769ce599c51SPaolo Abeni WRITE_ONCE(msk->rmem_released, 0); 277033d41c9cSPaolo Abeni msk->timer_ival = TCP_RTO_MIN; 27715684ab1aSPaolo Abeni msk->scaling_ratio = TCP_DEFAULT_SCALING_RATIO; 2772cec37a6eSPeter Krystad 27731b1b43eeSPaolo Abeni WRITE_ONCE(msk->first, NULL); 2774dc24f8b4SPaolo Abeni inet_csk(sk)->icsk_sync_mss = mptcp_sync_mss; 2775752e9067SGeliang Tang WRITE_ONCE(msk->csum_enabled, mptcp_is_checksum_enabled(sock_net(sk))); 27760530020aSGeliang Tang WRITE_ONCE(msk->allow_infinite_fallback, true); 27771e1d9d6fSPaolo Abeni msk->recovery = false; 27786f06b4d4SPaolo Abeni msk->subflow_id = 1; 27798ab183deSPaolo Abeni 27801b1c7a0eSPeter Krystad mptcp_pm_data_init(msk); 27811b1c7a0eSPeter Krystad 2782b51f9b80SPaolo Abeni /* re-use the csk retrans timer for MPTCP-level retrans */ 2783b51f9b80SPaolo Abeni timer_setup(&msk->sk.icsk_retransmit_timer, mptcp_retransmit_timer, 0); 2784f6909dc1SPaolo Abeni timer_setup(&sk->sk_timer, mptcp_tout_timer, 0); 2785f870fa0bSMat Martineau } 2786f870fa0bSMat Martineau 2787b29fcfb5SPaolo Abeni static void mptcp_ca_reset(struct sock *sk) 2788784325e9SMatthieu Baerts { 278920b5759fSPaolo Abeni struct inet_connection_sock *icsk = inet_csk(sk); 2790b29fcfb5SPaolo Abeni 2791b29fcfb5SPaolo Abeni tcp_assign_congestion_control(sk); 2792b29fcfb5SPaolo Abeni strcpy(mptcp_sk(sk)->ca_name, icsk->icsk_ca_ops->name); 2793b29fcfb5SPaolo Abeni 2794b29fcfb5SPaolo Abeni /* no need to keep a reference to the ops, the name will suffice */ 2795b29fcfb5SPaolo Abeni tcp_cleanup_congestion_control(sk); 2796b29fcfb5SPaolo Abeni icsk->icsk_ca_ops = NULL; 2797b29fcfb5SPaolo Abeni } 2798b29fcfb5SPaolo Abeni 2799b29fcfb5SPaolo Abeni static int mptcp_init_sock(struct sock *sk) 2800b29fcfb5SPaolo Abeni { 2801fc518953SFlorian Westphal struct net *net = sock_net(sk); 28021730b2b2SGeliang Tang int ret; 280318b683bfSPaolo Abeni 2804e2636917SKuniyuki Iwashima __mptcp_init_sock(sk); 2805b6c08380SGeliang Tang 2806fc518953SFlorian Westphal if (!mptcp_is_enabled(net)) 2807fc518953SFlorian Westphal return -ENOPROTOOPT; 2808fc518953SFlorian Westphal 2809fc518953SFlorian Westphal if (unlikely(!net->mib.mptcp_statistics) && !mptcp_mib_alloc(net)) 2810fc518953SFlorian Westphal return -ENOMEM; 2811fc518953SFlorian Westphal 28121730b2b2SGeliang Tang ret = mptcp_init_sched(mptcp_sk(sk), 28131730b2b2SGeliang Tang mptcp_sched_find(mptcp_get_scheduler(net))); 28141730b2b2SGeliang Tang if (ret) 28151730b2b2SGeliang Tang return ret; 28161730b2b2SGeliang Tang 2817a5ef058dSPaolo Abeni set_bit(SOCK_CUSTOM_SOCKOPT, &sk->sk_socket->flags); 2818a5ef058dSPaolo Abeni 281920b5759fSPaolo Abeni /* fetch the ca name; do it outside __mptcp_init_sock(), so that clone will 282020b5759fSPaolo Abeni * propagate the correct value 282120b5759fSPaolo Abeni */ 2822b29fcfb5SPaolo Abeni mptcp_ca_reset(sk); 282320b5759fSPaolo Abeni 2824d027236cSPaolo Abeni sk_sockets_allocated_inc(sk); 2825a963853fSGeliang Tang sk->sk_rcvbuf = READ_ONCE(net->ipv4.sysctl_tcp_rmem[1]); 2826a963853fSGeliang Tang sk->sk_sndbuf = READ_ONCE(net->ipv4.sysctl_tcp_wmem[1]); 2827d027236cSPaolo Abeni 282818b683bfSPaolo Abeni return 0; 282918b683bfSPaolo Abeni } 283018b683bfSPaolo Abeni 283118b683bfSPaolo Abeni static void __mptcp_clear_xmit(struct sock *sk) 283218b683bfSPaolo Abeni { 283318b683bfSPaolo Abeni struct mptcp_sock *msk = mptcp_sk(sk); 283418b683bfSPaolo Abeni struct mptcp_data_frag *dtmp, *dfrag; 283518b683bfSPaolo Abeni 2836d9ca1de8SPaolo Abeni WRITE_ONCE(msk->first_pending, NULL); 283718b683bfSPaolo Abeni list_for_each_entry_safe(dfrag, dtmp, &msk->rtx_queue, list) 2838d027236cSPaolo Abeni dfrag_clear(sk, dfrag); 2839784325e9SMatthieu Baerts } 2840784325e9SMatthieu Baerts 284130e51b92SMenglong Dong void mptcp_cancel_work(struct sock *sk) 284280992017SPaolo Abeni { 284380992017SPaolo Abeni struct mptcp_sock *msk = mptcp_sk(sk); 284480992017SPaolo Abeni 2845b2771d24SPaolo Abeni if (cancel_work_sync(&msk->work)) 2846e16163b6SPaolo Abeni __sock_put(sk); 284780992017SPaolo Abeni } 284880992017SPaolo Abeni 2849d0876b22SGeliang Tang void mptcp_subflow_shutdown(struct sock *sk, struct sock *ssk, int how) 285021498490SPeter Krystad { 285121498490SPeter Krystad lock_sock(ssk); 285221498490SPeter Krystad 285321498490SPeter Krystad switch (ssk->sk_state) { 285421498490SPeter Krystad case TCP_LISTEN: 285521498490SPeter Krystad if (!(how & RCV_SHUTDOWN)) 285621498490SPeter Krystad break; 2857df561f66SGustavo A. R. Silva fallthrough; 285821498490SPeter Krystad case TCP_SYN_SENT: 2859c2b2ae39SPaolo Abeni WARN_ON_ONCE(tcp_disconnect(ssk, O_NONBLOCK)); 286021498490SPeter Krystad break; 286121498490SPeter Krystad default: 286243b54c6eSMat Martineau if (__mptcp_check_fallback(mptcp_sk(sk))) { 286343b54c6eSMat Martineau pr_debug("Fallback"); 286421498490SPeter Krystad ssk->sk_shutdown |= how; 286521498490SPeter Krystad tcp_shutdown(ssk, how); 286681c1d029SPaolo Abeni 286781c1d029SPaolo Abeni /* simulate the data_fin ack reception to let the state 286881c1d029SPaolo Abeni * machine move forward 286981c1d029SPaolo Abeni */ 287081c1d029SPaolo Abeni WRITE_ONCE(mptcp_sk(sk)->snd_una, mptcp_sk(sk)->snd_nxt); 287181c1d029SPaolo Abeni mptcp_schedule_work(sk); 287243b54c6eSMat Martineau } else { 287343b54c6eSMat Martineau pr_debug("Sending DATA_FIN on subflow %p", ssk); 287443b54c6eSMat Martineau tcp_send_ack(ssk); 2875f6909dc1SPaolo Abeni if (!mptcp_rtx_timer_pending(sk)) 2876f6909dc1SPaolo Abeni mptcp_reset_rtx_timer(sk); 287743b54c6eSMat Martineau } 287821498490SPeter Krystad break; 287921498490SPeter Krystad } 288021498490SPeter Krystad 288121498490SPeter Krystad release_sock(ssk); 288221498490SPeter Krystad } 288321498490SPeter Krystad 28846920b851SMat Martineau static const unsigned char new_state[16] = { 28856920b851SMat Martineau /* current state: new state: action: */ 28866920b851SMat Martineau [0 /* (Invalid) */] = TCP_CLOSE, 28876920b851SMat Martineau [TCP_ESTABLISHED] = TCP_FIN_WAIT1 | TCP_ACTION_FIN, 28886920b851SMat Martineau [TCP_SYN_SENT] = TCP_CLOSE, 28896920b851SMat Martineau [TCP_SYN_RECV] = TCP_FIN_WAIT1 | TCP_ACTION_FIN, 28906920b851SMat Martineau [TCP_FIN_WAIT1] = TCP_FIN_WAIT1, 28916920b851SMat Martineau [TCP_FIN_WAIT2] = TCP_FIN_WAIT2, 28926920b851SMat Martineau [TCP_TIME_WAIT] = TCP_CLOSE, /* should not happen ! */ 28936920b851SMat Martineau [TCP_CLOSE] = TCP_CLOSE, 28946920b851SMat Martineau [TCP_CLOSE_WAIT] = TCP_LAST_ACK | TCP_ACTION_FIN, 28956920b851SMat Martineau [TCP_LAST_ACK] = TCP_LAST_ACK, 28966920b851SMat Martineau [TCP_LISTEN] = TCP_CLOSE, 28976920b851SMat Martineau [TCP_CLOSING] = TCP_CLOSING, 28986920b851SMat Martineau [TCP_NEW_SYN_RECV] = TCP_CLOSE, /* should not happen ! */ 28996920b851SMat Martineau }; 29006920b851SMat Martineau 29016920b851SMat Martineau static int mptcp_close_state(struct sock *sk) 29026920b851SMat Martineau { 29036920b851SMat Martineau int next = (int)new_state[sk->sk_state]; 29046920b851SMat Martineau int ns = next & TCP_STATE_MASK; 29056920b851SMat Martineau 29066920b851SMat Martineau inet_sk_state_store(sk, ns); 29076920b851SMat Martineau 29086920b851SMat Martineau return next & TCP_ACTION_FIN; 29096920b851SMat Martineau } 29106920b851SMat Martineau 291181c1d029SPaolo Abeni static void mptcp_check_send_data_fin(struct sock *sk) 2912e16163b6SPaolo Abeni { 2913e16163b6SPaolo Abeni struct mptcp_subflow_context *subflow; 2914e16163b6SPaolo Abeni struct mptcp_sock *msk = mptcp_sk(sk); 2915e16163b6SPaolo Abeni 2916e16163b6SPaolo Abeni pr_debug("msk=%p snd_data_fin_enable=%d pending=%d snd_nxt=%llu write_seq=%llu", 2917e16163b6SPaolo Abeni msk, msk->snd_data_fin_enable, !!mptcp_send_head(sk), 2918e16163b6SPaolo Abeni msk->snd_nxt, msk->write_seq); 2919e16163b6SPaolo Abeni 2920e16163b6SPaolo Abeni /* we still need to enqueue subflows or not really shutting down, 2921e16163b6SPaolo Abeni * skip this 2922e16163b6SPaolo Abeni */ 2923e16163b6SPaolo Abeni if (!msk->snd_data_fin_enable || msk->snd_nxt + 1 != msk->write_seq || 2924e16163b6SPaolo Abeni mptcp_send_head(sk)) 2925e16163b6SPaolo Abeni return; 2926e16163b6SPaolo Abeni 2927e16163b6SPaolo Abeni WRITE_ONCE(msk->snd_nxt, msk->write_seq); 2928e16163b6SPaolo Abeni 2929e16163b6SPaolo Abeni mptcp_for_each_subflow(msk, subflow) { 2930e16163b6SPaolo Abeni struct sock *tcp_sk = mptcp_subflow_tcp_sock(subflow); 2931e16163b6SPaolo Abeni 2932e16163b6SPaolo Abeni mptcp_subflow_shutdown(sk, tcp_sk, SEND_SHUTDOWN); 2933e16163b6SPaolo Abeni } 2934e16163b6SPaolo Abeni } 2935e16163b6SPaolo Abeni 2936e16163b6SPaolo Abeni static void __mptcp_wr_shutdown(struct sock *sk) 2937e16163b6SPaolo Abeni { 2938e16163b6SPaolo Abeni struct mptcp_sock *msk = mptcp_sk(sk); 2939e16163b6SPaolo Abeni 2940e16163b6SPaolo Abeni pr_debug("msk=%p snd_data_fin_enable=%d shutdown=%x state=%d pending=%d", 2941e16163b6SPaolo Abeni msk, msk->snd_data_fin_enable, sk->sk_shutdown, sk->sk_state, 2942e16163b6SPaolo Abeni !!mptcp_send_head(sk)); 2943e16163b6SPaolo Abeni 2944e16163b6SPaolo Abeni /* will be ignored by fallback sockets */ 2945e16163b6SPaolo Abeni WRITE_ONCE(msk->write_seq, msk->write_seq + 1); 2946e16163b6SPaolo Abeni WRITE_ONCE(msk->snd_data_fin_enable, 1); 2947e16163b6SPaolo Abeni 294881c1d029SPaolo Abeni mptcp_check_send_data_fin(sk); 2949e16163b6SPaolo Abeni } 2950e16163b6SPaolo Abeni 2951e16163b6SPaolo Abeni static void __mptcp_destroy_sock(struct sock *sk) 2952f870fa0bSMat Martineau { 2953f870fa0bSMat Martineau struct mptcp_sock *msk = mptcp_sk(sk); 2954f870fa0bSMat Martineau 2955e16163b6SPaolo Abeni pr_debug("msk=%p", msk); 29562c22c06cSFlorian Westphal 29573abc05d9SFlorian Westphal might_sleep(); 29583abc05d9SFlorian Westphal 2959f6909dc1SPaolo Abeni mptcp_stop_rtx_timer(sk); 2960e16163b6SPaolo Abeni sk_stop_timer(sk, &sk->sk_timer); 2961e16163b6SPaolo Abeni msk->pm.status = 0; 29621730b2b2SGeliang Tang mptcp_release_sched(msk); 2963b2c5b614SFlorian Westphal 2964e16163b6SPaolo Abeni sk->sk_prot->destroy(sk); 2965e16163b6SPaolo Abeni 29666511882cSPaolo Abeni WARN_ON_ONCE(msk->rmem_fwd_alloc); 296787952603SPaolo Abeni WARN_ON_ONCE(msk->rmem_released); 2968e16163b6SPaolo Abeni sk_stream_kill_queues(sk); 2969e16163b6SPaolo Abeni xfrm_sk_free_policy(sk); 2970aa1fbd94SFlorian Westphal 2971e16163b6SPaolo Abeni sock_put(sk); 2972e16163b6SPaolo Abeni } 2973e16163b6SPaolo Abeni 297463740448SPaolo Abeni void __mptcp_unaccepted_force_close(struct sock *sk) 297563740448SPaolo Abeni { 297663740448SPaolo Abeni sock_set_flag(sk, SOCK_DEAD); 297763740448SPaolo Abeni mptcp_do_fastclose(sk); 297863740448SPaolo Abeni __mptcp_destroy_sock(sk); 297963740448SPaolo Abeni } 298063740448SPaolo Abeni 29815684ab1aSPaolo Abeni static __poll_t mptcp_check_readable(struct sock *sk) 2982d21f8348SPaolo Abeni { 29835684ab1aSPaolo Abeni return mptcp_epollin_ready(sk) ? EPOLLIN | EPOLLRDNORM : 0; 2984d21f8348SPaolo Abeni } 2985d21f8348SPaolo Abeni 298657fc0f1cSPaolo Abeni static void mptcp_check_listen_stop(struct sock *sk) 2987c558246eSMenglong Dong { 298857fc0f1cSPaolo Abeni struct sock *ssk; 298957fc0f1cSPaolo Abeni 299057fc0f1cSPaolo Abeni if (inet_sk_state_load(sk) != TCP_LISTEN) 299157fc0f1cSPaolo Abeni return; 299257fc0f1cSPaolo Abeni 2993c558246eSMenglong Dong sock_prot_inuse_add(sock_net(sk), sk->sk_prot, -1); 299457fc0f1cSPaolo Abeni ssk = mptcp_sk(sk)->first; 299557fc0f1cSPaolo Abeni if (WARN_ON_ONCE(!ssk || inet_sk_state_load(ssk) != TCP_LISTEN)) 299657fc0f1cSPaolo Abeni return; 299757fc0f1cSPaolo Abeni 299857fc0f1cSPaolo Abeni lock_sock_nested(ssk, SINGLE_DEPTH_NESTING); 29993fffa15bSPaolo Abeni tcp_set_state(ssk, TCP_CLOSE); 300057fc0f1cSPaolo Abeni mptcp_subflow_queue_clean(sk, ssk); 300157fc0f1cSPaolo Abeni inet_csk_listen_stop(ssk); 300257fc0f1cSPaolo Abeni mptcp_event_pm_listener(ssk, MPTCP_EVENT_LISTENER_CLOSED); 300357fc0f1cSPaolo Abeni release_sock(ssk); 3004c558246eSMenglong Dong } 3005c558246eSMenglong Dong 300626d3e21cSMenglong Dong bool __mptcp_close(struct sock *sk, long timeout) 3007e16163b6SPaolo Abeni { 3008e16163b6SPaolo Abeni struct mptcp_subflow_context *subflow; 300976a13b31SGeliang Tang struct mptcp_sock *msk = mptcp_sk(sk); 3010e16163b6SPaolo Abeni bool do_cancel_work = false; 3011d4e85922SPaolo Abeni int subflows_alive = 0; 3012e16163b6SPaolo Abeni 30136b9831bfSPaolo Abeni WRITE_ONCE(sk->sk_shutdown, SHUTDOWN_MASK); 3014e16163b6SPaolo Abeni 3015e16163b6SPaolo Abeni if ((1 << sk->sk_state) & (TCPF_LISTEN | TCPF_CLOSE)) { 301657fc0f1cSPaolo Abeni mptcp_check_listen_stop(sk); 3017e16163b6SPaolo Abeni inet_sk_state_store(sk, TCP_CLOSE); 3018e16163b6SPaolo Abeni goto cleanup; 3019e16163b6SPaolo Abeni } 3020e16163b6SPaolo Abeni 30215684ab1aSPaolo Abeni if (mptcp_data_avail(msk) || timeout < 0) { 30228d547809SPaolo Abeni /* If the msk has read data, or the caller explicitly ask it, 30238d547809SPaolo Abeni * do the MPTCP equivalent of TCP reset, aka MPTCP fastclose 30248d547809SPaolo Abeni */ 3025d21f8348SPaolo Abeni mptcp_do_fastclose(sk); 30268d547809SPaolo Abeni timeout = 0; 3027d21f8348SPaolo Abeni } else if (mptcp_close_state(sk)) { 3028e16163b6SPaolo Abeni __mptcp_wr_shutdown(sk); 3029d21f8348SPaolo Abeni } 3030e16163b6SPaolo Abeni 3031e16163b6SPaolo Abeni sk_stream_wait_close(sk, timeout); 3032e16163b6SPaolo Abeni 3033e16163b6SPaolo Abeni cleanup: 3034e16163b6SPaolo Abeni /* orphan all the subflows */ 303576a13b31SGeliang Tang mptcp_for_each_subflow(msk, subflow) { 3036e16163b6SPaolo Abeni struct sock *ssk = mptcp_subflow_tcp_sock(subflow); 303749054556SPaolo Abeni bool slow = lock_sock_fast_nested(ssk); 3038e16163b6SPaolo Abeni 3039d4e85922SPaolo Abeni subflows_alive += ssk->sk_state != TCP_CLOSE; 3040d4e85922SPaolo Abeni 304176a13b31SGeliang Tang /* since the close timeout takes precedence on the fail one, 304276a13b31SGeliang Tang * cancel the latter 304376a13b31SGeliang Tang */ 304476a13b31SGeliang Tang if (ssk == msk->first) 304576a13b31SGeliang Tang subflow->fail_tout = 0; 304676a13b31SGeliang Tang 3047fe948001SMenglong Dong /* detach from the parent socket, but allow data_ready to 3048fe948001SMenglong Dong * push incoming data into the mptcp stack, to properly ack it 3049fe948001SMenglong Dong */ 3050fe948001SMenglong Dong ssk->sk_socket = NULL; 3051fe948001SMenglong Dong ssk->sk_wq = NULL; 3052e16163b6SPaolo Abeni unlock_sock_fast(ssk, slow); 3053e16163b6SPaolo Abeni } 3054e16163b6SPaolo Abeni sock_orphan(sk); 3055e16163b6SPaolo Abeni 3056d4e85922SPaolo Abeni /* all the subflows are closed, only timeout can change the msk 3057d4e85922SPaolo Abeni * state, let's not keep resources busy for no reasons 3058d4e85922SPaolo Abeni */ 3059d4e85922SPaolo Abeni if (subflows_alive == 0) 3060d4e85922SPaolo Abeni inet_sk_state_store(sk, TCP_CLOSE); 3061d4e85922SPaolo Abeni 3062e16163b6SPaolo Abeni sock_hold(sk); 3063e16163b6SPaolo Abeni pr_debug("msk=%p state=%d", sk, sk->sk_state); 306400df24f1SGeliang Tang if (msk->token) 306576a13b31SGeliang Tang mptcp_event(MPTCP_EVENT_CLOSED, msk, NULL, GFP_KERNEL); 3066b29fcfb5SPaolo Abeni 3067e16163b6SPaolo Abeni if (sk->sk_state == TCP_CLOSE) { 3068e16163b6SPaolo Abeni __mptcp_destroy_sock(sk); 3069e16163b6SPaolo Abeni do_cancel_work = true; 3070e16163b6SPaolo Abeni } else { 307127e5ccc2SPaolo Abeni mptcp_start_tout_timer(sk); 3072e16163b6SPaolo Abeni } 307326d3e21cSMenglong Dong 307426d3e21cSMenglong Dong return do_cancel_work; 307526d3e21cSMenglong Dong } 307626d3e21cSMenglong Dong 307726d3e21cSMenglong Dong static void mptcp_close(struct sock *sk, long timeout) 307826d3e21cSMenglong Dong { 307926d3e21cSMenglong Dong bool do_cancel_work; 308026d3e21cSMenglong Dong 308126d3e21cSMenglong Dong lock_sock(sk); 308226d3e21cSMenglong Dong 308326d3e21cSMenglong Dong do_cancel_work = __mptcp_close(sk, timeout); 3084e16163b6SPaolo Abeni release_sock(sk); 3085e16163b6SPaolo Abeni if (do_cancel_work) 308680992017SPaolo Abeni mptcp_cancel_work(sk); 3087b911c97cSFlorian Westphal 3088e16163b6SPaolo Abeni sock_put(sk); 3089f870fa0bSMat Martineau } 3090f870fa0bSMat Martineau 30917e8b88ecSPaolo Abeni static void mptcp_copy_inaddrs(struct sock *msk, const struct sock *ssk) 3092cf7da0d6SPeter Krystad { 3093cf7da0d6SPeter Krystad #if IS_ENABLED(CONFIG_MPTCP_IPV6) 3094cf7da0d6SPeter Krystad const struct ipv6_pinfo *ssk6 = inet6_sk(ssk); 3095cf7da0d6SPeter Krystad struct ipv6_pinfo *msk6 = inet6_sk(msk); 3096cf7da0d6SPeter Krystad 3097cf7da0d6SPeter Krystad msk->sk_v6_daddr = ssk->sk_v6_daddr; 3098cf7da0d6SPeter Krystad msk->sk_v6_rcv_saddr = ssk->sk_v6_rcv_saddr; 3099cf7da0d6SPeter Krystad 3100cf7da0d6SPeter Krystad if (msk6 && ssk6) { 3101cf7da0d6SPeter Krystad msk6->saddr = ssk6->saddr; 3102cf7da0d6SPeter Krystad msk6->flow_label = ssk6->flow_label; 3103cf7da0d6SPeter Krystad } 3104cf7da0d6SPeter Krystad #endif 3105cf7da0d6SPeter Krystad 3106cf7da0d6SPeter Krystad inet_sk(msk)->inet_num = inet_sk(ssk)->inet_num; 3107cf7da0d6SPeter Krystad inet_sk(msk)->inet_dport = inet_sk(ssk)->inet_dport; 3108cf7da0d6SPeter Krystad inet_sk(msk)->inet_sport = inet_sk(ssk)->inet_sport; 3109cf7da0d6SPeter Krystad inet_sk(msk)->inet_daddr = inet_sk(ssk)->inet_daddr; 3110cf7da0d6SPeter Krystad inet_sk(msk)->inet_saddr = inet_sk(ssk)->inet_saddr; 3111cf7da0d6SPeter Krystad inet_sk(msk)->inet_rcv_saddr = inet_sk(ssk)->inet_rcv_saddr; 3112cf7da0d6SPeter Krystad } 3113cf7da0d6SPeter Krystad 311418b683bfSPaolo Abeni static int mptcp_disconnect(struct sock *sk, int flags) 311518b683bfSPaolo Abeni { 311676e2a55dSPaolo Abeni struct mptcp_sock *msk = mptcp_sk(sk); 311776e2a55dSPaolo Abeni 31187d803344SPaolo Abeni /* We are on the fastopen error path. We can't call straight into the 31197d803344SPaolo Abeni * subflows cleanup code due to lock nesting (we are already under 3120c2b2ae39SPaolo Abeni * msk->firstsocket lock). 31217d803344SPaolo Abeni */ 31227d803344SPaolo Abeni if (msk->fastopening) 3123c2b2ae39SPaolo Abeni return -EBUSY; 31247d803344SPaolo Abeni 312557fc0f1cSPaolo Abeni mptcp_check_listen_stop(sk); 3126b29fcfb5SPaolo Abeni inet_sk_state_store(sk, TCP_CLOSE); 3127b29fcfb5SPaolo Abeni 3128f6909dc1SPaolo Abeni mptcp_stop_rtx_timer(sk); 312927e5ccc2SPaolo Abeni mptcp_stop_tout_timer(sk); 3130b29fcfb5SPaolo Abeni 313100df24f1SGeliang Tang if (msk->token) 313200df24f1SGeliang Tang mptcp_event(MPTCP_EVENT_CLOSED, msk, NULL, GFP_KERNEL); 3133b29fcfb5SPaolo Abeni 3134c0bf3c6aSPaolo Abeni /* msk->subflow is still intact, the following will not free the first 3135c0bf3c6aSPaolo Abeni * subflow 3136c0bf3c6aSPaolo Abeni */ 3137c0bf3c6aSPaolo Abeni mptcp_destroy_common(msk, MPTCP_CF_FASTCLOSE); 3138e9d09bacSPaolo Abeni WRITE_ONCE(msk->flags, 0); 3139e9d09bacSPaolo Abeni msk->cb_flags = 0; 3140e9d09bacSPaolo Abeni msk->push_pending = 0; 3141b29fcfb5SPaolo Abeni msk->recovery = false; 3142b29fcfb5SPaolo Abeni msk->can_ack = false; 3143b29fcfb5SPaolo Abeni msk->fully_established = false; 3144b29fcfb5SPaolo Abeni msk->rcv_data_fin = false; 3145b29fcfb5SPaolo Abeni msk->snd_data_fin_enable = false; 3146b29fcfb5SPaolo Abeni msk->rcv_fastclose = false; 3147b29fcfb5SPaolo Abeni msk->use_64bit_ack = false; 31485684ab1aSPaolo Abeni msk->bytes_consumed = 0; 3149b29fcfb5SPaolo Abeni WRITE_ONCE(msk->csum_enabled, mptcp_is_checksum_enabled(sock_net(sk))); 3150b29fcfb5SPaolo Abeni mptcp_pm_data_reset(msk); 3151b29fcfb5SPaolo Abeni mptcp_ca_reset(sk); 315238967f42SPaolo Abeni msk->bytes_acked = 0; 315338967f42SPaolo Abeni msk->bytes_received = 0; 315438967f42SPaolo Abeni msk->bytes_sent = 0; 315538967f42SPaolo Abeni msk->bytes_retrans = 0; 3156b29fcfb5SPaolo Abeni 31576b9831bfSPaolo Abeni WRITE_ONCE(sk->sk_shutdown, 0); 3158b29fcfb5SPaolo Abeni sk_error_report(sk); 315942c556feSFlorian Westphal return 0; 316018b683bfSPaolo Abeni } 316118b683bfSPaolo Abeni 3162b0519de8SFlorian Westphal #if IS_ENABLED(CONFIG_MPTCP_IPV6) 3163b0519de8SFlorian Westphal static struct ipv6_pinfo *mptcp_inet6_sk(const struct sock *sk) 3164b0519de8SFlorian Westphal { 3165b0519de8SFlorian Westphal unsigned int offset = sizeof(struct mptcp6_sock) - sizeof(struct ipv6_pinfo); 3166b0519de8SFlorian Westphal 3167b0519de8SFlorian Westphal return (struct ipv6_pinfo *)(((u8 *)sk) + offset); 3168b0519de8SFlorian Westphal } 3169b0519de8SFlorian Westphal #endif 3170b0519de8SFlorian Westphal 31717e8b88ecSPaolo Abeni struct sock *mptcp_sk_clone_init(const struct sock *sk, 3172cfde141eSPaolo Abeni const struct mptcp_options_received *mp_opt, 31737e8b88ecSPaolo Abeni struct sock *ssk, 3174fca5c82cSPaolo Abeni struct request_sock *req) 3175b0519de8SFlorian Westphal { 317658b09919SPaolo Abeni struct mptcp_subflow_request_sock *subflow_req = mptcp_subflow_rsk(req); 3177b0519de8SFlorian Westphal struct sock *nsk = sk_clone_lock(sk, GFP_ATOMIC); 317858b09919SPaolo Abeni struct mptcp_sock *msk; 3179b0519de8SFlorian Westphal 3180b0519de8SFlorian Westphal if (!nsk) 3181b0519de8SFlorian Westphal return NULL; 3182b0519de8SFlorian Westphal 3183b0519de8SFlorian Westphal #if IS_ENABLED(CONFIG_MPTCP_IPV6) 3184b0519de8SFlorian Westphal if (nsk->sk_family == AF_INET6) 3185b0519de8SFlorian Westphal inet_sk(nsk)->pinet6 = mptcp_inet6_sk(nsk); 3186b0519de8SFlorian Westphal #endif 3187b0519de8SFlorian Westphal 318858b09919SPaolo Abeni __mptcp_init_sock(nsk); 318958b09919SPaolo Abeni 319058b09919SPaolo Abeni msk = mptcp_sk(nsk); 319158b09919SPaolo Abeni msk->local_key = subflow_req->local_key; 319258b09919SPaolo Abeni msk->token = subflow_req->token; 3193b6985b9bSPaolo Abeni msk->in_accept_queue = 1; 3194b93df08cSPaolo Abeni WRITE_ONCE(msk->fully_established, false); 319574c7dfbeSPaolo Abeni if (mp_opt->suboptions & OPTION_MPTCP_CSUMREQD) 319606251181SGeliang Tang WRITE_ONCE(msk->csum_enabled, true); 319758b09919SPaolo Abeni 319858b09919SPaolo Abeni msk->write_seq = subflow_req->idsn + 1; 3199eaa2ffabSPaolo Abeni msk->snd_nxt = msk->write_seq; 32007439d687SPaolo Abeni msk->snd_una = msk->write_seq; 32017439d687SPaolo Abeni msk->wnd_end = msk->snd_nxt + req->rsk_rcv_wnd; 3202df00b087SFlorian Westphal msk->setsockopt_seq = mptcp_sk(sk)->setsockopt_seq; 32031730b2b2SGeliang Tang mptcp_init_sched(msk, mptcp_sk(sk)->sched); 32046f8a612aSFlorian Westphal 32056f06b4d4SPaolo Abeni /* passive msk is created after the first/MPC subflow */ 32066f06b4d4SPaolo Abeni msk->subflow_id = 2; 32076f06b4d4SPaolo Abeni 32085e20087dSFlorian Westphal sock_reset_flag(nsk, SOCK_RCU_FREE); 32090c148460SPaolo Abeni security_inet_csk_clone(nsk, req); 32107e8b88ecSPaolo Abeni 32117e8b88ecSPaolo Abeni /* this can't race with mptcp_close(), as the msk is 32127e8b88ecSPaolo Abeni * not yet exposted to user-space 32137e8b88ecSPaolo Abeni */ 32147e8b88ecSPaolo Abeni inet_sk_state_store(nsk, TCP_ESTABLISHED); 32157e8b88ecSPaolo Abeni 32167e8b88ecSPaolo Abeni /* The msk maintain a ref to each subflow in the connections list */ 32177e8b88ecSPaolo Abeni WRITE_ONCE(msk->first, ssk); 32187e8b88ecSPaolo Abeni list_add(&mptcp_subflow_ctx(ssk)->node, &msk->conn_list); 32197e8b88ecSPaolo Abeni sock_hold(ssk); 32207e8b88ecSPaolo Abeni 32217e8b88ecSPaolo Abeni /* new mpc subflow takes ownership of the newly 32227e8b88ecSPaolo Abeni * created mptcp socket 32237e8b88ecSPaolo Abeni */ 32247e8b88ecSPaolo Abeni mptcp_token_accept(subflow_req, msk); 32257e8b88ecSPaolo Abeni 32267e8b88ecSPaolo Abeni /* set msk addresses early to ensure mptcp_pm_get_local_id() 32277e8b88ecSPaolo Abeni * uses the correct data 32287e8b88ecSPaolo Abeni */ 32297e8b88ecSPaolo Abeni mptcp_copy_inaddrs(nsk, ssk); 32308005184fSPaolo Abeni __mptcp_propagate_sndbuf(nsk, ssk); 32317e8b88ecSPaolo Abeni 32327e8b88ecSPaolo Abeni mptcp_rcv_space_init(msk, ssk); 323358b09919SPaolo Abeni bh_unlock_sock(nsk); 323458b09919SPaolo Abeni 3235b6985b9bSPaolo Abeni /* note: the newly allocated socket refcount is 2 now */ 3236b0519de8SFlorian Westphal return nsk; 3237b0519de8SFlorian Westphal } 3238b0519de8SFlorian Westphal 3239a6b118feSFlorian Westphal void mptcp_rcv_space_init(struct mptcp_sock *msk, const struct sock *ssk) 3240a6b118feSFlorian Westphal { 3241a6b118feSFlorian Westphal const struct tcp_sock *tp = tcp_sk(ssk); 3242a6b118feSFlorian Westphal 3243a6b118feSFlorian Westphal msk->rcvq_space.copied = 0; 3244a6b118feSFlorian Westphal msk->rcvq_space.rtt_us = 0; 3245a6b118feSFlorian Westphal 3246a6b118feSFlorian Westphal msk->rcvq_space.time = tp->tcp_mstamp; 3247a6b118feSFlorian Westphal 3248a6b118feSFlorian Westphal /* initial rcv_space offering made to peer */ 3249a6b118feSFlorian Westphal msk->rcvq_space.space = min_t(u32, tp->rcv_wnd, 3250a6b118feSFlorian Westphal TCP_INIT_CWND * tp->advmss); 3251a6b118feSFlorian Westphal if (msk->rcvq_space.space == 0) 3252a6b118feSFlorian Westphal msk->rcvq_space.space = TCP_INIT_CWND * TCP_MSS_DEFAULT; 32536f8a612aSFlorian Westphal 32547439d687SPaolo Abeni WRITE_ONCE(msk->wnd_end, msk->snd_nxt + tcp_sk(ssk)->snd_wnd); 3255a6b118feSFlorian Westphal } 3256a6b118feSFlorian Westphal 32571f6610b9SPaolo Abeni static struct sock *mptcp_accept(struct sock *ssk, int flags, int *err, 3258cf7da0d6SPeter Krystad bool kern) 3259cf7da0d6SPeter Krystad { 3260cf7da0d6SPeter Krystad struct sock *newsk; 3261cf7da0d6SPeter Krystad 32621f6610b9SPaolo Abeni pr_debug("ssk=%p, listener=%p", ssk, mptcp_subflow_ctx(ssk)); 32631f6610b9SPaolo Abeni newsk = inet_csk_accept(ssk, flags, err, kern); 3264cf7da0d6SPeter Krystad if (!newsk) 3265cf7da0d6SPeter Krystad return NULL; 3266cf7da0d6SPeter Krystad 32671f6610b9SPaolo Abeni pr_debug("newsk=%p, subflow is mptcp=%d", newsk, sk_is_mptcp(newsk)); 3268cf7da0d6SPeter Krystad if (sk_is_mptcp(newsk)) { 3269cf7da0d6SPeter Krystad struct mptcp_subflow_context *subflow; 3270cf7da0d6SPeter Krystad struct sock *new_mptcp_sock; 3271cf7da0d6SPeter Krystad 3272cf7da0d6SPeter Krystad subflow = mptcp_subflow_ctx(newsk); 327358b09919SPaolo Abeni new_mptcp_sock = subflow->conn; 327458b09919SPaolo Abeni 327558b09919SPaolo Abeni /* is_mptcp should be false if subflow->conn is missing, see 327658b09919SPaolo Abeni * subflow_syn_recv_sock() 327758b09919SPaolo Abeni */ 327858b09919SPaolo Abeni if (WARN_ON_ONCE(!new_mptcp_sock)) { 327958b09919SPaolo Abeni tcp_sk(newsk)->is_mptcp = 0; 3280d6692b3bSFlorian Westphal goto out; 328158b09919SPaolo Abeni } 328258b09919SPaolo Abeni 3283cf7da0d6SPeter Krystad newsk = new_mptcp_sock; 32841f6610b9SPaolo Abeni MPTCP_INC_STATS(sock_net(ssk), MPTCP_MIB_MPCAPABLEPASSIVEACK); 3285fc518953SFlorian Westphal } else { 32861f6610b9SPaolo Abeni MPTCP_INC_STATS(sock_net(ssk), 3287fc518953SFlorian Westphal MPTCP_MIB_MPCAPABLEPASSIVEFALLBACK); 3288cf7da0d6SPeter Krystad } 3289cf7da0d6SPeter Krystad 3290d6692b3bSFlorian Westphal out: 3291d6692b3bSFlorian Westphal newsk->sk_kern_sock = kern; 3292cf7da0d6SPeter Krystad return newsk; 3293cf7da0d6SPeter Krystad } 3294cf7da0d6SPeter Krystad 3295c0bf3c6aSPaolo Abeni void mptcp_destroy_common(struct mptcp_sock *msk, unsigned int flags) 32965c8c1640SGeliang Tang { 3297c0bf3c6aSPaolo Abeni struct mptcp_subflow_context *subflow, *tmp; 329887952603SPaolo Abeni struct sock *sk = (struct sock *)msk; 329987952603SPaolo Abeni 33006e628cd3SPaolo Abeni __mptcp_clear_xmit(sk); 33016e628cd3SPaolo Abeni 3302c0bf3c6aSPaolo Abeni /* join list will be eventually flushed (with rst) at sock lock release time */ 33035efbf6f7SMatthieu Baerts mptcp_for_each_subflow_safe(msk, subflow, tmp) 3304c0bf3c6aSPaolo Abeni __mptcp_close_ssk(sk, mptcp_subflow_tcp_sock(subflow), subflow, flags); 3305c0bf3c6aSPaolo Abeni 330687952603SPaolo Abeni /* move to sk_receive_queue, sk_stream_kill_queues will purge it */ 3307b29fcfb5SPaolo Abeni mptcp_data_lock(sk); 330887952603SPaolo Abeni skb_queue_splice_tail_init(&msk->receive_queue, &sk->sk_receive_queue); 33096511882cSPaolo Abeni __skb_queue_purge(&sk->sk_receive_queue); 33105c8c1640SGeliang Tang skb_rbtree_purge(&msk->out_of_order_queue); 3311b29fcfb5SPaolo Abeni mptcp_data_unlock(sk); 33126511882cSPaolo Abeni 33136511882cSPaolo Abeni /* move all the rx fwd alloc into the sk_mem_reclaim_final in 33146511882cSPaolo Abeni * inet_sock_destruct() will dispose it 33156511882cSPaolo Abeni */ 33165e6300e7SEric Dumazet sk_forward_alloc_add(sk, msk->rmem_fwd_alloc); 33179531e4a8SEric Dumazet WRITE_ONCE(msk->rmem_fwd_alloc, 0); 33185c8c1640SGeliang Tang mptcp_token_destroy(msk); 33195c8c1640SGeliang Tang mptcp_pm_free_anno_list(msk); 33204638de5aSKishen Maloor mptcp_free_local_addr_list(msk); 33215c8c1640SGeliang Tang } 33225c8c1640SGeliang Tang 332379c0949eSPeter Krystad static void mptcp_destroy(struct sock *sk) 332479c0949eSPeter Krystad { 3325c9fd9c5fSFlorian Westphal struct mptcp_sock *msk = mptcp_sk(sk); 3326c9fd9c5fSFlorian Westphal 332739880bd8SPaolo Abeni /* allow the following to close even the initial subflow */ 332839880bd8SPaolo Abeni msk->free_first = 1; 3329c0bf3c6aSPaolo Abeni mptcp_destroy_common(msk, 0); 3330d027236cSPaolo Abeni sk_sockets_allocated_dec(sk); 333179c0949eSPeter Krystad } 333279c0949eSPeter Krystad 33336e628cd3SPaolo Abeni void __mptcp_data_acked(struct sock *sk) 33346e628cd3SPaolo Abeni { 33356e628cd3SPaolo Abeni if (!sock_owned_by_user(sk)) 33366e628cd3SPaolo Abeni __mptcp_clean_una(sk); 33376e628cd3SPaolo Abeni else 3338e9d09bacSPaolo Abeni __set_bit(MPTCP_CLEAN_UNA, &mptcp_sk(sk)->cb_flags); 33396e628cd3SPaolo Abeni 33406e628cd3SPaolo Abeni if (mptcp_pending_data_fin_ack(sk)) 33416e628cd3SPaolo Abeni mptcp_schedule_work(sk); 33426e628cd3SPaolo Abeni } 33436e628cd3SPaolo Abeni 3344219d0499SPaolo Abeni void __mptcp_check_push(struct sock *sk, struct sock *ssk) 33456e628cd3SPaolo Abeni { 33466e628cd3SPaolo Abeni if (!mptcp_send_head(sk)) 33476e628cd3SPaolo Abeni return; 33486e628cd3SPaolo Abeni 334973a0052aSGeliang Tang if (!sock_owned_by_user(sk)) 335073a0052aSGeliang Tang __mptcp_subflow_push_pending(sk, ssk, false); 335173a0052aSGeliang Tang else 3352e9d09bacSPaolo Abeni __set_bit(MPTCP_PUSH_PENDING, &mptcp_sk(sk)->cb_flags); 33536e628cd3SPaolo Abeni } 33546e628cd3SPaolo Abeni 3355e9d09bacSPaolo Abeni #define MPTCP_FLAGS_PROCESS_CTX_NEED (BIT(MPTCP_PUSH_PENDING) | \ 3356e9d09bacSPaolo Abeni BIT(MPTCP_RETRANSMIT) | \ 3357e9d09bacSPaolo Abeni BIT(MPTCP_FLUSH_JOIN_LIST)) 3358e9d09bacSPaolo Abeni 3359e93da928SPaolo Abeni /* processes deferred events and flush wmem */ 336014c441b5SPaolo Abeni static void mptcp_release_cb(struct sock *sk) 3361e9d09bacSPaolo Abeni __must_hold(&sk->sk_lock.slock) 336214c441b5SPaolo Abeni { 3363e9d09bacSPaolo Abeni struct mptcp_sock *msk = mptcp_sk(sk); 33642d6f5a2bSPaolo Abeni 3365e9d09bacSPaolo Abeni for (;;) { 3366e9d09bacSPaolo Abeni unsigned long flags = (msk->cb_flags & MPTCP_FLAGS_PROCESS_CTX_NEED) | 3367e9d09bacSPaolo Abeni msk->push_pending; 336856a666c4SPaolo Abeni struct list_head join_list; 336956a666c4SPaolo Abeni 3370c2e6048fSPaolo Abeni if (!flags) 3371c2e6048fSPaolo Abeni break; 3372c2e6048fSPaolo Abeni 337356a666c4SPaolo Abeni INIT_LIST_HEAD(&join_list); 337456a666c4SPaolo Abeni list_splice_init(&msk->join_list, &join_list); 337556a666c4SPaolo Abeni 3376c2e6048fSPaolo Abeni /* the following actions acquire the subflow socket lock 33776e628cd3SPaolo Abeni * 33786e628cd3SPaolo Abeni * 1) can't be invoked in atomic scope 33796e628cd3SPaolo Abeni * 2) must avoid ABBA deadlock with msk socket spinlock: the RX 33806e628cd3SPaolo Abeni * datapath acquires the msk socket spinlock while helding 33816e628cd3SPaolo Abeni * the subflow socket lock 33826e628cd3SPaolo Abeni */ 3383e9d09bacSPaolo Abeni msk->push_pending = 0; 3384e9d09bacSPaolo Abeni msk->cb_flags &= ~flags; 33856e628cd3SPaolo Abeni spin_unlock_bh(&sk->sk_lock.slock); 338656a666c4SPaolo Abeni 33873e501490SPaolo Abeni if (flags & BIT(MPTCP_FLUSH_JOIN_LIST)) 338856a666c4SPaolo Abeni __mptcp_flush_join_list(sk, &join_list); 33892e5de7e0SDan Carpenter if (flags & BIT(MPTCP_PUSH_PENDING)) 3390c2e6048fSPaolo Abeni __mptcp_push_pending(sk, 0); 33912d6f5a2bSPaolo Abeni if (flags & BIT(MPTCP_RETRANSMIT)) 33922d6f5a2bSPaolo Abeni __mptcp_retrans(sk); 3393c2e6048fSPaolo Abeni 3394c2e6048fSPaolo Abeni cond_resched(); 33956e628cd3SPaolo Abeni spin_lock_bh(&sk->sk_lock.slock); 33966e628cd3SPaolo Abeni } 3397c2e6048fSPaolo Abeni 339865a569b0SPaolo Abeni if (__test_and_clear_bit(MPTCP_CLEAN_UNA, &msk->cb_flags)) 339965a569b0SPaolo Abeni __mptcp_clean_una_wakeup(sk); 340017ebf8a4SXiang Yang if (unlikely(msk->cb_flags)) { 3401490274b4SPaolo Abeni /* be sure to set the current sk state before tacking actions 340265a569b0SPaolo Abeni * depending on sk_state, that is processing MPTCP_ERROR_REPORT 3403490274b4SPaolo Abeni */ 3404e9d09bacSPaolo Abeni if (__test_and_clear_bit(MPTCP_CONNECTED, &msk->cb_flags)) 3405490274b4SPaolo Abeni __mptcp_set_connected(sk); 3406e9d09bacSPaolo Abeni if (__test_and_clear_bit(MPTCP_ERROR_REPORT, &msk->cb_flags)) 340715cc1045SPaolo Abeni __mptcp_error_report(sk); 34088005184fSPaolo Abeni if (__test_and_clear_bit(MPTCP_SYNC_SNDBUF, &msk->cb_flags)) 34098005184fSPaolo Abeni __mptcp_sync_sndbuf(sk); 341065a569b0SPaolo Abeni } 34116e628cd3SPaolo Abeni 341287952603SPaolo Abeni __mptcp_update_rmem(sk); 341314c441b5SPaolo Abeni } 341414c441b5SPaolo Abeni 3415bcd97734SPaolo Abeni /* MP_JOIN client subflow must wait for 4th ack before sending any data: 3416bcd97734SPaolo Abeni * TCP can't schedule delack timer before the subflow is fully established. 3417bcd97734SPaolo Abeni * MPTCP uses the delack timer to do 3rd ack retransmissions 3418bcd97734SPaolo Abeni */ 3419bcd97734SPaolo Abeni static void schedule_3rdack_retransmission(struct sock *ssk) 3420bcd97734SPaolo Abeni { 3421bcd97734SPaolo Abeni struct inet_connection_sock *icsk = inet_csk(ssk); 3422bcd97734SPaolo Abeni struct tcp_sock *tp = tcp_sk(ssk); 3423bcd97734SPaolo Abeni unsigned long timeout; 3424bcd97734SPaolo Abeni 3425bcd97734SPaolo Abeni if (mptcp_subflow_ctx(ssk)->fully_established) 3426bcd97734SPaolo Abeni return; 3427bcd97734SPaolo Abeni 3428bcd97734SPaolo Abeni /* reschedule with a timeout above RTT, as we must look only for drop */ 3429bcd97734SPaolo Abeni if (tp->srtt_us) 3430bcd97734SPaolo Abeni timeout = usecs_to_jiffies(tp->srtt_us >> (3 - 1)); 3431bcd97734SPaolo Abeni else 3432bcd97734SPaolo Abeni timeout = TCP_TIMEOUT_INIT; 3433bcd97734SPaolo Abeni timeout += jiffies; 3434bcd97734SPaolo Abeni 3435bcd97734SPaolo Abeni WARN_ON_ONCE(icsk->icsk_ack.pending & ICSK_ACK_TIMER); 3436bcd97734SPaolo Abeni icsk->icsk_ack.pending |= ICSK_ACK_SCHED | ICSK_ACK_TIMER; 3437bcd97734SPaolo Abeni icsk->icsk_ack.timeout = timeout; 3438bcd97734SPaolo Abeni sk_reset_timer(ssk, &icsk->icsk_delack_timer, timeout); 3439bcd97734SPaolo Abeni } 3440bcd97734SPaolo Abeni 3441a5efdbceSPaolo Abeni void mptcp_subflow_process_delegated(struct sock *ssk, long status) 3442b19bc294SPaolo Abeni { 3443b19bc294SPaolo Abeni struct mptcp_subflow_context *subflow = mptcp_subflow_ctx(ssk); 3444b19bc294SPaolo Abeni struct sock *sk = subflow->conn; 3445b19bc294SPaolo Abeni 3446a5efdbceSPaolo Abeni if (status & BIT(MPTCP_DELEGATE_SEND)) { 3447b19bc294SPaolo Abeni mptcp_data_lock(sk); 3448b19bc294SPaolo Abeni if (!sock_owned_by_user(sk)) 344973a0052aSGeliang Tang __mptcp_subflow_push_pending(sk, ssk, true); 3450b19bc294SPaolo Abeni else 3451e9d09bacSPaolo Abeni __set_bit(MPTCP_PUSH_PENDING, &mptcp_sk(sk)->cb_flags); 3452b19bc294SPaolo Abeni mptcp_data_unlock(sk); 3453bcd97734SPaolo Abeni } 34548005184fSPaolo Abeni if (status & BIT(MPTCP_DELEGATE_SNDBUF)) { 34558005184fSPaolo Abeni mptcp_data_lock(sk); 34568005184fSPaolo Abeni if (!sock_owned_by_user(sk)) 34578005184fSPaolo Abeni __mptcp_sync_sndbuf(sk); 34588005184fSPaolo Abeni else 34598005184fSPaolo Abeni __set_bit(MPTCP_SYNC_SNDBUF, &mptcp_sk(sk)->cb_flags); 34608005184fSPaolo Abeni mptcp_data_unlock(sk); 34618005184fSPaolo Abeni } 3462a5efdbceSPaolo Abeni if (status & BIT(MPTCP_DELEGATE_ACK)) 3463bcd97734SPaolo Abeni schedule_3rdack_retransmission(ssk); 3464b19bc294SPaolo Abeni } 3465b19bc294SPaolo Abeni 34662c5ebd00SPaolo Abeni static int mptcp_hash(struct sock *sk) 34672c5ebd00SPaolo Abeni { 34682c5ebd00SPaolo Abeni /* should never be called, 34692c5ebd00SPaolo Abeni * we hash the TCP subflows not the master socket 34702c5ebd00SPaolo Abeni */ 34712c5ebd00SPaolo Abeni WARN_ON_ONCE(1); 34722c5ebd00SPaolo Abeni return 0; 34732c5ebd00SPaolo Abeni } 34742c5ebd00SPaolo Abeni 34752c5ebd00SPaolo Abeni static void mptcp_unhash(struct sock *sk) 34762c5ebd00SPaolo Abeni { 34772c5ebd00SPaolo Abeni /* called from sk_common_release(), but nothing to do here */ 34782c5ebd00SPaolo Abeni } 34792c5ebd00SPaolo Abeni 3480cec37a6eSPeter Krystad static int mptcp_get_port(struct sock *sk, unsigned short snum) 3481f870fa0bSMat Martineau { 3482f870fa0bSMat Martineau struct mptcp_sock *msk = mptcp_sk(sk); 3483f870fa0bSMat Martineau 3484cfb63e50SPaolo Abeni pr_debug("msk=%p, ssk=%p", msk, msk->first); 3485cfb63e50SPaolo Abeni if (WARN_ON_ONCE(!msk->first)) 3486cec37a6eSPeter Krystad return -EINVAL; 3487f870fa0bSMat Martineau 3488cfb63e50SPaolo Abeni return inet_csk_get_port(msk->first, snum); 3489cec37a6eSPeter Krystad } 3490f870fa0bSMat Martineau 3491cec37a6eSPeter Krystad void mptcp_finish_connect(struct sock *ssk) 3492cec37a6eSPeter Krystad { 3493cec37a6eSPeter Krystad struct mptcp_subflow_context *subflow; 3494cec37a6eSPeter Krystad struct mptcp_sock *msk; 3495cec37a6eSPeter Krystad struct sock *sk; 3496f870fa0bSMat Martineau 3497cec37a6eSPeter Krystad subflow = mptcp_subflow_ctx(ssk); 3498cec37a6eSPeter Krystad sk = subflow->conn; 3499cec37a6eSPeter Krystad msk = mptcp_sk(sk); 3500cec37a6eSPeter Krystad 3501648ef4b8SMat Martineau pr_debug("msk=%p, token=%u", sk, subflow->token); 3502648ef4b8SMat Martineau 3503b3ea6b27SPaolo Abeni subflow->map_seq = subflow->iasn; 3504648ef4b8SMat Martineau subflow->map_subflow_seq = 1; 35056d0060f6SMat Martineau 3506cec37a6eSPeter Krystad /* the socket is not connected yet, no msk/subflow ops can access/race 3507cec37a6eSPeter Krystad * accessing the field below 3508cec37a6eSPeter Krystad */ 3509cec37a6eSPeter Krystad WRITE_ONCE(msk->local_key, subflow->local_key); 35106d0060f6SMat Martineau WRITE_ONCE(msk->write_seq, subflow->idsn + 1); 3511eaa2ffabSPaolo Abeni WRITE_ONCE(msk->snd_nxt, msk->write_seq); 35127439d687SPaolo Abeni WRITE_ONCE(msk->snd_una, msk->write_seq); 35131b1c7a0eSPeter Krystad 35146c714f1bSFlorian Westphal mptcp_pm_new_connection(msk, ssk, 0); 3515a6b118feSFlorian Westphal 3516a6b118feSFlorian Westphal mptcp_rcv_space_init(msk, ssk); 3517f870fa0bSMat Martineau } 3518f870fa0bSMat Martineau 3519866f26f2SPaolo Abeni void mptcp_sock_graft(struct sock *sk, struct socket *parent) 3520cf7da0d6SPeter Krystad { 3521cf7da0d6SPeter Krystad write_lock_bh(&sk->sk_callback_lock); 3522cf7da0d6SPeter Krystad rcu_assign_pointer(sk->sk_wq, &parent->wq); 3523cf7da0d6SPeter Krystad sk_set_socket(sk, parent); 3524cf7da0d6SPeter Krystad sk->sk_uid = SOCK_INODE(parent)->i_uid; 3525cf7da0d6SPeter Krystad write_unlock_bh(&sk->sk_callback_lock); 3526cf7da0d6SPeter Krystad } 3527cf7da0d6SPeter Krystad 3528e16163b6SPaolo Abeni bool mptcp_finish_join(struct sock *ssk) 3529f296234cSPeter Krystad { 3530e16163b6SPaolo Abeni struct mptcp_subflow_context *subflow = mptcp_subflow_ctx(ssk); 3531f296234cSPeter Krystad struct mptcp_sock *msk = mptcp_sk(subflow->conn); 3532f296234cSPeter Krystad struct sock *parent = (void *)msk; 35333e501490SPaolo Abeni bool ret = true; 3534f296234cSPeter Krystad 3535f296234cSPeter Krystad pr_debug("msk=%p, subflow=%p", msk, subflow); 3536f296234cSPeter Krystad 3537f296234cSPeter Krystad /* mptcp socket already closing? */ 3538dc87efdbSFlorian Westphal if (!mptcp_is_fully_established(parent)) { 3539dc87efdbSFlorian Westphal subflow->reset_reason = MPTCP_RST_EMPTCP; 3540f296234cSPeter Krystad return false; 3541dc87efdbSFlorian Westphal } 3542f296234cSPeter Krystad 354355b47ca7SPaolo Abeni /* active subflow, already present inside the conn_list */ 354455b47ca7SPaolo Abeni if (!list_empty(&subflow->node)) { 354555b47ca7SPaolo Abeni mptcp_subflow_joined(msk, ssk); 35468005184fSPaolo Abeni mptcp_propagate_sndbuf(parent, ssk); 354755b47ca7SPaolo Abeni return true; 354855b47ca7SPaolo Abeni } 3549f296234cSPeter Krystad 35503e501490SPaolo Abeni if (!mptcp_pm_allow_new_subflow(msk)) 35513e501490SPaolo Abeni goto err_prohibited; 355210f6d46cSPaolo Abeni 355355b47ca7SPaolo Abeni /* If we can't acquire msk socket lock here, let the release callback 35543e501490SPaolo Abeni * handle it 355510f6d46cSPaolo Abeni */ 35563e501490SPaolo Abeni mptcp_data_lock(parent); 35573e501490SPaolo Abeni if (!sock_owned_by_user(parent)) { 35583e501490SPaolo Abeni ret = __mptcp_finish_join(msk, ssk); 35593e501490SPaolo Abeni if (ret) { 3560e16163b6SPaolo Abeni sock_hold(ssk); 35613e501490SPaolo Abeni list_add_tail(&subflow->node, &msk->conn_list); 3562e16163b6SPaolo Abeni } 35633e501490SPaolo Abeni } else { 35643e501490SPaolo Abeni sock_hold(ssk); 35653e501490SPaolo Abeni list_add_tail(&subflow->node, &msk->join_list); 3566e9d09bacSPaolo Abeni __set_bit(MPTCP_FLUSH_JOIN_LIST, &msk->cb_flags); 35673e501490SPaolo Abeni } 35683e501490SPaolo Abeni mptcp_data_unlock(parent); 35693e501490SPaolo Abeni 3570dc87efdbSFlorian Westphal if (!ret) { 35713e501490SPaolo Abeni err_prohibited: 3572dc87efdbSFlorian Westphal subflow->reset_reason = MPTCP_RST_EPROHIBIT; 357310f6d46cSPaolo Abeni return false; 3574dc87efdbSFlorian Westphal } 357510f6d46cSPaolo Abeni 357610f6d46cSPaolo Abeni return true; 3577f296234cSPeter Krystad } 3578f296234cSPeter Krystad 357976e2a55dSPaolo Abeni static void mptcp_shutdown(struct sock *sk, int how) 358076e2a55dSPaolo Abeni { 358176e2a55dSPaolo Abeni pr_debug("sk=%p, how=%d", sk, how); 358276e2a55dSPaolo Abeni 358376e2a55dSPaolo Abeni if ((how & SEND_SHUTDOWN) && mptcp_close_state(sk)) 358476e2a55dSPaolo Abeni __mptcp_wr_shutdown(sk); 358576e2a55dSPaolo Abeni } 358676e2a55dSPaolo Abeni 35876511882cSPaolo Abeni static int mptcp_forward_alloc_get(const struct sock *sk) 35886511882cSPaolo Abeni { 35899531e4a8SEric Dumazet return READ_ONCE(sk->sk_forward_alloc) + 35909531e4a8SEric Dumazet READ_ONCE(mptcp_sk(sk)->rmem_fwd_alloc); 35916511882cSPaolo Abeni } 35926511882cSPaolo Abeni 3593644807e3SFlorian Westphal static int mptcp_ioctl_outq(const struct mptcp_sock *msk, u64 v) 3594644807e3SFlorian Westphal { 3595644807e3SFlorian Westphal const struct sock *sk = (void *)msk; 3596644807e3SFlorian Westphal u64 delta; 3597644807e3SFlorian Westphal 3598644807e3SFlorian Westphal if (sk->sk_state == TCP_LISTEN) 3599644807e3SFlorian Westphal return -EINVAL; 3600644807e3SFlorian Westphal 3601644807e3SFlorian Westphal if ((1 << sk->sk_state) & (TCPF_SYN_SENT | TCPF_SYN_RECV)) 3602644807e3SFlorian Westphal return 0; 3603644807e3SFlorian Westphal 3604644807e3SFlorian Westphal delta = msk->write_seq - v; 360507c2c7a3SPaolo Abeni if (__mptcp_check_fallback(msk) && msk->first) { 360607c2c7a3SPaolo Abeni struct tcp_sock *tp = tcp_sk(msk->first); 360707c2c7a3SPaolo Abeni 360807c2c7a3SPaolo Abeni /* the first subflow is disconnected after close - see 360907c2c7a3SPaolo Abeni * __mptcp_close_ssk(). tcp_disconnect() moves the write_seq 361007c2c7a3SPaolo Abeni * so ignore that status, too. 361107c2c7a3SPaolo Abeni */ 361207c2c7a3SPaolo Abeni if (!((1 << msk->first->sk_state) & 361307c2c7a3SPaolo Abeni (TCPF_SYN_SENT | TCPF_SYN_RECV | TCPF_CLOSE))) 361407c2c7a3SPaolo Abeni delta += READ_ONCE(tp->write_seq) - tp->snd_una; 361507c2c7a3SPaolo Abeni } 3616644807e3SFlorian Westphal if (delta > INT_MAX) 3617644807e3SFlorian Westphal delta = INT_MAX; 3618644807e3SFlorian Westphal 3619644807e3SFlorian Westphal return (int)delta; 3620644807e3SFlorian Westphal } 3621644807e3SFlorian Westphal 3622e1d001faSBreno Leitao static int mptcp_ioctl(struct sock *sk, int cmd, int *karg) 3623644807e3SFlorian Westphal { 3624644807e3SFlorian Westphal struct mptcp_sock *msk = mptcp_sk(sk); 3625644807e3SFlorian Westphal bool slow; 3626644807e3SFlorian Westphal 3627644807e3SFlorian Westphal switch (cmd) { 3628644807e3SFlorian Westphal case SIOCINQ: 3629644807e3SFlorian Westphal if (sk->sk_state == TCP_LISTEN) 3630644807e3SFlorian Westphal return -EINVAL; 3631644807e3SFlorian Westphal 3632644807e3SFlorian Westphal lock_sock(sk); 3633644807e3SFlorian Westphal __mptcp_move_skbs(msk); 3634e1d001faSBreno Leitao *karg = mptcp_inq_hint(sk); 3635644807e3SFlorian Westphal release_sock(sk); 3636644807e3SFlorian Westphal break; 3637644807e3SFlorian Westphal case SIOCOUTQ: 3638644807e3SFlorian Westphal slow = lock_sock_fast(sk); 3639e1d001faSBreno Leitao *karg = mptcp_ioctl_outq(msk, READ_ONCE(msk->snd_una)); 3640644807e3SFlorian Westphal unlock_sock_fast(sk, slow); 3641644807e3SFlorian Westphal break; 3642644807e3SFlorian Westphal case SIOCOUTQNSD: 3643644807e3SFlorian Westphal slow = lock_sock_fast(sk); 3644e1d001faSBreno Leitao *karg = mptcp_ioctl_outq(msk, msk->snd_nxt); 3645644807e3SFlorian Westphal unlock_sock_fast(sk, slow); 3646644807e3SFlorian Westphal break; 3647644807e3SFlorian Westphal default: 3648644807e3SFlorian Westphal return -ENOIOCTLCMD; 3649644807e3SFlorian Westphal } 3650644807e3SFlorian Westphal 3651e1d001faSBreno Leitao return 0; 3652644807e3SFlorian Westphal } 3653644807e3SFlorian Westphal 365454f1944eSPaolo Abeni static void mptcp_subflow_early_fallback(struct mptcp_sock *msk, 365554f1944eSPaolo Abeni struct mptcp_subflow_context *subflow) 365654f1944eSPaolo Abeni { 365754f1944eSPaolo Abeni subflow->request_mptcp = 0; 365854f1944eSPaolo Abeni __mptcp_do_fallback(msk); 365954f1944eSPaolo Abeni } 366054f1944eSPaolo Abeni 366154f1944eSPaolo Abeni static int mptcp_connect(struct sock *sk, struct sockaddr *uaddr, int addr_len) 366254f1944eSPaolo Abeni { 366354f1944eSPaolo Abeni struct mptcp_subflow_context *subflow; 366454f1944eSPaolo Abeni struct mptcp_sock *msk = mptcp_sk(sk); 366554f1944eSPaolo Abeni int err = -EINVAL; 3666ccae357cSPaolo Abeni struct sock *ssk; 366754f1944eSPaolo Abeni 36683f326a82SPaolo Abeni ssk = __mptcp_nmpc_sk(msk); 36693f326a82SPaolo Abeni if (IS_ERR(ssk)) 36703f326a82SPaolo Abeni return PTR_ERR(ssk); 367154f1944eSPaolo Abeni 367254f1944eSPaolo Abeni inet_sk_state_store(sk, TCP_SYN_SENT); 3673ccae357cSPaolo Abeni subflow = mptcp_subflow_ctx(ssk); 367454f1944eSPaolo Abeni #ifdef CONFIG_TCP_MD5SIG 367554f1944eSPaolo Abeni /* no MPTCP if MD5SIG is enabled on this socket or we may run out of 367654f1944eSPaolo Abeni * TCP option space. 367754f1944eSPaolo Abeni */ 3678ccae357cSPaolo Abeni if (rcu_access_pointer(tcp_sk(ssk)->md5sig_info)) 367954f1944eSPaolo Abeni mptcp_subflow_early_fallback(msk, subflow); 368054f1944eSPaolo Abeni #endif 3681ccae357cSPaolo Abeni if (subflow->request_mptcp && mptcp_token_new_connect(ssk)) { 3682ccae357cSPaolo Abeni MPTCP_INC_STATS(sock_net(ssk), MPTCP_MIB_TOKENFALLBACKINIT); 368354f1944eSPaolo Abeni mptcp_subflow_early_fallback(msk, subflow); 368454f1944eSPaolo Abeni } 368554f1944eSPaolo Abeni if (likely(!__mptcp_check_fallback(msk))) 368654f1944eSPaolo Abeni MPTCP_INC_STATS(sock_net(sk), MPTCP_MIB_MPCAPABLEACTIVE); 368754f1944eSPaolo Abeni 368854f1944eSPaolo Abeni /* if reaching here via the fastopen/sendmsg path, the caller already 368954f1944eSPaolo Abeni * acquired the subflow socket lock, too. 369054f1944eSPaolo Abeni */ 3691ccae357cSPaolo Abeni if (!msk->fastopening) 3692ccae357cSPaolo Abeni lock_sock(ssk); 3693ccae357cSPaolo Abeni 3694ccae357cSPaolo Abeni /* the following mirrors closely a very small chunk of code from 3695ccae357cSPaolo Abeni * __inet_stream_connect() 3696ccae357cSPaolo Abeni */ 3697ccae357cSPaolo Abeni if (ssk->sk_state != TCP_CLOSE) 3698ccae357cSPaolo Abeni goto out; 3699ccae357cSPaolo Abeni 3700ccae357cSPaolo Abeni if (BPF_CGROUP_PRE_CONNECT_ENABLED(ssk)) { 3701ccae357cSPaolo Abeni err = ssk->sk_prot->pre_connect(ssk, uaddr, addr_len); 3702ccae357cSPaolo Abeni if (err) 3703ccae357cSPaolo Abeni goto out; 3704ccae357cSPaolo Abeni } 3705ccae357cSPaolo Abeni 3706ccae357cSPaolo Abeni err = ssk->sk_prot->connect(ssk, uaddr, addr_len); 3707ccae357cSPaolo Abeni if (err < 0) 3708ccae357cSPaolo Abeni goto out; 3709ccae357cSPaolo Abeni 371008e39c0dSEric Dumazet inet_assign_bit(DEFER_CONNECT, sk, inet_test_bit(DEFER_CONNECT, ssk)); 3711ccae357cSPaolo Abeni 3712ccae357cSPaolo Abeni out: 3713ccae357cSPaolo Abeni if (!msk->fastopening) 3714ccae357cSPaolo Abeni release_sock(ssk); 371554f1944eSPaolo Abeni 371654f1944eSPaolo Abeni /* on successful connect, the msk state will be moved to established by 371754f1944eSPaolo Abeni * subflow_finish_connect() 371854f1944eSPaolo Abeni */ 3719ccae357cSPaolo Abeni if (unlikely(err)) { 3720131a6277SPaolo Abeni /* avoid leaving a dangling token in an unconnected socket */ 3721131a6277SPaolo Abeni mptcp_token_destroy(msk); 3722ccae357cSPaolo Abeni inet_sk_state_store(sk, TCP_CLOSE); 372354f1944eSPaolo Abeni return err; 372454f1944eSPaolo Abeni } 372554f1944eSPaolo Abeni 3726ccae357cSPaolo Abeni mptcp_copy_inaddrs(sk, ssk); 3727786fc124SPaolo Abeni return 0; 372854f1944eSPaolo Abeni } 372954f1944eSPaolo Abeni 3730f870fa0bSMat Martineau static struct proto mptcp_prot = { 3731f870fa0bSMat Martineau .name = "MPTCP", 3732f870fa0bSMat Martineau .owner = THIS_MODULE, 3733f870fa0bSMat Martineau .init = mptcp_init_sock, 373454f1944eSPaolo Abeni .connect = mptcp_connect, 373518b683bfSPaolo Abeni .disconnect = mptcp_disconnect, 3736f870fa0bSMat Martineau .close = mptcp_close, 3737cf7da0d6SPeter Krystad .accept = mptcp_accept, 3738717e79c8SPeter Krystad .setsockopt = mptcp_setsockopt, 3739717e79c8SPeter Krystad .getsockopt = mptcp_getsockopt, 374076e2a55dSPaolo Abeni .shutdown = mptcp_shutdown, 374179c0949eSPeter Krystad .destroy = mptcp_destroy, 3742f870fa0bSMat Martineau .sendmsg = mptcp_sendmsg, 3743644807e3SFlorian Westphal .ioctl = mptcp_ioctl, 3744f870fa0bSMat Martineau .recvmsg = mptcp_recvmsg, 374514c441b5SPaolo Abeni .release_cb = mptcp_release_cb, 37462c5ebd00SPaolo Abeni .hash = mptcp_hash, 37472c5ebd00SPaolo Abeni .unhash = mptcp_unhash, 3748cec37a6eSPeter Krystad .get_port = mptcp_get_port, 37496511882cSPaolo Abeni .forward_alloc_get = mptcp_forward_alloc_get, 3750d027236cSPaolo Abeni .sockets_allocated = &mptcp_sockets_allocated, 37510defbb0aSEric Dumazet 3752d027236cSPaolo Abeni .memory_allocated = &tcp_memory_allocated, 37530defbb0aSEric Dumazet .per_cpu_fw_alloc = &tcp_memory_per_cpu_fw_alloc, 37540defbb0aSEric Dumazet 3755d027236cSPaolo Abeni .memory_pressure = &tcp_memory_pressure, 3756d027236cSPaolo Abeni .sysctl_wmem_offset = offsetof(struct net, ipv4.sysctl_tcp_wmem), 3757989ef49bSPaolo Abeni .sysctl_rmem_offset = offsetof(struct net, ipv4.sysctl_tcp_rmem), 3758d027236cSPaolo Abeni .sysctl_mem = sysctl_tcp_mem, 3759f870fa0bSMat Martineau .obj_size = sizeof(struct mptcp_sock), 37602c5ebd00SPaolo Abeni .slab_flags = SLAB_TYPESAFE_BY_RCU, 3761f870fa0bSMat Martineau .no_autobind = true, 3762f870fa0bSMat Martineau }; 3763f870fa0bSMat Martineau 37642303f994SPeter Krystad static int mptcp_bind(struct socket *sock, struct sockaddr *uaddr, int addr_len) 37652303f994SPeter Krystad { 37662303f994SPeter Krystad struct mptcp_sock *msk = mptcp_sk(sock->sk); 37678cf2ebdcSPaolo Abeni struct sock *ssk, *sk = sock->sk; 37688cf2ebdcSPaolo Abeni int err = -EINVAL; 37692303f994SPeter Krystad 37708cf2ebdcSPaolo Abeni lock_sock(sk); 37713f326a82SPaolo Abeni ssk = __mptcp_nmpc_sk(msk); 37723f326a82SPaolo Abeni if (IS_ERR(ssk)) { 37733f326a82SPaolo Abeni err = PTR_ERR(ssk); 37742303f994SPeter Krystad goto unlock; 37752303f994SPeter Krystad } 37762303f994SPeter Krystad 37778cf2ebdcSPaolo Abeni if (sk->sk_family == AF_INET) 37788cf2ebdcSPaolo Abeni err = inet_bind_sk(ssk, uaddr, addr_len); 37798cf2ebdcSPaolo Abeni #if IS_ENABLED(CONFIG_MPTCP_IPV6) 37808cf2ebdcSPaolo Abeni else if (sk->sk_family == AF_INET6) 37818cf2ebdcSPaolo Abeni err = inet6_bind_sk(ssk, uaddr, addr_len); 37828cf2ebdcSPaolo Abeni #endif 3783cf7da0d6SPeter Krystad if (!err) 37848cf2ebdcSPaolo Abeni mptcp_copy_inaddrs(sk, ssk); 37852303f994SPeter Krystad 37862303f994SPeter Krystad unlock: 37878cf2ebdcSPaolo Abeni release_sock(sk); 37882303f994SPeter Krystad return err; 37892303f994SPeter Krystad } 37902303f994SPeter Krystad 3791cf7da0d6SPeter Krystad static int mptcp_listen(struct socket *sock, int backlog) 3792cf7da0d6SPeter Krystad { 3793cf7da0d6SPeter Krystad struct mptcp_sock *msk = mptcp_sk(sock->sk); 3794cfdcfeedSMenglong Dong struct sock *sk = sock->sk; 379540f56d0cSPaolo Abeni struct sock *ssk; 3796cf7da0d6SPeter Krystad int err; 3797cf7da0d6SPeter Krystad 3798cf7da0d6SPeter Krystad pr_debug("msk=%p", msk); 3799cf7da0d6SPeter Krystad 3800cfdcfeedSMenglong Dong lock_sock(sk); 38010226436aSPaolo Abeni 38020226436aSPaolo Abeni err = -EINVAL; 38030226436aSPaolo Abeni if (sock->state != SS_UNCONNECTED || sock->type != SOCK_STREAM) 38040226436aSPaolo Abeni goto unlock; 38050226436aSPaolo Abeni 38063f326a82SPaolo Abeni ssk = __mptcp_nmpc_sk(msk); 38073f326a82SPaolo Abeni if (IS_ERR(ssk)) { 38083f326a82SPaolo Abeni err = PTR_ERR(ssk); 3809cf7da0d6SPeter Krystad goto unlock; 3810cf7da0d6SPeter Krystad } 3811cf7da0d6SPeter Krystad 3812cfdcfeedSMenglong Dong inet_sk_state_store(sk, TCP_LISTEN); 3813cfdcfeedSMenglong Dong sock_set_flag(sk, SOCK_RCU_FREE); 38145e20087dSFlorian Westphal 381540f56d0cSPaolo Abeni lock_sock(ssk); 381640f56d0cSPaolo Abeni err = __inet_listen_sk(ssk, backlog); 381740f56d0cSPaolo Abeni release_sock(ssk); 381840f56d0cSPaolo Abeni inet_sk_state_store(sk, inet_sk_state_load(ssk)); 381940f56d0cSPaolo Abeni 3820c558246eSMenglong Dong if (!err) { 3821c558246eSMenglong Dong sock_prot_inuse_add(sock_net(sk), sk->sk_prot, 1); 382240f56d0cSPaolo Abeni mptcp_copy_inaddrs(sk, ssk); 382340f56d0cSPaolo Abeni mptcp_event_pm_listener(ssk, MPTCP_EVENT_LISTENER_CREATED); 382421d9b73aSPaolo Abeni } 3825f8c9dfbdSGeliang Tang 3826cf7da0d6SPeter Krystad unlock: 3827cfdcfeedSMenglong Dong release_sock(sk); 3828cf7da0d6SPeter Krystad return err; 3829cf7da0d6SPeter Krystad } 3830cf7da0d6SPeter Krystad 3831cf7da0d6SPeter Krystad static int mptcp_stream_accept(struct socket *sock, struct socket *newsock, 3832cf7da0d6SPeter Krystad int flags, bool kern) 3833cf7da0d6SPeter Krystad { 3834cf7da0d6SPeter Krystad struct mptcp_sock *msk = mptcp_sk(sock->sk); 38351f6610b9SPaolo Abeni struct sock *ssk, *newsk; 3836cf7da0d6SPeter Krystad int err; 3837cf7da0d6SPeter Krystad 3838cf7da0d6SPeter Krystad pr_debug("msk=%p", msk); 3839cf7da0d6SPeter Krystad 38405b825727SPaolo Abeni /* Buggy applications can call accept on socket states other then LISTEN 384161761231SPaolo Abeni * but no need to allocate the first subflow just to error out. 384261761231SPaolo Abeni */ 38431f6610b9SPaolo Abeni ssk = READ_ONCE(msk->first); 38441f6610b9SPaolo Abeni if (!ssk) 384571ba088cSPaolo Abeni return -EINVAL; 3846cf7da0d6SPeter Krystad 38471f6610b9SPaolo Abeni newsk = mptcp_accept(ssk, flags, &err, kern); 3848e76c8ef5SPaolo Abeni if (!newsk) 3849e76c8ef5SPaolo Abeni return err; 3850e76c8ef5SPaolo Abeni 3851e76c8ef5SPaolo Abeni lock_sock(newsk); 3852e76c8ef5SPaolo Abeni 3853e76c8ef5SPaolo Abeni __inet_accept(sock, newsock, newsk); 3854e76c8ef5SPaolo Abeni if (!mptcp_is_tcpsk(newsock->sk)) { 3855e76c8ef5SPaolo Abeni struct mptcp_sock *msk = mptcp_sk(newsk); 3856cf7da0d6SPeter Krystad struct mptcp_subflow_context *subflow; 38570397c6d8SPaolo Abeni 3858a5ef058dSPaolo Abeni set_bit(SOCK_CUSTOM_SOCKOPT, &newsock->flags); 3859b6985b9bSPaolo Abeni msk->in_accept_queue = 0; 3860a5ef058dSPaolo Abeni 3861cf7da0d6SPeter Krystad /* set ssk->sk_socket of accept()ed flows to mptcp socket. 3862cf7da0d6SPeter Krystad * This is needed so NOSPACE flag can be set from tcp stack. 3863cf7da0d6SPeter Krystad */ 3864190f8b06SGeliang Tang mptcp_for_each_subflow(msk, subflow) { 3865cf7da0d6SPeter Krystad struct sock *ssk = mptcp_subflow_tcp_sock(subflow); 3866cf7da0d6SPeter Krystad 3867cf7da0d6SPeter Krystad if (!ssk->sk_socket) 3868cf7da0d6SPeter Krystad mptcp_sock_graft(ssk, newsock); 3869cf7da0d6SPeter Krystad } 387063740448SPaolo Abeni 387163740448SPaolo Abeni /* Do late cleanup for the first subflow as necessary. Also 387263740448SPaolo Abeni * deal with bad peers not doing a complete shutdown. 387363740448SPaolo Abeni */ 387439880bd8SPaolo Abeni if (unlikely(inet_sk_state_load(msk->first) == TCP_CLOSE)) { 387563740448SPaolo Abeni __mptcp_close_ssk(newsk, msk->first, 387663740448SPaolo Abeni mptcp_subflow_ctx(msk->first), 0); 387739880bd8SPaolo Abeni if (unlikely(list_is_singular(&msk->conn_list))) 387863740448SPaolo Abeni inet_sk_state_store(newsk, TCP_CLOSE); 387963740448SPaolo Abeni } 3880cf7da0d6SPeter Krystad } 3881e76c8ef5SPaolo Abeni release_sock(newsk); 3882cf7da0d6SPeter Krystad 3883e76c8ef5SPaolo Abeni return 0; 3884cf7da0d6SPeter Krystad } 3885cf7da0d6SPeter Krystad 38868edf0864SFlorian Westphal static __poll_t mptcp_check_writeable(struct mptcp_sock *msk) 38878edf0864SFlorian Westphal { 38888edf0864SFlorian Westphal struct sock *sk = (struct sock *)msk; 38898edf0864SFlorian Westphal 38908edf0864SFlorian Westphal if (sk_stream_is_writeable(sk)) 38918edf0864SFlorian Westphal return EPOLLOUT | EPOLLWRNORM; 38928edf0864SFlorian Westphal 38935cf92bbaSPaolo Abeni mptcp_set_nospace(sk); 38946e628cd3SPaolo Abeni smp_mb__after_atomic(); /* msk->flags is changed by write_space cb */ 38956e628cd3SPaolo Abeni if (sk_stream_is_writeable(sk)) 38966e628cd3SPaolo Abeni return EPOLLOUT | EPOLLWRNORM; 38978edf0864SFlorian Westphal 38986e628cd3SPaolo Abeni return 0; 38998edf0864SFlorian Westphal } 39008edf0864SFlorian Westphal 39012303f994SPeter Krystad static __poll_t mptcp_poll(struct file *file, struct socket *sock, 39022303f994SPeter Krystad struct poll_table_struct *wait) 39032303f994SPeter Krystad { 39041891c4a0SFlorian Westphal struct sock *sk = sock->sk; 39058ab183deSPaolo Abeni struct mptcp_sock *msk; 39062303f994SPeter Krystad __poll_t mask = 0; 39076b9831bfSPaolo Abeni u8 shutdown; 39088a05661bSPaolo Abeni int state; 39092303f994SPeter Krystad 39101891c4a0SFlorian Westphal msk = mptcp_sk(sk); 39111891c4a0SFlorian Westphal sock_poll_wait(file, sock, wait); 39121891c4a0SFlorian Westphal 39138a05661bSPaolo Abeni state = inet_sk_state_load(sk); 39146719331cSPaolo Abeni pr_debug("msk=%p state=%d flags=%lx", msk, state, msk->flags); 391571ba088cSPaolo Abeni if (state == TCP_LISTEN) { 39165426a4efSPaolo Abeni struct sock *ssk = READ_ONCE(msk->first); 39175b825727SPaolo Abeni 39185426a4efSPaolo Abeni if (WARN_ON_ONCE(!ssk)) 391971ba088cSPaolo Abeni return 0; 392071ba088cSPaolo Abeni 39215426a4efSPaolo Abeni return inet_csk_listen_poll(ssk); 392271ba088cSPaolo Abeni } 39238a05661bSPaolo Abeni 39246b9831bfSPaolo Abeni shutdown = READ_ONCE(sk->sk_shutdown); 39256b9831bfSPaolo Abeni if (shutdown == SHUTDOWN_MASK || state == TCP_CLOSE) 39266b9831bfSPaolo Abeni mask |= EPOLLHUP; 39276b9831bfSPaolo Abeni if (shutdown & RCV_SHUTDOWN) 39286b9831bfSPaolo Abeni mask |= EPOLLIN | EPOLLRDNORM | EPOLLRDHUP; 39296b9831bfSPaolo Abeni 39308a05661bSPaolo Abeni if (state != TCP_SYN_SENT && state != TCP_SYN_RECV) { 39315684ab1aSPaolo Abeni mask |= mptcp_check_readable(sk); 39326b9831bfSPaolo Abeni if (shutdown & SEND_SHUTDOWN) 39336b9831bfSPaolo Abeni mask |= EPOLLOUT | EPOLLWRNORM; 39346b9831bfSPaolo Abeni else 39358edf0864SFlorian Westphal mask |= mptcp_check_writeable(msk); 393608e39c0dSEric Dumazet } else if (state == TCP_SYN_SENT && 393708e39c0dSEric Dumazet inet_test_bit(DEFER_CONNECT, sk)) { 3938a42cf9d1SBenjamin Hesmans /* cf tcp_poll() note about TFO */ 3939a42cf9d1SBenjamin Hesmans mask |= EPOLLOUT | EPOLLWRNORM; 39408a05661bSPaolo Abeni } 39411891c4a0SFlorian Westphal 3942d89e3ed7SPaolo Abeni /* This barrier is coupled with smp_wmb() in __mptcp_error_report() */ 394315cc1045SPaolo Abeni smp_rmb(); 39449ae8e5adSEric Dumazet if (READ_ONCE(sk->sk_err)) 394515cc1045SPaolo Abeni mask |= EPOLLERR; 394615cc1045SPaolo Abeni 39472303f994SPeter Krystad return mask; 39482303f994SPeter Krystad } 39492303f994SPeter Krystad 3950e42f1ac6SFlorian Westphal static const struct proto_ops mptcp_stream_ops = { 3951e42f1ac6SFlorian Westphal .family = PF_INET, 3952e42f1ac6SFlorian Westphal .owner = THIS_MODULE, 39530a3cc579SPaolo Abeni .release = inet_release, 3954e42f1ac6SFlorian Westphal .bind = mptcp_bind, 3955786fc124SPaolo Abeni .connect = inet_stream_connect, 3956e42f1ac6SFlorian Westphal .socketpair = sock_no_socketpair, 3957e42f1ac6SFlorian Westphal .accept = mptcp_stream_accept, 3958d2f77c53SPaolo Abeni .getname = inet_getname, 3959e42f1ac6SFlorian Westphal .poll = mptcp_poll, 3960e42f1ac6SFlorian Westphal .ioctl = inet_ioctl, 3961e42f1ac6SFlorian Westphal .gettstamp = sock_gettstamp, 3962e42f1ac6SFlorian Westphal .listen = mptcp_listen, 396376e2a55dSPaolo Abeni .shutdown = inet_shutdown, 3964e42f1ac6SFlorian Westphal .setsockopt = sock_common_setsockopt, 3965e42f1ac6SFlorian Westphal .getsockopt = sock_common_getsockopt, 3966e42f1ac6SFlorian Westphal .sendmsg = inet_sendmsg, 3967e42f1ac6SFlorian Westphal .recvmsg = inet_recvmsg, 3968e42f1ac6SFlorian Westphal .mmap = sock_no_mmap, 39695684ab1aSPaolo Abeni .set_rcvlowat = mptcp_set_rcvlowat, 3970e42f1ac6SFlorian Westphal }; 39712303f994SPeter Krystad 3972f870fa0bSMat Martineau static struct inet_protosw mptcp_protosw = { 3973f870fa0bSMat Martineau .type = SOCK_STREAM, 3974f870fa0bSMat Martineau .protocol = IPPROTO_MPTCP, 3975f870fa0bSMat Martineau .prot = &mptcp_prot, 39762303f994SPeter Krystad .ops = &mptcp_stream_ops, 39772303f994SPeter Krystad .flags = INET_PROTOSW_ICSK, 3978f870fa0bSMat Martineau }; 3979f870fa0bSMat Martineau 3980b19bc294SPaolo Abeni static int mptcp_napi_poll(struct napi_struct *napi, int budget) 3981b19bc294SPaolo Abeni { 3982b19bc294SPaolo Abeni struct mptcp_delegated_action *delegated; 3983b19bc294SPaolo Abeni struct mptcp_subflow_context *subflow; 3984b19bc294SPaolo Abeni int work_done = 0; 3985b19bc294SPaolo Abeni 3986b19bc294SPaolo Abeni delegated = container_of(napi, struct mptcp_delegated_action, napi); 3987b19bc294SPaolo Abeni while ((subflow = mptcp_subflow_delegated_next(delegated)) != NULL) { 3988b19bc294SPaolo Abeni struct sock *ssk = mptcp_subflow_tcp_sock(subflow); 3989b19bc294SPaolo Abeni 3990b19bc294SPaolo Abeni bh_lock_sock_nested(ssk); 3991a5efdbceSPaolo Abeni if (!sock_owned_by_user(ssk)) { 3992a5efdbceSPaolo Abeni mptcp_subflow_process_delegated(ssk, xchg(&subflow->delegated_status, 0)); 3993a5efdbceSPaolo Abeni } else { 3994a5efdbceSPaolo Abeni /* tcp_release_cb_override already processed 3995b19bc294SPaolo Abeni * the action or will do at next release_sock(). 3996b19bc294SPaolo Abeni * In both case must dequeue the subflow here - on the same 3997b19bc294SPaolo Abeni * CPU that scheduled it. 3998b19bc294SPaolo Abeni */ 3999a5efdbceSPaolo Abeni smp_wmb(); 4000a5efdbceSPaolo Abeni clear_bit(MPTCP_DELEGATE_SCHEDULED, &subflow->delegated_status); 4001a5efdbceSPaolo Abeni } 4002b19bc294SPaolo Abeni bh_unlock_sock(ssk); 4003b19bc294SPaolo Abeni sock_put(ssk); 4004b19bc294SPaolo Abeni 4005b19bc294SPaolo Abeni if (++work_done == budget) 4006b19bc294SPaolo Abeni return budget; 4007b19bc294SPaolo Abeni } 4008b19bc294SPaolo Abeni 4009b19bc294SPaolo Abeni /* always provide a 0 'work_done' argument, so that napi_complete_done 4010b19bc294SPaolo Abeni * will not try accessing the NULL napi->dev ptr 4011b19bc294SPaolo Abeni */ 4012b19bc294SPaolo Abeni napi_complete_done(napi, 0); 4013b19bc294SPaolo Abeni return work_done; 4014b19bc294SPaolo Abeni } 4015b19bc294SPaolo Abeni 4016d39dcecaSPaolo Abeni void __init mptcp_proto_init(void) 4017f870fa0bSMat Martineau { 4018b19bc294SPaolo Abeni struct mptcp_delegated_action *delegated; 4019b19bc294SPaolo Abeni int cpu; 4020b19bc294SPaolo Abeni 40212303f994SPeter Krystad mptcp_prot.h.hashinfo = tcp_prot.h.hashinfo; 40222303f994SPeter Krystad 4023d027236cSPaolo Abeni if (percpu_counter_init(&mptcp_sockets_allocated, 0, GFP_KERNEL)) 4024d027236cSPaolo Abeni panic("Failed to allocate MPTCP pcpu counter\n"); 4025d027236cSPaolo Abeni 4026b19bc294SPaolo Abeni init_dummy_netdev(&mptcp_napi_dev); 4027b19bc294SPaolo Abeni for_each_possible_cpu(cpu) { 4028b19bc294SPaolo Abeni delegated = per_cpu_ptr(&mptcp_delegated_actions, cpu); 4029b19bc294SPaolo Abeni INIT_LIST_HEAD(&delegated->head); 403016d083e2SJakub Kicinski netif_napi_add_tx(&mptcp_napi_dev, &delegated->napi, 403116d083e2SJakub Kicinski mptcp_napi_poll); 4032b19bc294SPaolo Abeni napi_enable(&delegated->napi); 4033b19bc294SPaolo Abeni } 4034b19bc294SPaolo Abeni 40352303f994SPeter Krystad mptcp_subflow_init(); 40361b1c7a0eSPeter Krystad mptcp_pm_init(); 4037ed1ad86bSGeliang Tang mptcp_sched_init(); 40382c5ebd00SPaolo Abeni mptcp_token_init(); 40392303f994SPeter Krystad 4040f870fa0bSMat Martineau if (proto_register(&mptcp_prot, 1) != 0) 4041f870fa0bSMat Martineau panic("Failed to register MPTCP proto.\n"); 4042f870fa0bSMat Martineau 4043f870fa0bSMat Martineau inet_register_protosw(&mptcp_protosw); 40446771bfd9SFlorian Westphal 40456771bfd9SFlorian Westphal BUILD_BUG_ON(sizeof(struct mptcp_skb_cb) > sizeof_field(struct sk_buff, cb)); 4046f870fa0bSMat Martineau } 4047f870fa0bSMat Martineau 4048f870fa0bSMat Martineau #if IS_ENABLED(CONFIG_MPTCP_IPV6) 4049e42f1ac6SFlorian Westphal static const struct proto_ops mptcp_v6_stream_ops = { 4050e42f1ac6SFlorian Westphal .family = PF_INET6, 4051e42f1ac6SFlorian Westphal .owner = THIS_MODULE, 40520a3cc579SPaolo Abeni .release = inet6_release, 4053e42f1ac6SFlorian Westphal .bind = mptcp_bind, 4054786fc124SPaolo Abeni .connect = inet_stream_connect, 4055e42f1ac6SFlorian Westphal .socketpair = sock_no_socketpair, 4056e42f1ac6SFlorian Westphal .accept = mptcp_stream_accept, 4057d2f77c53SPaolo Abeni .getname = inet6_getname, 4058e42f1ac6SFlorian Westphal .poll = mptcp_poll, 4059e42f1ac6SFlorian Westphal .ioctl = inet6_ioctl, 4060e42f1ac6SFlorian Westphal .gettstamp = sock_gettstamp, 4061e42f1ac6SFlorian Westphal .listen = mptcp_listen, 406276e2a55dSPaolo Abeni .shutdown = inet_shutdown, 4063e42f1ac6SFlorian Westphal .setsockopt = sock_common_setsockopt, 4064e42f1ac6SFlorian Westphal .getsockopt = sock_common_getsockopt, 4065e42f1ac6SFlorian Westphal .sendmsg = inet6_sendmsg, 4066e42f1ac6SFlorian Westphal .recvmsg = inet6_recvmsg, 4067e42f1ac6SFlorian Westphal .mmap = sock_no_mmap, 4068e42f1ac6SFlorian Westphal #ifdef CONFIG_COMPAT 40693986912fSChristoph Hellwig .compat_ioctl = inet6_compat_ioctl, 4070e42f1ac6SFlorian Westphal #endif 40715684ab1aSPaolo Abeni .set_rcvlowat = mptcp_set_rcvlowat, 4072e42f1ac6SFlorian Westphal }; 4073e42f1ac6SFlorian Westphal 4074f870fa0bSMat Martineau static struct proto mptcp_v6_prot; 4075f870fa0bSMat Martineau 4076f870fa0bSMat Martineau static struct inet_protosw mptcp_v6_protosw = { 4077f870fa0bSMat Martineau .type = SOCK_STREAM, 4078f870fa0bSMat Martineau .protocol = IPPROTO_MPTCP, 4079f870fa0bSMat Martineau .prot = &mptcp_v6_prot, 40802303f994SPeter Krystad .ops = &mptcp_v6_stream_ops, 4081f870fa0bSMat Martineau .flags = INET_PROTOSW_ICSK, 4082f870fa0bSMat Martineau }; 4083f870fa0bSMat Martineau 4084d39dcecaSPaolo Abeni int __init mptcp_proto_v6_init(void) 4085f870fa0bSMat Martineau { 4086f870fa0bSMat Martineau int err; 4087f870fa0bSMat Martineau 4088f870fa0bSMat Martineau mptcp_v6_prot = mptcp_prot; 4089f870fa0bSMat Martineau strcpy(mptcp_v6_prot.name, "MPTCPv6"); 4090f870fa0bSMat Martineau mptcp_v6_prot.slab = NULL; 4091b0519de8SFlorian Westphal mptcp_v6_prot.obj_size = sizeof(struct mptcp6_sock); 4092f5f80e32SEric Dumazet mptcp_v6_prot.ipv6_pinfo_offset = offsetof(struct mptcp6_sock, np); 4093f870fa0bSMat Martineau 4094f870fa0bSMat Martineau err = proto_register(&mptcp_v6_prot, 1); 4095f870fa0bSMat Martineau if (err) 4096f870fa0bSMat Martineau return err; 4097f870fa0bSMat Martineau 4098f870fa0bSMat Martineau err = inet6_register_protosw(&mptcp_v6_protosw); 4099f870fa0bSMat Martineau if (err) 4100f870fa0bSMat Martineau proto_unregister(&mptcp_v6_prot); 4101f870fa0bSMat Martineau 4102f870fa0bSMat Martineau return err; 4103f870fa0bSMat Martineau } 4104f870fa0bSMat Martineau #endif 4105