1f870fa0bSMat Martineau // SPDX-License-Identifier: GPL-2.0 2f870fa0bSMat Martineau /* Multipath TCP 3f870fa0bSMat Martineau * 4f870fa0bSMat Martineau * Copyright (c) 2017 - 2019, Intel Corporation. 5f870fa0bSMat Martineau */ 6f870fa0bSMat Martineau 7f870fa0bSMat Martineau #define pr_fmt(fmt) "MPTCP: " fmt 8f870fa0bSMat Martineau 9f870fa0bSMat Martineau #include <linux/kernel.h> 10f870fa0bSMat Martineau #include <linux/module.h> 11f870fa0bSMat Martineau #include <linux/netdevice.h> 127a6a6cbcSPaolo Abeni #include <linux/sched/signal.h> 137a6a6cbcSPaolo Abeni #include <linux/atomic.h> 14f870fa0bSMat Martineau #include <net/sock.h> 15f870fa0bSMat Martineau #include <net/inet_common.h> 16f870fa0bSMat Martineau #include <net/inet_hashtables.h> 17f870fa0bSMat Martineau #include <net/protocol.h> 18f870fa0bSMat Martineau #include <net/tcp.h> 193721b9b6SMat Martineau #include <net/tcp_states.h> 20cf7da0d6SPeter Krystad #if IS_ENABLED(CONFIG_MPTCP_IPV6) 21cf7da0d6SPeter Krystad #include <net/transp_v6.h> 22cf7da0d6SPeter Krystad #endif 23f870fa0bSMat Martineau #include <net/mptcp.h> 24e16163b6SPaolo Abeni #include <net/xfrm.h> 25644807e3SFlorian Westphal #include <asm/ioctls.h> 26f870fa0bSMat Martineau #include "protocol.h" 27fc518953SFlorian Westphal #include "mib.h" 28f870fa0bSMat Martineau 29e10a9892SGeliang Tang #define CREATE_TRACE_POINTS 30e10a9892SGeliang Tang #include <trace/events/mptcp.h> 31e10a9892SGeliang Tang 32b0519de8SFlorian Westphal #if IS_ENABLED(CONFIG_MPTCP_IPV6) 33b0519de8SFlorian Westphal struct mptcp6_sock { 34b0519de8SFlorian Westphal struct mptcp_sock msk; 35b0519de8SFlorian Westphal struct ipv6_pinfo np; 36b0519de8SFlorian Westphal }; 37b0519de8SFlorian Westphal #endif 38b0519de8SFlorian Westphal 39b7f653b2SFlorian Westphal enum { 40b7f653b2SFlorian Westphal MPTCP_CMSG_TS = BIT(0), 412c9e7765SFlorian Westphal MPTCP_CMSG_INQ = BIT(1), 42b7f653b2SFlorian Westphal }; 43b7f653b2SFlorian Westphal 4491b6d325SEric Dumazet static struct percpu_counter mptcp_sockets_allocated ____cacheline_aligned_in_smp; 45d027236cSPaolo Abeni 46e16163b6SPaolo Abeni static void __mptcp_destroy_sock(struct sock *sk); 4781c1d029SPaolo Abeni static void mptcp_check_send_data_fin(struct sock *sk); 48e16163b6SPaolo Abeni 49b19bc294SPaolo Abeni DEFINE_PER_CPU(struct mptcp_delegated_action, mptcp_delegated_actions); 50b19bc294SPaolo Abeni static struct net_device mptcp_napi_dev; 51b19bc294SPaolo Abeni 526f8a612aSFlorian Westphal /* Returns end sequence number of the receiver's advertised window */ 536f8a612aSFlorian Westphal static u64 mptcp_wnd_end(const struct mptcp_sock *msk) 546f8a612aSFlorian Westphal { 557439d687SPaolo Abeni return READ_ONCE(msk->wnd_end); 566f8a612aSFlorian Westphal } 576f8a612aSFlorian Westphal 588e2b8a9fSDavide Caratti static const struct proto_ops *mptcp_fallback_tcp_ops(const struct sock *sk) 590b4f33deSFlorian Westphal { 600b4f33deSFlorian Westphal #if IS_ENABLED(CONFIG_MPTCP_IPV6) 618e2b8a9fSDavide Caratti if (sk->sk_prot == &tcpv6_prot) 628e2b8a9fSDavide Caratti return &inet6_stream_ops; 630b4f33deSFlorian Westphal #endif 648e2b8a9fSDavide Caratti WARN_ON_ONCE(sk->sk_prot != &tcp_prot); 658e2b8a9fSDavide Caratti return &inet_stream_ops; 660b4f33deSFlorian Westphal } 670b4f33deSFlorian Westphal 68fa68018dSPaolo Abeni static int __mptcp_socket_create(struct mptcp_sock *msk) 692303f994SPeter Krystad { 702303f994SPeter Krystad struct mptcp_subflow_context *subflow; 712303f994SPeter Krystad struct sock *sk = (struct sock *)msk; 722303f994SPeter Krystad struct socket *ssock; 732303f994SPeter Krystad int err; 742303f994SPeter Krystad 756bc1fe7dSPaolo Abeni err = mptcp_subflow_create_socket(sk, sk->sk_family, &ssock); 762303f994SPeter Krystad if (err) 77fa68018dSPaolo Abeni return err; 782303f994SPeter Krystad 79b8dc6d6cSPaolo Abeni msk->scaling_ratio = tcp_sk(ssock->sk)->scaling_ratio; 801b1b43eeSPaolo Abeni WRITE_ONCE(msk->first, ssock->sk); 812303f994SPeter Krystad subflow = mptcp_subflow_ctx(ssock->sk); 82cec37a6eSPeter Krystad list_add(&subflow->node, &msk->conn_list); 83e16163b6SPaolo Abeni sock_hold(ssock->sk); 842303f994SPeter Krystad subflow->request_mptcp = 1; 856f06b4d4SPaolo Abeni subflow->subflow_id = msk->subflow_id++; 864cf86ae8SPaolo Abeni 874cf86ae8SPaolo Abeni /* This is the first subflow, always with id 0 */ 884cf86ae8SPaolo Abeni subflow->local_id_valid = 1; 89866f26f2SPaolo Abeni mptcp_sock_graft(msk->first, sk->sk_socket); 9039880bd8SPaolo Abeni iput(SOCK_INODE(ssock)); 91e1ff9e82SDavide Caratti 92fa68018dSPaolo Abeni return 0; 932303f994SPeter Krystad } 942303f994SPeter Krystad 95ddb1a072SPaolo Abeni /* If the MPC handshake is not started, returns the first subflow, 96ddb1a072SPaolo Abeni * eventually allocating it. 97ddb1a072SPaolo Abeni */ 983f326a82SPaolo Abeni struct sock *__mptcp_nmpc_sk(struct mptcp_sock *msk) 99ddb1a072SPaolo Abeni { 100ddb1a072SPaolo Abeni struct sock *sk = (struct sock *)msk; 101ddb1a072SPaolo Abeni int ret; 102ddb1a072SPaolo Abeni 103ddb1a072SPaolo Abeni if (!((1 << sk->sk_state) & (TCPF_CLOSE | TCPF_LISTEN))) 104ddb1a072SPaolo Abeni return ERR_PTR(-EINVAL); 105ddb1a072SPaolo Abeni 1063f326a82SPaolo Abeni if (!msk->first) { 107ddb1a072SPaolo Abeni ret = __mptcp_socket_create(msk); 108ddb1a072SPaolo Abeni if (ret) 109ddb1a072SPaolo Abeni return ERR_PTR(ret); 110ddb1a072SPaolo Abeni } 111ddb1a072SPaolo Abeni 1123f326a82SPaolo Abeni return msk->first; 113ddb1a072SPaolo Abeni } 114ddb1a072SPaolo Abeni 115ab174ad8SPaolo Abeni static void mptcp_drop(struct sock *sk, struct sk_buff *skb) 116ab174ad8SPaolo Abeni { 117ab174ad8SPaolo Abeni sk_drops_add(sk, skb); 118ab174ad8SPaolo Abeni __kfree_skb(skb); 119ab174ad8SPaolo Abeni } 120ab174ad8SPaolo Abeni 1219531e4a8SEric Dumazet static void mptcp_rmem_fwd_alloc_add(struct sock *sk, int size) 1229531e4a8SEric Dumazet { 1239531e4a8SEric Dumazet WRITE_ONCE(mptcp_sk(sk)->rmem_fwd_alloc, 1249531e4a8SEric Dumazet mptcp_sk(sk)->rmem_fwd_alloc + size); 1259531e4a8SEric Dumazet } 1269531e4a8SEric Dumazet 1276511882cSPaolo Abeni static void mptcp_rmem_charge(struct sock *sk, int size) 1286511882cSPaolo Abeni { 1299531e4a8SEric Dumazet mptcp_rmem_fwd_alloc_add(sk, -size); 1306511882cSPaolo Abeni } 1316511882cSPaolo Abeni 1328268ed4cSPaolo Abeni static bool mptcp_try_coalesce(struct sock *sk, struct sk_buff *to, 1338268ed4cSPaolo Abeni struct sk_buff *from) 1348268ed4cSPaolo Abeni { 1358268ed4cSPaolo Abeni bool fragstolen; 1368268ed4cSPaolo Abeni int delta; 1378268ed4cSPaolo Abeni 1388268ed4cSPaolo Abeni if (MPTCP_SKB_CB(from)->offset || 1398268ed4cSPaolo Abeni !skb_try_coalesce(to, from, &fragstolen, &delta)) 1408268ed4cSPaolo Abeni return false; 1418268ed4cSPaolo Abeni 14206242e44SPaolo Abeni pr_debug("colesced seq %llx into %llx new len %d new end seq %llx", 14306242e44SPaolo Abeni MPTCP_SKB_CB(from)->map_seq, MPTCP_SKB_CB(to)->map_seq, 14406242e44SPaolo Abeni to->len, MPTCP_SKB_CB(from)->end_seq); 145ab174ad8SPaolo Abeni MPTCP_SKB_CB(to)->end_seq = MPTCP_SKB_CB(from)->end_seq; 1467288ff6eSPaolo Abeni 1477288ff6eSPaolo Abeni /* note the fwd memory can reach a negative value after accounting 1487288ff6eSPaolo Abeni * for the delta, but the later skb free will restore a non 1497288ff6eSPaolo Abeni * negative one 1507288ff6eSPaolo Abeni */ 1518268ed4cSPaolo Abeni atomic_add(delta, &sk->sk_rmem_alloc); 1526511882cSPaolo Abeni mptcp_rmem_charge(sk, delta); 1537288ff6eSPaolo Abeni kfree_skb_partial(from, fragstolen); 1547288ff6eSPaolo Abeni 1558268ed4cSPaolo Abeni return true; 1568268ed4cSPaolo Abeni } 1578268ed4cSPaolo Abeni 158ab174ad8SPaolo Abeni static bool mptcp_ooo_try_coalesce(struct mptcp_sock *msk, struct sk_buff *to, 159ab174ad8SPaolo Abeni struct sk_buff *from) 1606771bfd9SFlorian Westphal { 161ab174ad8SPaolo Abeni if (MPTCP_SKB_CB(from)->map_seq != MPTCP_SKB_CB(to)->end_seq) 162ab174ad8SPaolo Abeni return false; 163ab174ad8SPaolo Abeni 164ab174ad8SPaolo Abeni return mptcp_try_coalesce((struct sock *)msk, to, from); 165ab174ad8SPaolo Abeni } 166ab174ad8SPaolo Abeni 1676511882cSPaolo Abeni static void __mptcp_rmem_reclaim(struct sock *sk, int amount) 1686511882cSPaolo Abeni { 169100fdd1fSEric Dumazet amount >>= PAGE_SHIFT; 1709531e4a8SEric Dumazet mptcp_rmem_charge(sk, amount << PAGE_SHIFT); 1716511882cSPaolo Abeni __sk_mem_reduce_allocated(sk, amount); 1726511882cSPaolo Abeni } 1736511882cSPaolo Abeni 1746511882cSPaolo Abeni static void mptcp_rmem_uncharge(struct sock *sk, int size) 1756511882cSPaolo Abeni { 1766511882cSPaolo Abeni struct mptcp_sock *msk = mptcp_sk(sk); 1776511882cSPaolo Abeni int reclaimable; 1786511882cSPaolo Abeni 1799531e4a8SEric Dumazet mptcp_rmem_fwd_alloc_add(sk, size); 1806511882cSPaolo Abeni reclaimable = msk->rmem_fwd_alloc - sk_unused_reserved_mem(sk); 1816511882cSPaolo Abeni 1826511882cSPaolo Abeni /* see sk_mem_uncharge() for the rationale behind the following schema */ 183d24141feSPaolo Abeni if (unlikely(reclaimable >= PAGE_SIZE)) 184d24141feSPaolo Abeni __mptcp_rmem_reclaim(sk, reclaimable); 1856511882cSPaolo Abeni } 1866511882cSPaolo Abeni 1876511882cSPaolo Abeni static void mptcp_rfree(struct sk_buff *skb) 1886511882cSPaolo Abeni { 1896511882cSPaolo Abeni unsigned int len = skb->truesize; 1906511882cSPaolo Abeni struct sock *sk = skb->sk; 1916511882cSPaolo Abeni 1926511882cSPaolo Abeni atomic_sub(len, &sk->sk_rmem_alloc); 1936511882cSPaolo Abeni mptcp_rmem_uncharge(sk, len); 1946511882cSPaolo Abeni } 1956511882cSPaolo Abeni 19636b122baSDmytro Shytyi void mptcp_set_owner_r(struct sk_buff *skb, struct sock *sk) 1976511882cSPaolo Abeni { 1986511882cSPaolo Abeni skb_orphan(skb); 1996511882cSPaolo Abeni skb->sk = sk; 2006511882cSPaolo Abeni skb->destructor = mptcp_rfree; 2016511882cSPaolo Abeni atomic_add(skb->truesize, &sk->sk_rmem_alloc); 2026511882cSPaolo Abeni mptcp_rmem_charge(sk, skb->truesize); 2036511882cSPaolo Abeni } 2046511882cSPaolo Abeni 205ab174ad8SPaolo Abeni /* "inspired" by tcp_data_queue_ofo(), main differences: 206ab174ad8SPaolo Abeni * - use mptcp seqs 207ab174ad8SPaolo Abeni * - don't cope with sacks 208ab174ad8SPaolo Abeni */ 209ab174ad8SPaolo Abeni static void mptcp_data_queue_ofo(struct mptcp_sock *msk, struct sk_buff *skb) 210ab174ad8SPaolo Abeni { 211ab174ad8SPaolo Abeni struct sock *sk = (struct sock *)msk; 212ab174ad8SPaolo Abeni struct rb_node **p, *parent; 213ab174ad8SPaolo Abeni u64 seq, end_seq, max_seq; 214ab174ad8SPaolo Abeni struct sk_buff *skb1; 215ab174ad8SPaolo Abeni 216ab174ad8SPaolo Abeni seq = MPTCP_SKB_CB(skb)->map_seq; 217ab174ad8SPaolo Abeni end_seq = MPTCP_SKB_CB(skb)->end_seq; 218f3589be0SPaolo Abeni max_seq = atomic64_read(&msk->rcv_wnd_sent); 219ab174ad8SPaolo Abeni 22006242e44SPaolo Abeni pr_debug("msk=%p seq=%llx limit=%llx empty=%d", msk, seq, max_seq, 22106242e44SPaolo Abeni RB_EMPTY_ROOT(&msk->out_of_order_queue)); 222fa3fe2b1SFlorian Westphal if (after64(end_seq, max_seq)) { 223ab174ad8SPaolo Abeni /* out of window */ 224ab174ad8SPaolo Abeni mptcp_drop(sk, skb); 225fa3fe2b1SFlorian Westphal pr_debug("oow by %lld, rcv_wnd_sent %llu\n", 226fa3fe2b1SFlorian Westphal (unsigned long long)end_seq - (unsigned long)max_seq, 227f3589be0SPaolo Abeni (unsigned long long)atomic64_read(&msk->rcv_wnd_sent)); 22806242e44SPaolo Abeni MPTCP_INC_STATS(sock_net(sk), MPTCP_MIB_NODSSWINDOW); 229ab174ad8SPaolo Abeni return; 230ab174ad8SPaolo Abeni } 231ab174ad8SPaolo Abeni 232ab174ad8SPaolo Abeni p = &msk->out_of_order_queue.rb_node; 23306242e44SPaolo Abeni MPTCP_INC_STATS(sock_net(sk), MPTCP_MIB_OFOQUEUE); 234ab174ad8SPaolo Abeni if (RB_EMPTY_ROOT(&msk->out_of_order_queue)) { 235ab174ad8SPaolo Abeni rb_link_node(&skb->rbnode, NULL, p); 236ab174ad8SPaolo Abeni rb_insert_color(&skb->rbnode, &msk->out_of_order_queue); 237ab174ad8SPaolo Abeni msk->ooo_last_skb = skb; 238ab174ad8SPaolo Abeni goto end; 239ab174ad8SPaolo Abeni } 240ab174ad8SPaolo Abeni 241ab174ad8SPaolo Abeni /* with 2 subflows, adding at end of ooo queue is quite likely 242ab174ad8SPaolo Abeni * Use of ooo_last_skb avoids the O(Log(N)) rbtree lookup. 243ab174ad8SPaolo Abeni */ 24406242e44SPaolo Abeni if (mptcp_ooo_try_coalesce(msk, msk->ooo_last_skb, skb)) { 24506242e44SPaolo Abeni MPTCP_INC_STATS(sock_net(sk), MPTCP_MIB_OFOMERGE); 24606242e44SPaolo Abeni MPTCP_INC_STATS(sock_net(sk), MPTCP_MIB_OFOQUEUETAIL); 247ab174ad8SPaolo Abeni return; 24806242e44SPaolo Abeni } 249ab174ad8SPaolo Abeni 250ab174ad8SPaolo Abeni /* Can avoid an rbtree lookup if we are adding skb after ooo_last_skb */ 251ab174ad8SPaolo Abeni if (!before64(seq, MPTCP_SKB_CB(msk->ooo_last_skb)->end_seq)) { 25206242e44SPaolo Abeni MPTCP_INC_STATS(sock_net(sk), MPTCP_MIB_OFOQUEUETAIL); 253ab174ad8SPaolo Abeni parent = &msk->ooo_last_skb->rbnode; 254ab174ad8SPaolo Abeni p = &parent->rb_right; 255ab174ad8SPaolo Abeni goto insert; 256ab174ad8SPaolo Abeni } 257ab174ad8SPaolo Abeni 258ab174ad8SPaolo Abeni /* Find place to insert this segment. Handle overlaps on the way. */ 259ab174ad8SPaolo Abeni parent = NULL; 260ab174ad8SPaolo Abeni while (*p) { 261ab174ad8SPaolo Abeni parent = *p; 262ab174ad8SPaolo Abeni skb1 = rb_to_skb(parent); 263ab174ad8SPaolo Abeni if (before64(seq, MPTCP_SKB_CB(skb1)->map_seq)) { 264ab174ad8SPaolo Abeni p = &parent->rb_left; 265ab174ad8SPaolo Abeni continue; 266ab174ad8SPaolo Abeni } 267ab174ad8SPaolo Abeni if (before64(seq, MPTCP_SKB_CB(skb1)->end_seq)) { 268ab174ad8SPaolo Abeni if (!after64(end_seq, MPTCP_SKB_CB(skb1)->end_seq)) { 269ab174ad8SPaolo Abeni /* All the bits are present. Drop. */ 270ab174ad8SPaolo Abeni mptcp_drop(sk, skb); 27106242e44SPaolo Abeni MPTCP_INC_STATS(sock_net(sk), MPTCP_MIB_DUPDATA); 272ab174ad8SPaolo Abeni return; 273ab174ad8SPaolo Abeni } 274ab174ad8SPaolo Abeni if (after64(seq, MPTCP_SKB_CB(skb1)->map_seq)) { 275ab174ad8SPaolo Abeni /* partial overlap: 276ab174ad8SPaolo Abeni * | skb | 277ab174ad8SPaolo Abeni * | skb1 | 278ab174ad8SPaolo Abeni * continue traversing 279ab174ad8SPaolo Abeni */ 280ab174ad8SPaolo Abeni } else { 281ab174ad8SPaolo Abeni /* skb's seq == skb1's seq and skb covers skb1. 282ab174ad8SPaolo Abeni * Replace skb1 with skb. 283ab174ad8SPaolo Abeni */ 284ab174ad8SPaolo Abeni rb_replace_node(&skb1->rbnode, &skb->rbnode, 285ab174ad8SPaolo Abeni &msk->out_of_order_queue); 286ab174ad8SPaolo Abeni mptcp_drop(sk, skb1); 28706242e44SPaolo Abeni MPTCP_INC_STATS(sock_net(sk), MPTCP_MIB_DUPDATA); 288ab174ad8SPaolo Abeni goto merge_right; 289ab174ad8SPaolo Abeni } 290ab174ad8SPaolo Abeni } else if (mptcp_ooo_try_coalesce(msk, skb1, skb)) { 29106242e44SPaolo Abeni MPTCP_INC_STATS(sock_net(sk), MPTCP_MIB_OFOMERGE); 292ab174ad8SPaolo Abeni return; 293ab174ad8SPaolo Abeni } 294ab174ad8SPaolo Abeni p = &parent->rb_right; 295ab174ad8SPaolo Abeni } 29606242e44SPaolo Abeni 297ab174ad8SPaolo Abeni insert: 298ab174ad8SPaolo Abeni /* Insert segment into RB tree. */ 299ab174ad8SPaolo Abeni rb_link_node(&skb->rbnode, parent, p); 300ab174ad8SPaolo Abeni rb_insert_color(&skb->rbnode, &msk->out_of_order_queue); 301ab174ad8SPaolo Abeni 302ab174ad8SPaolo Abeni merge_right: 303ab174ad8SPaolo Abeni /* Remove other segments covered by skb. */ 304ab174ad8SPaolo Abeni while ((skb1 = skb_rb_next(skb)) != NULL) { 305ab174ad8SPaolo Abeni if (before64(end_seq, MPTCP_SKB_CB(skb1)->end_seq)) 306ab174ad8SPaolo Abeni break; 307ab174ad8SPaolo Abeni rb_erase(&skb1->rbnode, &msk->out_of_order_queue); 308ab174ad8SPaolo Abeni mptcp_drop(sk, skb1); 30906242e44SPaolo Abeni MPTCP_INC_STATS(sock_net(sk), MPTCP_MIB_DUPDATA); 310ab174ad8SPaolo Abeni } 311ab174ad8SPaolo Abeni /* If there is no skb after us, we are the last_skb ! */ 312ab174ad8SPaolo Abeni if (!skb1) 313ab174ad8SPaolo Abeni msk->ooo_last_skb = skb; 314ab174ad8SPaolo Abeni 315ab174ad8SPaolo Abeni end: 316ab174ad8SPaolo Abeni skb_condense(skb); 3176511882cSPaolo Abeni mptcp_set_owner_r(skb, sk); 3186511882cSPaolo Abeni } 3196511882cSPaolo Abeni 3206511882cSPaolo Abeni static bool mptcp_rmem_schedule(struct sock *sk, struct sock *ssk, int size) 3216511882cSPaolo Abeni { 3226511882cSPaolo Abeni struct mptcp_sock *msk = mptcp_sk(sk); 3236511882cSPaolo Abeni int amt, amount; 3246511882cSPaolo Abeni 32569d93daeSPaolo Abeni if (size <= msk->rmem_fwd_alloc) 3266511882cSPaolo Abeni return true; 3276511882cSPaolo Abeni 32869d93daeSPaolo Abeni size -= msk->rmem_fwd_alloc; 3296511882cSPaolo Abeni amt = sk_mem_pages(size); 330100fdd1fSEric Dumazet amount = amt << PAGE_SHIFT; 3314aaa1685SPaolo Abeni if (!__sk_mem_raise_allocated(sk, size, amt, SK_MEM_RECV)) 3326511882cSPaolo Abeni return false; 3336511882cSPaolo Abeni 3349531e4a8SEric Dumazet mptcp_rmem_fwd_alloc_add(sk, amount); 3356511882cSPaolo Abeni return true; 336ab174ad8SPaolo Abeni } 337ab174ad8SPaolo Abeni 338ab174ad8SPaolo Abeni static bool __mptcp_move_skb(struct mptcp_sock *msk, struct sock *ssk, 339ab174ad8SPaolo Abeni struct sk_buff *skb, unsigned int offset, 340ab174ad8SPaolo Abeni size_t copy_len) 341ab174ad8SPaolo Abeni { 342ab174ad8SPaolo Abeni struct mptcp_subflow_context *subflow = mptcp_subflow_ctx(ssk); 3436771bfd9SFlorian Westphal struct sock *sk = (struct sock *)msk; 3444e637c70SFlorian Westphal struct sk_buff *tail; 345b7f653b2SFlorian Westphal bool has_rxtstamp; 3466771bfd9SFlorian Westphal 3476771bfd9SFlorian Westphal __skb_unlink(skb, &ssk->sk_receive_queue); 3484e637c70SFlorian Westphal 3494e637c70SFlorian Westphal skb_ext_reset(skb); 3504e637c70SFlorian Westphal skb_orphan(skb); 3514e637c70SFlorian Westphal 3529c3f94e1SPaolo Abeni /* try to fetch required memory from subflow */ 3536511882cSPaolo Abeni if (!mptcp_rmem_schedule(sk, ssk, skb->truesize)) 3549c3f94e1SPaolo Abeni goto drop; 35572f96132SPaolo Abeni 356b7f653b2SFlorian Westphal has_rxtstamp = TCP_SKB_CB(skb)->has_rxtstamp; 357b7f653b2SFlorian Westphal 358ab174ad8SPaolo Abeni /* the skb map_seq accounts for the skb offset: 359ab174ad8SPaolo Abeni * mptcp_subflow_get_mapped_dsn() is based on the current tp->copied_seq 360ab174ad8SPaolo Abeni * value 361ab174ad8SPaolo Abeni */ 362ab174ad8SPaolo Abeni MPTCP_SKB_CB(skb)->map_seq = mptcp_subflow_get_mapped_dsn(subflow); 363ab174ad8SPaolo Abeni MPTCP_SKB_CB(skb)->end_seq = MPTCP_SKB_CB(skb)->map_seq + copy_len; 364ab174ad8SPaolo Abeni MPTCP_SKB_CB(skb)->offset = offset; 365b7f653b2SFlorian Westphal MPTCP_SKB_CB(skb)->has_rxtstamp = has_rxtstamp; 366ab174ad8SPaolo Abeni 367ab174ad8SPaolo Abeni if (MPTCP_SKB_CB(skb)->map_seq == msk->ack_seq) { 368ab174ad8SPaolo Abeni /* in sequence */ 36938967f42SPaolo Abeni msk->bytes_received += copy_len; 370917944daSMat Martineau WRITE_ONCE(msk->ack_seq, msk->ack_seq + copy_len); 3714e637c70SFlorian Westphal tail = skb_peek_tail(&sk->sk_receive_queue); 3728268ed4cSPaolo Abeni if (tail && mptcp_try_coalesce(sk, tail, skb)) 373ab174ad8SPaolo Abeni return true; 3744e637c70SFlorian Westphal 3756511882cSPaolo Abeni mptcp_set_owner_r(skb, sk); 3766771bfd9SFlorian Westphal __skb_queue_tail(&sk->sk_receive_queue, skb); 377ab174ad8SPaolo Abeni return true; 378ab174ad8SPaolo Abeni } else if (after64(MPTCP_SKB_CB(skb)->map_seq, msk->ack_seq)) { 379ab174ad8SPaolo Abeni mptcp_data_queue_ofo(msk, skb); 380ab174ad8SPaolo Abeni return false; 381ab174ad8SPaolo Abeni } 382ab174ad8SPaolo Abeni 383ab174ad8SPaolo Abeni /* old data, keep it simple and drop the whole pkt, sender 384ab174ad8SPaolo Abeni * will retransmit as needed, if needed. 385ab174ad8SPaolo Abeni */ 38606242e44SPaolo Abeni MPTCP_INC_STATS(sock_net(sk), MPTCP_MIB_DUPDATA); 3879c3f94e1SPaolo Abeni drop: 388ab174ad8SPaolo Abeni mptcp_drop(sk, skb); 389ab174ad8SPaolo Abeni return false; 3906771bfd9SFlorian Westphal } 3916771bfd9SFlorian Westphal 392f6909dc1SPaolo Abeni static void mptcp_stop_rtx_timer(struct sock *sk) 39316a9a9daSMat Martineau { 39416a9a9daSMat Martineau struct inet_connection_sock *icsk = inet_csk(sk); 39516a9a9daSMat Martineau 39616a9a9daSMat Martineau sk_stop_timer(sk, &icsk->icsk_retransmit_timer); 39716a9a9daSMat Martineau mptcp_sk(sk)->timer_ival = 0; 39816a9a9daSMat Martineau } 39916a9a9daSMat Martineau 400e16163b6SPaolo Abeni static void mptcp_close_wake_up(struct sock *sk) 401e16163b6SPaolo Abeni { 402e16163b6SPaolo Abeni if (sock_flag(sk, SOCK_DEAD)) 403e16163b6SPaolo Abeni return; 404e16163b6SPaolo Abeni 405e16163b6SPaolo Abeni sk->sk_state_change(sk); 406e16163b6SPaolo Abeni if (sk->sk_shutdown == SHUTDOWN_MASK || 407e16163b6SPaolo Abeni sk->sk_state == TCP_CLOSE) 408e16163b6SPaolo Abeni sk_wake_async(sk, SOCK_WAKE_WAITD, POLL_HUP); 409e16163b6SPaolo Abeni else 410e16163b6SPaolo Abeni sk_wake_async(sk, SOCK_WAKE_WAITD, POLL_IN); 411e16163b6SPaolo Abeni } 412e16163b6SPaolo Abeni 4136e628cd3SPaolo Abeni static bool mptcp_pending_data_fin_ack(struct sock *sk) 4146e628cd3SPaolo Abeni { 4156e628cd3SPaolo Abeni struct mptcp_sock *msk = mptcp_sk(sk); 4166e628cd3SPaolo Abeni 41781c1d029SPaolo Abeni return ((1 << sk->sk_state) & 4186e628cd3SPaolo Abeni (TCPF_FIN_WAIT1 | TCPF_CLOSING | TCPF_LAST_ACK)) && 4196e628cd3SPaolo Abeni msk->write_seq == READ_ONCE(msk->snd_una); 4206e628cd3SPaolo Abeni } 4216e628cd3SPaolo Abeni 42216a9a9daSMat Martineau static void mptcp_check_data_fin_ack(struct sock *sk) 42316a9a9daSMat Martineau { 42416a9a9daSMat Martineau struct mptcp_sock *msk = mptcp_sk(sk); 42516a9a9daSMat Martineau 42616a9a9daSMat Martineau /* Look for an acknowledged DATA_FIN */ 4276e628cd3SPaolo Abeni if (mptcp_pending_data_fin_ack(sk)) { 42816a9a9daSMat Martineau WRITE_ONCE(msk->snd_data_fin_enable, 0); 42916a9a9daSMat Martineau 43016a9a9daSMat Martineau switch (sk->sk_state) { 43116a9a9daSMat Martineau case TCP_FIN_WAIT1: 432c693a851SGeliang Tang mptcp_set_state(sk, TCP_FIN_WAIT2); 43316a9a9daSMat Martineau break; 43416a9a9daSMat Martineau case TCP_CLOSING: 43516a9a9daSMat Martineau case TCP_LAST_ACK: 436c693a851SGeliang Tang mptcp_set_state(sk, TCP_CLOSE); 43716a9a9daSMat Martineau break; 43816a9a9daSMat Martineau } 43916a9a9daSMat Martineau 440e16163b6SPaolo Abeni mptcp_close_wake_up(sk); 44116a9a9daSMat Martineau } 44216a9a9daSMat Martineau } 44316a9a9daSMat Martineau 4443721b9b6SMat Martineau static bool mptcp_pending_data_fin(struct sock *sk, u64 *seq) 4453721b9b6SMat Martineau { 4463721b9b6SMat Martineau struct mptcp_sock *msk = mptcp_sk(sk); 4473721b9b6SMat Martineau 4483721b9b6SMat Martineau if (READ_ONCE(msk->rcv_data_fin) && 4493721b9b6SMat Martineau ((1 << sk->sk_state) & 4503721b9b6SMat Martineau (TCPF_ESTABLISHED | TCPF_FIN_WAIT1 | TCPF_FIN_WAIT2))) { 4513721b9b6SMat Martineau u64 rcv_data_fin_seq = READ_ONCE(msk->rcv_data_fin_seq); 4523721b9b6SMat Martineau 4533721b9b6SMat Martineau if (msk->ack_seq == rcv_data_fin_seq) { 4543721b9b6SMat Martineau if (seq) 4553721b9b6SMat Martineau *seq = rcv_data_fin_seq; 4563721b9b6SMat Martineau 4573721b9b6SMat Martineau return true; 4583721b9b6SMat Martineau } 4593721b9b6SMat Martineau } 4603721b9b6SMat Martineau 4613721b9b6SMat Martineau return false; 4623721b9b6SMat Martineau } 4633721b9b6SMat Martineau 464403a40f2SEric Dumazet static void mptcp_set_datafin_timeout(struct sock *sk) 4656477dd39SMat Martineau { 4666477dd39SMat Martineau struct inet_connection_sock *icsk = inet_csk(sk); 467877d11f0SMat Martineau u32 retransmits; 4686477dd39SMat Martineau 469877d11f0SMat Martineau retransmits = min_t(u32, icsk->icsk_retransmits, 470877d11f0SMat Martineau ilog2(TCP_RTO_MAX / TCP_RTO_MIN)); 471877d11f0SMat Martineau 472877d11f0SMat Martineau mptcp_sk(sk)->timer_ival = TCP_RTO_MIN << retransmits; 4736477dd39SMat Martineau } 4746477dd39SMat Martineau 47533d41c9cSPaolo Abeni static void __mptcp_set_timeout(struct sock *sk, long tout) 4763721b9b6SMat Martineau { 4773721b9b6SMat Martineau mptcp_sk(sk)->timer_ival = tout > 0 ? tout : TCP_RTO_MIN; 4783721b9b6SMat Martineau } 4793721b9b6SMat Martineau 48033d41c9cSPaolo Abeni static long mptcp_timeout_from_subflow(const struct mptcp_subflow_context *subflow) 48133d41c9cSPaolo Abeni { 48233d41c9cSPaolo Abeni const struct sock *ssk = mptcp_subflow_tcp_sock(subflow); 48333d41c9cSPaolo Abeni 48471b7dec2SPaolo Abeni return inet_csk(ssk)->icsk_pending && !subflow->stale_count ? 48571b7dec2SPaolo Abeni inet_csk(ssk)->icsk_timeout - jiffies : 0; 48633d41c9cSPaolo Abeni } 48733d41c9cSPaolo Abeni 48833d41c9cSPaolo Abeni static void mptcp_set_timeout(struct sock *sk) 48933d41c9cSPaolo Abeni { 49033d41c9cSPaolo Abeni struct mptcp_subflow_context *subflow; 49133d41c9cSPaolo Abeni long tout = 0; 49233d41c9cSPaolo Abeni 49333d41c9cSPaolo Abeni mptcp_for_each_subflow(mptcp_sk(sk), subflow) 49433d41c9cSPaolo Abeni tout = max(tout, mptcp_timeout_from_subflow(subflow)); 49533d41c9cSPaolo Abeni __mptcp_set_timeout(sk, tout); 49633d41c9cSPaolo Abeni } 49733d41c9cSPaolo Abeni 49876a13b31SGeliang Tang static inline bool tcp_can_send_ack(const struct sock *ssk) 499fd897679SPaolo Abeni { 500fd897679SPaolo Abeni return !((1 << inet_sk_state_load(ssk)) & 50120bc80b6SPaolo Abeni (TCPF_SYN_SENT | TCPF_SYN_RECV | TCPF_TIME_WAIT | TCPF_CLOSE | TCPF_LISTEN)); 502fd897679SPaolo Abeni } 503fd897679SPaolo Abeni 504a6574302SMat Martineau void __mptcp_subflow_send_ack(struct sock *ssk) 505a6574302SMat Martineau { 506a6574302SMat Martineau if (tcp_can_send_ack(ssk)) 507a6574302SMat Martineau tcp_send_ack(ssk); 508a6574302SMat Martineau } 509a6574302SMat Martineau 510f5360e9bSPaolo Abeni static void mptcp_subflow_send_ack(struct sock *ssk) 5117ed90803SPaolo Abeni { 51275e908c3SPaolo Abeni bool slow; 5137ed90803SPaolo Abeni 51475e908c3SPaolo Abeni slow = lock_sock_fast(ssk); 515a6574302SMat Martineau __mptcp_subflow_send_ack(ssk); 51675e908c3SPaolo Abeni unlock_sock_fast(ssk, slow); 517fd897679SPaolo Abeni } 518340fa666SMat Martineau 519340fa666SMat Martineau static void mptcp_send_ack(struct mptcp_sock *msk) 520340fa666SMat Martineau { 521340fa666SMat Martineau struct mptcp_subflow_context *subflow; 522340fa666SMat Martineau 523340fa666SMat Martineau mptcp_for_each_subflow(msk, subflow) 524340fa666SMat Martineau mptcp_subflow_send_ack(mptcp_subflow_tcp_sock(subflow)); 525ea4ca586SPaolo Abeni } 526ea4ca586SPaolo Abeni 527fde56eeaSPaolo Abeni static void mptcp_subflow_cleanup_rbuf(struct sock *ssk) 528fd897679SPaolo Abeni { 52975e908c3SPaolo Abeni bool slow; 530fd897679SPaolo Abeni 53175e908c3SPaolo Abeni slow = lock_sock_fast(ssk); 532fde56eeaSPaolo Abeni if (tcp_can_send_ack(ssk)) 533fd897679SPaolo Abeni tcp_cleanup_rbuf(ssk, 1); 53475e908c3SPaolo Abeni unlock_sock_fast(ssk, slow); 535fde56eeaSPaolo Abeni } 536fde56eeaSPaolo Abeni 537fde56eeaSPaolo Abeni static bool mptcp_subflow_could_cleanup(const struct sock *ssk, bool rx_empty) 538fde56eeaSPaolo Abeni { 539fde56eeaSPaolo Abeni const struct inet_connection_sock *icsk = inet_csk(ssk); 540c4512c63SMatthieu Baerts u8 ack_pending = READ_ONCE(icsk->icsk_ack.pending); 541fde56eeaSPaolo Abeni const struct tcp_sock *tp = tcp_sk(ssk); 542fde56eeaSPaolo Abeni 543fde56eeaSPaolo Abeni return (ack_pending & ICSK_ACK_SCHED) && 544fde56eeaSPaolo Abeni ((READ_ONCE(tp->rcv_nxt) - READ_ONCE(tp->rcv_wup) > 545fde56eeaSPaolo Abeni READ_ONCE(icsk->icsk_ack.rcv_mss)) || 546fde56eeaSPaolo Abeni (rx_empty && ack_pending & 547fde56eeaSPaolo Abeni (ICSK_ACK_PUSHED2 | ICSK_ACK_PUSHED))); 548ea4ca586SPaolo Abeni } 549fd897679SPaolo Abeni 550fd897679SPaolo Abeni static void mptcp_cleanup_rbuf(struct mptcp_sock *msk) 551fd897679SPaolo Abeni { 552e3859603SPaolo Abeni int old_space = READ_ONCE(msk->old_wspace); 553fd897679SPaolo Abeni struct mptcp_subflow_context *subflow; 554e3859603SPaolo Abeni struct sock *sk = (struct sock *)msk; 555fde56eeaSPaolo Abeni int space = __mptcp_space(sk); 556fde56eeaSPaolo Abeni bool cleanup, rx_empty; 557e3859603SPaolo Abeni 558fde56eeaSPaolo Abeni cleanup = (space > 0) && (space >= (old_space << 1)); 559ce599c51SPaolo Abeni rx_empty = !__mptcp_rmem(sk); 560fd897679SPaolo Abeni 561fd897679SPaolo Abeni mptcp_for_each_subflow(msk, subflow) { 562fd897679SPaolo Abeni struct sock *ssk = mptcp_subflow_tcp_sock(subflow); 563fd897679SPaolo Abeni 564fde56eeaSPaolo Abeni if (cleanup || mptcp_subflow_could_cleanup(ssk, rx_empty)) 565fde56eeaSPaolo Abeni mptcp_subflow_cleanup_rbuf(ssk); 5667ed90803SPaolo Abeni } 5677ed90803SPaolo Abeni } 5687ed90803SPaolo Abeni 5697ed90803SPaolo Abeni static bool mptcp_check_data_fin(struct sock *sk) 5703721b9b6SMat Martineau { 5713721b9b6SMat Martineau struct mptcp_sock *msk = mptcp_sk(sk); 5723721b9b6SMat Martineau u64 rcv_data_fin_seq; 5737ed90803SPaolo Abeni bool ret = false; 5743721b9b6SMat Martineau 5753721b9b6SMat Martineau /* Need to ack a DATA_FIN received from a peer while this side 5763721b9b6SMat Martineau * of the connection is in ESTABLISHED, FIN_WAIT1, or FIN_WAIT2. 5773721b9b6SMat Martineau * msk->rcv_data_fin was set when parsing the incoming options 5783721b9b6SMat Martineau * at the subflow level and the msk lock was not held, so this 5793721b9b6SMat Martineau * is the first opportunity to act on the DATA_FIN and change 5803721b9b6SMat Martineau * the msk state. 5813721b9b6SMat Martineau * 5823721b9b6SMat Martineau * If we are caught up to the sequence number of the incoming 5833721b9b6SMat Martineau * DATA_FIN, send the DATA_ACK now and do state transition. If 5843721b9b6SMat Martineau * not caught up, do nothing and let the recv code send DATA_ACK 5853721b9b6SMat Martineau * when catching up. 5863721b9b6SMat Martineau */ 5873721b9b6SMat Martineau 5883721b9b6SMat Martineau if (mptcp_pending_data_fin(sk, &rcv_data_fin_seq)) { 589917944daSMat Martineau WRITE_ONCE(msk->ack_seq, msk->ack_seq + 1); 5903721b9b6SMat Martineau WRITE_ONCE(msk->rcv_data_fin, 0); 5913721b9b6SMat Martineau 5926b9831bfSPaolo Abeni WRITE_ONCE(sk->sk_shutdown, sk->sk_shutdown | RCV_SHUTDOWN); 59316a9a9daSMat Martineau smp_mb__before_atomic(); /* SHUTDOWN must be visible first */ 5943721b9b6SMat Martineau 5953721b9b6SMat Martineau switch (sk->sk_state) { 5963721b9b6SMat Martineau case TCP_ESTABLISHED: 597c693a851SGeliang Tang mptcp_set_state(sk, TCP_CLOSE_WAIT); 5983721b9b6SMat Martineau break; 5993721b9b6SMat Martineau case TCP_FIN_WAIT1: 600c693a851SGeliang Tang mptcp_set_state(sk, TCP_CLOSING); 6013721b9b6SMat Martineau break; 6023721b9b6SMat Martineau case TCP_FIN_WAIT2: 603c693a851SGeliang Tang mptcp_set_state(sk, TCP_CLOSE); 6043721b9b6SMat Martineau break; 6053721b9b6SMat Martineau default: 6063721b9b6SMat Martineau /* Other states not expected */ 6073721b9b6SMat Martineau WARN_ON_ONCE(1); 6083721b9b6SMat Martineau break; 6093721b9b6SMat Martineau } 6103721b9b6SMat Martineau 6117ed90803SPaolo Abeni ret = true; 61281c1d029SPaolo Abeni if (!__mptcp_check_fallback(msk)) 613fd897679SPaolo Abeni mptcp_send_ack(msk); 614e16163b6SPaolo Abeni mptcp_close_wake_up(sk); 6153721b9b6SMat Martineau } 6167ed90803SPaolo Abeni return ret; 6173721b9b6SMat Martineau } 6183721b9b6SMat Martineau 6196771bfd9SFlorian Westphal static bool __mptcp_move_skbs_from_subflow(struct mptcp_sock *msk, 6206771bfd9SFlorian Westphal struct sock *ssk, 6216771bfd9SFlorian Westphal unsigned int *bytes) 6226771bfd9SFlorian Westphal { 6236771bfd9SFlorian Westphal struct mptcp_subflow_context *subflow = mptcp_subflow_ctx(ssk); 624600911ffSFlorian Westphal struct sock *sk = (struct sock *)msk; 6256771bfd9SFlorian Westphal unsigned int moved = 0; 6266771bfd9SFlorian Westphal bool more_data_avail; 6276771bfd9SFlorian Westphal struct tcp_sock *tp; 6286771bfd9SFlorian Westphal bool done = false; 62913c7ba0cSFlorian Westphal int sk_rbuf; 63013c7ba0cSFlorian Westphal 63113c7ba0cSFlorian Westphal sk_rbuf = READ_ONCE(sk->sk_rcvbuf); 63213c7ba0cSFlorian Westphal 63313c7ba0cSFlorian Westphal if (!(sk->sk_userlocks & SOCK_RCVBUF_LOCK)) { 63413c7ba0cSFlorian Westphal int ssk_rbuf = READ_ONCE(ssk->sk_rcvbuf); 63513c7ba0cSFlorian Westphal 63613c7ba0cSFlorian Westphal if (unlikely(ssk_rbuf > sk_rbuf)) { 63713c7ba0cSFlorian Westphal WRITE_ONCE(sk->sk_rcvbuf, ssk_rbuf); 63813c7ba0cSFlorian Westphal sk_rbuf = ssk_rbuf; 63913c7ba0cSFlorian Westphal } 64013c7ba0cSFlorian Westphal } 641600911ffSFlorian Westphal 642ab174ad8SPaolo Abeni pr_debug("msk=%p ssk=%p", msk, ssk); 6436771bfd9SFlorian Westphal tp = tcp_sk(ssk); 6446771bfd9SFlorian Westphal do { 6456771bfd9SFlorian Westphal u32 map_remaining, offset; 6466771bfd9SFlorian Westphal u32 seq = tp->copied_seq; 6476771bfd9SFlorian Westphal struct sk_buff *skb; 6486771bfd9SFlorian Westphal bool fin; 6496771bfd9SFlorian Westphal 6506771bfd9SFlorian Westphal /* try to move as much data as available */ 6516771bfd9SFlorian Westphal map_remaining = subflow->map_data_len - 6526771bfd9SFlorian Westphal mptcp_subflow_get_map_offset(subflow); 6536771bfd9SFlorian Westphal 6546771bfd9SFlorian Westphal skb = skb_peek(&ssk->sk_receive_queue); 655d9fb8c50SPaolo Abeni if (!skb) { 656d89e3ed7SPaolo Abeni /* With racing move_skbs_to_msk() and __mptcp_move_skbs(), 657d89e3ed7SPaolo Abeni * a different CPU can have already processed the pending 658d89e3ed7SPaolo Abeni * data, stop here or we can enter an infinite loop 659d9fb8c50SPaolo Abeni */ 660d9fb8c50SPaolo Abeni if (!moved) 661d9fb8c50SPaolo Abeni done = true; 6626771bfd9SFlorian Westphal break; 663d9fb8c50SPaolo Abeni } 6646771bfd9SFlorian Westphal 665e1ff9e82SDavide Caratti if (__mptcp_check_fallback(msk)) { 666d89e3ed7SPaolo Abeni /* Under fallback skbs have no MPTCP extension and TCP could 667d89e3ed7SPaolo Abeni * collapse them between the dummy map creation and the 668d89e3ed7SPaolo Abeni * current dequeue. Be sure to adjust the map size. 669e1ff9e82SDavide Caratti */ 670e1ff9e82SDavide Caratti map_remaining = skb->len; 671e1ff9e82SDavide Caratti subflow->map_data_len = skb->len; 672e1ff9e82SDavide Caratti } 673e1ff9e82SDavide Caratti 6746771bfd9SFlorian Westphal offset = seq - TCP_SKB_CB(skb)->seq; 6756771bfd9SFlorian Westphal fin = TCP_SKB_CB(skb)->tcp_flags & TCPHDR_FIN; 6766771bfd9SFlorian Westphal if (fin) { 6776771bfd9SFlorian Westphal done = true; 6786771bfd9SFlorian Westphal seq++; 6796771bfd9SFlorian Westphal } 6806771bfd9SFlorian Westphal 6816771bfd9SFlorian Westphal if (offset < skb->len) { 6826771bfd9SFlorian Westphal size_t len = skb->len - offset; 6836771bfd9SFlorian Westphal 6846771bfd9SFlorian Westphal if (tp->urg_data) 6856771bfd9SFlorian Westphal done = true; 6866771bfd9SFlorian Westphal 687ab174ad8SPaolo Abeni if (__mptcp_move_skb(msk, ssk, skb, offset, len)) 6886771bfd9SFlorian Westphal moved += len; 689ab174ad8SPaolo Abeni seq += len; 6906771bfd9SFlorian Westphal 6916771bfd9SFlorian Westphal if (WARN_ON_ONCE(map_remaining < len)) 6926771bfd9SFlorian Westphal break; 6936771bfd9SFlorian Westphal } else { 6946771bfd9SFlorian Westphal WARN_ON_ONCE(!fin); 6956771bfd9SFlorian Westphal sk_eat_skb(ssk, skb); 6966771bfd9SFlorian Westphal done = true; 6976771bfd9SFlorian Westphal } 6986771bfd9SFlorian Westphal 6996771bfd9SFlorian Westphal WRITE_ONCE(tp->copied_seq, seq); 7006771bfd9SFlorian Westphal more_data_avail = mptcp_subflow_data_available(ssk); 701600911ffSFlorian Westphal 70213c7ba0cSFlorian Westphal if (atomic_read(&sk->sk_rmem_alloc) > sk_rbuf) { 703600911ffSFlorian Westphal done = true; 704600911ffSFlorian Westphal break; 705600911ffSFlorian Westphal } 7066771bfd9SFlorian Westphal } while (more_data_avail); 7076771bfd9SFlorian Westphal 7086719331cSPaolo Abeni *bytes += moved; 7096771bfd9SFlorian Westphal return done; 7106771bfd9SFlorian Westphal } 7116771bfd9SFlorian Westphal 71287952603SPaolo Abeni static bool __mptcp_ofo_queue(struct mptcp_sock *msk) 713ab174ad8SPaolo Abeni { 714ab174ad8SPaolo Abeni struct sock *sk = (struct sock *)msk; 715ab174ad8SPaolo Abeni struct sk_buff *skb, *tail; 716ab174ad8SPaolo Abeni bool moved = false; 717ab174ad8SPaolo Abeni struct rb_node *p; 718ab174ad8SPaolo Abeni u64 end_seq; 719ab174ad8SPaolo Abeni 720ab174ad8SPaolo Abeni p = rb_first(&msk->out_of_order_queue); 72106242e44SPaolo Abeni pr_debug("msk=%p empty=%d", msk, RB_EMPTY_ROOT(&msk->out_of_order_queue)); 722ab174ad8SPaolo Abeni while (p) { 723ab174ad8SPaolo Abeni skb = rb_to_skb(p); 724ab174ad8SPaolo Abeni if (after64(MPTCP_SKB_CB(skb)->map_seq, msk->ack_seq)) 725ab174ad8SPaolo Abeni break; 726ab174ad8SPaolo Abeni 727ab174ad8SPaolo Abeni p = rb_next(p); 728ab174ad8SPaolo Abeni rb_erase(&skb->rbnode, &msk->out_of_order_queue); 729ab174ad8SPaolo Abeni 730ab174ad8SPaolo Abeni if (unlikely(!after64(MPTCP_SKB_CB(skb)->end_seq, 731ab174ad8SPaolo Abeni msk->ack_seq))) { 732ab174ad8SPaolo Abeni mptcp_drop(sk, skb); 73306242e44SPaolo Abeni MPTCP_INC_STATS(sock_net(sk), MPTCP_MIB_DUPDATA); 734ab174ad8SPaolo Abeni continue; 735ab174ad8SPaolo Abeni } 736ab174ad8SPaolo Abeni 737ab174ad8SPaolo Abeni end_seq = MPTCP_SKB_CB(skb)->end_seq; 738ab174ad8SPaolo Abeni tail = skb_peek_tail(&sk->sk_receive_queue); 739ab174ad8SPaolo Abeni if (!tail || !mptcp_ooo_try_coalesce(msk, tail, skb)) { 740ab174ad8SPaolo Abeni int delta = msk->ack_seq - MPTCP_SKB_CB(skb)->map_seq; 741ab174ad8SPaolo Abeni 742ab174ad8SPaolo Abeni /* skip overlapping data, if any */ 74306242e44SPaolo Abeni pr_debug("uncoalesced seq=%llx ack seq=%llx delta=%d", 74406242e44SPaolo Abeni MPTCP_SKB_CB(skb)->map_seq, msk->ack_seq, 74506242e44SPaolo Abeni delta); 746ab174ad8SPaolo Abeni MPTCP_SKB_CB(skb)->offset += delta; 7472c9e7765SFlorian Westphal MPTCP_SKB_CB(skb)->map_seq += delta; 748ab174ad8SPaolo Abeni __skb_queue_tail(&sk->sk_receive_queue, skb); 749ab174ad8SPaolo Abeni } 75038967f42SPaolo Abeni msk->bytes_received += end_seq - msk->ack_seq; 751ab174ad8SPaolo Abeni msk->ack_seq = end_seq; 752ab174ad8SPaolo Abeni moved = true; 753ab174ad8SPaolo Abeni } 754ab174ad8SPaolo Abeni return moved; 755ab174ad8SPaolo Abeni } 756ab174ad8SPaolo Abeni 7579f1a9881SPaolo Abeni static bool __mptcp_subflow_error_report(struct sock *sk, struct sock *ssk) 758d5fbeff1SPaolo Abeni { 759d5fbeff1SPaolo Abeni int err = sock_error(ssk); 760d5fbeff1SPaolo Abeni int ssk_state; 761d5fbeff1SPaolo Abeni 762d5fbeff1SPaolo Abeni if (!err) 7639f1a9881SPaolo Abeni return false; 764d5fbeff1SPaolo Abeni 765d5fbeff1SPaolo Abeni /* only propagate errors on fallen-back sockets or 766d5fbeff1SPaolo Abeni * on MPC connect 767d5fbeff1SPaolo Abeni */ 7689f1a9881SPaolo Abeni if (sk->sk_state != TCP_SYN_SENT && !__mptcp_check_fallback(mptcp_sk(sk))) 7699f1a9881SPaolo Abeni return false; 770d5fbeff1SPaolo Abeni 771d5fbeff1SPaolo Abeni /* We need to propagate only transition to CLOSE state. 772d5fbeff1SPaolo Abeni * Orphaned socket will see such state change via 773d5fbeff1SPaolo Abeni * subflow_sched_work_if_closed() and that path will properly 774d5fbeff1SPaolo Abeni * destroy the msk as needed. 775d5fbeff1SPaolo Abeni */ 776d5fbeff1SPaolo Abeni ssk_state = inet_sk_state_load(ssk); 777d5fbeff1SPaolo Abeni if (ssk_state == TCP_CLOSE && !sock_flag(sk, SOCK_DEAD)) 778c693a851SGeliang Tang mptcp_set_state(sk, ssk_state); 779d5fbeff1SPaolo Abeni WRITE_ONCE(sk->sk_err, -err); 780d5fbeff1SPaolo Abeni 781d5fbeff1SPaolo Abeni /* This barrier is coupled with smp_rmb() in mptcp_poll() */ 782d5fbeff1SPaolo Abeni smp_wmb(); 783d5fbeff1SPaolo Abeni sk_error_report(sk); 7849f1a9881SPaolo Abeni return true; 785d5fbeff1SPaolo Abeni } 7869f1a9881SPaolo Abeni 7879f1a9881SPaolo Abeni void __mptcp_error_report(struct sock *sk) 7889f1a9881SPaolo Abeni { 7899f1a9881SPaolo Abeni struct mptcp_subflow_context *subflow; 7909f1a9881SPaolo Abeni struct mptcp_sock *msk = mptcp_sk(sk); 7919f1a9881SPaolo Abeni 7929f1a9881SPaolo Abeni mptcp_for_each_subflow(msk, subflow) 7939f1a9881SPaolo Abeni if (__mptcp_subflow_error_report(sk, mptcp_subflow_tcp_sock(subflow))) 7949f1a9881SPaolo Abeni break; 795d5fbeff1SPaolo Abeni } 796d5fbeff1SPaolo Abeni 7972e52213cSFlorian Westphal /* In most cases we will be able to lock the mptcp socket. If its already 7982e52213cSFlorian Westphal * owned, we need to defer to the work queue to avoid ABBA deadlock. 7992e52213cSFlorian Westphal */ 80099d1055cSPaolo Abeni static bool move_skbs_to_msk(struct mptcp_sock *msk, struct sock *ssk) 8012e52213cSFlorian Westphal { 8022e52213cSFlorian Westphal struct sock *sk = (struct sock *)msk; 8032e52213cSFlorian Westphal unsigned int moved = 0; 8042e52213cSFlorian Westphal 8052e52213cSFlorian Westphal __mptcp_move_skbs_from_subflow(msk, ssk, &moved); 80687952603SPaolo Abeni __mptcp_ofo_queue(msk); 807499ada50SPaolo Abeni if (unlikely(ssk->sk_err)) { 808499ada50SPaolo Abeni if (!sock_owned_by_user(sk)) 809499ada50SPaolo Abeni __mptcp_error_report(sk); 810499ada50SPaolo Abeni else 811e9d09bacSPaolo Abeni __set_bit(MPTCP_ERROR_REPORT, &msk->cb_flags); 812499ada50SPaolo Abeni } 813ab174ad8SPaolo Abeni 814ab174ad8SPaolo Abeni /* If the moves have caught up with the DATA_FIN sequence number 815ab174ad8SPaolo Abeni * it's time to ack the DATA_FIN and change socket state, but 816ab174ad8SPaolo Abeni * this is not a good place to change state. Let the workqueue 817ab174ad8SPaolo Abeni * do it. 818ab174ad8SPaolo Abeni */ 819ba8f48f7SPaolo Abeni if (mptcp_pending_data_fin(sk, NULL)) 820ba8f48f7SPaolo Abeni mptcp_schedule_work(sk); 82199d1055cSPaolo Abeni return moved > 0; 8222e52213cSFlorian Westphal } 8232e52213cSFlorian Westphal 8242e52213cSFlorian Westphal void mptcp_data_ready(struct sock *sk, struct sock *ssk) 825101f6f85SFlorian Westphal { 8266719331cSPaolo Abeni struct mptcp_subflow_context *subflow = mptcp_subflow_ctx(ssk); 827101f6f85SFlorian Westphal struct mptcp_sock *msk = mptcp_sk(sk); 82813c7ba0cSFlorian Westphal int sk_rbuf, ssk_rbuf; 829101f6f85SFlorian Westphal 830d7b1bfd0SPaolo Abeni /* The peer can send data while we are shutting down this 831d7b1bfd0SPaolo Abeni * subflow at msk destruction time, but we must avoid enqueuing 832d7b1bfd0SPaolo Abeni * more data to the msk receive queue 833d7b1bfd0SPaolo Abeni */ 834d7b1bfd0SPaolo Abeni if (unlikely(subflow->disposable)) 835d7b1bfd0SPaolo Abeni return; 836d7b1bfd0SPaolo Abeni 83713c7ba0cSFlorian Westphal ssk_rbuf = READ_ONCE(ssk->sk_rcvbuf); 83813c7ba0cSFlorian Westphal sk_rbuf = READ_ONCE(sk->sk_rcvbuf); 83913c7ba0cSFlorian Westphal if (unlikely(ssk_rbuf > sk_rbuf)) 84013c7ba0cSFlorian Westphal sk_rbuf = ssk_rbuf; 84113c7ba0cSFlorian Westphal 84299d1055cSPaolo Abeni /* over limit? can't append more skbs to msk, Also, no need to wake-up*/ 843ce599c51SPaolo Abeni if (__mptcp_rmem(sk) > sk_rbuf) { 844ce599c51SPaolo Abeni MPTCP_INC_STATS(sock_net(sk), MPTCP_MIB_RCVPRUNED); 84599d1055cSPaolo Abeni return; 846ce599c51SPaolo Abeni } 8472e52213cSFlorian Westphal 84899d1055cSPaolo Abeni /* Wake-up the reader only for in-sequence data */ 84999d1055cSPaolo Abeni mptcp_data_lock(sk); 8505684ab1aSPaolo Abeni if (move_skbs_to_msk(msk, ssk) && mptcp_epollin_ready(sk)) 851101f6f85SFlorian Westphal sk->sk_data_ready(sk); 85299d1055cSPaolo Abeni mptcp_data_unlock(sk); 85399d1055cSPaolo Abeni } 854101f6f85SFlorian Westphal 85555b47ca7SPaolo Abeni static void mptcp_subflow_joined(struct mptcp_sock *msk, struct sock *ssk) 85655b47ca7SPaolo Abeni { 85755b47ca7SPaolo Abeni mptcp_subflow_ctx(ssk)->map_seq = READ_ONCE(msk->ack_seq); 85855b47ca7SPaolo Abeni WRITE_ONCE(msk->allow_infinite_fallback, false); 85955b47ca7SPaolo Abeni mptcp_event(MPTCP_EVENT_SUB_ESTABLISHED, msk, ssk, GFP_ATOMIC); 86055b47ca7SPaolo Abeni } 86155b47ca7SPaolo Abeni 8623e501490SPaolo Abeni static bool __mptcp_finish_join(struct mptcp_sock *msk, struct sock *ssk) 863ec3edaa7SPeter Krystad { 8643e501490SPaolo Abeni struct sock *sk = (struct sock *)msk; 8655cf92bbaSPaolo Abeni 8663e501490SPaolo Abeni if (sk->sk_state != TCP_ESTABLISHED) 86778962489SFlorian Westphal return false; 868ec3edaa7SPeter Krystad 8693e501490SPaolo Abeni /* attach to msk socket only after we are sure we will deal with it 8703e501490SPaolo Abeni * at close time 8713e501490SPaolo Abeni */ 8723e501490SPaolo Abeni if (sk->sk_socket && !ssk->sk_socket) 8733e501490SPaolo Abeni mptcp_sock_graft(ssk, sk->sk_socket); 87478962489SFlorian Westphal 8756f06b4d4SPaolo Abeni mptcp_subflow_ctx(ssk)->subflow_id = msk->subflow_id++; 8763e501490SPaolo Abeni mptcp_sockopt_sync_locked(msk, ssk); 87755b47ca7SPaolo Abeni mptcp_subflow_joined(msk, ssk); 87827e5ccc2SPaolo Abeni mptcp_stop_tout_timer(sk); 8798005184fSPaolo Abeni __mptcp_propagate_sndbuf(sk, ssk); 8803e501490SPaolo Abeni return true; 88178962489SFlorian Westphal } 88278962489SFlorian Westphal 88356a666c4SPaolo Abeni static void __mptcp_flush_join_list(struct sock *sk, struct list_head *join_list) 88478962489SFlorian Westphal { 8853e501490SPaolo Abeni struct mptcp_subflow_context *tmp, *subflow; 8863e501490SPaolo Abeni struct mptcp_sock *msk = mptcp_sk(sk); 88778962489SFlorian Westphal 88856a666c4SPaolo Abeni list_for_each_entry_safe(subflow, tmp, join_list, node) { 8893e501490SPaolo Abeni struct sock *ssk = mptcp_subflow_tcp_sock(subflow); 8903e501490SPaolo Abeni bool slow = lock_sock_fast(ssk); 8913e501490SPaolo Abeni 8923e501490SPaolo Abeni list_move_tail(&subflow->node, &msk->conn_list); 8933e501490SPaolo Abeni if (!__mptcp_finish_join(msk, ssk)) 8943e501490SPaolo Abeni mptcp_subflow_reset(ssk); 8953e501490SPaolo Abeni unlock_sock_fast(ssk, slow); 89678962489SFlorian Westphal } 897ec3edaa7SPeter Krystad } 898ec3edaa7SPeter Krystad 899f6909dc1SPaolo Abeni static bool mptcp_rtx_timer_pending(struct sock *sk) 900b51f9b80SPaolo Abeni { 901b51f9b80SPaolo Abeni return timer_pending(&inet_csk(sk)->icsk_retransmit_timer); 902b51f9b80SPaolo Abeni } 903b51f9b80SPaolo Abeni 904f6909dc1SPaolo Abeni static void mptcp_reset_rtx_timer(struct sock *sk) 905b51f9b80SPaolo Abeni { 906b51f9b80SPaolo Abeni struct inet_connection_sock *icsk = inet_csk(sk); 907b51f9b80SPaolo Abeni unsigned long tout; 908b51f9b80SPaolo Abeni 909e16163b6SPaolo Abeni /* prevent rescheduling on close */ 910e16163b6SPaolo Abeni if (unlikely(inet_sk_state_load(sk) == TCP_CLOSE)) 911e16163b6SPaolo Abeni return; 912e16163b6SPaolo Abeni 91333d41c9cSPaolo Abeni tout = mptcp_sk(sk)->timer_ival; 914b51f9b80SPaolo Abeni sk_reset_timer(sk, &icsk->icsk_retransmit_timer, jiffies + tout); 915b51f9b80SPaolo Abeni } 916b51f9b80SPaolo Abeni 917ba8f48f7SPaolo Abeni bool mptcp_schedule_work(struct sock *sk) 918ba8f48f7SPaolo Abeni { 919ba8f48f7SPaolo Abeni if (inet_sk_state_load(sk) != TCP_CLOSE && 920ba8f48f7SPaolo Abeni schedule_work(&mptcp_sk(sk)->work)) { 921ba8f48f7SPaolo Abeni /* each subflow already holds a reference to the sk, and the 922ba8f48f7SPaolo Abeni * workqueue is invoked by a subflow, so sk can't go away here. 923ba8f48f7SPaolo Abeni */ 924ba8f48f7SPaolo Abeni sock_hold(sk); 925ba8f48f7SPaolo Abeni return true; 926ba8f48f7SPaolo Abeni } 927ba8f48f7SPaolo Abeni return false; 928ba8f48f7SPaolo Abeni } 929ba8f48f7SPaolo Abeni 9307a6a6cbcSPaolo Abeni static struct sock *mptcp_subflow_recv_lookup(const struct mptcp_sock *msk) 9317a6a6cbcSPaolo Abeni { 9327a6a6cbcSPaolo Abeni struct mptcp_subflow_context *subflow; 9337a6a6cbcSPaolo Abeni 934109cdeb8SGeliang Tang msk_owned_by_me(msk); 9357a6a6cbcSPaolo Abeni 9367a6a6cbcSPaolo Abeni mptcp_for_each_subflow(msk, subflow) { 93799d1055cSPaolo Abeni if (READ_ONCE(subflow->data_avail)) 9387a6a6cbcSPaolo Abeni return mptcp_subflow_tcp_sock(subflow); 9397a6a6cbcSPaolo Abeni } 9407a6a6cbcSPaolo Abeni 9417a6a6cbcSPaolo Abeni return NULL; 9427a6a6cbcSPaolo Abeni } 9437a6a6cbcSPaolo Abeni 9443f8e0aaeSPaolo Abeni static bool mptcp_skb_can_collapse_to(u64 write_seq, 94557040755SPaolo Abeni const struct sk_buff *skb, 94657040755SPaolo Abeni const struct mptcp_ext *mpext) 9476d0060f6SMat Martineau { 94857040755SPaolo Abeni if (!tcp_skb_can_collapse_to(skb)) 94957040755SPaolo Abeni return false; 95057040755SPaolo Abeni 9515a369ca6SPaolo Abeni /* can collapse only if MPTCP level sequence is in order and this 9525a369ca6SPaolo Abeni * mapping has not been xmitted yet 9535a369ca6SPaolo Abeni */ 9545a369ca6SPaolo Abeni return mpext && mpext->data_seq + mpext->data_len == write_seq && 9555a369ca6SPaolo Abeni !mpext->frozen; 95657040755SPaolo Abeni } 95757040755SPaolo Abeni 95829249eacSPaolo Abeni /* we can append data to the given data frag if: 95929249eacSPaolo Abeni * - there is space available in the backing page_frag 96029249eacSPaolo Abeni * - the data frag tail matches the current page_frag free offset 96129249eacSPaolo Abeni * - the data frag end sequence number matches the current write seq 96229249eacSPaolo Abeni */ 96318b683bfSPaolo Abeni static bool mptcp_frag_can_collapse_to(const struct mptcp_sock *msk, 96418b683bfSPaolo Abeni const struct page_frag *pfrag, 96518b683bfSPaolo Abeni const struct mptcp_data_frag *df) 96618b683bfSPaolo Abeni { 96718b683bfSPaolo Abeni return df && pfrag->page == df->page && 968d9ca1de8SPaolo Abeni pfrag->size - pfrag->offset > 0 && 96929249eacSPaolo Abeni pfrag->offset == (df->offset + df->data_len) && 97018b683bfSPaolo Abeni df->data_seq + df->data_len == msk->write_seq; 97118b683bfSPaolo Abeni } 97218b683bfSPaolo Abeni 973d027236cSPaolo Abeni static void dfrag_uncharge(struct sock *sk, int len) 97418b683bfSPaolo Abeni { 975d027236cSPaolo Abeni sk_mem_uncharge(sk, len); 9767948f6ccSFlorian Westphal sk_wmem_queued_add(sk, -len); 977d027236cSPaolo Abeni } 978d027236cSPaolo Abeni 979d027236cSPaolo Abeni static void dfrag_clear(struct sock *sk, struct mptcp_data_frag *dfrag) 980d027236cSPaolo Abeni { 981d027236cSPaolo Abeni int len = dfrag->data_len + dfrag->overhead; 982d027236cSPaolo Abeni 98318b683bfSPaolo Abeni list_del(&dfrag->list); 984d027236cSPaolo Abeni dfrag_uncharge(sk, len); 98518b683bfSPaolo Abeni put_page(dfrag->page); 98618b683bfSPaolo Abeni } 98718b683bfSPaolo Abeni 9886e628cd3SPaolo Abeni static void __mptcp_clean_una(struct sock *sk) 98918b683bfSPaolo Abeni { 99018b683bfSPaolo Abeni struct mptcp_sock *msk = mptcp_sk(sk); 99118b683bfSPaolo Abeni struct mptcp_data_frag *dtmp, *dfrag; 992e1ff9e82SDavide Caratti u64 snd_una; 993e1ff9e82SDavide Caratti 9947439d687SPaolo Abeni snd_una = msk->snd_una; 99518b683bfSPaolo Abeni list_for_each_entry_safe(dfrag, dtmp, &msk->rtx_queue, list) { 99618b683bfSPaolo Abeni if (after64(dfrag->data_seq + dfrag->data_len, snd_una)) 99718b683bfSPaolo Abeni break; 99818b683bfSPaolo Abeni 9991e1d9d6fSPaolo Abeni if (unlikely(dfrag == msk->first_pending)) { 10001e1d9d6fSPaolo Abeni /* in recovery mode can see ack after the current snd head */ 10011e1d9d6fSPaolo Abeni if (WARN_ON_ONCE(!msk->recovery)) 1002d9ca1de8SPaolo Abeni break; 10031e1d9d6fSPaolo Abeni 10041e1d9d6fSPaolo Abeni WRITE_ONCE(msk->first_pending, mptcp_send_next(sk)); 10051e1d9d6fSPaolo Abeni } 10061e1d9d6fSPaolo Abeni 1007d027236cSPaolo Abeni dfrag_clear(sk, dfrag); 1008d027236cSPaolo Abeni } 1009d027236cSPaolo Abeni 10107948f6ccSFlorian Westphal dfrag = mptcp_rtx_head(sk); 10117948f6ccSFlorian Westphal if (dfrag && after64(snd_una, dfrag->data_seq)) { 101253eb4c38SPaolo Abeni u64 delta = snd_una - dfrag->data_seq; 101353eb4c38SPaolo Abeni 10141e1d9d6fSPaolo Abeni /* prevent wrap around in recovery mode */ 10151e1d9d6fSPaolo Abeni if (unlikely(delta > dfrag->already_sent)) { 10161e1d9d6fSPaolo Abeni if (WARN_ON_ONCE(!msk->recovery)) 101753eb4c38SPaolo Abeni goto out; 10181e1d9d6fSPaolo Abeni if (WARN_ON_ONCE(delta > dfrag->data_len)) 10191e1d9d6fSPaolo Abeni goto out; 10201e1d9d6fSPaolo Abeni dfrag->already_sent += delta - dfrag->already_sent; 10211e1d9d6fSPaolo Abeni } 10227948f6ccSFlorian Westphal 10237948f6ccSFlorian Westphal dfrag->data_seq += delta; 102453eb4c38SPaolo Abeni dfrag->offset += delta; 10257948f6ccSFlorian Westphal dfrag->data_len -= delta; 1026d9ca1de8SPaolo Abeni dfrag->already_sent -= delta; 10277948f6ccSFlorian Westphal 10287948f6ccSFlorian Westphal dfrag_uncharge(sk, delta); 10297948f6ccSFlorian Westphal } 10307948f6ccSFlorian Westphal 10311e1d9d6fSPaolo Abeni /* all retransmitted data acked, recovery completed */ 10321e1d9d6fSPaolo Abeni if (unlikely(msk->recovery) && after64(msk->snd_una, msk->recovery_snd_nxt)) 10331e1d9d6fSPaolo Abeni msk->recovery = false; 10341e1d9d6fSPaolo Abeni 103553eb4c38SPaolo Abeni out: 10363241a9c0SFlorian Westphal if (snd_una == READ_ONCE(msk->snd_nxt) && 10373241a9c0SFlorian Westphal snd_una == READ_ONCE(msk->write_seq)) { 1038f6909dc1SPaolo Abeni if (mptcp_rtx_timer_pending(sk) && !mptcp_data_fin_enabled(msk)) 1039f6909dc1SPaolo Abeni mptcp_stop_rtx_timer(sk); 10406e628cd3SPaolo Abeni } else { 1041f6909dc1SPaolo Abeni mptcp_reset_rtx_timer(sk); 10426e628cd3SPaolo Abeni } 10436e628cd3SPaolo Abeni } 10446e628cd3SPaolo Abeni 1045417789dfSPaolo Abeni static void __mptcp_clean_una_wakeup(struct sock *sk) 1046417789dfSPaolo Abeni { 1047765ff425SPaolo Abeni lockdep_assert_held_once(&sk->sk_lock.slock); 1048765ff425SPaolo Abeni 1049417789dfSPaolo Abeni __mptcp_clean_una(sk); 1050417789dfSPaolo Abeni mptcp_write_space(sk); 1051417789dfSPaolo Abeni } 1052417789dfSPaolo Abeni 1053b5941f06SPaolo Abeni static void mptcp_clean_una_wakeup(struct sock *sk) 1054b5941f06SPaolo Abeni { 1055b5941f06SPaolo Abeni mptcp_data_lock(sk); 1056b5941f06SPaolo Abeni __mptcp_clean_una_wakeup(sk); 1057b5941f06SPaolo Abeni mptcp_data_unlock(sk); 1058b5941f06SPaolo Abeni } 1059b5941f06SPaolo Abeni 1060724cfd2eSPaolo Abeni static void mptcp_enter_memory_pressure(struct sock *sk) 106118b683bfSPaolo Abeni { 1062d9ca1de8SPaolo Abeni struct mptcp_subflow_context *subflow; 1063d9ca1de8SPaolo Abeni struct mptcp_sock *msk = mptcp_sk(sk); 1064d9ca1de8SPaolo Abeni bool first = true; 1065d9ca1de8SPaolo Abeni 1066d9ca1de8SPaolo Abeni mptcp_for_each_subflow(msk, subflow) { 1067d9ca1de8SPaolo Abeni struct sock *ssk = mptcp_subflow_tcp_sock(subflow); 1068d9ca1de8SPaolo Abeni 1069d9ca1de8SPaolo Abeni if (first) 1070d9ca1de8SPaolo Abeni tcp_enter_memory_pressure(ssk); 1071d9ca1de8SPaolo Abeni sk_stream_moderate_sndbuf(ssk); 10728005184fSPaolo Abeni 1073d9ca1de8SPaolo Abeni first = false; 1074d9ca1de8SPaolo Abeni } 10758005184fSPaolo Abeni __mptcp_sync_sndbuf(sk); 1076724cfd2eSPaolo Abeni } 1077724cfd2eSPaolo Abeni 1078724cfd2eSPaolo Abeni /* ensure we get enough memory for the frag hdr, beyond some minimal amount of 1079724cfd2eSPaolo Abeni * data 1080724cfd2eSPaolo Abeni */ 1081724cfd2eSPaolo Abeni static bool mptcp_page_frag_refill(struct sock *sk, struct page_frag *pfrag) 1082724cfd2eSPaolo Abeni { 1083724cfd2eSPaolo Abeni if (likely(skb_page_frag_refill(32U + sizeof(struct mptcp_data_frag), 1084724cfd2eSPaolo Abeni pfrag, sk->sk_allocation))) 1085724cfd2eSPaolo Abeni return true; 1086724cfd2eSPaolo Abeni 1087724cfd2eSPaolo Abeni mptcp_enter_memory_pressure(sk); 108818b683bfSPaolo Abeni return false; 108918b683bfSPaolo Abeni } 109018b683bfSPaolo Abeni 109118b683bfSPaolo Abeni static struct mptcp_data_frag * 109218b683bfSPaolo Abeni mptcp_carve_data_frag(const struct mptcp_sock *msk, struct page_frag *pfrag, 109318b683bfSPaolo Abeni int orig_offset) 109418b683bfSPaolo Abeni { 109518b683bfSPaolo Abeni int offset = ALIGN(orig_offset, sizeof(long)); 109618b683bfSPaolo Abeni struct mptcp_data_frag *dfrag; 109718b683bfSPaolo Abeni 109818b683bfSPaolo Abeni dfrag = (struct mptcp_data_frag *)(page_to_virt(pfrag->page) + offset); 109918b683bfSPaolo Abeni dfrag->data_len = 0; 110018b683bfSPaolo Abeni dfrag->data_seq = msk->write_seq; 110118b683bfSPaolo Abeni dfrag->overhead = offset - orig_offset + sizeof(struct mptcp_data_frag); 110218b683bfSPaolo Abeni dfrag->offset = offset + sizeof(struct mptcp_data_frag); 1103d9ca1de8SPaolo Abeni dfrag->already_sent = 0; 110418b683bfSPaolo Abeni dfrag->page = pfrag->page; 110518b683bfSPaolo Abeni 110618b683bfSPaolo Abeni return dfrag; 110718b683bfSPaolo Abeni } 110818b683bfSPaolo Abeni 1109caf971dfSPaolo Abeni struct mptcp_sendmsg_info { 1110caf971dfSPaolo Abeni int mss_now; 1111caf971dfSPaolo Abeni int size_goal; 1112d9ca1de8SPaolo Abeni u16 limit; 1113d9ca1de8SPaolo Abeni u16 sent; 1114d9ca1de8SPaolo Abeni unsigned int flags; 11151094c6feSPaolo Abeni bool data_lock_held; 1116caf971dfSPaolo Abeni }; 1117caf971dfSPaolo Abeni 1118b713d006SPaolo Abeni static int mptcp_check_allowed_size(const struct mptcp_sock *msk, struct sock *ssk, 1119b713d006SPaolo Abeni u64 data_seq, int avail_size) 11206f8a612aSFlorian Westphal { 11216f8a612aSFlorian Westphal u64 window_end = mptcp_wnd_end(msk); 1122b713d006SPaolo Abeni u64 mptcp_snd_wnd; 11236f8a612aSFlorian Westphal 11246f8a612aSFlorian Westphal if (__mptcp_check_fallback(msk)) 11256f8a612aSFlorian Westphal return avail_size; 11266f8a612aSFlorian Westphal 1127b713d006SPaolo Abeni mptcp_snd_wnd = window_end - data_seq; 1128b713d006SPaolo Abeni avail_size = min_t(unsigned int, mptcp_snd_wnd, avail_size); 11296f8a612aSFlorian Westphal 113092be2f52SPaolo Abeni if (unlikely(tcp_sk(ssk)->snd_wnd < mptcp_snd_wnd)) { 1131b713d006SPaolo Abeni tcp_sk(ssk)->snd_wnd = min_t(u64, U32_MAX, mptcp_snd_wnd); 113292be2f52SPaolo Abeni MPTCP_INC_STATS(sock_net(ssk), MPTCP_MIB_SNDWNDSHARED); 113392be2f52SPaolo Abeni } 11346f8a612aSFlorian Westphal 11356f8a612aSFlorian Westphal return avail_size; 11366f8a612aSFlorian Westphal } 11376f8a612aSFlorian Westphal 1138724cfd2eSPaolo Abeni static bool __mptcp_add_ext(struct sk_buff *skb, gfp_t gfp) 1139724cfd2eSPaolo Abeni { 1140724cfd2eSPaolo Abeni struct skb_ext *mpext = __skb_ext_alloc(gfp); 1141724cfd2eSPaolo Abeni 1142724cfd2eSPaolo Abeni if (!mpext) 1143724cfd2eSPaolo Abeni return false; 1144724cfd2eSPaolo Abeni __skb_ext_set(skb, SKB_EXT_MPTCP, mpext); 1145724cfd2eSPaolo Abeni return true; 1146724cfd2eSPaolo Abeni } 1147724cfd2eSPaolo Abeni 11486e628cd3SPaolo Abeni static struct sk_buff *__mptcp_do_alloc_tx_skb(struct sock *sk, gfp_t gfp) 1149724cfd2eSPaolo Abeni { 1150724cfd2eSPaolo Abeni struct sk_buff *skb; 1151724cfd2eSPaolo Abeni 11526e628cd3SPaolo Abeni skb = alloc_skb_fclone(MAX_TCP_HEADER, gfp); 1153724cfd2eSPaolo Abeni if (likely(skb)) { 11546e628cd3SPaolo Abeni if (likely(__mptcp_add_ext(skb, gfp))) { 1155724cfd2eSPaolo Abeni skb_reserve(skb, MAX_TCP_HEADER); 1156a52fe46eSEric Dumazet skb->ip_summed = CHECKSUM_PARTIAL; 1157f70cad10SPaolo Abeni INIT_LIST_HEAD(&skb->tcp_tsorted_anchor); 1158724cfd2eSPaolo Abeni return skb; 1159724cfd2eSPaolo Abeni } 1160724cfd2eSPaolo Abeni __kfree_skb(skb); 1161724cfd2eSPaolo Abeni } else { 1162724cfd2eSPaolo Abeni mptcp_enter_memory_pressure(sk); 1163724cfd2eSPaolo Abeni } 1164724cfd2eSPaolo Abeni return NULL; 1165724cfd2eSPaolo Abeni } 1166724cfd2eSPaolo Abeni 1167f70cad10SPaolo Abeni static struct sk_buff *__mptcp_alloc_tx_skb(struct sock *sk, struct sock *ssk, gfp_t gfp) 1168724cfd2eSPaolo Abeni { 1169724cfd2eSPaolo Abeni struct sk_buff *skb; 1170724cfd2eSPaolo Abeni 11716e628cd3SPaolo Abeni skb = __mptcp_do_alloc_tx_skb(sk, gfp); 1172724cfd2eSPaolo Abeni if (!skb) 1173f70cad10SPaolo Abeni return NULL; 1174724cfd2eSPaolo Abeni 1175724cfd2eSPaolo Abeni if (likely(sk_wmem_schedule(ssk, skb->truesize))) { 1176f70cad10SPaolo Abeni tcp_skb_entail(ssk, skb); 1177f70cad10SPaolo Abeni return skb; 1178724cfd2eSPaolo Abeni } 11793ef3905aSYonglong Li tcp_skb_tsorted_anchor_cleanup(skb); 1180724cfd2eSPaolo Abeni kfree_skb(skb); 1181f70cad10SPaolo Abeni return NULL; 1182724cfd2eSPaolo Abeni } 1183724cfd2eSPaolo Abeni 1184f70cad10SPaolo Abeni static struct sk_buff *mptcp_alloc_tx_skb(struct sock *sk, struct sock *ssk, bool data_lock_held) 1185724cfd2eSPaolo Abeni { 11861094c6feSPaolo Abeni gfp_t gfp = data_lock_held ? GFP_ATOMIC : sk->sk_allocation; 1187724cfd2eSPaolo Abeni 11881094c6feSPaolo Abeni return __mptcp_alloc_tx_skb(sk, ssk, gfp); 1189724cfd2eSPaolo Abeni } 1190724cfd2eSPaolo Abeni 1191d0cc2987SGeliang Tang /* note: this always recompute the csum on the whole skb, even 1192d0cc2987SGeliang Tang * if we just appended a single frag. More status info needed 1193d0cc2987SGeliang Tang */ 1194d0cc2987SGeliang Tang static void mptcp_update_data_checksum(struct sk_buff *skb, int added) 1195d0cc2987SGeliang Tang { 1196d0cc2987SGeliang Tang struct mptcp_ext *mpext = mptcp_get_ext(skb); 1197d0cc2987SGeliang Tang __wsum csum = ~csum_unfold(mpext->csum); 1198d0cc2987SGeliang Tang int offset = skb->len - added; 1199d0cc2987SGeliang Tang 1200d0cc2987SGeliang Tang mpext->csum = csum_fold(csum_block_add(csum, skb_checksum(skb, offset, added, 0), offset)); 1201d0cc2987SGeliang Tang } 1202d0cc2987SGeliang Tang 12031e39e5a3SGeliang Tang static void mptcp_update_infinite_map(struct mptcp_sock *msk, 12041e39e5a3SGeliang Tang struct sock *ssk, 12051e39e5a3SGeliang Tang struct mptcp_ext *mpext) 12061e39e5a3SGeliang Tang { 12071e39e5a3SGeliang Tang if (!mpext) 12081e39e5a3SGeliang Tang return; 12091e39e5a3SGeliang Tang 12101e39e5a3SGeliang Tang mpext->infinite_map = 1; 12111e39e5a3SGeliang Tang mpext->data_len = 0; 12121e39e5a3SGeliang Tang 1213104125b8SGeliang Tang MPTCP_INC_STATS(sock_net(ssk), MPTCP_MIB_INFINITEMAPTX); 12141e39e5a3SGeliang Tang mptcp_subflow_ctx(ssk)->send_infinite_map = 0; 12151e39e5a3SGeliang Tang pr_fallback(msk); 1216d51991e2SPaolo Abeni mptcp_do_fallback(ssk); 12171e39e5a3SGeliang Tang } 12181e39e5a3SGeliang Tang 12199fce92f0SPaolo Abeni #define MPTCP_MAX_GSO_SIZE (GSO_LEGACY_MAX_SIZE - (MAX_TCP_HEADER + 1)) 12209fce92f0SPaolo Abeni 122157040755SPaolo Abeni static int mptcp_sendmsg_frag(struct sock *sk, struct sock *ssk, 1222d9ca1de8SPaolo Abeni struct mptcp_data_frag *dfrag, 1223caf971dfSPaolo Abeni struct mptcp_sendmsg_info *info) 122457040755SPaolo Abeni { 1225d9ca1de8SPaolo Abeni u64 data_seq = dfrag->data_seq + info->sent; 1226f70cad10SPaolo Abeni int offset = dfrag->offset + info->sent; 12276d0060f6SMat Martineau struct mptcp_sock *msk = mptcp_sk(sk); 12286f8a612aSFlorian Westphal bool zero_window_probe = false; 12296d0060f6SMat Martineau struct mptcp_ext *mpext = NULL; 1230f70cad10SPaolo Abeni bool can_coalesce = false; 1231f70cad10SPaolo Abeni bool reuse_skb = true; 1232f70cad10SPaolo Abeni struct sk_buff *skb; 1233f70cad10SPaolo Abeni size_t copy; 1234f70cad10SPaolo Abeni int i; 12356d0060f6SMat Martineau 1236e4b61351SGeliang Tang pr_debug("msk=%p ssk=%p sending dfrag at seq=%llu len=%u already sent=%u", 1237d9ca1de8SPaolo Abeni msk, ssk, dfrag->data_seq, dfrag->data_len, info->sent); 12386d0060f6SMat Martineau 1239f70cad10SPaolo Abeni if (WARN_ON_ONCE(info->sent > info->limit || 1240f70cad10SPaolo Abeni info->limit > dfrag->data_len)) 1241f70cad10SPaolo Abeni return 0; 1242f70cad10SPaolo Abeni 1243c886d702SPaolo Abeni if (unlikely(!__tcp_can_send(ssk))) 1244c886d702SPaolo Abeni return -EAGAIN; 1245c886d702SPaolo Abeni 1246d9ca1de8SPaolo Abeni /* compute send limit */ 12479fce92f0SPaolo Abeni if (unlikely(ssk->sk_gso_max_size > MPTCP_MAX_GSO_SIZE)) 12489fce92f0SPaolo Abeni ssk->sk_gso_max_size = MPTCP_MAX_GSO_SIZE; 1249d9ca1de8SPaolo Abeni info->mss_now = tcp_send_mss(ssk, &info->size_goal, info->flags); 1250f70cad10SPaolo Abeni copy = info->size_goal; 1251f70cad10SPaolo Abeni 125257040755SPaolo Abeni skb = tcp_write_queue_tail(ssk); 1253f70cad10SPaolo Abeni if (skb && copy > skb->len) { 125457040755SPaolo Abeni /* Limit the write to the size available in the 125557040755SPaolo Abeni * current skb, if any, so that we create at most a new skb. 125657040755SPaolo Abeni * Explicitly tells TCP internals to avoid collapsing on later 125757040755SPaolo Abeni * queue management operation, to avoid breaking the ext <-> 125857040755SPaolo Abeni * SSN association set here 125957040755SPaolo Abeni */ 1260a16c054bSGeliang Tang mpext = mptcp_get_ext(skb); 12611094c6feSPaolo Abeni if (!mptcp_skb_can_collapse_to(data_seq, skb, mpext)) { 126257040755SPaolo Abeni TCP_SKB_CB(skb)->eor = 1; 12631094c6feSPaolo Abeni goto alloc_skb; 12641094c6feSPaolo Abeni } 12651094c6feSPaolo Abeni 1266f70cad10SPaolo Abeni i = skb_shinfo(skb)->nr_frags; 1267f70cad10SPaolo Abeni can_coalesce = skb_can_coalesce(skb, i, dfrag->page, offset); 1268657b991aSKuniyuki Iwashima if (!can_coalesce && i >= READ_ONCE(sysctl_max_skb_frags)) { 1269f70cad10SPaolo Abeni tcp_mark_push(tcp_sk(ssk), skb); 1270f70cad10SPaolo Abeni goto alloc_skb; 127115e6ca97SPaolo Abeni } 127218b683bfSPaolo Abeni 1273f70cad10SPaolo Abeni copy -= skb->len; 1274f70cad10SPaolo Abeni } else { 12751094c6feSPaolo Abeni alloc_skb: 1276f70cad10SPaolo Abeni skb = mptcp_alloc_tx_skb(sk, ssk, info->data_lock_held); 1277f70cad10SPaolo Abeni if (!skb) 1278f70cad10SPaolo Abeni return -ENOMEM; 1279f70cad10SPaolo Abeni 1280f70cad10SPaolo Abeni i = skb_shinfo(skb)->nr_frags; 1281f70cad10SPaolo Abeni reuse_skb = false; 1282a16c054bSGeliang Tang mpext = mptcp_get_ext(skb); 1283f70cad10SPaolo Abeni } 12841094c6feSPaolo Abeni 12856f8a612aSFlorian Westphal /* Zero window and all data acked? Probe. */ 1286b713d006SPaolo Abeni copy = mptcp_check_allowed_size(msk, ssk, data_seq, copy); 1287f70cad10SPaolo Abeni if (copy == 0) { 12887439d687SPaolo Abeni u64 snd_una = READ_ONCE(msk->snd_una); 12897439d687SPaolo Abeni 129072377ab2SPaolo Abeni if (snd_una != msk->snd_nxt || tcp_write_queue_tail(ssk)) { 129127728ba8SEric Dumazet tcp_remove_empty_skb(ssk); 12926f8a612aSFlorian Westphal return 0; 12936f8a612aSFlorian Westphal } 12946f8a612aSFlorian Westphal 1295f70cad10SPaolo Abeni zero_window_probe = true; 1296f70cad10SPaolo Abeni data_seq = snd_una - 1; 1297f70cad10SPaolo Abeni copy = 1; 1298f70cad10SPaolo Abeni } 1299f70cad10SPaolo Abeni 1300f70cad10SPaolo Abeni copy = min_t(size_t, copy, info->limit - info->sent); 1301f70cad10SPaolo Abeni if (!sk_wmem_schedule(ssk, copy)) { 130227728ba8SEric Dumazet tcp_remove_empty_skb(ssk); 1303e2223995SPaolo Abeni return -ENOMEM; 130435759383SFlorian Westphal } 130518b683bfSPaolo Abeni 1306f70cad10SPaolo Abeni if (can_coalesce) { 1307f70cad10SPaolo Abeni skb_frag_size_add(&skb_shinfo(skb)->frags[i - 1], copy); 1308f70cad10SPaolo Abeni } else { 1309f70cad10SPaolo Abeni get_page(dfrag->page); 1310f70cad10SPaolo Abeni skb_fill_page_desc(skb, i, dfrag->page, offset, copy); 131157040755SPaolo Abeni } 131257040755SPaolo Abeni 1313f70cad10SPaolo Abeni skb->len += copy; 1314f70cad10SPaolo Abeni skb->data_len += copy; 1315f70cad10SPaolo Abeni skb->truesize += copy; 1316f70cad10SPaolo Abeni sk_wmem_queued_add(ssk, copy); 1317f70cad10SPaolo Abeni sk_mem_charge(ssk, copy); 1318f70cad10SPaolo Abeni WRITE_ONCE(tcp_sk(ssk)->write_seq, tcp_sk(ssk)->write_seq + copy); 1319f70cad10SPaolo Abeni TCP_SKB_CB(skb)->end_seq += copy; 1320f70cad10SPaolo Abeni tcp_skb_pcount_set(skb, 0); 1321f70cad10SPaolo Abeni 1322f70cad10SPaolo Abeni /* on skb reuse we just need to update the DSS len */ 1323f70cad10SPaolo Abeni if (reuse_skb) { 1324f70cad10SPaolo Abeni TCP_SKB_CB(skb)->tcp_flags &= ~TCPHDR_PSH; 1325f70cad10SPaolo Abeni mpext->data_len += copy; 1326f70cad10SPaolo Abeni goto out; 1327724cfd2eSPaolo Abeni } 13286d0060f6SMat Martineau 13296d0060f6SMat Martineau memset(mpext, 0, sizeof(*mpext)); 1330d9ca1de8SPaolo Abeni mpext->data_seq = data_seq; 13316d0060f6SMat Martineau mpext->subflow_seq = mptcp_subflow_ctx(ssk)->rel_write_seq; 1332f70cad10SPaolo Abeni mpext->data_len = copy; 13336d0060f6SMat Martineau mpext->use_map = 1; 13346d0060f6SMat Martineau mpext->dsn64 = 1; 13356d0060f6SMat Martineau 13366d0060f6SMat Martineau pr_debug("data_seq=%llu subflow_seq=%u data_len=%u dsn64=%d", 13376d0060f6SMat Martineau mpext->data_seq, mpext->subflow_seq, mpext->data_len, 13386d0060f6SMat Martineau mpext->dsn64); 13396d0060f6SMat Martineau 13406f8a612aSFlorian Westphal if (zero_window_probe) { 1341f70cad10SPaolo Abeni mptcp_subflow_ctx(ssk)->rel_write_seq += copy; 13426f8a612aSFlorian Westphal mpext->frozen = 1; 1343d0cc2987SGeliang Tang if (READ_ONCE(msk->csum_enabled)) 1344f70cad10SPaolo Abeni mptcp_update_data_checksum(skb, copy); 13456f8a612aSFlorian Westphal tcp_push_pending_frames(ssk); 1346d0cc2987SGeliang Tang return 0; 13476f8a612aSFlorian Westphal } 134857040755SPaolo Abeni out: 1349d0cc2987SGeliang Tang if (READ_ONCE(msk->csum_enabled)) 1350f70cad10SPaolo Abeni mptcp_update_data_checksum(skb, copy); 13511e39e5a3SGeliang Tang if (mptcp_subflow_ctx(ssk)->send_infinite_map) 13521e39e5a3SGeliang Tang mptcp_update_infinite_map(msk, ssk, mpext); 13530eb4e7eeSGeliang Tang trace_mptcp_sendmsg_frag(mpext); 1354f70cad10SPaolo Abeni mptcp_subflow_ctx(ssk)->rel_write_seq += copy; 1355f70cad10SPaolo Abeni return copy; 13566d0060f6SMat Martineau } 13576d0060f6SMat Martineau 1358d5f49190SPaolo Abeni #define MPTCP_SEND_BURST_SIZE ((1 << 16) - \ 1359d5f49190SPaolo Abeni sizeof(struct tcphdr) - \ 1360d5f49190SPaolo Abeni MAX_TCP_OPTION_SPACE - \ 1361d5f49190SPaolo Abeni sizeof(struct ipv6hdr) - \ 1362d5f49190SPaolo Abeni sizeof(struct frag_hdr)) 1363d5f49190SPaolo Abeni 1364d5f49190SPaolo Abeni struct subflow_send_info { 1365d5f49190SPaolo Abeni struct sock *ssk; 13663ce0852cSPaolo Abeni u64 linger_time; 1367d5f49190SPaolo Abeni }; 1368d5f49190SPaolo Abeni 1369ff5a0b42SPaolo Abeni void mptcp_subflow_set_active(struct mptcp_subflow_context *subflow) 1370ff5a0b42SPaolo Abeni { 1371ff5a0b42SPaolo Abeni if (!subflow->stale) 1372ff5a0b42SPaolo Abeni return; 1373ff5a0b42SPaolo Abeni 1374ff5a0b42SPaolo Abeni subflow->stale = 0; 1375fc1b4e3bSPaolo Abeni MPTCP_INC_STATS(sock_net(mptcp_subflow_tcp_sock(subflow)), MPTCP_MIB_SUBFLOWRECOVER); 1376ff5a0b42SPaolo Abeni } 1377ff5a0b42SPaolo Abeni 1378ff5a0b42SPaolo Abeni bool mptcp_subflow_active(struct mptcp_subflow_context *subflow) 1379ff5a0b42SPaolo Abeni { 1380ff5a0b42SPaolo Abeni if (unlikely(subflow->stale)) { 1381ff5a0b42SPaolo Abeni u32 rcv_tstamp = READ_ONCE(tcp_sk(mptcp_subflow_tcp_sock(subflow))->rcv_tstamp); 1382ff5a0b42SPaolo Abeni 1383ff5a0b42SPaolo Abeni if (subflow->stale_rcv_tstamp == rcv_tstamp) 1384ff5a0b42SPaolo Abeni return false; 1385ff5a0b42SPaolo Abeni 1386ff5a0b42SPaolo Abeni mptcp_subflow_set_active(subflow); 1387ff5a0b42SPaolo Abeni } 1388ff5a0b42SPaolo Abeni return __mptcp_subflow_active(subflow); 1389ff5a0b42SPaolo Abeni } 1390ff5a0b42SPaolo Abeni 13913ce0852cSPaolo Abeni #define SSK_MODE_ACTIVE 0 13923ce0852cSPaolo Abeni #define SSK_MODE_BACKUP 1 13933ce0852cSPaolo Abeni #define SSK_MODE_MAX 2 13943ce0852cSPaolo Abeni 139533d41c9cSPaolo Abeni /* implement the mptcp packet scheduler; 139633d41c9cSPaolo Abeni * returns the subflow that will transmit the next DSS 139733d41c9cSPaolo Abeni * additionally updates the rtx timeout 139833d41c9cSPaolo Abeni */ 139907336a87SGeliang Tang struct sock *mptcp_subflow_get_send(struct mptcp_sock *msk) 1400f296234cSPeter Krystad { 14013ce0852cSPaolo Abeni struct subflow_send_info send_info[SSK_MODE_MAX]; 1402f296234cSPeter Krystad struct mptcp_subflow_context *subflow; 140333d41c9cSPaolo Abeni struct sock *sk = (struct sock *)msk; 14043ce0852cSPaolo Abeni u32 pace, burst, wmem; 1405d5f49190SPaolo Abeni int i, nr_active = 0; 1406d5f49190SPaolo Abeni struct sock *ssk; 14073ce0852cSPaolo Abeni u64 linger_time; 140833d41c9cSPaolo Abeni long tout = 0; 1409f296234cSPeter Krystad 1410d5f49190SPaolo Abeni /* pick the subflow with the lower wmem/wspace ratio */ 14113ce0852cSPaolo Abeni for (i = 0; i < SSK_MODE_MAX; ++i) { 1412d5f49190SPaolo Abeni send_info[i].ssk = NULL; 14133ce0852cSPaolo Abeni send_info[i].linger_time = -1; 1414d5f49190SPaolo Abeni } 14153ce0852cSPaolo Abeni 1416d5f49190SPaolo Abeni mptcp_for_each_subflow(msk, subflow) { 1417e10a9892SGeliang Tang trace_mptcp_subflow_get_send(subflow); 1418d5f49190SPaolo Abeni ssk = mptcp_subflow_tcp_sock(subflow); 1419d5f49190SPaolo Abeni if (!mptcp_subflow_active(subflow)) 1420f296234cSPeter Krystad continue; 1421d5f49190SPaolo Abeni 142233d41c9cSPaolo Abeni tout = max(tout, mptcp_timeout_from_subflow(subflow)); 1423d5f49190SPaolo Abeni nr_active += !subflow->backup; 14243ce0852cSPaolo Abeni pace = subflow->avg_pacing_rate; 14253ce0852cSPaolo Abeni if (unlikely(!pace)) { 14263ce0852cSPaolo Abeni /* init pacing rate from socket */ 14273ce0852cSPaolo Abeni subflow->avg_pacing_rate = READ_ONCE(ssk->sk_pacing_rate); 14283ce0852cSPaolo Abeni pace = subflow->avg_pacing_rate; 1429d5f49190SPaolo Abeni if (!pace) 1430d5f49190SPaolo Abeni continue; 14313ce0852cSPaolo Abeni } 1432d5f49190SPaolo Abeni 14333ce0852cSPaolo Abeni linger_time = div_u64((u64)READ_ONCE(ssk->sk_wmem_queued) << 32, pace); 14343ce0852cSPaolo Abeni if (linger_time < send_info[subflow->backup].linger_time) { 1435d5f49190SPaolo Abeni send_info[subflow->backup].ssk = ssk; 14363ce0852cSPaolo Abeni send_info[subflow->backup].linger_time = linger_time; 1437d5f49190SPaolo Abeni } 1438f296234cSPeter Krystad } 143933d41c9cSPaolo Abeni __mptcp_set_timeout(sk, tout); 1440f296234cSPeter Krystad 1441d5f49190SPaolo Abeni /* pick the best backup if no other subflow is active */ 1442d5f49190SPaolo Abeni if (!nr_active) 14433ce0852cSPaolo Abeni send_info[SSK_MODE_ACTIVE].ssk = send_info[SSK_MODE_BACKUP].ssk; 1444d5f49190SPaolo Abeni 14453ce0852cSPaolo Abeni /* According to the blest algorithm, to avoid HoL blocking for the 14463ce0852cSPaolo Abeni * faster flow, we need to: 14473ce0852cSPaolo Abeni * - estimate the faster flow linger time 14483ce0852cSPaolo Abeni * - use the above to estimate the amount of byte transferred 14493ce0852cSPaolo Abeni * by the faster flow 14503ce0852cSPaolo Abeni * - check that the amount of queued data is greter than the above, 14513ce0852cSPaolo Abeni * otherwise do not use the picked, slower, subflow 14523ce0852cSPaolo Abeni * We select the subflow with the shorter estimated time to flush 14533ce0852cSPaolo Abeni * the queued mem, which basically ensure the above. We just need 14543ce0852cSPaolo Abeni * to check that subflow has a non empty cwin. 14553ce0852cSPaolo Abeni */ 14563ce0852cSPaolo Abeni ssk = send_info[SSK_MODE_ACTIVE].ssk; 1457b713d006SPaolo Abeni if (!ssk || !sk_stream_memory_free(ssk)) 1458d5f49190SPaolo Abeni return NULL; 14593ce0852cSPaolo Abeni 1460b713d006SPaolo Abeni burst = min_t(int, MPTCP_SEND_BURST_SIZE, mptcp_wnd_end(msk) - msk->snd_nxt); 14613ce0852cSPaolo Abeni wmem = READ_ONCE(ssk->sk_wmem_queued); 1462ebc1e08fSGeliang Tang if (!burst) 1463b713d006SPaolo Abeni return ssk; 1464b713d006SPaolo Abeni 14653ce0852cSPaolo Abeni subflow = mptcp_subflow_ctx(ssk); 14663ce0852cSPaolo Abeni subflow->avg_pacing_rate = div_u64((u64)subflow->avg_pacing_rate * wmem + 14673ce0852cSPaolo Abeni READ_ONCE(ssk->sk_pacing_rate) * burst, 14683ce0852cSPaolo Abeni burst + wmem); 14693ce0852cSPaolo Abeni msk->snd_burst = burst; 14703ce0852cSPaolo Abeni return ssk; 1471f296234cSPeter Krystad } 1472f296234cSPeter Krystad 1473b8e0def3SGeliang Tang static void mptcp_push_release(struct sock *ssk, struct mptcp_sendmsg_info *info) 1474d9ca1de8SPaolo Abeni { 1475d9ca1de8SPaolo Abeni tcp_push(ssk, 0, info->mss_now, tcp_sk(ssk)->nonagle, info->size_goal); 1476d9ca1de8SPaolo Abeni release_sock(ssk); 1477d9ca1de8SPaolo Abeni } 1478d9ca1de8SPaolo Abeni 14790d199e43SFlorian Westphal static void mptcp_update_post_push(struct mptcp_sock *msk, 14800d199e43SFlorian Westphal struct mptcp_data_frag *dfrag, 14810d199e43SFlorian Westphal u32 sent) 14820d199e43SFlorian Westphal { 14830d199e43SFlorian Westphal u64 snd_nxt_new = dfrag->data_seq; 14840d199e43SFlorian Westphal 14850d199e43SFlorian Westphal dfrag->already_sent += sent; 14860d199e43SFlorian Westphal 14870d199e43SFlorian Westphal msk->snd_burst -= sent; 14880d199e43SFlorian Westphal 14890d199e43SFlorian Westphal snd_nxt_new += dfrag->already_sent; 14900d199e43SFlorian Westphal 14910d199e43SFlorian Westphal /* snd_nxt_new can be smaller than snd_nxt in case mptcp 14920d199e43SFlorian Westphal * is recovering after a failover. In that event, this re-sends 14930d199e43SFlorian Westphal * old segments. 14940d199e43SFlorian Westphal * 14950d199e43SFlorian Westphal * Thus compute snd_nxt_new candidate based on 14960d199e43SFlorian Westphal * the dfrag->data_seq that was sent and the data 14970d199e43SFlorian Westphal * that has been handed to the subflow for transmission 14980d199e43SFlorian Westphal * and skip update in case it was old dfrag. 14990d199e43SFlorian Westphal */ 150038967f42SPaolo Abeni if (likely(after64(snd_nxt_new, msk->snd_nxt))) { 150138967f42SPaolo Abeni msk->bytes_sent += snd_nxt_new - msk->snd_nxt; 15020d199e43SFlorian Westphal msk->snd_nxt = snd_nxt_new; 15030d199e43SFlorian Westphal } 150438967f42SPaolo Abeni } 15050d199e43SFlorian Westphal 15068b38217aSMaxim Galaganov void mptcp_check_and_set_pending(struct sock *sk) 15073241a9c0SFlorian Westphal { 1508e9d09bacSPaolo Abeni if (mptcp_send_head(sk)) 1509e9d09bacSPaolo Abeni mptcp_sk(sk)->push_pending |= BIT(MPTCP_PUSH_PENDING); 15103241a9c0SFlorian Westphal } 15113241a9c0SFlorian Westphal 1512c5b4297dSGeliang Tang static int __subflow_push_pending(struct sock *sk, struct sock *ssk, 1513c5b4297dSGeliang Tang struct mptcp_sendmsg_info *info) 1514c5b4297dSGeliang Tang { 1515c5b4297dSGeliang Tang struct mptcp_sock *msk = mptcp_sk(sk); 1516c5b4297dSGeliang Tang struct mptcp_data_frag *dfrag; 1517c5b4297dSGeliang Tang int len, copied = 0, err = 0; 1518c5b4297dSGeliang Tang 1519c5b4297dSGeliang Tang while ((dfrag = mptcp_send_head(sk))) { 1520c5b4297dSGeliang Tang info->sent = dfrag->already_sent; 1521c5b4297dSGeliang Tang info->limit = dfrag->data_len; 1522c5b4297dSGeliang Tang len = dfrag->data_len - dfrag->already_sent; 1523c5b4297dSGeliang Tang while (len > 0) { 1524c5b4297dSGeliang Tang int ret = 0; 1525c5b4297dSGeliang Tang 1526c5b4297dSGeliang Tang ret = mptcp_sendmsg_frag(sk, ssk, dfrag, info); 1527c5b4297dSGeliang Tang if (ret <= 0) { 1528c5b4297dSGeliang Tang err = copied ? : ret; 1529c5b4297dSGeliang Tang goto out; 1530c5b4297dSGeliang Tang } 1531c5b4297dSGeliang Tang 1532c5b4297dSGeliang Tang info->sent += ret; 1533c5b4297dSGeliang Tang copied += ret; 1534c5b4297dSGeliang Tang len -= ret; 1535c5b4297dSGeliang Tang 1536c5b4297dSGeliang Tang mptcp_update_post_push(msk, dfrag, ret); 1537c5b4297dSGeliang Tang } 1538c5b4297dSGeliang Tang WRITE_ONCE(msk->first_pending, mptcp_send_next(sk)); 1539c5b4297dSGeliang Tang 1540c5b4297dSGeliang Tang if (msk->snd_burst <= 0 || 1541c5b4297dSGeliang Tang !sk_stream_memory_free(ssk) || 1542c5b4297dSGeliang Tang !mptcp_subflow_active(mptcp_subflow_ctx(ssk))) { 1543c5b4297dSGeliang Tang err = copied; 1544c5b4297dSGeliang Tang goto out; 1545c5b4297dSGeliang Tang } 1546c5b4297dSGeliang Tang mptcp_set_timeout(sk); 1547c5b4297dSGeliang Tang } 1548c5b4297dSGeliang Tang err = copied; 1549c5b4297dSGeliang Tang 1550c5b4297dSGeliang Tang out: 1551c5b4297dSGeliang Tang return err; 1552c5b4297dSGeliang Tang } 1553c5b4297dSGeliang Tang 1554ff5a0b42SPaolo Abeni void __mptcp_push_pending(struct sock *sk, unsigned int flags) 1555d9ca1de8SPaolo Abeni { 1556d9ca1de8SPaolo Abeni struct sock *prev_ssk = NULL, *ssk = NULL; 1557d9ca1de8SPaolo Abeni struct mptcp_sock *msk = mptcp_sk(sk); 1558d9ca1de8SPaolo Abeni struct mptcp_sendmsg_info info = { 1559d9ca1de8SPaolo Abeni .flags = flags, 1560d9ca1de8SPaolo Abeni }; 15610522b424SGeliang Tang bool do_check_data_fin = false; 15620fa1b378SGeliang Tang int push_count = 1; 1563d9ca1de8SPaolo Abeni 15640fa1b378SGeliang Tang while (mptcp_send_head(sk) && (push_count > 0)) { 15650fa1b378SGeliang Tang struct mptcp_subflow_context *subflow; 1566d9ca1de8SPaolo Abeni int ret = 0; 1567d9ca1de8SPaolo Abeni 15680fa1b378SGeliang Tang if (mptcp_sched_get_send(msk)) 15690fa1b378SGeliang Tang break; 1570d9ca1de8SPaolo Abeni 15710fa1b378SGeliang Tang push_count = 0; 15720fa1b378SGeliang Tang 15730fa1b378SGeliang Tang mptcp_for_each_subflow(msk, subflow) { 15740fa1b378SGeliang Tang if (READ_ONCE(subflow->scheduled)) { 15750fa1b378SGeliang Tang mptcp_subflow_set_scheduled(subflow, false); 15760fa1b378SGeliang Tang 15770fa1b378SGeliang Tang prev_ssk = ssk; 15780fa1b378SGeliang Tang ssk = mptcp_subflow_tcp_sock(subflow); 15790fa1b378SGeliang Tang if (ssk != prev_ssk) { 15809758f40eSPaolo Abeni /* First check. If the ssk has changed since 15819758f40eSPaolo Abeni * the last round, release prev_ssk 1582d9ca1de8SPaolo Abeni */ 15830fa1b378SGeliang Tang if (prev_ssk) 1584b8e0def3SGeliang Tang mptcp_push_release(prev_ssk, &info); 1585d9ca1de8SPaolo Abeni 15869758f40eSPaolo Abeni /* Need to lock the new subflow only if different 15879758f40eSPaolo Abeni * from the previous one, otherwise we are still 15889758f40eSPaolo Abeni * helding the relevant lock 15899758f40eSPaolo Abeni */ 1590d9ca1de8SPaolo Abeni lock_sock(ssk); 15910fa1b378SGeliang Tang } 15920fa1b378SGeliang Tang 15930fa1b378SGeliang Tang push_count++; 1594d9ca1de8SPaolo Abeni 1595c5b4297dSGeliang Tang ret = __subflow_push_pending(sk, ssk, &info); 1596d9ca1de8SPaolo Abeni if (ret <= 0) { 15970fa1b378SGeliang Tang if (ret != -EAGAIN || 15980fa1b378SGeliang Tang (1 << ssk->sk_state) & 15990fa1b378SGeliang Tang (TCPF_FIN_WAIT1 | TCPF_FIN_WAIT2 | TCPF_CLOSE)) 16000fa1b378SGeliang Tang push_count--; 1601c886d702SPaolo Abeni continue; 1602d9ca1de8SPaolo Abeni } 16030522b424SGeliang Tang do_check_data_fin = true; 1604d9ca1de8SPaolo Abeni } 16050fa1b378SGeliang Tang } 16060fa1b378SGeliang Tang } 1607d9ca1de8SPaolo Abeni 1608d9ca1de8SPaolo Abeni /* at this point we held the socket lock for the last subflow we used */ 1609d9ca1de8SPaolo Abeni if (ssk) 1610b8e0def3SGeliang Tang mptcp_push_release(ssk, &info); 1611d9ca1de8SPaolo Abeni 161233d41c9cSPaolo Abeni /* ensure the rtx timer is running */ 1613f6909dc1SPaolo Abeni if (!mptcp_rtx_timer_pending(sk)) 1614f6909dc1SPaolo Abeni mptcp_reset_rtx_timer(sk); 16150522b424SGeliang Tang if (do_check_data_fin) 161681c1d029SPaolo Abeni mptcp_check_send_data_fin(sk); 1617d9ca1de8SPaolo Abeni } 1618d9ca1de8SPaolo Abeni 161973a0052aSGeliang Tang static void __mptcp_subflow_push_pending(struct sock *sk, struct sock *ssk, bool first) 16206e628cd3SPaolo Abeni { 16216e628cd3SPaolo Abeni struct mptcp_sock *msk = mptcp_sk(sk); 16221094c6feSPaolo Abeni struct mptcp_sendmsg_info info = { 16231094c6feSPaolo Abeni .data_lock_held = true, 16241094c6feSPaolo Abeni }; 16250fa1b378SGeliang Tang bool keep_pushing = true; 1626b19bc294SPaolo Abeni struct sock *xmit_ssk; 1627c5b4297dSGeliang Tang int copied = 0; 16286e628cd3SPaolo Abeni 16296e628cd3SPaolo Abeni info.flags = 0; 16300fa1b378SGeliang Tang while (mptcp_send_head(sk) && keep_pushing) { 16310fa1b378SGeliang Tang struct mptcp_subflow_context *subflow = mptcp_subflow_ctx(ssk); 16326e628cd3SPaolo Abeni int ret = 0; 16336e628cd3SPaolo Abeni 163473a0052aSGeliang Tang /* check for a different subflow usage only after 1635b19bc294SPaolo Abeni * spooling the first chunk of data 1636b19bc294SPaolo Abeni */ 16370fa1b378SGeliang Tang if (first) { 16380fa1b378SGeliang Tang mptcp_subflow_set_scheduled(subflow, false); 1639c5b4297dSGeliang Tang ret = __subflow_push_pending(sk, ssk, &info); 1640b19bc294SPaolo Abeni first = false; 1641c5b4297dSGeliang Tang if (ret <= 0) 1642c5b4297dSGeliang Tang break; 1643c5b4297dSGeliang Tang copied += ret; 16440fa1b378SGeliang Tang continue; 16450fa1b378SGeliang Tang } 16460fa1b378SGeliang Tang 16470fa1b378SGeliang Tang if (mptcp_sched_get_send(msk)) 16480fa1b378SGeliang Tang goto out; 16490fa1b378SGeliang Tang 16500fa1b378SGeliang Tang if (READ_ONCE(subflow->scheduled)) { 16510fa1b378SGeliang Tang mptcp_subflow_set_scheduled(subflow, false); 16520fa1b378SGeliang Tang ret = __subflow_push_pending(sk, ssk, &info); 16530fa1b378SGeliang Tang if (ret <= 0) 16540fa1b378SGeliang Tang keep_pushing = false; 16550fa1b378SGeliang Tang copied += ret; 16560fa1b378SGeliang Tang } 16570fa1b378SGeliang Tang 16580fa1b378SGeliang Tang mptcp_for_each_subflow(msk, subflow) { 16590fa1b378SGeliang Tang if (READ_ONCE(subflow->scheduled)) { 16600fa1b378SGeliang Tang xmit_ssk = mptcp_subflow_tcp_sock(subflow); 16610fa1b378SGeliang Tang if (xmit_ssk != ssk) { 16620fa1b378SGeliang Tang mptcp_subflow_delegate(subflow, 16630fa1b378SGeliang Tang MPTCP_DELEGATE_SEND); 16640fa1b378SGeliang Tang keep_pushing = false; 16650fa1b378SGeliang Tang } 16660fa1b378SGeliang Tang } 16670fa1b378SGeliang Tang } 16686e628cd3SPaolo Abeni } 16696e628cd3SPaolo Abeni 16706e628cd3SPaolo Abeni out: 16716e628cd3SPaolo Abeni /* __mptcp_alloc_tx_skb could have released some wmem and we are 16726e628cd3SPaolo Abeni * not going to flush it via release_sock() 16736e628cd3SPaolo Abeni */ 16746e628cd3SPaolo Abeni if (copied) { 16756e628cd3SPaolo Abeni tcp_push(ssk, 0, info.mss_now, tcp_sk(ssk)->nonagle, 16766e628cd3SPaolo Abeni info.size_goal); 1677f6909dc1SPaolo Abeni if (!mptcp_rtx_timer_pending(sk)) 1678f6909dc1SPaolo Abeni mptcp_reset_rtx_timer(sk); 1679d09d818eSPaolo Abeni 16806e628cd3SPaolo Abeni if (msk->snd_data_fin_enable && 16816e628cd3SPaolo Abeni msk->snd_nxt + 1 == msk->write_seq) 16826e628cd3SPaolo Abeni mptcp_schedule_work(sk); 16836e628cd3SPaolo Abeni } 16846e628cd3SPaolo Abeni } 16856e628cd3SPaolo Abeni 16865cf92bbaSPaolo Abeni static void mptcp_set_nospace(struct sock *sk) 16875cf92bbaSPaolo Abeni { 16885cf92bbaSPaolo Abeni /* enable autotune */ 16895cf92bbaSPaolo Abeni set_bit(SOCK_NOSPACE, &sk->sk_socket->flags); 16905cf92bbaSPaolo Abeni 16915cf92bbaSPaolo Abeni /* will be cleared on avail space */ 16925cf92bbaSPaolo Abeni set_bit(MPTCP_NOSPACE, &mptcp_sk(sk)->flags); 16935cf92bbaSPaolo Abeni } 16945cf92bbaSPaolo Abeni 16957d803344SPaolo Abeni static int mptcp_disconnect(struct sock *sk, int flags); 16967d803344SPaolo Abeni 1697a2702a07SPaolo Abeni static int mptcp_sendmsg_fastopen(struct sock *sk, struct msghdr *msg, 1698fa9e5746SPaolo Abeni size_t len, int *copied_syn) 1699fa9e5746SPaolo Abeni { 1700fa9e5746SPaolo Abeni unsigned int saved_flags = msg->msg_flags; 1701fa9e5746SPaolo Abeni struct mptcp_sock *msk = mptcp_sk(sk); 1702a2702a07SPaolo Abeni struct sock *ssk; 1703fa9e5746SPaolo Abeni int ret; 1704fa9e5746SPaolo Abeni 1705a2702a07SPaolo Abeni /* on flags based fastopen the mptcp is supposed to create the 1706a2702a07SPaolo Abeni * first subflow right now. Otherwise we are in the defer_connect 1707a2702a07SPaolo Abeni * path, and the first subflow must be already present. 1708a2702a07SPaolo Abeni * Since the defer_connect flag is cleared after the first succsful 1709a2702a07SPaolo Abeni * fastopen attempt, no need to check for additional subflow status. 1710a2702a07SPaolo Abeni */ 1711ddb1a072SPaolo Abeni if (msg->msg_flags & MSG_FASTOPEN) { 17123f326a82SPaolo Abeni ssk = __mptcp_nmpc_sk(msk); 17133f326a82SPaolo Abeni if (IS_ERR(ssk)) 17143f326a82SPaolo Abeni return PTR_ERR(ssk); 1715ddb1a072SPaolo Abeni } 1716a2702a07SPaolo Abeni if (!msk->first) 1717a2702a07SPaolo Abeni return -EINVAL; 1718a2702a07SPaolo Abeni 1719a2702a07SPaolo Abeni ssk = msk->first; 1720a2702a07SPaolo Abeni 1721fa9e5746SPaolo Abeni lock_sock(ssk); 1722fa9e5746SPaolo Abeni msg->msg_flags |= MSG_DONTWAIT; 17237d803344SPaolo Abeni msk->fastopening = 1; 1724fa9e5746SPaolo Abeni ret = tcp_sendmsg_fastopen(ssk, msg, copied_syn, len, NULL); 17257d803344SPaolo Abeni msk->fastopening = 0; 1726fa9e5746SPaolo Abeni msg->msg_flags = saved_flags; 1727fa9e5746SPaolo Abeni release_sock(ssk); 1728fa9e5746SPaolo Abeni 1729fa9e5746SPaolo Abeni /* do the blocking bits of inet_stream_connect outside the ssk socket lock */ 1730fa9e5746SPaolo Abeni if (ret == -EINPROGRESS && !(msg->msg_flags & MSG_DONTWAIT)) { 1731fa9e5746SPaolo Abeni ret = __inet_stream_connect(sk->sk_socket, msg->msg_name, 1732fa9e5746SPaolo Abeni msg->msg_namelen, msg->msg_flags, 1); 1733fa9e5746SPaolo Abeni 1734fa9e5746SPaolo Abeni /* Keep the same behaviour of plain TCP: zero the copied bytes in 1735fa9e5746SPaolo Abeni * case of any error, except timeout or signal 1736fa9e5746SPaolo Abeni */ 1737fa9e5746SPaolo Abeni if (ret && ret != -EINPROGRESS && ret != -ERESTARTSYS && ret != -EINTR) 1738fa9e5746SPaolo Abeni *copied_syn = 0; 17397d803344SPaolo Abeni } else if (ret && ret != -EINPROGRESS) { 1740c2b2ae39SPaolo Abeni /* The disconnect() op called by tcp_sendmsg_fastopen()/ 1741c2b2ae39SPaolo Abeni * __inet_stream_connect() can fail, due to looking check, 1742c2b2ae39SPaolo Abeni * see mptcp_disconnect(). 1743c2b2ae39SPaolo Abeni * Attempt it again outside the problematic scope. 1744c2b2ae39SPaolo Abeni */ 1745c2b2ae39SPaolo Abeni if (!mptcp_disconnect(sk, 0)) 1746c2b2ae39SPaolo Abeni sk->sk_socket->state = SS_UNCONNECTED; 1747fa9e5746SPaolo Abeni } 174808e39c0dSEric Dumazet inet_clear_bit(DEFER_CONNECT, sk); 1749fa9e5746SPaolo Abeni 1750fa9e5746SPaolo Abeni return ret; 1751fa9e5746SPaolo Abeni } 1752fa9e5746SPaolo Abeni 17530ffe8e74SPaolo Abeni static int do_copy_data_nocache(struct sock *sk, int copy, 17540ffe8e74SPaolo Abeni struct iov_iter *from, char *to) 17550ffe8e74SPaolo Abeni { 17560ffe8e74SPaolo Abeni if (sk->sk_route_caps & NETIF_F_NOCACHE_COPY) { 17570ffe8e74SPaolo Abeni if (!copy_from_iter_full_nocache(to, copy, from)) 17580ffe8e74SPaolo Abeni return -EFAULT; 17590ffe8e74SPaolo Abeni } else if (!copy_from_iter_full(to, copy, from)) { 17600ffe8e74SPaolo Abeni return -EFAULT; 17610ffe8e74SPaolo Abeni } 17620ffe8e74SPaolo Abeni return 0; 17630ffe8e74SPaolo Abeni } 17640ffe8e74SPaolo Abeni 1765f870fa0bSMat Martineau static int mptcp_sendmsg(struct sock *sk, struct msghdr *msg, size_t len) 1766f870fa0bSMat Martineau { 1767f870fa0bSMat Martineau struct mptcp_sock *msk = mptcp_sk(sk); 176817091708SFlorian Westphal struct page_frag *pfrag; 17696d0060f6SMat Martineau size_t copied = 0; 1770caf971dfSPaolo Abeni int ret = 0; 17716d0060f6SMat Martineau long timeo; 1772f870fa0bSMat Martineau 1773987858e5SPaolo Abeni /* silently ignore everything else */ 17741e777f39SDmytro Shytyi msg->msg_flags &= MSG_MORE | MSG_DONTWAIT | MSG_NOSIGNAL | MSG_FASTOPEN; 1775987858e5SPaolo Abeni 17766511882cSPaolo Abeni lock_sock(sk); 17771954b860SMat Martineau 177808e39c0dSEric Dumazet if (unlikely(inet_test_bit(DEFER_CONNECT, sk) || 177908e39c0dSEric Dumazet msg->msg_flags & MSG_FASTOPEN)) { 1780d98a82a6SDmytro Shytyi int copied_syn = 0; 1781d98a82a6SDmytro Shytyi 1782a2702a07SPaolo Abeni ret = mptcp_sendmsg_fastopen(sk, msg, len, &copied_syn); 1783d98a82a6SDmytro Shytyi copied += copied_syn; 1784fa9e5746SPaolo Abeni if (ret == -EINPROGRESS && copied_syn > 0) 1785d98a82a6SDmytro Shytyi goto out; 1786fa9e5746SPaolo Abeni else if (ret) 178769800e51SPaolo Abeni goto do_error; 1788d98a82a6SDmytro Shytyi } 1789d98a82a6SDmytro Shytyi 17901954b860SMat Martineau timeo = sock_sndtimeo(sk, msg->msg_flags & MSG_DONTWAIT); 17911954b860SMat Martineau 17921954b860SMat Martineau if ((1 << sk->sk_state) & ~(TCPF_ESTABLISHED | TCPF_CLOSE_WAIT)) { 17931954b860SMat Martineau ret = sk_stream_wait_connect(sk, &timeo); 17941954b860SMat Martineau if (ret) 179569800e51SPaolo Abeni goto do_error; 17961954b860SMat Martineau } 17971954b860SMat Martineau 179869800e51SPaolo Abeni ret = -EPIPE; 179969800e51SPaolo Abeni if (unlikely(sk->sk_err || (sk->sk_shutdown & SEND_SHUTDOWN))) 180069800e51SPaolo Abeni goto do_error; 180169800e51SPaolo Abeni 180217091708SFlorian Westphal pfrag = sk_page_frag(sk); 180318b683bfSPaolo Abeni 1804d9ca1de8SPaolo Abeni while (msg_data_left(msg)) { 1805724cfd2eSPaolo Abeni int total_ts, frag_truesize = 0; 1806d9ca1de8SPaolo Abeni struct mptcp_data_frag *dfrag; 1807d9ca1de8SPaolo Abeni bool dfrag_collapsed; 1808d9ca1de8SPaolo Abeni size_t psize, offset; 1809d9ca1de8SPaolo Abeni 1810d9ca1de8SPaolo Abeni /* reuse tail pfrag, if possible, or carve a new one from the 1811d9ca1de8SPaolo Abeni * page allocator 1812fb529e62SFlorian Westphal */ 1813d9ca1de8SPaolo Abeni dfrag = mptcp_pending_tail(sk); 1814d9ca1de8SPaolo Abeni dfrag_collapsed = mptcp_frag_can_collapse_to(msk, pfrag, dfrag); 1815d9ca1de8SPaolo Abeni if (!dfrag_collapsed) { 1816d9ca1de8SPaolo Abeni if (!sk_stream_memory_free(sk)) 1817d9ca1de8SPaolo Abeni goto wait_for_memory; 18186e628cd3SPaolo Abeni 1819d9ca1de8SPaolo Abeni if (!mptcp_page_frag_refill(sk, pfrag)) 1820d9ca1de8SPaolo Abeni goto wait_for_memory; 1821d9ca1de8SPaolo Abeni 1822d9ca1de8SPaolo Abeni dfrag = mptcp_carve_data_frag(msk, pfrag, pfrag->offset); 1823d9ca1de8SPaolo Abeni frag_truesize = dfrag->overhead; 1824fb529e62SFlorian Westphal } 1825fb529e62SFlorian Westphal 1826d9ca1de8SPaolo Abeni /* we do not bound vs wspace, to allow a single packet. 1827d9ca1de8SPaolo Abeni * memory accounting will prevent execessive memory usage 1828d9ca1de8SPaolo Abeni * anyway 1829d9ca1de8SPaolo Abeni */ 1830d9ca1de8SPaolo Abeni offset = dfrag->offset + dfrag->data_len; 1831d9ca1de8SPaolo Abeni psize = pfrag->size - offset; 1832d9ca1de8SPaolo Abeni psize = min_t(size_t, psize, msg_data_left(msg)); 1833724cfd2eSPaolo Abeni total_ts = psize + frag_truesize; 18348ce568edSPaolo Abeni 18356511882cSPaolo Abeni if (!sk_wmem_schedule(sk, total_ts)) 1836d9ca1de8SPaolo Abeni goto wait_for_memory; 1837d9ca1de8SPaolo Abeni 18380ffe8e74SPaolo Abeni ret = do_copy_data_nocache(sk, psize, &msg->msg_iter, 18390ffe8e74SPaolo Abeni page_address(dfrag->page) + offset); 18400ffe8e74SPaolo Abeni if (ret) 184169800e51SPaolo Abeni goto do_error; 1842d9ca1de8SPaolo Abeni 1843d9ca1de8SPaolo Abeni /* data successfully copied into the write queue */ 18445e6300e7SEric Dumazet sk_forward_alloc_add(sk, -total_ts); 1845d9ca1de8SPaolo Abeni copied += psize; 1846d9ca1de8SPaolo Abeni dfrag->data_len += psize; 1847d9ca1de8SPaolo Abeni frag_truesize += psize; 1848d9ca1de8SPaolo Abeni pfrag->offset += frag_truesize; 1849d9ca1de8SPaolo Abeni WRITE_ONCE(msk->write_seq, msk->write_seq + psize); 1850d9ca1de8SPaolo Abeni 1851d9ca1de8SPaolo Abeni /* charge data on mptcp pending queue to the msk socket 1852d9ca1de8SPaolo Abeni * Note: we charge such data both to sk and ssk 1853d9ca1de8SPaolo Abeni */ 1854d9ca1de8SPaolo Abeni sk_wmem_queued_add(sk, frag_truesize); 1855d9ca1de8SPaolo Abeni if (!dfrag_collapsed) { 1856d9ca1de8SPaolo Abeni get_page(dfrag->page); 1857d9ca1de8SPaolo Abeni list_add_tail(&dfrag->list, &msk->rtx_queue); 1858d9ca1de8SPaolo Abeni if (!msk->first_pending) 1859d9ca1de8SPaolo Abeni WRITE_ONCE(msk->first_pending, dfrag); 1860d9ca1de8SPaolo Abeni } 1861e4b61351SGeliang Tang pr_debug("msk=%p dfrag at seq=%llu len=%u sent=%u new=%d", msk, 1862d9ca1de8SPaolo Abeni dfrag->data_seq, dfrag->data_len, dfrag->already_sent, 1863d9ca1de8SPaolo Abeni !dfrag_collapsed); 1864d9ca1de8SPaolo Abeni 1865d9ca1de8SPaolo Abeni continue; 1866d9ca1de8SPaolo Abeni 1867d9ca1de8SPaolo Abeni wait_for_memory: 18685cf92bbaSPaolo Abeni mptcp_set_nospace(sk); 1869c2e6048fSPaolo Abeni __mptcp_push_pending(sk, msg->msg_flags); 1870f296234cSPeter Krystad ret = sk_stream_wait_memory(sk, &timeo); 1871f296234cSPeter Krystad if (ret) 187269800e51SPaolo Abeni goto do_error; 1873cec37a6eSPeter Krystad } 1874cec37a6eSPeter Krystad 187513e16037SPaolo Abeni if (copied) 1876c2e6048fSPaolo Abeni __mptcp_push_pending(sk, msg->msg_flags); 1877da51aef5SPaolo Abeni 18781954b860SMat Martineau out: 1879cec37a6eSPeter Krystad release_sock(sk); 188069800e51SPaolo Abeni return copied; 188169800e51SPaolo Abeni 188269800e51SPaolo Abeni do_error: 188369800e51SPaolo Abeni if (copied) 188469800e51SPaolo Abeni goto out; 188569800e51SPaolo Abeni 188669800e51SPaolo Abeni copied = sk_stream_error(sk, msg->msg_flags, ret); 188769800e51SPaolo Abeni goto out; 1888f870fa0bSMat Martineau } 1889f870fa0bSMat Martineau 18906771bfd9SFlorian Westphal static int __mptcp_recvmsg_mskq(struct mptcp_sock *msk, 18916771bfd9SFlorian Westphal struct msghdr *msg, 1892b7f653b2SFlorian Westphal size_t len, int flags, 1893b7f653b2SFlorian Westphal struct scm_timestamping_internal *tss, 1894b7f653b2SFlorian Westphal int *cmsg_flags) 18956771bfd9SFlorian Westphal { 1896ca4fb892SYonglong Li struct sk_buff *skb, *tmp; 18976771bfd9SFlorian Westphal int copied = 0; 18986771bfd9SFlorian Westphal 1899ca4fb892SYonglong Li skb_queue_walk_safe(&msk->receive_queue, skb, tmp) { 19006771bfd9SFlorian Westphal u32 offset = MPTCP_SKB_CB(skb)->offset; 19016771bfd9SFlorian Westphal u32 data_len = skb->len - offset; 19026771bfd9SFlorian Westphal u32 count = min_t(size_t, len - copied, data_len); 19036771bfd9SFlorian Westphal int err; 19046771bfd9SFlorian Westphal 1905d976092cSPaolo Abeni if (!(flags & MSG_TRUNC)) { 19066771bfd9SFlorian Westphal err = skb_copy_datagram_msg(skb, offset, msg, count); 19076771bfd9SFlorian Westphal if (unlikely(err < 0)) { 19086771bfd9SFlorian Westphal if (!copied) 19096771bfd9SFlorian Westphal return err; 19106771bfd9SFlorian Westphal break; 19116771bfd9SFlorian Westphal } 1912d976092cSPaolo Abeni } 19136771bfd9SFlorian Westphal 1914b7f653b2SFlorian Westphal if (MPTCP_SKB_CB(skb)->has_rxtstamp) { 1915b7f653b2SFlorian Westphal tcp_update_recv_tstamps(skb, tss); 1916b7f653b2SFlorian Westphal *cmsg_flags |= MPTCP_CMSG_TS; 1917b7f653b2SFlorian Westphal } 1918b7f653b2SFlorian Westphal 19196771bfd9SFlorian Westphal copied += count; 19206771bfd9SFlorian Westphal 19216771bfd9SFlorian Westphal if (count < data_len) { 19222c9e7765SFlorian Westphal if (!(flags & MSG_PEEK)) { 19236771bfd9SFlorian Westphal MPTCP_SKB_CB(skb)->offset += count; 19242c9e7765SFlorian Westphal MPTCP_SKB_CB(skb)->map_seq += count; 19255684ab1aSPaolo Abeni msk->bytes_consumed += count; 19262c9e7765SFlorian Westphal } 19276771bfd9SFlorian Westphal break; 19286771bfd9SFlorian Westphal } 19296771bfd9SFlorian Westphal 1930ca4fb892SYonglong Li if (!(flags & MSG_PEEK)) { 193187952603SPaolo Abeni /* we will bulk release the skb memory later */ 193287952603SPaolo Abeni skb->destructor = NULL; 1933ce599c51SPaolo Abeni WRITE_ONCE(msk->rmem_released, msk->rmem_released + skb->truesize); 193487952603SPaolo Abeni __skb_unlink(skb, &msk->receive_queue); 19356771bfd9SFlorian Westphal __kfree_skb(skb); 19365684ab1aSPaolo Abeni msk->bytes_consumed += count; 1937ca4fb892SYonglong Li } 19386771bfd9SFlorian Westphal 19396771bfd9SFlorian Westphal if (copied >= len) 19406771bfd9SFlorian Westphal break; 19416771bfd9SFlorian Westphal } 19426771bfd9SFlorian Westphal 19436771bfd9SFlorian Westphal return copied; 19446771bfd9SFlorian Westphal } 19456771bfd9SFlorian Westphal 1946a6b118feSFlorian Westphal /* receive buffer autotuning. See tcp_rcv_space_adjust for more information. 1947a6b118feSFlorian Westphal * 1948a6b118feSFlorian Westphal * Only difference: Use highest rtt estimate of the subflows in use. 1949a6b118feSFlorian Westphal */ 1950a6b118feSFlorian Westphal static void mptcp_rcv_space_adjust(struct mptcp_sock *msk, int copied) 1951a6b118feSFlorian Westphal { 1952a6b118feSFlorian Westphal struct mptcp_subflow_context *subflow; 1953a6b118feSFlorian Westphal struct sock *sk = (struct sock *)msk; 1954b8dc6d6cSPaolo Abeni u8 scaling_ratio = U8_MAX; 1955a6b118feSFlorian Westphal u32 time, advmss = 1; 1956a6b118feSFlorian Westphal u64 rtt_us, mstamp; 1957a6b118feSFlorian Westphal 1958109cdeb8SGeliang Tang msk_owned_by_me(msk); 1959a6b118feSFlorian Westphal 1960a6b118feSFlorian Westphal if (copied <= 0) 1961a6b118feSFlorian Westphal return; 1962a6b118feSFlorian Westphal 1963a6b118feSFlorian Westphal msk->rcvq_space.copied += copied; 1964a6b118feSFlorian Westphal 1965a6b118feSFlorian Westphal mstamp = div_u64(tcp_clock_ns(), NSEC_PER_USEC); 1966a6b118feSFlorian Westphal time = tcp_stamp_us_delta(mstamp, msk->rcvq_space.time); 1967a6b118feSFlorian Westphal 1968a6b118feSFlorian Westphal rtt_us = msk->rcvq_space.rtt_us; 1969a6b118feSFlorian Westphal if (rtt_us && time < (rtt_us >> 3)) 1970a6b118feSFlorian Westphal return; 1971a6b118feSFlorian Westphal 1972a6b118feSFlorian Westphal rtt_us = 0; 1973a6b118feSFlorian Westphal mptcp_for_each_subflow(msk, subflow) { 1974a6b118feSFlorian Westphal const struct tcp_sock *tp; 1975a6b118feSFlorian Westphal u64 sf_rtt_us; 1976a6b118feSFlorian Westphal u32 sf_advmss; 1977a6b118feSFlorian Westphal 1978a6b118feSFlorian Westphal tp = tcp_sk(mptcp_subflow_tcp_sock(subflow)); 1979a6b118feSFlorian Westphal 1980a6b118feSFlorian Westphal sf_rtt_us = READ_ONCE(tp->rcv_rtt_est.rtt_us); 1981a6b118feSFlorian Westphal sf_advmss = READ_ONCE(tp->advmss); 1982a6b118feSFlorian Westphal 1983a6b118feSFlorian Westphal rtt_us = max(sf_rtt_us, rtt_us); 1984a6b118feSFlorian Westphal advmss = max(sf_advmss, advmss); 1985b8dc6d6cSPaolo Abeni scaling_ratio = min(tp->scaling_ratio, scaling_ratio); 1986a6b118feSFlorian Westphal } 1987a6b118feSFlorian Westphal 1988a6b118feSFlorian Westphal msk->rcvq_space.rtt_us = rtt_us; 1989b8dc6d6cSPaolo Abeni msk->scaling_ratio = scaling_ratio; 1990a6b118feSFlorian Westphal if (time < (rtt_us >> 3) || rtt_us == 0) 1991a6b118feSFlorian Westphal return; 1992a6b118feSFlorian Westphal 1993a6b118feSFlorian Westphal if (msk->rcvq_space.copied <= msk->rcvq_space.space) 1994a6b118feSFlorian Westphal goto new_measure; 1995a6b118feSFlorian Westphal 199678047648SKuniyuki Iwashima if (READ_ONCE(sock_net(sk)->ipv4.sysctl_tcp_moderate_rcvbuf) && 1997a6b118feSFlorian Westphal !(sk->sk_userlocks & SOCK_RCVBUF_LOCK)) { 1998a6b118feSFlorian Westphal u64 rcvwin, grow; 1999b8dc6d6cSPaolo Abeni int rcvbuf; 2000a6b118feSFlorian Westphal 2001a6b118feSFlorian Westphal rcvwin = ((u64)msk->rcvq_space.copied << 1) + 16 * advmss; 2002a6b118feSFlorian Westphal 2003a6b118feSFlorian Westphal grow = rcvwin * (msk->rcvq_space.copied - msk->rcvq_space.space); 2004a6b118feSFlorian Westphal 2005a6b118feSFlorian Westphal do_div(grow, msk->rcvq_space.space); 2006a6b118feSFlorian Westphal rcvwin += (grow << 1); 2007a6b118feSFlorian Westphal 2008b8dc6d6cSPaolo Abeni rcvbuf = min_t(u64, __tcp_space_from_win(scaling_ratio, rcvwin), 200902739545SKuniyuki Iwashima READ_ONCE(sock_net(sk)->ipv4.sysctl_tcp_rmem[2])); 2010a6b118feSFlorian Westphal 2011a6b118feSFlorian Westphal if (rcvbuf > sk->sk_rcvbuf) { 2012a6b118feSFlorian Westphal u32 window_clamp; 2013a6b118feSFlorian Westphal 2014b8dc6d6cSPaolo Abeni window_clamp = __tcp_win_from_space(scaling_ratio, rcvbuf); 2015a6b118feSFlorian Westphal WRITE_ONCE(sk->sk_rcvbuf, rcvbuf); 2016a6b118feSFlorian Westphal 2017a6b118feSFlorian Westphal /* Make subflows follow along. If we do not do this, we 2018a6b118feSFlorian Westphal * get drops at subflow level if skbs can't be moved to 2019a6b118feSFlorian Westphal * the mptcp rx queue fast enough (announced rcv_win can 2020a6b118feSFlorian Westphal * exceed ssk->sk_rcvbuf). 2021a6b118feSFlorian Westphal */ 2022a6b118feSFlorian Westphal mptcp_for_each_subflow(msk, subflow) { 2023a6b118feSFlorian Westphal struct sock *ssk; 2024c76c6956SPaolo Abeni bool slow; 2025a6b118feSFlorian Westphal 2026a6b118feSFlorian Westphal ssk = mptcp_subflow_tcp_sock(subflow); 2027c76c6956SPaolo Abeni slow = lock_sock_fast(ssk); 2028a6b118feSFlorian Westphal WRITE_ONCE(ssk->sk_rcvbuf, rcvbuf); 2029a6b118feSFlorian Westphal tcp_sk(ssk)->window_clamp = window_clamp; 2030c76c6956SPaolo Abeni tcp_cleanup_rbuf(ssk, 1); 2031c76c6956SPaolo Abeni unlock_sock_fast(ssk, slow); 2032a6b118feSFlorian Westphal } 2033a6b118feSFlorian Westphal } 2034a6b118feSFlorian Westphal } 2035a6b118feSFlorian Westphal 2036a6b118feSFlorian Westphal msk->rcvq_space.space = msk->rcvq_space.copied; 2037a6b118feSFlorian Westphal new_measure: 2038a6b118feSFlorian Westphal msk->rcvq_space.copied = 0; 2039a6b118feSFlorian Westphal msk->rcvq_space.time = mstamp; 2040a6b118feSFlorian Westphal } 2041a6b118feSFlorian Westphal 204287952603SPaolo Abeni static void __mptcp_update_rmem(struct sock *sk) 204387952603SPaolo Abeni { 204487952603SPaolo Abeni struct mptcp_sock *msk = mptcp_sk(sk); 204587952603SPaolo Abeni 204687952603SPaolo Abeni if (!msk->rmem_released) 204787952603SPaolo Abeni return; 204887952603SPaolo Abeni 204987952603SPaolo Abeni atomic_sub(msk->rmem_released, &sk->sk_rmem_alloc); 20506511882cSPaolo Abeni mptcp_rmem_uncharge(sk, msk->rmem_released); 2051ce599c51SPaolo Abeni WRITE_ONCE(msk->rmem_released, 0); 205287952603SPaolo Abeni } 205387952603SPaolo Abeni 205487952603SPaolo Abeni static void __mptcp_splice_receive_queue(struct sock *sk) 205587952603SPaolo Abeni { 205687952603SPaolo Abeni struct mptcp_sock *msk = mptcp_sk(sk); 205787952603SPaolo Abeni 205887952603SPaolo Abeni skb_queue_splice_tail_init(&sk->sk_receive_queue, &msk->receive_queue); 205987952603SPaolo Abeni } 206087952603SPaolo Abeni 2061e3859603SPaolo Abeni static bool __mptcp_move_skbs(struct mptcp_sock *msk) 20626771bfd9SFlorian Westphal { 206387952603SPaolo Abeni struct sock *sk = (struct sock *)msk; 20646771bfd9SFlorian Westphal unsigned int moved = 0; 206587952603SPaolo Abeni bool ret, done; 2066d5f49190SPaolo Abeni 20676771bfd9SFlorian Westphal do { 20686771bfd9SFlorian Westphal struct sock *ssk = mptcp_subflow_recv_lookup(msk); 206965f49fe7SFlorian Westphal bool slowpath; 20706771bfd9SFlorian Westphal 207187952603SPaolo Abeni /* we can have data pending in the subflows only if the msk 207287952603SPaolo Abeni * receive buffer was full at subflow_data_ready() time, 207387952603SPaolo Abeni * that is an unlikely slow path. 207487952603SPaolo Abeni */ 207587952603SPaolo Abeni if (likely(!ssk)) 20766771bfd9SFlorian Westphal break; 20776771bfd9SFlorian Westphal 207865f49fe7SFlorian Westphal slowpath = lock_sock_fast(ssk); 207987952603SPaolo Abeni mptcp_data_lock(sk); 2080e3859603SPaolo Abeni __mptcp_update_rmem(sk); 20816771bfd9SFlorian Westphal done = __mptcp_move_skbs_from_subflow(msk, ssk, &moved); 208287952603SPaolo Abeni mptcp_data_unlock(sk); 2083499ada50SPaolo Abeni 2084499ada50SPaolo Abeni if (unlikely(ssk->sk_err)) 2085499ada50SPaolo Abeni __mptcp_error_report(sk); 208665f49fe7SFlorian Westphal unlock_sock_fast(ssk, slowpath); 20876771bfd9SFlorian Westphal } while (!done); 20886771bfd9SFlorian Westphal 208987952603SPaolo Abeni /* acquire the data lock only if some input data is pending */ 209087952603SPaolo Abeni ret = moved > 0; 209187952603SPaolo Abeni if (!RB_EMPTY_ROOT(&msk->out_of_order_queue) || 209287952603SPaolo Abeni !skb_queue_empty_lockless(&sk->sk_receive_queue)) { 209387952603SPaolo Abeni mptcp_data_lock(sk); 209487952603SPaolo Abeni __mptcp_update_rmem(sk); 209587952603SPaolo Abeni ret |= __mptcp_ofo_queue(msk); 209687952603SPaolo Abeni __mptcp_splice_receive_queue(sk); 209787952603SPaolo Abeni mptcp_data_unlock(sk); 2098ab174ad8SPaolo Abeni } 209987952603SPaolo Abeni if (ret) 210087952603SPaolo Abeni mptcp_check_data_fin((struct sock *)msk); 210187952603SPaolo Abeni return !skb_queue_empty(&msk->receive_queue); 21026771bfd9SFlorian Westphal } 21036771bfd9SFlorian Westphal 21042c9e7765SFlorian Westphal static unsigned int mptcp_inq_hint(const struct sock *sk) 21052c9e7765SFlorian Westphal { 21062c9e7765SFlorian Westphal const struct mptcp_sock *msk = mptcp_sk(sk); 21072c9e7765SFlorian Westphal const struct sk_buff *skb; 21082c9e7765SFlorian Westphal 21092c9e7765SFlorian Westphal skb = skb_peek(&msk->receive_queue); 21102c9e7765SFlorian Westphal if (skb) { 21112c9e7765SFlorian Westphal u64 hint_val = msk->ack_seq - MPTCP_SKB_CB(skb)->map_seq; 21122c9e7765SFlorian Westphal 21132c9e7765SFlorian Westphal if (hint_val >= INT_MAX) 21142c9e7765SFlorian Westphal return INT_MAX; 21152c9e7765SFlorian Westphal 21162c9e7765SFlorian Westphal return (unsigned int)hint_val; 21172c9e7765SFlorian Westphal } 21182c9e7765SFlorian Westphal 21192c9e7765SFlorian Westphal if (sk->sk_state == TCP_CLOSE || (sk->sk_shutdown & RCV_SHUTDOWN)) 21202c9e7765SFlorian Westphal return 1; 21212c9e7765SFlorian Westphal 21222c9e7765SFlorian Westphal return 0; 21232c9e7765SFlorian Westphal } 21242c9e7765SFlorian Westphal 2125f870fa0bSMat Martineau static int mptcp_recvmsg(struct sock *sk, struct msghdr *msg, size_t len, 2126ec095263SOliver Hartkopp int flags, int *addr_len) 2127f870fa0bSMat Martineau { 2128f870fa0bSMat Martineau struct mptcp_sock *msk = mptcp_sk(sk); 2129b7f653b2SFlorian Westphal struct scm_timestamping_internal tss; 2130b7f653b2SFlorian Westphal int copied = 0, cmsg_flags = 0; 21317a6a6cbcSPaolo Abeni int target; 21327a6a6cbcSPaolo Abeni long timeo; 2133f870fa0bSMat Martineau 2134cb9d80f4SPaolo Abeni /* MSG_ERRQUEUE is really a no-op till we support IP_RECVERR */ 2135cb9d80f4SPaolo Abeni if (unlikely(flags & MSG_ERRQUEUE)) 2136cb9d80f4SPaolo Abeni return inet_recv_error(sk, msg, len, addr_len); 2137cb9d80f4SPaolo Abeni 21386511882cSPaolo Abeni lock_sock(sk); 2139fd897679SPaolo Abeni if (unlikely(sk->sk_state == TCP_LISTEN)) { 2140fd897679SPaolo Abeni copied = -ENOTCONN; 2141fd897679SPaolo Abeni goto out_err; 2142fd897679SPaolo Abeni } 2143fd897679SPaolo Abeni 2144ec095263SOliver Hartkopp timeo = sock_rcvtimeo(sk, flags & MSG_DONTWAIT); 21457a6a6cbcSPaolo Abeni 21467a6a6cbcSPaolo Abeni len = min_t(size_t, len, INT_MAX); 21477a6a6cbcSPaolo Abeni target = sock_rcvlowat(sk, flags & MSG_WAITALL, len); 21487a6a6cbcSPaolo Abeni 21492c9e7765SFlorian Westphal if (unlikely(msk->recvmsg_inq)) 21502c9e7765SFlorian Westphal cmsg_flags = MPTCP_CMSG_INQ; 21512c9e7765SFlorian Westphal 215205e3eceaSEric Dumazet while (copied < len) { 2153e3859603SPaolo Abeni int bytes_read; 21547a6a6cbcSPaolo Abeni 2155b7f653b2SFlorian Westphal bytes_read = __mptcp_recvmsg_mskq(msk, msg, len - copied, flags, &tss, &cmsg_flags); 21566771bfd9SFlorian Westphal if (unlikely(bytes_read < 0)) { 21577a6a6cbcSPaolo Abeni if (!copied) 21587a6a6cbcSPaolo Abeni copied = bytes_read; 21596771bfd9SFlorian Westphal goto out_err; 2160cec37a6eSPeter Krystad } 2161cec37a6eSPeter Krystad 21627a6a6cbcSPaolo Abeni copied += bytes_read; 21637a6a6cbcSPaolo Abeni 2164ea4ca586SPaolo Abeni /* be sure to advertise window change */ 2165fd897679SPaolo Abeni mptcp_cleanup_rbuf(msk); 2166ea4ca586SPaolo Abeni 2167e3859603SPaolo Abeni if (skb_queue_empty(&msk->receive_queue) && __mptcp_move_skbs(msk)) 2168e3859603SPaolo Abeni continue; 2169e3859603SPaolo Abeni 21707a6a6cbcSPaolo Abeni /* only the master socket status is relevant here. The exit 21717a6a6cbcSPaolo Abeni * conditions mirror closely tcp_recvmsg() 21727a6a6cbcSPaolo Abeni */ 21737a6a6cbcSPaolo Abeni if (copied >= target) 21747a6a6cbcSPaolo Abeni break; 21757a6a6cbcSPaolo Abeni 21767a6a6cbcSPaolo Abeni if (copied) { 21777a6a6cbcSPaolo Abeni if (sk->sk_err || 21787a6a6cbcSPaolo Abeni sk->sk_state == TCP_CLOSE || 21797a6a6cbcSPaolo Abeni (sk->sk_shutdown & RCV_SHUTDOWN) || 21807a6a6cbcSPaolo Abeni !timeo || 21817a6a6cbcSPaolo Abeni signal_pending(current)) 21827a6a6cbcSPaolo Abeni break; 21837a6a6cbcSPaolo Abeni } else { 21847a6a6cbcSPaolo Abeni if (sk->sk_err) { 21857a6a6cbcSPaolo Abeni copied = sock_error(sk); 21867a6a6cbcSPaolo Abeni break; 21877a6a6cbcSPaolo Abeni } 21887a6a6cbcSPaolo Abeni 218987952603SPaolo Abeni if (sk->sk_shutdown & RCV_SHUTDOWN) { 219087952603SPaolo Abeni /* race breaker: the shutdown could be after the 219187952603SPaolo Abeni * previous receive queue check 219287952603SPaolo Abeni */ 2193e3859603SPaolo Abeni if (__mptcp_move_skbs(msk)) 219487952603SPaolo Abeni continue; 21957a6a6cbcSPaolo Abeni break; 219687952603SPaolo Abeni } 21977a6a6cbcSPaolo Abeni 21987a6a6cbcSPaolo Abeni if (sk->sk_state == TCP_CLOSE) { 21997a6a6cbcSPaolo Abeni copied = -ENOTCONN; 22007a6a6cbcSPaolo Abeni break; 22017a6a6cbcSPaolo Abeni } 22027a6a6cbcSPaolo Abeni 22037a6a6cbcSPaolo Abeni if (!timeo) { 22047a6a6cbcSPaolo Abeni copied = -EAGAIN; 22057a6a6cbcSPaolo Abeni break; 22067a6a6cbcSPaolo Abeni } 22077a6a6cbcSPaolo Abeni 22087a6a6cbcSPaolo Abeni if (signal_pending(current)) { 22097a6a6cbcSPaolo Abeni copied = sock_intr_errno(timeo); 22107a6a6cbcSPaolo Abeni break; 22117a6a6cbcSPaolo Abeni } 22127a6a6cbcSPaolo Abeni } 22137a6a6cbcSPaolo Abeni 22147a6a6cbcSPaolo Abeni pr_debug("block timeout %ld", timeo); 2215612f71d7SPaolo Abeni sk_wait_data(sk, &timeo, NULL); 22167a6a6cbcSPaolo Abeni } 22173c90e377SPaolo Abeni 22186771bfd9SFlorian Westphal out_err: 2219b7f653b2SFlorian Westphal if (cmsg_flags && copied >= 0) { 2220b7f653b2SFlorian Westphal if (cmsg_flags & MPTCP_CMSG_TS) 2221b7f653b2SFlorian Westphal tcp_recv_timestamp(msg, sk, &tss); 22222c9e7765SFlorian Westphal 22232c9e7765SFlorian Westphal if (cmsg_flags & MPTCP_CMSG_INQ) { 22242c9e7765SFlorian Westphal unsigned int inq = mptcp_inq_hint(sk); 22252c9e7765SFlorian Westphal 22262c9e7765SFlorian Westphal put_cmsg(msg, SOL_TCP, TCP_CM_INQ, sizeof(inq), &inq); 22272c9e7765SFlorian Westphal } 2228b7f653b2SFlorian Westphal } 2229b7f653b2SFlorian Westphal 2230612f71d7SPaolo Abeni pr_debug("msk=%p rx queue empty=%d:%d copied=%d", 2231612f71d7SPaolo Abeni msk, skb_queue_empty_lockless(&sk->sk_receive_queue), 2232612f71d7SPaolo Abeni skb_queue_empty(&msk->receive_queue), copied); 2233ca4fb892SYonglong Li if (!(flags & MSG_PEEK)) 2234a6b118feSFlorian Westphal mptcp_rcv_space_adjust(msk, copied); 2235a6b118feSFlorian Westphal 2236cec37a6eSPeter Krystad release_sock(sk); 2237cec37a6eSPeter Krystad return copied; 2238cec37a6eSPeter Krystad } 2239cec37a6eSPeter Krystad 2240b51f9b80SPaolo Abeni static void mptcp_retransmit_timer(struct timer_list *t) 2241b51f9b80SPaolo Abeni { 2242b51f9b80SPaolo Abeni struct inet_connection_sock *icsk = from_timer(icsk, t, 2243b51f9b80SPaolo Abeni icsk_retransmit_timer); 2244b51f9b80SPaolo Abeni struct sock *sk = &icsk->icsk_inet.sk; 22452d6f5a2bSPaolo Abeni struct mptcp_sock *msk = mptcp_sk(sk); 2246b51f9b80SPaolo Abeni 2247b51f9b80SPaolo Abeni bh_lock_sock(sk); 2248b51f9b80SPaolo Abeni if (!sock_owned_by_user(sk)) { 22492d6f5a2bSPaolo Abeni /* we need a process context to retransmit */ 22502d6f5a2bSPaolo Abeni if (!test_and_set_bit(MPTCP_WORK_RTX, &msk->flags)) 22512d6f5a2bSPaolo Abeni mptcp_schedule_work(sk); 2252b51f9b80SPaolo Abeni } else { 2253b51f9b80SPaolo Abeni /* delegate our work to tcp_release_cb() */ 2254e9d09bacSPaolo Abeni __set_bit(MPTCP_RETRANSMIT, &msk->cb_flags); 2255b51f9b80SPaolo Abeni } 2256b51f9b80SPaolo Abeni bh_unlock_sock(sk); 2257b51f9b80SPaolo Abeni sock_put(sk); 2258b51f9b80SPaolo Abeni } 2259b51f9b80SPaolo Abeni 2260f6909dc1SPaolo Abeni static void mptcp_tout_timer(struct timer_list *t) 2261e16163b6SPaolo Abeni { 2262e16163b6SPaolo Abeni struct sock *sk = from_timer(sk, t, sk_timer); 2263e16163b6SPaolo Abeni 2264e16163b6SPaolo Abeni mptcp_schedule_work(sk); 2265b6d69fc8SFlorian Westphal sock_put(sk); 2266e16163b6SPaolo Abeni } 2267e16163b6SPaolo Abeni 22683b1d6210SPaolo Abeni /* Find an idle subflow. Return NULL if there is unacked data at tcp 22693b1d6210SPaolo Abeni * level. 22703b1d6210SPaolo Abeni * 22713b1d6210SPaolo Abeni * A backup subflow is returned only if that is the only kind available. 22723b1d6210SPaolo Abeni */ 227307336a87SGeliang Tang struct sock *mptcp_subflow_get_retrans(struct mptcp_sock *msk) 22743b1d6210SPaolo Abeni { 227571b7dec2SPaolo Abeni struct sock *backup = NULL, *pick = NULL; 22763b1d6210SPaolo Abeni struct mptcp_subflow_context *subflow; 227771b7dec2SPaolo Abeni int min_stale_count = INT_MAX; 22783b1d6210SPaolo Abeni 22793b1d6210SPaolo Abeni mptcp_for_each_subflow(msk, subflow) { 22803b1d6210SPaolo Abeni struct sock *ssk = mptcp_subflow_tcp_sock(subflow); 22813b1d6210SPaolo Abeni 2282ff5a0b42SPaolo Abeni if (!__mptcp_subflow_active(subflow)) 2283d5f49190SPaolo Abeni continue; 2284d5f49190SPaolo Abeni 228571b7dec2SPaolo Abeni /* still data outstanding at TCP level? skip this */ 228671b7dec2SPaolo Abeni if (!tcp_rtx_and_write_queues_empty(ssk)) { 228771b7dec2SPaolo Abeni mptcp_pm_subflow_chk_stale(msk, ssk); 228871b7dec2SPaolo Abeni min_stale_count = min_t(int, min_stale_count, subflow->stale_count); 2289860975c6SFlorian Westphal continue; 2290860975c6SFlorian Westphal } 22913b1d6210SPaolo Abeni 22923b1d6210SPaolo Abeni if (subflow->backup) { 22933b1d6210SPaolo Abeni if (!backup) 22943b1d6210SPaolo Abeni backup = ssk; 22953b1d6210SPaolo Abeni continue; 22963b1d6210SPaolo Abeni } 22973b1d6210SPaolo Abeni 229871b7dec2SPaolo Abeni if (!pick) 229971b7dec2SPaolo Abeni pick = ssk; 23003b1d6210SPaolo Abeni } 23013b1d6210SPaolo Abeni 230271b7dec2SPaolo Abeni if (pick) 230371b7dec2SPaolo Abeni return pick; 230471b7dec2SPaolo Abeni 230571b7dec2SPaolo Abeni /* use backup only if there are no progresses anywhere */ 230671b7dec2SPaolo Abeni return min_stale_count > 1 ? backup : NULL; 23073b1d6210SPaolo Abeni } 23083b1d6210SPaolo Abeni 23091e1d9d6fSPaolo Abeni bool __mptcp_retransmit_pending_data(struct sock *sk) 23101e1d9d6fSPaolo Abeni { 23111e1d9d6fSPaolo Abeni struct mptcp_data_frag *cur, *rtx_head; 23121e1d9d6fSPaolo Abeni struct mptcp_sock *msk = mptcp_sk(sk); 23131e1d9d6fSPaolo Abeni 231400df24f1SGeliang Tang if (__mptcp_check_fallback(msk)) 23151e1d9d6fSPaolo Abeni return false; 23161e1d9d6fSPaolo Abeni 23171e1d9d6fSPaolo Abeni /* the closing socket has some data untransmitted and/or unacked: 23181e1d9d6fSPaolo Abeni * some data in the mptcp rtx queue has not really xmitted yet. 23191e1d9d6fSPaolo Abeni * keep it simple and re-inject the whole mptcp level rtx queue 23201e1d9d6fSPaolo Abeni */ 23211e1d9d6fSPaolo Abeni mptcp_data_lock(sk); 23221e1d9d6fSPaolo Abeni __mptcp_clean_una_wakeup(sk); 23231e1d9d6fSPaolo Abeni rtx_head = mptcp_rtx_head(sk); 23241e1d9d6fSPaolo Abeni if (!rtx_head) { 23251e1d9d6fSPaolo Abeni mptcp_data_unlock(sk); 23261e1d9d6fSPaolo Abeni return false; 23271e1d9d6fSPaolo Abeni } 23281e1d9d6fSPaolo Abeni 23291e1d9d6fSPaolo Abeni msk->recovery_snd_nxt = msk->snd_nxt; 23301e1d9d6fSPaolo Abeni msk->recovery = true; 23311e1d9d6fSPaolo Abeni mptcp_data_unlock(sk); 23321e1d9d6fSPaolo Abeni 23331e1d9d6fSPaolo Abeni msk->first_pending = rtx_head; 23341e1d9d6fSPaolo Abeni msk->snd_burst = 0; 23351e1d9d6fSPaolo Abeni 23361e1d9d6fSPaolo Abeni /* be sure to clear the "sent status" on all re-injected fragments */ 23371e1d9d6fSPaolo Abeni list_for_each_entry(cur, &msk->rtx_queue, list) { 23381e1d9d6fSPaolo Abeni if (!cur->already_sent) 23391e1d9d6fSPaolo Abeni break; 23401e1d9d6fSPaolo Abeni cur->already_sent = 0; 23411e1d9d6fSPaolo Abeni } 23421e1d9d6fSPaolo Abeni 23431e1d9d6fSPaolo Abeni return true; 23441e1d9d6fSPaolo Abeni } 23451e1d9d6fSPaolo Abeni 2346b29fcfb5SPaolo Abeni /* flags for __mptcp_close_ssk() */ 2347b29fcfb5SPaolo Abeni #define MPTCP_CF_PUSH BIT(1) 2348b29fcfb5SPaolo Abeni #define MPTCP_CF_FASTCLOSE BIT(2) 2349b29fcfb5SPaolo Abeni 235014c56686SGeliang Tang /* be sure to send a reset only if the caller asked for it, also 235114c56686SGeliang Tang * clean completely the subflow status when the subflow reaches 235214c56686SGeliang Tang * TCP_CLOSE state 235314c56686SGeliang Tang */ 235414c56686SGeliang Tang static void __mptcp_subflow_disconnect(struct sock *ssk, 235514c56686SGeliang Tang struct mptcp_subflow_context *subflow, 235614c56686SGeliang Tang unsigned int flags) 235714c56686SGeliang Tang { 235814c56686SGeliang Tang if (((1 << ssk->sk_state) & (TCPF_CLOSE | TCPF_LISTEN)) || 235914c56686SGeliang Tang (flags & MPTCP_CF_FASTCLOSE)) { 236014c56686SGeliang Tang /* The MPTCP code never wait on the subflow sockets, TCP-level 236114c56686SGeliang Tang * disconnect should never fail 236214c56686SGeliang Tang */ 236314c56686SGeliang Tang WARN_ON_ONCE(tcp_disconnect(ssk, 0)); 236414c56686SGeliang Tang mptcp_subflow_ctx_reset(subflow); 236514c56686SGeliang Tang } else { 236614c56686SGeliang Tang tcp_shutdown(ssk, SEND_SHUTDOWN); 236714c56686SGeliang Tang } 236814c56686SGeliang Tang } 236914c56686SGeliang Tang 2370cec37a6eSPeter Krystad /* subflow sockets can be either outgoing (connect) or incoming 2371cec37a6eSPeter Krystad * (accept). 2372cec37a6eSPeter Krystad * 2373cec37a6eSPeter Krystad * Outgoing subflows use in-kernel sockets. 2374cec37a6eSPeter Krystad * Incoming subflows do not have their own 'struct socket' allocated, 2375cec37a6eSPeter Krystad * so we need to use tcp_close() after detaching them from the mptcp 2376cec37a6eSPeter Krystad * parent socket. 2377cec37a6eSPeter Krystad */ 2378a141e02eSFlorian Westphal static void __mptcp_close_ssk(struct sock *sk, struct sock *ssk, 2379b29fcfb5SPaolo Abeni struct mptcp_subflow_context *subflow, 2380b29fcfb5SPaolo Abeni unsigned int flags) 2381cec37a6eSPeter Krystad { 2382e0be4931SFlorian Westphal struct mptcp_sock *msk = mptcp_sk(sk); 238363740448SPaolo Abeni bool dispose_it, need_push = false; 238463740448SPaolo Abeni 238563740448SPaolo Abeni /* If the first subflow moved to a close state before accept, e.g. due 238627e5ccc2SPaolo Abeni * to an incoming reset or listener shutdown, the subflow socket is 238727e5ccc2SPaolo Abeni * already deleted by inet_child_forget() and the mptcp socket can't 238827e5ccc2SPaolo Abeni * survive too. 238963740448SPaolo Abeni */ 239027e5ccc2SPaolo Abeni if (msk->in_accept_queue && msk->first == ssk && 239127e5ccc2SPaolo Abeni (sock_flag(sk, SOCK_DEAD) || sock_flag(ssk, SOCK_DEAD))) { 239263740448SPaolo Abeni /* ensure later check in mptcp_worker() will dispose the msk */ 239363740448SPaolo Abeni sock_set_flag(sk, SOCK_DEAD); 2394d866ae9aSPaolo Abeni mptcp_set_close_tout(sk, tcp_jiffies32 - (mptcp_close_timeout(sk) + 1)); 239563740448SPaolo Abeni lock_sock_nested(ssk, SINGLE_DEPTH_NESTING); 239663740448SPaolo Abeni mptcp_subflow_drop_ctx(ssk); 239763740448SPaolo Abeni goto out_release; 239863740448SPaolo Abeni } 2399e0be4931SFlorian Westphal 240039880bd8SPaolo Abeni dispose_it = msk->free_first || ssk != msk->first; 2401b29fcfb5SPaolo Abeni if (dispose_it) 2402cec37a6eSPeter Krystad list_del(&subflow->node); 2403cec37a6eSPeter Krystad 24043f8b2667SPaolo Abeni lock_sock_nested(ssk, SINGLE_DEPTH_NESTING); 2405e16163b6SPaolo Abeni 2406ff18f9efSPaolo Abeni if ((flags & MPTCP_CF_FASTCLOSE) && !__mptcp_check_fallback(msk)) { 240714c56686SGeliang Tang /* be sure to force the tcp_close path 2408d21f8348SPaolo Abeni * to generate the egress reset 2409d21f8348SPaolo Abeni */ 2410d21f8348SPaolo Abeni ssk->sk_lingertime = 0; 2411d21f8348SPaolo Abeni sock_set_flag(ssk, SOCK_LINGER); 2412b29fcfb5SPaolo Abeni subflow->send_fastclose = 1; 2413d21f8348SPaolo Abeni } 2414b29fcfb5SPaolo Abeni 2415b29fcfb5SPaolo Abeni need_push = (flags & MPTCP_CF_PUSH) && __mptcp_retransmit_pending_data(sk); 2416b29fcfb5SPaolo Abeni if (!dispose_it) { 241714c56686SGeliang Tang __mptcp_subflow_disconnect(ssk, subflow, flags); 2418b29fcfb5SPaolo Abeni release_sock(ssk); 2419b29fcfb5SPaolo Abeni 2420b29fcfb5SPaolo Abeni goto out; 2421b29fcfb5SPaolo Abeni } 2422b29fcfb5SPaolo Abeni 2423d7b1bfd0SPaolo Abeni subflow->disposable = 1; 2424d7b1bfd0SPaolo Abeni 2425e16163b6SPaolo Abeni /* if ssk hit tcp_done(), tcp_cleanup_ulp() cleared the related ops 2426e16163b6SPaolo Abeni * the ssk has been already destroyed, we just need to release the 2427e16163b6SPaolo Abeni * reference owned by msk; 2428e16163b6SPaolo Abeni */ 2429e16163b6SPaolo Abeni if (!inet_csk(ssk)->icsk_ulp_ops) { 2430b6985b9bSPaolo Abeni WARN_ON_ONCE(!sock_flag(ssk, SOCK_DEAD)); 2431e16163b6SPaolo Abeni kfree_rcu(subflow, rcu); 2432e16163b6SPaolo Abeni } else { 2433d7b1bfd0SPaolo Abeni /* otherwise tcp will dispose of the ssk and subflow ctx */ 2434e16163b6SPaolo Abeni __tcp_close(ssk, 0); 2435e16163b6SPaolo Abeni 2436e16163b6SPaolo Abeni /* close acquired an extra ref */ 2437e16163b6SPaolo Abeni __sock_put(ssk); 2438e16163b6SPaolo Abeni } 243963740448SPaolo Abeni 244063740448SPaolo Abeni out_release: 24419f1a9881SPaolo Abeni __mptcp_subflow_error_report(sk, ssk); 2442e16163b6SPaolo Abeni release_sock(ssk); 2443e16163b6SPaolo Abeni 2444e16163b6SPaolo Abeni sock_put(ssk); 2445e0be4931SFlorian Westphal 2446c8fe62f0SFlorian Westphal if (ssk == msk->first) 24471b1b43eeSPaolo Abeni WRITE_ONCE(msk->first, NULL); 2448c8fe62f0SFlorian Westphal 2449b29fcfb5SPaolo Abeni out: 24508005184fSPaolo Abeni __mptcp_sync_sndbuf(sk); 24511e1d9d6fSPaolo Abeni if (need_push) 24521e1d9d6fSPaolo Abeni __mptcp_push_pending(sk, 0); 245327e5ccc2SPaolo Abeni 245427e5ccc2SPaolo Abeni /* Catch every 'all subflows closed' scenario, including peers silently 245527e5ccc2SPaolo Abeni * closing them, e.g. due to timeout. 245627e5ccc2SPaolo Abeni * For established sockets, allow an additional timeout before closing, 245727e5ccc2SPaolo Abeni * as the protocol can still create more subflows. 245827e5ccc2SPaolo Abeni */ 245927e5ccc2SPaolo Abeni if (list_is_singular(&msk->conn_list) && msk->first && 246027e5ccc2SPaolo Abeni inet_sk_state_load(msk->first) == TCP_CLOSE) { 246127e5ccc2SPaolo Abeni if (sk->sk_state != TCP_ESTABLISHED || 246227e5ccc2SPaolo Abeni msk->in_accept_queue || sock_flag(sk, SOCK_DEAD)) { 2463c693a851SGeliang Tang mptcp_set_state(sk, TCP_CLOSE); 246427e5ccc2SPaolo Abeni mptcp_close_wake_up(sk); 246527e5ccc2SPaolo Abeni } else { 246627e5ccc2SPaolo Abeni mptcp_start_tout_timer(sk); 246727e5ccc2SPaolo Abeni } 246827e5ccc2SPaolo Abeni } 2469f870fa0bSMat Martineau } 2470f870fa0bSMat Martineau 2471a141e02eSFlorian Westphal void mptcp_close_ssk(struct sock *sk, struct sock *ssk, 2472a141e02eSFlorian Westphal struct mptcp_subflow_context *subflow) 2473a141e02eSFlorian Westphal { 2474b911c97cSFlorian Westphal if (sk->sk_state == TCP_ESTABLISHED) 2475b911c97cSFlorian Westphal mptcp_event(MPTCP_EVENT_SUB_CLOSED, mptcp_sk(sk), ssk, GFP_KERNEL); 2476a88c9e49SPaolo Abeni 2477a88c9e49SPaolo Abeni /* subflow aborted before reaching the fully_established status 2478a88c9e49SPaolo Abeni * attempt the creation of the next subflow 2479a88c9e49SPaolo Abeni */ 248074cbb0c6SGeliang Tang mptcp_pm_subflow_check_next(mptcp_sk(sk), subflow); 2481a88c9e49SPaolo Abeni 2482b29fcfb5SPaolo Abeni __mptcp_close_ssk(sk, ssk, subflow, MPTCP_CF_PUSH); 2483a141e02eSFlorian Westphal } 2484a141e02eSFlorian Westphal 2485dc24f8b4SPaolo Abeni static unsigned int mptcp_sync_mss(struct sock *sk, u32 pmtu) 2486dc24f8b4SPaolo Abeni { 2487dc24f8b4SPaolo Abeni return 0; 2488dc24f8b4SPaolo Abeni } 2489dc24f8b4SPaolo Abeni 2490b6985b9bSPaolo Abeni static void __mptcp_close_subflow(struct sock *sk) 24910e4f35d7SPaolo Abeni { 24920e4f35d7SPaolo Abeni struct mptcp_subflow_context *subflow, *tmp; 2493b6985b9bSPaolo Abeni struct mptcp_sock *msk = mptcp_sk(sk); 24940e4f35d7SPaolo Abeni 24953abc05d9SFlorian Westphal might_sleep(); 24963abc05d9SFlorian Westphal 24975efbf6f7SMatthieu Baerts mptcp_for_each_subflow_safe(msk, subflow, tmp) { 24980e4f35d7SPaolo Abeni struct sock *ssk = mptcp_subflow_tcp_sock(subflow); 24990e4f35d7SPaolo Abeni 25000e4f35d7SPaolo Abeni if (inet_sk_state_load(ssk) != TCP_CLOSE) 25010e4f35d7SPaolo Abeni continue; 25020e4f35d7SPaolo Abeni 250340947e13SFlorian Westphal /* 'subflow_data_ready' will re-sched once rx queue is empty */ 250440947e13SFlorian Westphal if (!skb_queue_empty_lockless(&ssk->sk_receive_queue)) 250540947e13SFlorian Westphal continue; 250640947e13SFlorian Westphal 2507b6985b9bSPaolo Abeni mptcp_close_ssk(sk, ssk, subflow); 2508b6985b9bSPaolo Abeni } 2509b6985b9bSPaolo Abeni 25100e4f35d7SPaolo Abeni } 25110e4f35d7SPaolo Abeni 251227e5ccc2SPaolo Abeni static bool mptcp_close_tout_expired(const struct sock *sk) 2513e16163b6SPaolo Abeni { 251427e5ccc2SPaolo Abeni if (!inet_csk(sk)->icsk_mtup.probe_timestamp || 251527e5ccc2SPaolo Abeni sk->sk_state == TCP_CLOSE) 2516e16163b6SPaolo Abeni return false; 251727e5ccc2SPaolo Abeni 251827e5ccc2SPaolo Abeni return time_after32(tcp_jiffies32, 2519d866ae9aSPaolo Abeni inet_csk(sk)->icsk_mtup.probe_timestamp + mptcp_close_timeout(sk)); 2520e16163b6SPaolo Abeni } 2521e16163b6SPaolo Abeni 252250c504a2SFlorian Westphal static void mptcp_check_fastclose(struct mptcp_sock *msk) 252350c504a2SFlorian Westphal { 252450c504a2SFlorian Westphal struct mptcp_subflow_context *subflow, *tmp; 252580638684SGeliang Tang struct sock *sk = (struct sock *)msk; 252650c504a2SFlorian Westphal 252750c504a2SFlorian Westphal if (likely(!READ_ONCE(msk->rcv_fastclose))) 252850c504a2SFlorian Westphal return; 252950c504a2SFlorian Westphal 253050c504a2SFlorian Westphal mptcp_token_destroy(msk); 253150c504a2SFlorian Westphal 25325efbf6f7SMatthieu Baerts mptcp_for_each_subflow_safe(msk, subflow, tmp) { 253350c504a2SFlorian Westphal struct sock *tcp_sk = mptcp_subflow_tcp_sock(subflow); 253475e908c3SPaolo Abeni bool slow; 253550c504a2SFlorian Westphal 253675e908c3SPaolo Abeni slow = lock_sock_fast(tcp_sk); 253750c504a2SFlorian Westphal if (tcp_sk->sk_state != TCP_CLOSE) { 253850c504a2SFlorian Westphal tcp_send_active_reset(tcp_sk, GFP_ATOMIC); 253950c504a2SFlorian Westphal tcp_set_state(tcp_sk, TCP_CLOSE); 254050c504a2SFlorian Westphal } 254175e908c3SPaolo Abeni unlock_sock_fast(tcp_sk, slow); 254250c504a2SFlorian Westphal } 254350c504a2SFlorian Westphal 254469800e51SPaolo Abeni /* Mirror the tcp_reset() error propagation */ 254569800e51SPaolo Abeni switch (sk->sk_state) { 254669800e51SPaolo Abeni case TCP_SYN_SENT: 25479ae8e5adSEric Dumazet WRITE_ONCE(sk->sk_err, ECONNREFUSED); 254869800e51SPaolo Abeni break; 254969800e51SPaolo Abeni case TCP_CLOSE_WAIT: 25509ae8e5adSEric Dumazet WRITE_ONCE(sk->sk_err, EPIPE); 255169800e51SPaolo Abeni break; 255269800e51SPaolo Abeni case TCP_CLOSE: 255369800e51SPaolo Abeni return; 255469800e51SPaolo Abeni default: 25559ae8e5adSEric Dumazet WRITE_ONCE(sk->sk_err, ECONNRESET); 255669800e51SPaolo Abeni } 255769800e51SPaolo Abeni 2558c693a851SGeliang Tang mptcp_set_state(sk, TCP_CLOSE); 25596b9831bfSPaolo Abeni WRITE_ONCE(sk->sk_shutdown, SHUTDOWN_MASK); 256050c504a2SFlorian Westphal smp_mb__before_atomic(); /* SHUTDOWN must be visible first */ 256150c504a2SFlorian Westphal set_bit(MPTCP_WORK_CLOSE_SUBFLOW, &msk->flags); 256250c504a2SFlorian Westphal 256369800e51SPaolo Abeni /* the calling mptcp_worker will properly destroy the socket */ 256469800e51SPaolo Abeni if (sock_flag(sk, SOCK_DEAD)) 256569800e51SPaolo Abeni return; 256669800e51SPaolo Abeni 256769800e51SPaolo Abeni sk->sk_state_change(sk); 256869800e51SPaolo Abeni sk_error_report(sk); 256950c504a2SFlorian Westphal } 257050c504a2SFlorian Westphal 25712948d0a1SPaolo Abeni static void __mptcp_retrans(struct sock *sk) 257280992017SPaolo Abeni { 25732948d0a1SPaolo Abeni struct mptcp_sock *msk = mptcp_sk(sk); 2574ee2708aeSGeliang Tang struct mptcp_subflow_context *subflow; 2575caf971dfSPaolo Abeni struct mptcp_sendmsg_info info = {}; 25763b1d6210SPaolo Abeni struct mptcp_data_frag *dfrag; 25772948d0a1SPaolo Abeni struct sock *ssk; 2578ee2708aeSGeliang Tang int ret, err; 2579ee2708aeSGeliang Tang u16 len = 0; 25802948d0a1SPaolo Abeni 2581b5941f06SPaolo Abeni mptcp_clean_una_wakeup(sk); 25823241a9c0SFlorian Westphal 25833241a9c0SFlorian Westphal /* first check ssk: need to kick "stale" logic */ 2584ee2708aeSGeliang Tang err = mptcp_sched_get_retrans(msk); 25852948d0a1SPaolo Abeni dfrag = mptcp_rtx_head(sk); 25866477dd39SMat Martineau if (!dfrag) { 25876477dd39SMat Martineau if (mptcp_data_fin_enabled(msk)) { 25886477dd39SMat Martineau struct inet_connection_sock *icsk = inet_csk(sk); 25896477dd39SMat Martineau 25906477dd39SMat Martineau icsk->icsk_retransmits++; 25916477dd39SMat Martineau mptcp_set_datafin_timeout(sk); 25926477dd39SMat Martineau mptcp_send_ack(msk); 25936477dd39SMat Martineau 25946477dd39SMat Martineau goto reset_timer; 25956477dd39SMat Martineau } 25966477dd39SMat Martineau 25973241a9c0SFlorian Westphal if (!mptcp_send_head(sk)) 25982948d0a1SPaolo Abeni return; 25993241a9c0SFlorian Westphal 26003241a9c0SFlorian Westphal goto reset_timer; 26016477dd39SMat Martineau } 26022948d0a1SPaolo Abeni 2603ee2708aeSGeliang Tang if (err) 26042948d0a1SPaolo Abeni goto reset_timer; 26052948d0a1SPaolo Abeni 2606ee2708aeSGeliang Tang mptcp_for_each_subflow(msk, subflow) { 2607ee2708aeSGeliang Tang if (READ_ONCE(subflow->scheduled)) { 2608ee2708aeSGeliang Tang u16 copied = 0; 2609ee2708aeSGeliang Tang 2610ee2708aeSGeliang Tang mptcp_subflow_set_scheduled(subflow, false); 2611ee2708aeSGeliang Tang 2612ee2708aeSGeliang Tang ssk = mptcp_subflow_tcp_sock(subflow); 2613ee2708aeSGeliang Tang 26142948d0a1SPaolo Abeni lock_sock(ssk); 26152948d0a1SPaolo Abeni 26162948d0a1SPaolo Abeni /* limit retransmission to the bytes already sent on some subflows */ 26172948d0a1SPaolo Abeni info.sent = 0; 2618ee2708aeSGeliang Tang info.limit = READ_ONCE(msk->csum_enabled) ? dfrag->data_len : 2619ee2708aeSGeliang Tang dfrag->already_sent; 26204e14867dSPaolo Abeni while (info.sent < info.limit) { 26212948d0a1SPaolo Abeni ret = mptcp_sendmsg_frag(sk, ssk, dfrag, &info); 26222948d0a1SPaolo Abeni if (ret <= 0) 26232948d0a1SPaolo Abeni break; 26242948d0a1SPaolo Abeni 26252948d0a1SPaolo Abeni MPTCP_INC_STATS(sock_net(sk), MPTCP_MIB_RETRANSSEGS); 26262948d0a1SPaolo Abeni copied += ret; 26272948d0a1SPaolo Abeni info.sent += ret; 26282948d0a1SPaolo Abeni } 26294e14867dSPaolo Abeni if (copied) { 2630ee2708aeSGeliang Tang len = max(copied, len); 26312948d0a1SPaolo Abeni tcp_push(ssk, 0, info.mss_now, tcp_sk(ssk)->nonagle, 26322948d0a1SPaolo Abeni info.size_goal); 26330530020aSGeliang Tang WRITE_ONCE(msk->allow_infinite_fallback, false); 26344e14867dSPaolo Abeni } 26352948d0a1SPaolo Abeni 26362948d0a1SPaolo Abeni release_sock(ssk); 2637ee2708aeSGeliang Tang } 2638ee2708aeSGeliang Tang } 2639ee2708aeSGeliang Tang 2640ee2708aeSGeliang Tang msk->bytes_retrans += len; 2641ee2708aeSGeliang Tang dfrag->already_sent = max(dfrag->already_sent, len); 26422948d0a1SPaolo Abeni 26432948d0a1SPaolo Abeni reset_timer: 26443241a9c0SFlorian Westphal mptcp_check_and_set_pending(sk); 26453241a9c0SFlorian Westphal 2646f6909dc1SPaolo Abeni if (!mptcp_rtx_timer_pending(sk)) 2647f6909dc1SPaolo Abeni mptcp_reset_rtx_timer(sk); 26482948d0a1SPaolo Abeni } 26492948d0a1SPaolo Abeni 265076a13b31SGeliang Tang /* schedule the timeout timer for the relevant event: either close timeout 265176a13b31SGeliang Tang * or mp_fail timeout. The close timeout takes precedence on the mp_fail one 265276a13b31SGeliang Tang */ 2653f6909dc1SPaolo Abeni void mptcp_reset_tout_timer(struct mptcp_sock *msk, unsigned long fail_tout) 265476a13b31SGeliang Tang { 265576a13b31SGeliang Tang struct sock *sk = (struct sock *)msk; 265676a13b31SGeliang Tang unsigned long timeout, close_timeout; 265776a13b31SGeliang Tang 265827e5ccc2SPaolo Abeni if (!fail_tout && !inet_csk(sk)->icsk_mtup.probe_timestamp) 265976a13b31SGeliang Tang return; 266076a13b31SGeliang Tang 266127e5ccc2SPaolo Abeni close_timeout = inet_csk(sk)->icsk_mtup.probe_timestamp - tcp_jiffies32 + jiffies + 2662d866ae9aSPaolo Abeni mptcp_close_timeout(sk); 266376a13b31SGeliang Tang 266476a13b31SGeliang Tang /* the close timeout takes precedence on the fail one, and here at least one of 266576a13b31SGeliang Tang * them is active 266676a13b31SGeliang Tang */ 266727e5ccc2SPaolo Abeni timeout = inet_csk(sk)->icsk_mtup.probe_timestamp ? close_timeout : fail_tout; 266876a13b31SGeliang Tang 266976a13b31SGeliang Tang sk_reset_timer(sk, &sk->sk_timer, timeout); 267076a13b31SGeliang Tang } 267176a13b31SGeliang Tang 267249fa1919SGeliang Tang static void mptcp_mp_fail_no_response(struct mptcp_sock *msk) 267349fa1919SGeliang Tang { 267476a13b31SGeliang Tang struct sock *ssk = msk->first; 267549fa1919SGeliang Tang bool slow; 267649fa1919SGeliang Tang 267776a13b31SGeliang Tang if (!ssk) 267876a13b31SGeliang Tang return; 267976a13b31SGeliang Tang 268049fa1919SGeliang Tang pr_debug("MP_FAIL doesn't respond, reset the subflow"); 268149fa1919SGeliang Tang 268249fa1919SGeliang Tang slow = lock_sock_fast(ssk); 268349fa1919SGeliang Tang mptcp_subflow_reset(ssk); 268476a13b31SGeliang Tang WRITE_ONCE(mptcp_subflow_ctx(ssk)->fail_tout, 0); 268549fa1919SGeliang Tang unlock_sock_fast(ssk, slow); 268649fa1919SGeliang Tang } 268749fa1919SGeliang Tang 2688d21f8348SPaolo Abeni static void mptcp_do_fastclose(struct sock *sk) 2689d21f8348SPaolo Abeni { 2690d21f8348SPaolo Abeni struct mptcp_subflow_context *subflow, *tmp; 2691d21f8348SPaolo Abeni struct mptcp_sock *msk = mptcp_sk(sk); 2692d21f8348SPaolo Abeni 2693c693a851SGeliang Tang mptcp_set_state(sk, TCP_CLOSE); 2694d21f8348SPaolo Abeni mptcp_for_each_subflow_safe(msk, subflow, tmp) 2695d21f8348SPaolo Abeni __mptcp_close_ssk(sk, mptcp_subflow_tcp_sock(subflow), 2696d21f8348SPaolo Abeni subflow, MPTCP_CF_FASTCLOSE); 2697d21f8348SPaolo Abeni } 2698d21f8348SPaolo Abeni 26992948d0a1SPaolo Abeni static void mptcp_worker(struct work_struct *work) 27002948d0a1SPaolo Abeni { 27012948d0a1SPaolo Abeni struct mptcp_sock *msk = container_of(work, struct mptcp_sock, work); 270280638684SGeliang Tang struct sock *sk = (struct sock *)msk; 270376a13b31SGeliang Tang unsigned long fail_tout; 27042948d0a1SPaolo Abeni int state; 270580992017SPaolo Abeni 270680992017SPaolo Abeni lock_sock(sk); 2707e16163b6SPaolo Abeni state = sk->sk_state; 2708d6a04437SPaolo Abeni if (unlikely((1 << state) & (TCPF_CLOSE | TCPF_LISTEN))) 2709e16163b6SPaolo Abeni goto unlock; 2710e16163b6SPaolo Abeni 271150c504a2SFlorian Westphal mptcp_check_fastclose(msk); 271250c504a2SFlorian Westphal 2713e9801430SFlorian Westphal mptcp_pm_nl_work(msk); 2714b416268bSFlorian Westphal 271581c1d029SPaolo Abeni mptcp_check_send_data_fin(sk); 271681c1d029SPaolo Abeni mptcp_check_data_fin_ack(sk); 271743b54c6eSMat Martineau mptcp_check_data_fin(sk); 271843b54c6eSMat Martineau 2719b6985b9bSPaolo Abeni if (test_and_clear_bit(MPTCP_WORK_CLOSE_SUBFLOW, &msk->flags)) 2720b6985b9bSPaolo Abeni __mptcp_close_subflow(sk); 2721b6985b9bSPaolo Abeni 272227e5ccc2SPaolo Abeni if (mptcp_close_tout_expired(sk)) { 2723d21f8348SPaolo Abeni mptcp_do_fastclose(sk); 272427e5ccc2SPaolo Abeni mptcp_close_wake_up(sk); 272527e5ccc2SPaolo Abeni } 2726bbd49d11SPaolo Abeni 272727e5ccc2SPaolo Abeni if (sock_flag(sk, SOCK_DEAD) && sk->sk_state == TCP_CLOSE) { 2728e16163b6SPaolo Abeni __mptcp_destroy_sock(sk); 2729e16163b6SPaolo Abeni goto unlock; 2730e16163b6SPaolo Abeni } 2731e16163b6SPaolo Abeni 27322948d0a1SPaolo Abeni if (test_and_clear_bit(MPTCP_WORK_RTX, &msk->flags)) 27332948d0a1SPaolo Abeni __mptcp_retrans(sk); 27343b1d6210SPaolo Abeni 273576a13b31SGeliang Tang fail_tout = msk->first ? READ_ONCE(mptcp_subflow_ctx(msk->first)->fail_tout) : 0; 273676a13b31SGeliang Tang if (fail_tout && time_after(jiffies, fail_tout)) 273749fa1919SGeliang Tang mptcp_mp_fail_no_response(msk); 273849fa1919SGeliang Tang 27393b1d6210SPaolo Abeni unlock: 274080992017SPaolo Abeni release_sock(sk); 274180992017SPaolo Abeni sock_put(sk); 274280992017SPaolo Abeni } 274380992017SPaolo Abeni 2744e2636917SKuniyuki Iwashima static void __mptcp_init_sock(struct sock *sk) 2745f870fa0bSMat Martineau { 2746cec37a6eSPeter Krystad struct mptcp_sock *msk = mptcp_sk(sk); 2747cec37a6eSPeter Krystad 2748cec37a6eSPeter Krystad INIT_LIST_HEAD(&msk->conn_list); 2749ec3edaa7SPeter Krystad INIT_LIST_HEAD(&msk->join_list); 275018b683bfSPaolo Abeni INIT_LIST_HEAD(&msk->rtx_queue); 275180992017SPaolo Abeni INIT_WORK(&msk->work, mptcp_worker); 275287952603SPaolo Abeni __skb_queue_head_init(&msk->receive_queue); 2753ab174ad8SPaolo Abeni msk->out_of_order_queue = RB_ROOT; 2754f0e6a4cfSPaolo Abeni msk->first_pending = NULL; 27556511882cSPaolo Abeni msk->rmem_fwd_alloc = 0; 2756ce599c51SPaolo Abeni WRITE_ONCE(msk->rmem_released, 0); 275733d41c9cSPaolo Abeni msk->timer_ival = TCP_RTO_MIN; 27585684ab1aSPaolo Abeni msk->scaling_ratio = TCP_DEFAULT_SCALING_RATIO; 2759cec37a6eSPeter Krystad 27601b1b43eeSPaolo Abeni WRITE_ONCE(msk->first, NULL); 2761dc24f8b4SPaolo Abeni inet_csk(sk)->icsk_sync_mss = mptcp_sync_mss; 2762752e9067SGeliang Tang WRITE_ONCE(msk->csum_enabled, mptcp_is_checksum_enabled(sock_net(sk))); 27630530020aSGeliang Tang WRITE_ONCE(msk->allow_infinite_fallback, true); 27641e1d9d6fSPaolo Abeni msk->recovery = false; 27656f06b4d4SPaolo Abeni msk->subflow_id = 1; 27668ab183deSPaolo Abeni 27671b1c7a0eSPeter Krystad mptcp_pm_data_init(msk); 27681b1c7a0eSPeter Krystad 2769b51f9b80SPaolo Abeni /* re-use the csk retrans timer for MPTCP-level retrans */ 2770b51f9b80SPaolo Abeni timer_setup(&msk->sk.icsk_retransmit_timer, mptcp_retransmit_timer, 0); 2771f6909dc1SPaolo Abeni timer_setup(&sk->sk_timer, mptcp_tout_timer, 0); 2772f870fa0bSMat Martineau } 2773f870fa0bSMat Martineau 2774b29fcfb5SPaolo Abeni static void mptcp_ca_reset(struct sock *sk) 2775784325e9SMatthieu Baerts { 277620b5759fSPaolo Abeni struct inet_connection_sock *icsk = inet_csk(sk); 2777b29fcfb5SPaolo Abeni 2778b29fcfb5SPaolo Abeni tcp_assign_congestion_control(sk); 2779b29fcfb5SPaolo Abeni strcpy(mptcp_sk(sk)->ca_name, icsk->icsk_ca_ops->name); 2780b29fcfb5SPaolo Abeni 2781b29fcfb5SPaolo Abeni /* no need to keep a reference to the ops, the name will suffice */ 2782b29fcfb5SPaolo Abeni tcp_cleanup_congestion_control(sk); 2783b29fcfb5SPaolo Abeni icsk->icsk_ca_ops = NULL; 2784b29fcfb5SPaolo Abeni } 2785b29fcfb5SPaolo Abeni 2786b29fcfb5SPaolo Abeni static int mptcp_init_sock(struct sock *sk) 2787b29fcfb5SPaolo Abeni { 2788fc518953SFlorian Westphal struct net *net = sock_net(sk); 27891730b2b2SGeliang Tang int ret; 279018b683bfSPaolo Abeni 2791e2636917SKuniyuki Iwashima __mptcp_init_sock(sk); 2792b6c08380SGeliang Tang 2793fc518953SFlorian Westphal if (!mptcp_is_enabled(net)) 2794fc518953SFlorian Westphal return -ENOPROTOOPT; 2795fc518953SFlorian Westphal 2796fc518953SFlorian Westphal if (unlikely(!net->mib.mptcp_statistics) && !mptcp_mib_alloc(net)) 2797fc518953SFlorian Westphal return -ENOMEM; 2798fc518953SFlorian Westphal 27991730b2b2SGeliang Tang ret = mptcp_init_sched(mptcp_sk(sk), 28001730b2b2SGeliang Tang mptcp_sched_find(mptcp_get_scheduler(net))); 28011730b2b2SGeliang Tang if (ret) 28021730b2b2SGeliang Tang return ret; 28031730b2b2SGeliang Tang 2804a5ef058dSPaolo Abeni set_bit(SOCK_CUSTOM_SOCKOPT, &sk->sk_socket->flags); 2805a5ef058dSPaolo Abeni 280620b5759fSPaolo Abeni /* fetch the ca name; do it outside __mptcp_init_sock(), so that clone will 280720b5759fSPaolo Abeni * propagate the correct value 280820b5759fSPaolo Abeni */ 2809b29fcfb5SPaolo Abeni mptcp_ca_reset(sk); 281020b5759fSPaolo Abeni 2811d027236cSPaolo Abeni sk_sockets_allocated_inc(sk); 2812a963853fSGeliang Tang sk->sk_rcvbuf = READ_ONCE(net->ipv4.sysctl_tcp_rmem[1]); 2813a963853fSGeliang Tang sk->sk_sndbuf = READ_ONCE(net->ipv4.sysctl_tcp_wmem[1]); 2814d027236cSPaolo Abeni 281518b683bfSPaolo Abeni return 0; 281618b683bfSPaolo Abeni } 281718b683bfSPaolo Abeni 281818b683bfSPaolo Abeni static void __mptcp_clear_xmit(struct sock *sk) 281918b683bfSPaolo Abeni { 282018b683bfSPaolo Abeni struct mptcp_sock *msk = mptcp_sk(sk); 282118b683bfSPaolo Abeni struct mptcp_data_frag *dtmp, *dfrag; 282218b683bfSPaolo Abeni 2823d9ca1de8SPaolo Abeni WRITE_ONCE(msk->first_pending, NULL); 282418b683bfSPaolo Abeni list_for_each_entry_safe(dfrag, dtmp, &msk->rtx_queue, list) 2825d027236cSPaolo Abeni dfrag_clear(sk, dfrag); 2826784325e9SMatthieu Baerts } 2827784325e9SMatthieu Baerts 282830e51b92SMenglong Dong void mptcp_cancel_work(struct sock *sk) 282980992017SPaolo Abeni { 283080992017SPaolo Abeni struct mptcp_sock *msk = mptcp_sk(sk); 283180992017SPaolo Abeni 2832b2771d24SPaolo Abeni if (cancel_work_sync(&msk->work)) 2833e16163b6SPaolo Abeni __sock_put(sk); 283480992017SPaolo Abeni } 283580992017SPaolo Abeni 2836d0876b22SGeliang Tang void mptcp_subflow_shutdown(struct sock *sk, struct sock *ssk, int how) 283721498490SPeter Krystad { 283821498490SPeter Krystad lock_sock(ssk); 283921498490SPeter Krystad 284021498490SPeter Krystad switch (ssk->sk_state) { 284121498490SPeter Krystad case TCP_LISTEN: 284221498490SPeter Krystad if (!(how & RCV_SHUTDOWN)) 284321498490SPeter Krystad break; 2844df561f66SGustavo A. R. Silva fallthrough; 284521498490SPeter Krystad case TCP_SYN_SENT: 2846c2b2ae39SPaolo Abeni WARN_ON_ONCE(tcp_disconnect(ssk, O_NONBLOCK)); 284721498490SPeter Krystad break; 284821498490SPeter Krystad default: 284943b54c6eSMat Martineau if (__mptcp_check_fallback(mptcp_sk(sk))) { 285043b54c6eSMat Martineau pr_debug("Fallback"); 285121498490SPeter Krystad ssk->sk_shutdown |= how; 285221498490SPeter Krystad tcp_shutdown(ssk, how); 285381c1d029SPaolo Abeni 285481c1d029SPaolo Abeni /* simulate the data_fin ack reception to let the state 285581c1d029SPaolo Abeni * machine move forward 285681c1d029SPaolo Abeni */ 285781c1d029SPaolo Abeni WRITE_ONCE(mptcp_sk(sk)->snd_una, mptcp_sk(sk)->snd_nxt); 285881c1d029SPaolo Abeni mptcp_schedule_work(sk); 285943b54c6eSMat Martineau } else { 286043b54c6eSMat Martineau pr_debug("Sending DATA_FIN on subflow %p", ssk); 286143b54c6eSMat Martineau tcp_send_ack(ssk); 2862f6909dc1SPaolo Abeni if (!mptcp_rtx_timer_pending(sk)) 2863f6909dc1SPaolo Abeni mptcp_reset_rtx_timer(sk); 286443b54c6eSMat Martineau } 286521498490SPeter Krystad break; 286621498490SPeter Krystad } 286721498490SPeter Krystad 286821498490SPeter Krystad release_sock(ssk); 286921498490SPeter Krystad } 287021498490SPeter Krystad 2871d9cd27b8SGeliang Tang void mptcp_set_state(struct sock *sk, int state) 2872d9cd27b8SGeliang Tang { 2873d9cd27b8SGeliang Tang int oldstate = sk->sk_state; 2874d9cd27b8SGeliang Tang 2875d9cd27b8SGeliang Tang switch (state) { 2876d9cd27b8SGeliang Tang case TCP_ESTABLISHED: 2877d9cd27b8SGeliang Tang if (oldstate != TCP_ESTABLISHED) 2878d9cd27b8SGeliang Tang MPTCP_INC_STATS(sock_net(sk), MPTCP_MIB_CURRESTAB); 2879d9cd27b8SGeliang Tang break; 2880d9cd27b8SGeliang Tang 2881d9cd27b8SGeliang Tang default: 2882d9cd27b8SGeliang Tang if (oldstate == TCP_ESTABLISHED) 2883d9cd27b8SGeliang Tang MPTCP_DEC_STATS(sock_net(sk), MPTCP_MIB_CURRESTAB); 2884d9cd27b8SGeliang Tang } 2885d9cd27b8SGeliang Tang 2886d9cd27b8SGeliang Tang inet_sk_state_store(sk, state); 2887d9cd27b8SGeliang Tang } 2888d9cd27b8SGeliang Tang 28896920b851SMat Martineau static const unsigned char new_state[16] = { 28906920b851SMat Martineau /* current state: new state: action: */ 28916920b851SMat Martineau [0 /* (Invalid) */] = TCP_CLOSE, 28926920b851SMat Martineau [TCP_ESTABLISHED] = TCP_FIN_WAIT1 | TCP_ACTION_FIN, 28936920b851SMat Martineau [TCP_SYN_SENT] = TCP_CLOSE, 28946920b851SMat Martineau [TCP_SYN_RECV] = TCP_FIN_WAIT1 | TCP_ACTION_FIN, 28956920b851SMat Martineau [TCP_FIN_WAIT1] = TCP_FIN_WAIT1, 28966920b851SMat Martineau [TCP_FIN_WAIT2] = TCP_FIN_WAIT2, 28976920b851SMat Martineau [TCP_TIME_WAIT] = TCP_CLOSE, /* should not happen ! */ 28986920b851SMat Martineau [TCP_CLOSE] = TCP_CLOSE, 28996920b851SMat Martineau [TCP_CLOSE_WAIT] = TCP_LAST_ACK | TCP_ACTION_FIN, 29006920b851SMat Martineau [TCP_LAST_ACK] = TCP_LAST_ACK, 29016920b851SMat Martineau [TCP_LISTEN] = TCP_CLOSE, 29026920b851SMat Martineau [TCP_CLOSING] = TCP_CLOSING, 29036920b851SMat Martineau [TCP_NEW_SYN_RECV] = TCP_CLOSE, /* should not happen ! */ 29046920b851SMat Martineau }; 29056920b851SMat Martineau 29066920b851SMat Martineau static int mptcp_close_state(struct sock *sk) 29076920b851SMat Martineau { 29086920b851SMat Martineau int next = (int)new_state[sk->sk_state]; 29096920b851SMat Martineau int ns = next & TCP_STATE_MASK; 29106920b851SMat Martineau 2911c693a851SGeliang Tang mptcp_set_state(sk, ns); 29126920b851SMat Martineau 29136920b851SMat Martineau return next & TCP_ACTION_FIN; 29146920b851SMat Martineau } 29156920b851SMat Martineau 291681c1d029SPaolo Abeni static void mptcp_check_send_data_fin(struct sock *sk) 2917e16163b6SPaolo Abeni { 2918e16163b6SPaolo Abeni struct mptcp_subflow_context *subflow; 2919e16163b6SPaolo Abeni struct mptcp_sock *msk = mptcp_sk(sk); 2920e16163b6SPaolo Abeni 2921e16163b6SPaolo Abeni pr_debug("msk=%p snd_data_fin_enable=%d pending=%d snd_nxt=%llu write_seq=%llu", 2922e16163b6SPaolo Abeni msk, msk->snd_data_fin_enable, !!mptcp_send_head(sk), 2923e16163b6SPaolo Abeni msk->snd_nxt, msk->write_seq); 2924e16163b6SPaolo Abeni 2925e16163b6SPaolo Abeni /* we still need to enqueue subflows or not really shutting down, 2926e16163b6SPaolo Abeni * skip this 2927e16163b6SPaolo Abeni */ 2928e16163b6SPaolo Abeni if (!msk->snd_data_fin_enable || msk->snd_nxt + 1 != msk->write_seq || 2929e16163b6SPaolo Abeni mptcp_send_head(sk)) 2930e16163b6SPaolo Abeni return; 2931e16163b6SPaolo Abeni 2932e16163b6SPaolo Abeni WRITE_ONCE(msk->snd_nxt, msk->write_seq); 2933e16163b6SPaolo Abeni 2934e16163b6SPaolo Abeni mptcp_for_each_subflow(msk, subflow) { 2935e16163b6SPaolo Abeni struct sock *tcp_sk = mptcp_subflow_tcp_sock(subflow); 2936e16163b6SPaolo Abeni 2937e16163b6SPaolo Abeni mptcp_subflow_shutdown(sk, tcp_sk, SEND_SHUTDOWN); 2938e16163b6SPaolo Abeni } 2939e16163b6SPaolo Abeni } 2940e16163b6SPaolo Abeni 2941e16163b6SPaolo Abeni static void __mptcp_wr_shutdown(struct sock *sk) 2942e16163b6SPaolo Abeni { 2943e16163b6SPaolo Abeni struct mptcp_sock *msk = mptcp_sk(sk); 2944e16163b6SPaolo Abeni 2945e16163b6SPaolo Abeni pr_debug("msk=%p snd_data_fin_enable=%d shutdown=%x state=%d pending=%d", 2946e16163b6SPaolo Abeni msk, msk->snd_data_fin_enable, sk->sk_shutdown, sk->sk_state, 2947e16163b6SPaolo Abeni !!mptcp_send_head(sk)); 2948e16163b6SPaolo Abeni 2949e16163b6SPaolo Abeni /* will be ignored by fallback sockets */ 2950e16163b6SPaolo Abeni WRITE_ONCE(msk->write_seq, msk->write_seq + 1); 2951e16163b6SPaolo Abeni WRITE_ONCE(msk->snd_data_fin_enable, 1); 2952e16163b6SPaolo Abeni 295381c1d029SPaolo Abeni mptcp_check_send_data_fin(sk); 2954e16163b6SPaolo Abeni } 2955e16163b6SPaolo Abeni 2956e16163b6SPaolo Abeni static void __mptcp_destroy_sock(struct sock *sk) 2957f870fa0bSMat Martineau { 2958f870fa0bSMat Martineau struct mptcp_sock *msk = mptcp_sk(sk); 2959f870fa0bSMat Martineau 2960e16163b6SPaolo Abeni pr_debug("msk=%p", msk); 29612c22c06cSFlorian Westphal 29623abc05d9SFlorian Westphal might_sleep(); 29633abc05d9SFlorian Westphal 2964f6909dc1SPaolo Abeni mptcp_stop_rtx_timer(sk); 2965e16163b6SPaolo Abeni sk_stop_timer(sk, &sk->sk_timer); 2966e16163b6SPaolo Abeni msk->pm.status = 0; 29671730b2b2SGeliang Tang mptcp_release_sched(msk); 2968b2c5b614SFlorian Westphal 2969e16163b6SPaolo Abeni sk->sk_prot->destroy(sk); 2970e16163b6SPaolo Abeni 29716511882cSPaolo Abeni WARN_ON_ONCE(msk->rmem_fwd_alloc); 297287952603SPaolo Abeni WARN_ON_ONCE(msk->rmem_released); 2973e16163b6SPaolo Abeni sk_stream_kill_queues(sk); 2974e16163b6SPaolo Abeni xfrm_sk_free_policy(sk); 2975aa1fbd94SFlorian Westphal 2976e16163b6SPaolo Abeni sock_put(sk); 2977e16163b6SPaolo Abeni } 2978e16163b6SPaolo Abeni 297963740448SPaolo Abeni void __mptcp_unaccepted_force_close(struct sock *sk) 298063740448SPaolo Abeni { 298163740448SPaolo Abeni sock_set_flag(sk, SOCK_DEAD); 298263740448SPaolo Abeni mptcp_do_fastclose(sk); 298363740448SPaolo Abeni __mptcp_destroy_sock(sk); 298463740448SPaolo Abeni } 298563740448SPaolo Abeni 29865684ab1aSPaolo Abeni static __poll_t mptcp_check_readable(struct sock *sk) 2987d21f8348SPaolo Abeni { 29885684ab1aSPaolo Abeni return mptcp_epollin_ready(sk) ? EPOLLIN | EPOLLRDNORM : 0; 2989d21f8348SPaolo Abeni } 2990d21f8348SPaolo Abeni 299157fc0f1cSPaolo Abeni static void mptcp_check_listen_stop(struct sock *sk) 2992c558246eSMenglong Dong { 299357fc0f1cSPaolo Abeni struct sock *ssk; 299457fc0f1cSPaolo Abeni 299557fc0f1cSPaolo Abeni if (inet_sk_state_load(sk) != TCP_LISTEN) 299657fc0f1cSPaolo Abeni return; 299757fc0f1cSPaolo Abeni 2998c558246eSMenglong Dong sock_prot_inuse_add(sock_net(sk), sk->sk_prot, -1); 299957fc0f1cSPaolo Abeni ssk = mptcp_sk(sk)->first; 300057fc0f1cSPaolo Abeni if (WARN_ON_ONCE(!ssk || inet_sk_state_load(ssk) != TCP_LISTEN)) 300157fc0f1cSPaolo Abeni return; 300257fc0f1cSPaolo Abeni 300357fc0f1cSPaolo Abeni lock_sock_nested(ssk, SINGLE_DEPTH_NESTING); 30043fffa15bSPaolo Abeni tcp_set_state(ssk, TCP_CLOSE); 300557fc0f1cSPaolo Abeni mptcp_subflow_queue_clean(sk, ssk); 300657fc0f1cSPaolo Abeni inet_csk_listen_stop(ssk); 300757fc0f1cSPaolo Abeni mptcp_event_pm_listener(ssk, MPTCP_EVENT_LISTENER_CLOSED); 300857fc0f1cSPaolo Abeni release_sock(ssk); 3009c558246eSMenglong Dong } 3010c558246eSMenglong Dong 301126d3e21cSMenglong Dong bool __mptcp_close(struct sock *sk, long timeout) 3012e16163b6SPaolo Abeni { 3013e16163b6SPaolo Abeni struct mptcp_subflow_context *subflow; 301476a13b31SGeliang Tang struct mptcp_sock *msk = mptcp_sk(sk); 3015e16163b6SPaolo Abeni bool do_cancel_work = false; 3016d4e85922SPaolo Abeni int subflows_alive = 0; 3017e16163b6SPaolo Abeni 30186b9831bfSPaolo Abeni WRITE_ONCE(sk->sk_shutdown, SHUTDOWN_MASK); 3019e16163b6SPaolo Abeni 3020e16163b6SPaolo Abeni if ((1 << sk->sk_state) & (TCPF_LISTEN | TCPF_CLOSE)) { 302157fc0f1cSPaolo Abeni mptcp_check_listen_stop(sk); 3022c693a851SGeliang Tang mptcp_set_state(sk, TCP_CLOSE); 3023e16163b6SPaolo Abeni goto cleanup; 3024e16163b6SPaolo Abeni } 3025e16163b6SPaolo Abeni 30265684ab1aSPaolo Abeni if (mptcp_data_avail(msk) || timeout < 0) { 30278d547809SPaolo Abeni /* If the msk has read data, or the caller explicitly ask it, 30288d547809SPaolo Abeni * do the MPTCP equivalent of TCP reset, aka MPTCP fastclose 30298d547809SPaolo Abeni */ 3030d21f8348SPaolo Abeni mptcp_do_fastclose(sk); 30318d547809SPaolo Abeni timeout = 0; 3032d21f8348SPaolo Abeni } else if (mptcp_close_state(sk)) { 3033e16163b6SPaolo Abeni __mptcp_wr_shutdown(sk); 3034d21f8348SPaolo Abeni } 3035e16163b6SPaolo Abeni 3036e16163b6SPaolo Abeni sk_stream_wait_close(sk, timeout); 3037e16163b6SPaolo Abeni 3038e16163b6SPaolo Abeni cleanup: 3039e16163b6SPaolo Abeni /* orphan all the subflows */ 304076a13b31SGeliang Tang mptcp_for_each_subflow(msk, subflow) { 3041e16163b6SPaolo Abeni struct sock *ssk = mptcp_subflow_tcp_sock(subflow); 304249054556SPaolo Abeni bool slow = lock_sock_fast_nested(ssk); 3043e16163b6SPaolo Abeni 3044d4e85922SPaolo Abeni subflows_alive += ssk->sk_state != TCP_CLOSE; 3045d4e85922SPaolo Abeni 304676a13b31SGeliang Tang /* since the close timeout takes precedence on the fail one, 304776a13b31SGeliang Tang * cancel the latter 304876a13b31SGeliang Tang */ 304976a13b31SGeliang Tang if (ssk == msk->first) 305076a13b31SGeliang Tang subflow->fail_tout = 0; 305176a13b31SGeliang Tang 3052fe948001SMenglong Dong /* detach from the parent socket, but allow data_ready to 3053fe948001SMenglong Dong * push incoming data into the mptcp stack, to properly ack it 3054fe948001SMenglong Dong */ 3055fe948001SMenglong Dong ssk->sk_socket = NULL; 3056fe948001SMenglong Dong ssk->sk_wq = NULL; 3057e16163b6SPaolo Abeni unlock_sock_fast(ssk, slow); 3058e16163b6SPaolo Abeni } 3059e16163b6SPaolo Abeni sock_orphan(sk); 3060e16163b6SPaolo Abeni 3061d4e85922SPaolo Abeni /* all the subflows are closed, only timeout can change the msk 3062d4e85922SPaolo Abeni * state, let's not keep resources busy for no reasons 3063d4e85922SPaolo Abeni */ 3064d4e85922SPaolo Abeni if (subflows_alive == 0) 3065c693a851SGeliang Tang mptcp_set_state(sk, TCP_CLOSE); 3066d4e85922SPaolo Abeni 3067e16163b6SPaolo Abeni sock_hold(sk); 3068e16163b6SPaolo Abeni pr_debug("msk=%p state=%d", sk, sk->sk_state); 306900df24f1SGeliang Tang if (msk->token) 307076a13b31SGeliang Tang mptcp_event(MPTCP_EVENT_CLOSED, msk, NULL, GFP_KERNEL); 3071b29fcfb5SPaolo Abeni 3072e16163b6SPaolo Abeni if (sk->sk_state == TCP_CLOSE) { 3073e16163b6SPaolo Abeni __mptcp_destroy_sock(sk); 3074e16163b6SPaolo Abeni do_cancel_work = true; 3075e16163b6SPaolo Abeni } else { 307627e5ccc2SPaolo Abeni mptcp_start_tout_timer(sk); 3077e16163b6SPaolo Abeni } 307826d3e21cSMenglong Dong 307926d3e21cSMenglong Dong return do_cancel_work; 308026d3e21cSMenglong Dong } 308126d3e21cSMenglong Dong 308226d3e21cSMenglong Dong static void mptcp_close(struct sock *sk, long timeout) 308326d3e21cSMenglong Dong { 308426d3e21cSMenglong Dong bool do_cancel_work; 308526d3e21cSMenglong Dong 308626d3e21cSMenglong Dong lock_sock(sk); 308726d3e21cSMenglong Dong 308826d3e21cSMenglong Dong do_cancel_work = __mptcp_close(sk, timeout); 3089e16163b6SPaolo Abeni release_sock(sk); 3090e16163b6SPaolo Abeni if (do_cancel_work) 309180992017SPaolo Abeni mptcp_cancel_work(sk); 3092b911c97cSFlorian Westphal 3093e16163b6SPaolo Abeni sock_put(sk); 3094f870fa0bSMat Martineau } 3095f870fa0bSMat Martineau 30967e8b88ecSPaolo Abeni static void mptcp_copy_inaddrs(struct sock *msk, const struct sock *ssk) 3097cf7da0d6SPeter Krystad { 3098cf7da0d6SPeter Krystad #if IS_ENABLED(CONFIG_MPTCP_IPV6) 3099cf7da0d6SPeter Krystad const struct ipv6_pinfo *ssk6 = inet6_sk(ssk); 3100cf7da0d6SPeter Krystad struct ipv6_pinfo *msk6 = inet6_sk(msk); 3101cf7da0d6SPeter Krystad 3102cf7da0d6SPeter Krystad msk->sk_v6_daddr = ssk->sk_v6_daddr; 3103cf7da0d6SPeter Krystad msk->sk_v6_rcv_saddr = ssk->sk_v6_rcv_saddr; 3104cf7da0d6SPeter Krystad 3105cf7da0d6SPeter Krystad if (msk6 && ssk6) { 3106cf7da0d6SPeter Krystad msk6->saddr = ssk6->saddr; 3107cf7da0d6SPeter Krystad msk6->flow_label = ssk6->flow_label; 3108cf7da0d6SPeter Krystad } 3109cf7da0d6SPeter Krystad #endif 3110cf7da0d6SPeter Krystad 3111cf7da0d6SPeter Krystad inet_sk(msk)->inet_num = inet_sk(ssk)->inet_num; 3112cf7da0d6SPeter Krystad inet_sk(msk)->inet_dport = inet_sk(ssk)->inet_dport; 3113cf7da0d6SPeter Krystad inet_sk(msk)->inet_sport = inet_sk(ssk)->inet_sport; 3114cf7da0d6SPeter Krystad inet_sk(msk)->inet_daddr = inet_sk(ssk)->inet_daddr; 3115cf7da0d6SPeter Krystad inet_sk(msk)->inet_saddr = inet_sk(ssk)->inet_saddr; 3116cf7da0d6SPeter Krystad inet_sk(msk)->inet_rcv_saddr = inet_sk(ssk)->inet_rcv_saddr; 3117cf7da0d6SPeter Krystad } 3118cf7da0d6SPeter Krystad 311918b683bfSPaolo Abeni static int mptcp_disconnect(struct sock *sk, int flags) 312018b683bfSPaolo Abeni { 312176e2a55dSPaolo Abeni struct mptcp_sock *msk = mptcp_sk(sk); 312276e2a55dSPaolo Abeni 31237d803344SPaolo Abeni /* We are on the fastopen error path. We can't call straight into the 31247d803344SPaolo Abeni * subflows cleanup code due to lock nesting (we are already under 3125c2b2ae39SPaolo Abeni * msk->firstsocket lock). 31267d803344SPaolo Abeni */ 31277d803344SPaolo Abeni if (msk->fastopening) 3128c2b2ae39SPaolo Abeni return -EBUSY; 31297d803344SPaolo Abeni 313057fc0f1cSPaolo Abeni mptcp_check_listen_stop(sk); 3131c693a851SGeliang Tang mptcp_set_state(sk, TCP_CLOSE); 3132b29fcfb5SPaolo Abeni 3133f6909dc1SPaolo Abeni mptcp_stop_rtx_timer(sk); 313427e5ccc2SPaolo Abeni mptcp_stop_tout_timer(sk); 3135b29fcfb5SPaolo Abeni 313600df24f1SGeliang Tang if (msk->token) 313700df24f1SGeliang Tang mptcp_event(MPTCP_EVENT_CLOSED, msk, NULL, GFP_KERNEL); 3138b29fcfb5SPaolo Abeni 3139c0bf3c6aSPaolo Abeni /* msk->subflow is still intact, the following will not free the first 3140c0bf3c6aSPaolo Abeni * subflow 3141c0bf3c6aSPaolo Abeni */ 3142c0bf3c6aSPaolo Abeni mptcp_destroy_common(msk, MPTCP_CF_FASTCLOSE); 3143e9d09bacSPaolo Abeni WRITE_ONCE(msk->flags, 0); 3144e9d09bacSPaolo Abeni msk->cb_flags = 0; 3145e9d09bacSPaolo Abeni msk->push_pending = 0; 3146b29fcfb5SPaolo Abeni msk->recovery = false; 3147b29fcfb5SPaolo Abeni msk->can_ack = false; 3148b29fcfb5SPaolo Abeni msk->fully_established = false; 3149b29fcfb5SPaolo Abeni msk->rcv_data_fin = false; 3150b29fcfb5SPaolo Abeni msk->snd_data_fin_enable = false; 3151b29fcfb5SPaolo Abeni msk->rcv_fastclose = false; 3152b29fcfb5SPaolo Abeni msk->use_64bit_ack = false; 31535684ab1aSPaolo Abeni msk->bytes_consumed = 0; 3154b29fcfb5SPaolo Abeni WRITE_ONCE(msk->csum_enabled, mptcp_is_checksum_enabled(sock_net(sk))); 3155b29fcfb5SPaolo Abeni mptcp_pm_data_reset(msk); 3156b29fcfb5SPaolo Abeni mptcp_ca_reset(sk); 315738967f42SPaolo Abeni msk->bytes_acked = 0; 315838967f42SPaolo Abeni msk->bytes_received = 0; 315938967f42SPaolo Abeni msk->bytes_sent = 0; 316038967f42SPaolo Abeni msk->bytes_retrans = 0; 3161b29fcfb5SPaolo Abeni 31626b9831bfSPaolo Abeni WRITE_ONCE(sk->sk_shutdown, 0); 3163b29fcfb5SPaolo Abeni sk_error_report(sk); 316442c556feSFlorian Westphal return 0; 316518b683bfSPaolo Abeni } 316618b683bfSPaolo Abeni 3167b0519de8SFlorian Westphal #if IS_ENABLED(CONFIG_MPTCP_IPV6) 3168b0519de8SFlorian Westphal static struct ipv6_pinfo *mptcp_inet6_sk(const struct sock *sk) 3169b0519de8SFlorian Westphal { 3170b0519de8SFlorian Westphal unsigned int offset = sizeof(struct mptcp6_sock) - sizeof(struct ipv6_pinfo); 3171b0519de8SFlorian Westphal 3172b0519de8SFlorian Westphal return (struct ipv6_pinfo *)(((u8 *)sk) + offset); 3173b0519de8SFlorian Westphal } 3174b0519de8SFlorian Westphal #endif 3175b0519de8SFlorian Westphal 31767e8b88ecSPaolo Abeni struct sock *mptcp_sk_clone_init(const struct sock *sk, 3177cfde141eSPaolo Abeni const struct mptcp_options_received *mp_opt, 31787e8b88ecSPaolo Abeni struct sock *ssk, 3179fca5c82cSPaolo Abeni struct request_sock *req) 3180b0519de8SFlorian Westphal { 318158b09919SPaolo Abeni struct mptcp_subflow_request_sock *subflow_req = mptcp_subflow_rsk(req); 3182b0519de8SFlorian Westphal struct sock *nsk = sk_clone_lock(sk, GFP_ATOMIC); 318358b09919SPaolo Abeni struct mptcp_sock *msk; 3184b0519de8SFlorian Westphal 3185b0519de8SFlorian Westphal if (!nsk) 3186b0519de8SFlorian Westphal return NULL; 3187b0519de8SFlorian Westphal 3188b0519de8SFlorian Westphal #if IS_ENABLED(CONFIG_MPTCP_IPV6) 3189b0519de8SFlorian Westphal if (nsk->sk_family == AF_INET6) 3190b0519de8SFlorian Westphal inet_sk(nsk)->pinet6 = mptcp_inet6_sk(nsk); 3191b0519de8SFlorian Westphal #endif 3192b0519de8SFlorian Westphal 319358b09919SPaolo Abeni __mptcp_init_sock(nsk); 319458b09919SPaolo Abeni 319558b09919SPaolo Abeni msk = mptcp_sk(nsk); 3196*1c09d7cbSPaolo Abeni WRITE_ONCE(msk->local_key, subflow_req->local_key); 319758b09919SPaolo Abeni msk->token = subflow_req->token; 3198b6985b9bSPaolo Abeni msk->in_accept_queue = 1; 3199b93df08cSPaolo Abeni WRITE_ONCE(msk->fully_established, false); 320074c7dfbeSPaolo Abeni if (mp_opt->suboptions & OPTION_MPTCP_CSUMREQD) 320106251181SGeliang Tang WRITE_ONCE(msk->csum_enabled, true); 320258b09919SPaolo Abeni 320358b09919SPaolo Abeni msk->write_seq = subflow_req->idsn + 1; 3204eaa2ffabSPaolo Abeni msk->snd_nxt = msk->write_seq; 32057439d687SPaolo Abeni msk->snd_una = msk->write_seq; 32067439d687SPaolo Abeni msk->wnd_end = msk->snd_nxt + req->rsk_rcv_wnd; 3207df00b087SFlorian Westphal msk->setsockopt_seq = mptcp_sk(sk)->setsockopt_seq; 32081730b2b2SGeliang Tang mptcp_init_sched(msk, mptcp_sk(sk)->sched); 32096f8a612aSFlorian Westphal 32106f06b4d4SPaolo Abeni /* passive msk is created after the first/MPC subflow */ 32116f06b4d4SPaolo Abeni msk->subflow_id = 2; 32126f06b4d4SPaolo Abeni 32135e20087dSFlorian Westphal sock_reset_flag(nsk, SOCK_RCU_FREE); 32140c148460SPaolo Abeni security_inet_csk_clone(nsk, req); 32157e8b88ecSPaolo Abeni 32167e8b88ecSPaolo Abeni /* this can't race with mptcp_close(), as the msk is 32177e8b88ecSPaolo Abeni * not yet exposted to user-space 32187e8b88ecSPaolo Abeni */ 3219c693a851SGeliang Tang mptcp_set_state(nsk, TCP_ESTABLISHED); 32207e8b88ecSPaolo Abeni 32217e8b88ecSPaolo Abeni /* The msk maintain a ref to each subflow in the connections list */ 32227e8b88ecSPaolo Abeni WRITE_ONCE(msk->first, ssk); 32237e8b88ecSPaolo Abeni list_add(&mptcp_subflow_ctx(ssk)->node, &msk->conn_list); 32247e8b88ecSPaolo Abeni sock_hold(ssk); 32257e8b88ecSPaolo Abeni 32267e8b88ecSPaolo Abeni /* new mpc subflow takes ownership of the newly 32277e8b88ecSPaolo Abeni * created mptcp socket 32287e8b88ecSPaolo Abeni */ 32297e8b88ecSPaolo Abeni mptcp_token_accept(subflow_req, msk); 32307e8b88ecSPaolo Abeni 32317e8b88ecSPaolo Abeni /* set msk addresses early to ensure mptcp_pm_get_local_id() 32327e8b88ecSPaolo Abeni * uses the correct data 32337e8b88ecSPaolo Abeni */ 32347e8b88ecSPaolo Abeni mptcp_copy_inaddrs(nsk, ssk); 32358005184fSPaolo Abeni __mptcp_propagate_sndbuf(nsk, ssk); 32367e8b88ecSPaolo Abeni 32377e8b88ecSPaolo Abeni mptcp_rcv_space_init(msk, ssk); 323858b09919SPaolo Abeni bh_unlock_sock(nsk); 323958b09919SPaolo Abeni 3240b6985b9bSPaolo Abeni /* note: the newly allocated socket refcount is 2 now */ 3241b0519de8SFlorian Westphal return nsk; 3242b0519de8SFlorian Westphal } 3243b0519de8SFlorian Westphal 3244a6b118feSFlorian Westphal void mptcp_rcv_space_init(struct mptcp_sock *msk, const struct sock *ssk) 3245a6b118feSFlorian Westphal { 3246a6b118feSFlorian Westphal const struct tcp_sock *tp = tcp_sk(ssk); 3247a6b118feSFlorian Westphal 3248a6b118feSFlorian Westphal msk->rcvq_space.copied = 0; 3249a6b118feSFlorian Westphal msk->rcvq_space.rtt_us = 0; 3250a6b118feSFlorian Westphal 3251a6b118feSFlorian Westphal msk->rcvq_space.time = tp->tcp_mstamp; 3252a6b118feSFlorian Westphal 3253a6b118feSFlorian Westphal /* initial rcv_space offering made to peer */ 3254a6b118feSFlorian Westphal msk->rcvq_space.space = min_t(u32, tp->rcv_wnd, 3255a6b118feSFlorian Westphal TCP_INIT_CWND * tp->advmss); 3256a6b118feSFlorian Westphal if (msk->rcvq_space.space == 0) 3257a6b118feSFlorian Westphal msk->rcvq_space.space = TCP_INIT_CWND * TCP_MSS_DEFAULT; 32586f8a612aSFlorian Westphal 32597439d687SPaolo Abeni WRITE_ONCE(msk->wnd_end, msk->snd_nxt + tcp_sk(ssk)->snd_wnd); 3260a6b118feSFlorian Westphal } 3261a6b118feSFlorian Westphal 3262c0bf3c6aSPaolo Abeni void mptcp_destroy_common(struct mptcp_sock *msk, unsigned int flags) 32635c8c1640SGeliang Tang { 3264c0bf3c6aSPaolo Abeni struct mptcp_subflow_context *subflow, *tmp; 326587952603SPaolo Abeni struct sock *sk = (struct sock *)msk; 326687952603SPaolo Abeni 32676e628cd3SPaolo Abeni __mptcp_clear_xmit(sk); 32686e628cd3SPaolo Abeni 3269c0bf3c6aSPaolo Abeni /* join list will be eventually flushed (with rst) at sock lock release time */ 32705efbf6f7SMatthieu Baerts mptcp_for_each_subflow_safe(msk, subflow, tmp) 3271c0bf3c6aSPaolo Abeni __mptcp_close_ssk(sk, mptcp_subflow_tcp_sock(subflow), subflow, flags); 3272c0bf3c6aSPaolo Abeni 327387952603SPaolo Abeni /* move to sk_receive_queue, sk_stream_kill_queues will purge it */ 3274b29fcfb5SPaolo Abeni mptcp_data_lock(sk); 327587952603SPaolo Abeni skb_queue_splice_tail_init(&msk->receive_queue, &sk->sk_receive_queue); 32766511882cSPaolo Abeni __skb_queue_purge(&sk->sk_receive_queue); 32775c8c1640SGeliang Tang skb_rbtree_purge(&msk->out_of_order_queue); 3278b29fcfb5SPaolo Abeni mptcp_data_unlock(sk); 32796511882cSPaolo Abeni 32806511882cSPaolo Abeni /* move all the rx fwd alloc into the sk_mem_reclaim_final in 32816511882cSPaolo Abeni * inet_sock_destruct() will dispose it 32826511882cSPaolo Abeni */ 32835e6300e7SEric Dumazet sk_forward_alloc_add(sk, msk->rmem_fwd_alloc); 32849531e4a8SEric Dumazet WRITE_ONCE(msk->rmem_fwd_alloc, 0); 32855c8c1640SGeliang Tang mptcp_token_destroy(msk); 32865c8c1640SGeliang Tang mptcp_pm_free_anno_list(msk); 32874638de5aSKishen Maloor mptcp_free_local_addr_list(msk); 32885c8c1640SGeliang Tang } 32895c8c1640SGeliang Tang 329079c0949eSPeter Krystad static void mptcp_destroy(struct sock *sk) 329179c0949eSPeter Krystad { 3292c9fd9c5fSFlorian Westphal struct mptcp_sock *msk = mptcp_sk(sk); 3293c9fd9c5fSFlorian Westphal 329439880bd8SPaolo Abeni /* allow the following to close even the initial subflow */ 329539880bd8SPaolo Abeni msk->free_first = 1; 3296c0bf3c6aSPaolo Abeni mptcp_destroy_common(msk, 0); 3297d027236cSPaolo Abeni sk_sockets_allocated_dec(sk); 329879c0949eSPeter Krystad } 329979c0949eSPeter Krystad 33006e628cd3SPaolo Abeni void __mptcp_data_acked(struct sock *sk) 33016e628cd3SPaolo Abeni { 33026e628cd3SPaolo Abeni if (!sock_owned_by_user(sk)) 33036e628cd3SPaolo Abeni __mptcp_clean_una(sk); 33046e628cd3SPaolo Abeni else 3305e9d09bacSPaolo Abeni __set_bit(MPTCP_CLEAN_UNA, &mptcp_sk(sk)->cb_flags); 33066e628cd3SPaolo Abeni 33076e628cd3SPaolo Abeni if (mptcp_pending_data_fin_ack(sk)) 33086e628cd3SPaolo Abeni mptcp_schedule_work(sk); 33096e628cd3SPaolo Abeni } 33106e628cd3SPaolo Abeni 3311219d0499SPaolo Abeni void __mptcp_check_push(struct sock *sk, struct sock *ssk) 33126e628cd3SPaolo Abeni { 33136e628cd3SPaolo Abeni if (!mptcp_send_head(sk)) 33146e628cd3SPaolo Abeni return; 33156e628cd3SPaolo Abeni 331673a0052aSGeliang Tang if (!sock_owned_by_user(sk)) 331773a0052aSGeliang Tang __mptcp_subflow_push_pending(sk, ssk, false); 331873a0052aSGeliang Tang else 3319e9d09bacSPaolo Abeni __set_bit(MPTCP_PUSH_PENDING, &mptcp_sk(sk)->cb_flags); 33206e628cd3SPaolo Abeni } 33216e628cd3SPaolo Abeni 3322e9d09bacSPaolo Abeni #define MPTCP_FLAGS_PROCESS_CTX_NEED (BIT(MPTCP_PUSH_PENDING) | \ 3323e9d09bacSPaolo Abeni BIT(MPTCP_RETRANSMIT) | \ 3324e9d09bacSPaolo Abeni BIT(MPTCP_FLUSH_JOIN_LIST)) 3325e9d09bacSPaolo Abeni 3326e93da928SPaolo Abeni /* processes deferred events and flush wmem */ 332714c441b5SPaolo Abeni static void mptcp_release_cb(struct sock *sk) 3328e9d09bacSPaolo Abeni __must_hold(&sk->sk_lock.slock) 332914c441b5SPaolo Abeni { 3330e9d09bacSPaolo Abeni struct mptcp_sock *msk = mptcp_sk(sk); 33312d6f5a2bSPaolo Abeni 3332e9d09bacSPaolo Abeni for (;;) { 3333e9d09bacSPaolo Abeni unsigned long flags = (msk->cb_flags & MPTCP_FLAGS_PROCESS_CTX_NEED) | 3334e9d09bacSPaolo Abeni msk->push_pending; 333556a666c4SPaolo Abeni struct list_head join_list; 333656a666c4SPaolo Abeni 3337c2e6048fSPaolo Abeni if (!flags) 3338c2e6048fSPaolo Abeni break; 3339c2e6048fSPaolo Abeni 334056a666c4SPaolo Abeni INIT_LIST_HEAD(&join_list); 334156a666c4SPaolo Abeni list_splice_init(&msk->join_list, &join_list); 334256a666c4SPaolo Abeni 3343c2e6048fSPaolo Abeni /* the following actions acquire the subflow socket lock 33446e628cd3SPaolo Abeni * 33456e628cd3SPaolo Abeni * 1) can't be invoked in atomic scope 33466e628cd3SPaolo Abeni * 2) must avoid ABBA deadlock with msk socket spinlock: the RX 33476e628cd3SPaolo Abeni * datapath acquires the msk socket spinlock while helding 33486e628cd3SPaolo Abeni * the subflow socket lock 33496e628cd3SPaolo Abeni */ 3350e9d09bacSPaolo Abeni msk->push_pending = 0; 3351e9d09bacSPaolo Abeni msk->cb_flags &= ~flags; 33526e628cd3SPaolo Abeni spin_unlock_bh(&sk->sk_lock.slock); 335356a666c4SPaolo Abeni 33543e501490SPaolo Abeni if (flags & BIT(MPTCP_FLUSH_JOIN_LIST)) 335556a666c4SPaolo Abeni __mptcp_flush_join_list(sk, &join_list); 33562e5de7e0SDan Carpenter if (flags & BIT(MPTCP_PUSH_PENDING)) 3357c2e6048fSPaolo Abeni __mptcp_push_pending(sk, 0); 33582d6f5a2bSPaolo Abeni if (flags & BIT(MPTCP_RETRANSMIT)) 33592d6f5a2bSPaolo Abeni __mptcp_retrans(sk); 3360c2e6048fSPaolo Abeni 3361c2e6048fSPaolo Abeni cond_resched(); 33626e628cd3SPaolo Abeni spin_lock_bh(&sk->sk_lock.slock); 33636e628cd3SPaolo Abeni } 3364c2e6048fSPaolo Abeni 336565a569b0SPaolo Abeni if (__test_and_clear_bit(MPTCP_CLEAN_UNA, &msk->cb_flags)) 336665a569b0SPaolo Abeni __mptcp_clean_una_wakeup(sk); 336717ebf8a4SXiang Yang if (unlikely(msk->cb_flags)) { 33684fd19a30SPaolo Abeni /* be sure to sync the msk state before taking actions 3369d109a776SPaolo Abeni * depending on sk_state (MPTCP_ERROR_REPORT) 3370d109a776SPaolo Abeni * On sk release avoid actions depending on the first subflow 3371490274b4SPaolo Abeni */ 33724fd19a30SPaolo Abeni if (__test_and_clear_bit(MPTCP_SYNC_STATE, &msk->cb_flags) && msk->first) 33734fd19a30SPaolo Abeni __mptcp_sync_state(sk, msk->pending_state); 3374e9d09bacSPaolo Abeni if (__test_and_clear_bit(MPTCP_ERROR_REPORT, &msk->cb_flags)) 337515cc1045SPaolo Abeni __mptcp_error_report(sk); 33768005184fSPaolo Abeni if (__test_and_clear_bit(MPTCP_SYNC_SNDBUF, &msk->cb_flags)) 33778005184fSPaolo Abeni __mptcp_sync_sndbuf(sk); 337865a569b0SPaolo Abeni } 33796e628cd3SPaolo Abeni 338087952603SPaolo Abeni __mptcp_update_rmem(sk); 338114c441b5SPaolo Abeni } 338214c441b5SPaolo Abeni 3383bcd97734SPaolo Abeni /* MP_JOIN client subflow must wait for 4th ack before sending any data: 3384bcd97734SPaolo Abeni * TCP can't schedule delack timer before the subflow is fully established. 3385bcd97734SPaolo Abeni * MPTCP uses the delack timer to do 3rd ack retransmissions 3386bcd97734SPaolo Abeni */ 3387bcd97734SPaolo Abeni static void schedule_3rdack_retransmission(struct sock *ssk) 3388bcd97734SPaolo Abeni { 3389bcd97734SPaolo Abeni struct inet_connection_sock *icsk = inet_csk(ssk); 3390bcd97734SPaolo Abeni struct tcp_sock *tp = tcp_sk(ssk); 3391bcd97734SPaolo Abeni unsigned long timeout; 3392bcd97734SPaolo Abeni 3393bcd97734SPaolo Abeni if (mptcp_subflow_ctx(ssk)->fully_established) 3394bcd97734SPaolo Abeni return; 3395bcd97734SPaolo Abeni 3396bcd97734SPaolo Abeni /* reschedule with a timeout above RTT, as we must look only for drop */ 3397bcd97734SPaolo Abeni if (tp->srtt_us) 3398bcd97734SPaolo Abeni timeout = usecs_to_jiffies(tp->srtt_us >> (3 - 1)); 3399bcd97734SPaolo Abeni else 3400bcd97734SPaolo Abeni timeout = TCP_TIMEOUT_INIT; 3401bcd97734SPaolo Abeni timeout += jiffies; 3402bcd97734SPaolo Abeni 3403bcd97734SPaolo Abeni WARN_ON_ONCE(icsk->icsk_ack.pending & ICSK_ACK_TIMER); 3404bcd97734SPaolo Abeni icsk->icsk_ack.pending |= ICSK_ACK_SCHED | ICSK_ACK_TIMER; 3405bcd97734SPaolo Abeni icsk->icsk_ack.timeout = timeout; 3406bcd97734SPaolo Abeni sk_reset_timer(ssk, &icsk->icsk_delack_timer, timeout); 3407bcd97734SPaolo Abeni } 3408bcd97734SPaolo Abeni 3409a5efdbceSPaolo Abeni void mptcp_subflow_process_delegated(struct sock *ssk, long status) 3410b19bc294SPaolo Abeni { 3411b19bc294SPaolo Abeni struct mptcp_subflow_context *subflow = mptcp_subflow_ctx(ssk); 3412b19bc294SPaolo Abeni struct sock *sk = subflow->conn; 3413b19bc294SPaolo Abeni 3414a5efdbceSPaolo Abeni if (status & BIT(MPTCP_DELEGATE_SEND)) { 3415b19bc294SPaolo Abeni mptcp_data_lock(sk); 3416b19bc294SPaolo Abeni if (!sock_owned_by_user(sk)) 341773a0052aSGeliang Tang __mptcp_subflow_push_pending(sk, ssk, true); 3418b19bc294SPaolo Abeni else 3419e9d09bacSPaolo Abeni __set_bit(MPTCP_PUSH_PENDING, &mptcp_sk(sk)->cb_flags); 3420b19bc294SPaolo Abeni mptcp_data_unlock(sk); 3421bcd97734SPaolo Abeni } 34228005184fSPaolo Abeni if (status & BIT(MPTCP_DELEGATE_SNDBUF)) { 34238005184fSPaolo Abeni mptcp_data_lock(sk); 34248005184fSPaolo Abeni if (!sock_owned_by_user(sk)) 34258005184fSPaolo Abeni __mptcp_sync_sndbuf(sk); 34268005184fSPaolo Abeni else 34278005184fSPaolo Abeni __set_bit(MPTCP_SYNC_SNDBUF, &mptcp_sk(sk)->cb_flags); 34288005184fSPaolo Abeni mptcp_data_unlock(sk); 34298005184fSPaolo Abeni } 3430a5efdbceSPaolo Abeni if (status & BIT(MPTCP_DELEGATE_ACK)) 3431bcd97734SPaolo Abeni schedule_3rdack_retransmission(ssk); 3432b19bc294SPaolo Abeni } 3433b19bc294SPaolo Abeni 34342c5ebd00SPaolo Abeni static int mptcp_hash(struct sock *sk) 34352c5ebd00SPaolo Abeni { 34362c5ebd00SPaolo Abeni /* should never be called, 34372c5ebd00SPaolo Abeni * we hash the TCP subflows not the master socket 34382c5ebd00SPaolo Abeni */ 34392c5ebd00SPaolo Abeni WARN_ON_ONCE(1); 34402c5ebd00SPaolo Abeni return 0; 34412c5ebd00SPaolo Abeni } 34422c5ebd00SPaolo Abeni 34432c5ebd00SPaolo Abeni static void mptcp_unhash(struct sock *sk) 34442c5ebd00SPaolo Abeni { 34452c5ebd00SPaolo Abeni /* called from sk_common_release(), but nothing to do here */ 34462c5ebd00SPaolo Abeni } 34472c5ebd00SPaolo Abeni 3448cec37a6eSPeter Krystad static int mptcp_get_port(struct sock *sk, unsigned short snum) 3449f870fa0bSMat Martineau { 3450f870fa0bSMat Martineau struct mptcp_sock *msk = mptcp_sk(sk); 3451f870fa0bSMat Martineau 3452cfb63e50SPaolo Abeni pr_debug("msk=%p, ssk=%p", msk, msk->first); 3453cfb63e50SPaolo Abeni if (WARN_ON_ONCE(!msk->first)) 3454cec37a6eSPeter Krystad return -EINVAL; 3455f870fa0bSMat Martineau 3456cfb63e50SPaolo Abeni return inet_csk_get_port(msk->first, snum); 3457cec37a6eSPeter Krystad } 3458f870fa0bSMat Martineau 3459cec37a6eSPeter Krystad void mptcp_finish_connect(struct sock *ssk) 3460cec37a6eSPeter Krystad { 3461cec37a6eSPeter Krystad struct mptcp_subflow_context *subflow; 3462cec37a6eSPeter Krystad struct mptcp_sock *msk; 3463cec37a6eSPeter Krystad struct sock *sk; 3464f870fa0bSMat Martineau 3465cec37a6eSPeter Krystad subflow = mptcp_subflow_ctx(ssk); 3466cec37a6eSPeter Krystad sk = subflow->conn; 3467cec37a6eSPeter Krystad msk = mptcp_sk(sk); 3468cec37a6eSPeter Krystad 3469648ef4b8SMat Martineau pr_debug("msk=%p, token=%u", sk, subflow->token); 3470648ef4b8SMat Martineau 3471b3ea6b27SPaolo Abeni subflow->map_seq = subflow->iasn; 3472648ef4b8SMat Martineau subflow->map_subflow_seq = 1; 34736d0060f6SMat Martineau 3474cec37a6eSPeter Krystad /* the socket is not connected yet, no msk/subflow ops can access/race 3475cec37a6eSPeter Krystad * accessing the field below 3476cec37a6eSPeter Krystad */ 3477cec37a6eSPeter Krystad WRITE_ONCE(msk->local_key, subflow->local_key); 34786d0060f6SMat Martineau WRITE_ONCE(msk->write_seq, subflow->idsn + 1); 3479eaa2ffabSPaolo Abeni WRITE_ONCE(msk->snd_nxt, msk->write_seq); 34807439d687SPaolo Abeni WRITE_ONCE(msk->snd_una, msk->write_seq); 34811b1c7a0eSPeter Krystad 34826c714f1bSFlorian Westphal mptcp_pm_new_connection(msk, ssk, 0); 3483a6b118feSFlorian Westphal 3484a6b118feSFlorian Westphal mptcp_rcv_space_init(msk, ssk); 3485f870fa0bSMat Martineau } 3486f870fa0bSMat Martineau 3487866f26f2SPaolo Abeni void mptcp_sock_graft(struct sock *sk, struct socket *parent) 3488cf7da0d6SPeter Krystad { 3489cf7da0d6SPeter Krystad write_lock_bh(&sk->sk_callback_lock); 3490cf7da0d6SPeter Krystad rcu_assign_pointer(sk->sk_wq, &parent->wq); 3491cf7da0d6SPeter Krystad sk_set_socket(sk, parent); 3492cf7da0d6SPeter Krystad sk->sk_uid = SOCK_INODE(parent)->i_uid; 3493cf7da0d6SPeter Krystad write_unlock_bh(&sk->sk_callback_lock); 3494cf7da0d6SPeter Krystad } 3495cf7da0d6SPeter Krystad 3496e16163b6SPaolo Abeni bool mptcp_finish_join(struct sock *ssk) 3497f296234cSPeter Krystad { 3498e16163b6SPaolo Abeni struct mptcp_subflow_context *subflow = mptcp_subflow_ctx(ssk); 3499f296234cSPeter Krystad struct mptcp_sock *msk = mptcp_sk(subflow->conn); 3500f296234cSPeter Krystad struct sock *parent = (void *)msk; 35013e501490SPaolo Abeni bool ret = true; 3502f296234cSPeter Krystad 3503f296234cSPeter Krystad pr_debug("msk=%p, subflow=%p", msk, subflow); 3504f296234cSPeter Krystad 3505f296234cSPeter Krystad /* mptcp socket already closing? */ 3506dc87efdbSFlorian Westphal if (!mptcp_is_fully_established(parent)) { 3507dc87efdbSFlorian Westphal subflow->reset_reason = MPTCP_RST_EMPTCP; 3508f296234cSPeter Krystad return false; 3509dc87efdbSFlorian Westphal } 3510f296234cSPeter Krystad 351155b47ca7SPaolo Abeni /* active subflow, already present inside the conn_list */ 351255b47ca7SPaolo Abeni if (!list_empty(&subflow->node)) { 351355b47ca7SPaolo Abeni mptcp_subflow_joined(msk, ssk); 35148005184fSPaolo Abeni mptcp_propagate_sndbuf(parent, ssk); 351555b47ca7SPaolo Abeni return true; 351655b47ca7SPaolo Abeni } 3517f296234cSPeter Krystad 35183e501490SPaolo Abeni if (!mptcp_pm_allow_new_subflow(msk)) 35193e501490SPaolo Abeni goto err_prohibited; 352010f6d46cSPaolo Abeni 352155b47ca7SPaolo Abeni /* If we can't acquire msk socket lock here, let the release callback 35223e501490SPaolo Abeni * handle it 352310f6d46cSPaolo Abeni */ 35243e501490SPaolo Abeni mptcp_data_lock(parent); 35253e501490SPaolo Abeni if (!sock_owned_by_user(parent)) { 35263e501490SPaolo Abeni ret = __mptcp_finish_join(msk, ssk); 35273e501490SPaolo Abeni if (ret) { 3528e16163b6SPaolo Abeni sock_hold(ssk); 35293e501490SPaolo Abeni list_add_tail(&subflow->node, &msk->conn_list); 3530e16163b6SPaolo Abeni } 35313e501490SPaolo Abeni } else { 35323e501490SPaolo Abeni sock_hold(ssk); 35333e501490SPaolo Abeni list_add_tail(&subflow->node, &msk->join_list); 3534e9d09bacSPaolo Abeni __set_bit(MPTCP_FLUSH_JOIN_LIST, &msk->cb_flags); 35353e501490SPaolo Abeni } 35363e501490SPaolo Abeni mptcp_data_unlock(parent); 35373e501490SPaolo Abeni 3538dc87efdbSFlorian Westphal if (!ret) { 35393e501490SPaolo Abeni err_prohibited: 3540dc87efdbSFlorian Westphal subflow->reset_reason = MPTCP_RST_EPROHIBIT; 354110f6d46cSPaolo Abeni return false; 3542dc87efdbSFlorian Westphal } 354310f6d46cSPaolo Abeni 354410f6d46cSPaolo Abeni return true; 3545f296234cSPeter Krystad } 3546f296234cSPeter Krystad 354776e2a55dSPaolo Abeni static void mptcp_shutdown(struct sock *sk, int how) 354876e2a55dSPaolo Abeni { 354976e2a55dSPaolo Abeni pr_debug("sk=%p, how=%d", sk, how); 355076e2a55dSPaolo Abeni 355176e2a55dSPaolo Abeni if ((how & SEND_SHUTDOWN) && mptcp_close_state(sk)) 355276e2a55dSPaolo Abeni __mptcp_wr_shutdown(sk); 355376e2a55dSPaolo Abeni } 355476e2a55dSPaolo Abeni 35556511882cSPaolo Abeni static int mptcp_forward_alloc_get(const struct sock *sk) 35566511882cSPaolo Abeni { 35579531e4a8SEric Dumazet return READ_ONCE(sk->sk_forward_alloc) + 35589531e4a8SEric Dumazet READ_ONCE(mptcp_sk(sk)->rmem_fwd_alloc); 35596511882cSPaolo Abeni } 35606511882cSPaolo Abeni 3561644807e3SFlorian Westphal static int mptcp_ioctl_outq(const struct mptcp_sock *msk, u64 v) 3562644807e3SFlorian Westphal { 3563644807e3SFlorian Westphal const struct sock *sk = (void *)msk; 3564644807e3SFlorian Westphal u64 delta; 3565644807e3SFlorian Westphal 3566644807e3SFlorian Westphal if (sk->sk_state == TCP_LISTEN) 3567644807e3SFlorian Westphal return -EINVAL; 3568644807e3SFlorian Westphal 3569644807e3SFlorian Westphal if ((1 << sk->sk_state) & (TCPF_SYN_SENT | TCPF_SYN_RECV)) 3570644807e3SFlorian Westphal return 0; 3571644807e3SFlorian Westphal 3572644807e3SFlorian Westphal delta = msk->write_seq - v; 357307c2c7a3SPaolo Abeni if (__mptcp_check_fallback(msk) && msk->first) { 357407c2c7a3SPaolo Abeni struct tcp_sock *tp = tcp_sk(msk->first); 357507c2c7a3SPaolo Abeni 357607c2c7a3SPaolo Abeni /* the first subflow is disconnected after close - see 357707c2c7a3SPaolo Abeni * __mptcp_close_ssk(). tcp_disconnect() moves the write_seq 357807c2c7a3SPaolo Abeni * so ignore that status, too. 357907c2c7a3SPaolo Abeni */ 358007c2c7a3SPaolo Abeni if (!((1 << msk->first->sk_state) & 358107c2c7a3SPaolo Abeni (TCPF_SYN_SENT | TCPF_SYN_RECV | TCPF_CLOSE))) 358207c2c7a3SPaolo Abeni delta += READ_ONCE(tp->write_seq) - tp->snd_una; 358307c2c7a3SPaolo Abeni } 3584644807e3SFlorian Westphal if (delta > INT_MAX) 3585644807e3SFlorian Westphal delta = INT_MAX; 3586644807e3SFlorian Westphal 3587644807e3SFlorian Westphal return (int)delta; 3588644807e3SFlorian Westphal } 3589644807e3SFlorian Westphal 3590e1d001faSBreno Leitao static int mptcp_ioctl(struct sock *sk, int cmd, int *karg) 3591644807e3SFlorian Westphal { 3592644807e3SFlorian Westphal struct mptcp_sock *msk = mptcp_sk(sk); 3593644807e3SFlorian Westphal bool slow; 3594644807e3SFlorian Westphal 3595644807e3SFlorian Westphal switch (cmd) { 3596644807e3SFlorian Westphal case SIOCINQ: 3597644807e3SFlorian Westphal if (sk->sk_state == TCP_LISTEN) 3598644807e3SFlorian Westphal return -EINVAL; 3599644807e3SFlorian Westphal 3600644807e3SFlorian Westphal lock_sock(sk); 3601644807e3SFlorian Westphal __mptcp_move_skbs(msk); 3602e1d001faSBreno Leitao *karg = mptcp_inq_hint(sk); 3603644807e3SFlorian Westphal release_sock(sk); 3604644807e3SFlorian Westphal break; 3605644807e3SFlorian Westphal case SIOCOUTQ: 3606644807e3SFlorian Westphal slow = lock_sock_fast(sk); 3607e1d001faSBreno Leitao *karg = mptcp_ioctl_outq(msk, READ_ONCE(msk->snd_una)); 3608644807e3SFlorian Westphal unlock_sock_fast(sk, slow); 3609644807e3SFlorian Westphal break; 3610644807e3SFlorian Westphal case SIOCOUTQNSD: 3611644807e3SFlorian Westphal slow = lock_sock_fast(sk); 3612e1d001faSBreno Leitao *karg = mptcp_ioctl_outq(msk, msk->snd_nxt); 3613644807e3SFlorian Westphal unlock_sock_fast(sk, slow); 3614644807e3SFlorian Westphal break; 3615644807e3SFlorian Westphal default: 3616644807e3SFlorian Westphal return -ENOIOCTLCMD; 3617644807e3SFlorian Westphal } 3618644807e3SFlorian Westphal 3619e1d001faSBreno Leitao return 0; 3620644807e3SFlorian Westphal } 3621644807e3SFlorian Westphal 362254f1944eSPaolo Abeni static void mptcp_subflow_early_fallback(struct mptcp_sock *msk, 362354f1944eSPaolo Abeni struct mptcp_subflow_context *subflow) 362454f1944eSPaolo Abeni { 362554f1944eSPaolo Abeni subflow->request_mptcp = 0; 362654f1944eSPaolo Abeni __mptcp_do_fallback(msk); 362754f1944eSPaolo Abeni } 362854f1944eSPaolo Abeni 362954f1944eSPaolo Abeni static int mptcp_connect(struct sock *sk, struct sockaddr *uaddr, int addr_len) 363054f1944eSPaolo Abeni { 363154f1944eSPaolo Abeni struct mptcp_subflow_context *subflow; 363254f1944eSPaolo Abeni struct mptcp_sock *msk = mptcp_sk(sk); 363354f1944eSPaolo Abeni int err = -EINVAL; 3634ccae357cSPaolo Abeni struct sock *ssk; 363554f1944eSPaolo Abeni 36363f326a82SPaolo Abeni ssk = __mptcp_nmpc_sk(msk); 36373f326a82SPaolo Abeni if (IS_ERR(ssk)) 36383f326a82SPaolo Abeni return PTR_ERR(ssk); 363954f1944eSPaolo Abeni 3640c693a851SGeliang Tang mptcp_set_state(sk, TCP_SYN_SENT); 3641ccae357cSPaolo Abeni subflow = mptcp_subflow_ctx(ssk); 364254f1944eSPaolo Abeni #ifdef CONFIG_TCP_MD5SIG 364354f1944eSPaolo Abeni /* no MPTCP if MD5SIG is enabled on this socket or we may run out of 364454f1944eSPaolo Abeni * TCP option space. 364554f1944eSPaolo Abeni */ 3646ccae357cSPaolo Abeni if (rcu_access_pointer(tcp_sk(ssk)->md5sig_info)) 364754f1944eSPaolo Abeni mptcp_subflow_early_fallback(msk, subflow); 364854f1944eSPaolo Abeni #endif 3649ccae357cSPaolo Abeni if (subflow->request_mptcp && mptcp_token_new_connect(ssk)) { 3650ccae357cSPaolo Abeni MPTCP_INC_STATS(sock_net(ssk), MPTCP_MIB_TOKENFALLBACKINIT); 365154f1944eSPaolo Abeni mptcp_subflow_early_fallback(msk, subflow); 365254f1944eSPaolo Abeni } 365354f1944eSPaolo Abeni if (likely(!__mptcp_check_fallback(msk))) 365454f1944eSPaolo Abeni MPTCP_INC_STATS(sock_net(sk), MPTCP_MIB_MPCAPABLEACTIVE); 365554f1944eSPaolo Abeni 365654f1944eSPaolo Abeni /* if reaching here via the fastopen/sendmsg path, the caller already 365754f1944eSPaolo Abeni * acquired the subflow socket lock, too. 365854f1944eSPaolo Abeni */ 3659ccae357cSPaolo Abeni if (!msk->fastopening) 3660ccae357cSPaolo Abeni lock_sock(ssk); 3661ccae357cSPaolo Abeni 3662ccae357cSPaolo Abeni /* the following mirrors closely a very small chunk of code from 3663ccae357cSPaolo Abeni * __inet_stream_connect() 3664ccae357cSPaolo Abeni */ 3665ccae357cSPaolo Abeni if (ssk->sk_state != TCP_CLOSE) 3666ccae357cSPaolo Abeni goto out; 3667ccae357cSPaolo Abeni 3668ccae357cSPaolo Abeni if (BPF_CGROUP_PRE_CONNECT_ENABLED(ssk)) { 3669ccae357cSPaolo Abeni err = ssk->sk_prot->pre_connect(ssk, uaddr, addr_len); 3670ccae357cSPaolo Abeni if (err) 3671ccae357cSPaolo Abeni goto out; 3672ccae357cSPaolo Abeni } 3673ccae357cSPaolo Abeni 3674ccae357cSPaolo Abeni err = ssk->sk_prot->connect(ssk, uaddr, addr_len); 3675ccae357cSPaolo Abeni if (err < 0) 3676ccae357cSPaolo Abeni goto out; 3677ccae357cSPaolo Abeni 367808e39c0dSEric Dumazet inet_assign_bit(DEFER_CONNECT, sk, inet_test_bit(DEFER_CONNECT, ssk)); 3679ccae357cSPaolo Abeni 3680ccae357cSPaolo Abeni out: 3681ccae357cSPaolo Abeni if (!msk->fastopening) 3682ccae357cSPaolo Abeni release_sock(ssk); 368354f1944eSPaolo Abeni 368454f1944eSPaolo Abeni /* on successful connect, the msk state will be moved to established by 368554f1944eSPaolo Abeni * subflow_finish_connect() 368654f1944eSPaolo Abeni */ 3687ccae357cSPaolo Abeni if (unlikely(err)) { 3688131a6277SPaolo Abeni /* avoid leaving a dangling token in an unconnected socket */ 3689131a6277SPaolo Abeni mptcp_token_destroy(msk); 3690c693a851SGeliang Tang mptcp_set_state(sk, TCP_CLOSE); 369154f1944eSPaolo Abeni return err; 369254f1944eSPaolo Abeni } 369354f1944eSPaolo Abeni 3694ccae357cSPaolo Abeni mptcp_copy_inaddrs(sk, ssk); 3695786fc124SPaolo Abeni return 0; 369654f1944eSPaolo Abeni } 369754f1944eSPaolo Abeni 3698f870fa0bSMat Martineau static struct proto mptcp_prot = { 3699f870fa0bSMat Martineau .name = "MPTCP", 3700f870fa0bSMat Martineau .owner = THIS_MODULE, 3701f870fa0bSMat Martineau .init = mptcp_init_sock, 370254f1944eSPaolo Abeni .connect = mptcp_connect, 370318b683bfSPaolo Abeni .disconnect = mptcp_disconnect, 3704f870fa0bSMat Martineau .close = mptcp_close, 3705717e79c8SPeter Krystad .setsockopt = mptcp_setsockopt, 3706717e79c8SPeter Krystad .getsockopt = mptcp_getsockopt, 370776e2a55dSPaolo Abeni .shutdown = mptcp_shutdown, 370879c0949eSPeter Krystad .destroy = mptcp_destroy, 3709f870fa0bSMat Martineau .sendmsg = mptcp_sendmsg, 3710644807e3SFlorian Westphal .ioctl = mptcp_ioctl, 3711f870fa0bSMat Martineau .recvmsg = mptcp_recvmsg, 371214c441b5SPaolo Abeni .release_cb = mptcp_release_cb, 37132c5ebd00SPaolo Abeni .hash = mptcp_hash, 37142c5ebd00SPaolo Abeni .unhash = mptcp_unhash, 3715cec37a6eSPeter Krystad .get_port = mptcp_get_port, 37166511882cSPaolo Abeni .forward_alloc_get = mptcp_forward_alloc_get, 3717d027236cSPaolo Abeni .sockets_allocated = &mptcp_sockets_allocated, 37180defbb0aSEric Dumazet 3719d027236cSPaolo Abeni .memory_allocated = &tcp_memory_allocated, 37200defbb0aSEric Dumazet .per_cpu_fw_alloc = &tcp_memory_per_cpu_fw_alloc, 37210defbb0aSEric Dumazet 3722d027236cSPaolo Abeni .memory_pressure = &tcp_memory_pressure, 3723d027236cSPaolo Abeni .sysctl_wmem_offset = offsetof(struct net, ipv4.sysctl_tcp_wmem), 3724989ef49bSPaolo Abeni .sysctl_rmem_offset = offsetof(struct net, ipv4.sysctl_tcp_rmem), 3725d027236cSPaolo Abeni .sysctl_mem = sysctl_tcp_mem, 3726f870fa0bSMat Martineau .obj_size = sizeof(struct mptcp_sock), 37272c5ebd00SPaolo Abeni .slab_flags = SLAB_TYPESAFE_BY_RCU, 3728f870fa0bSMat Martineau .no_autobind = true, 3729f870fa0bSMat Martineau }; 3730f870fa0bSMat Martineau 37312303f994SPeter Krystad static int mptcp_bind(struct socket *sock, struct sockaddr *uaddr, int addr_len) 37322303f994SPeter Krystad { 37332303f994SPeter Krystad struct mptcp_sock *msk = mptcp_sk(sock->sk); 37348cf2ebdcSPaolo Abeni struct sock *ssk, *sk = sock->sk; 37358cf2ebdcSPaolo Abeni int err = -EINVAL; 37362303f994SPeter Krystad 37378cf2ebdcSPaolo Abeni lock_sock(sk); 37383f326a82SPaolo Abeni ssk = __mptcp_nmpc_sk(msk); 37393f326a82SPaolo Abeni if (IS_ERR(ssk)) { 37403f326a82SPaolo Abeni err = PTR_ERR(ssk); 37412303f994SPeter Krystad goto unlock; 37422303f994SPeter Krystad } 37432303f994SPeter Krystad 37448cf2ebdcSPaolo Abeni if (sk->sk_family == AF_INET) 37458cf2ebdcSPaolo Abeni err = inet_bind_sk(ssk, uaddr, addr_len); 37468cf2ebdcSPaolo Abeni #if IS_ENABLED(CONFIG_MPTCP_IPV6) 37478cf2ebdcSPaolo Abeni else if (sk->sk_family == AF_INET6) 37488cf2ebdcSPaolo Abeni err = inet6_bind_sk(ssk, uaddr, addr_len); 37498cf2ebdcSPaolo Abeni #endif 3750cf7da0d6SPeter Krystad if (!err) 37518cf2ebdcSPaolo Abeni mptcp_copy_inaddrs(sk, ssk); 37522303f994SPeter Krystad 37532303f994SPeter Krystad unlock: 37548cf2ebdcSPaolo Abeni release_sock(sk); 37552303f994SPeter Krystad return err; 37562303f994SPeter Krystad } 37572303f994SPeter Krystad 3758cf7da0d6SPeter Krystad static int mptcp_listen(struct socket *sock, int backlog) 3759cf7da0d6SPeter Krystad { 3760cf7da0d6SPeter Krystad struct mptcp_sock *msk = mptcp_sk(sock->sk); 3761cfdcfeedSMenglong Dong struct sock *sk = sock->sk; 376240f56d0cSPaolo Abeni struct sock *ssk; 3763cf7da0d6SPeter Krystad int err; 3764cf7da0d6SPeter Krystad 3765cf7da0d6SPeter Krystad pr_debug("msk=%p", msk); 3766cf7da0d6SPeter Krystad 3767cfdcfeedSMenglong Dong lock_sock(sk); 37680226436aSPaolo Abeni 37690226436aSPaolo Abeni err = -EINVAL; 37700226436aSPaolo Abeni if (sock->state != SS_UNCONNECTED || sock->type != SOCK_STREAM) 37710226436aSPaolo Abeni goto unlock; 37720226436aSPaolo Abeni 37733f326a82SPaolo Abeni ssk = __mptcp_nmpc_sk(msk); 37743f326a82SPaolo Abeni if (IS_ERR(ssk)) { 37753f326a82SPaolo Abeni err = PTR_ERR(ssk); 3776cf7da0d6SPeter Krystad goto unlock; 3777cf7da0d6SPeter Krystad } 3778cf7da0d6SPeter Krystad 3779c693a851SGeliang Tang mptcp_set_state(sk, TCP_LISTEN); 3780cfdcfeedSMenglong Dong sock_set_flag(sk, SOCK_RCU_FREE); 37815e20087dSFlorian Westphal 378240f56d0cSPaolo Abeni lock_sock(ssk); 378340f56d0cSPaolo Abeni err = __inet_listen_sk(ssk, backlog); 378440f56d0cSPaolo Abeni release_sock(ssk); 3785c693a851SGeliang Tang mptcp_set_state(sk, inet_sk_state_load(ssk)); 378640f56d0cSPaolo Abeni 3787c558246eSMenglong Dong if (!err) { 3788c558246eSMenglong Dong sock_prot_inuse_add(sock_net(sk), sk->sk_prot, 1); 378940f56d0cSPaolo Abeni mptcp_copy_inaddrs(sk, ssk); 379040f56d0cSPaolo Abeni mptcp_event_pm_listener(ssk, MPTCP_EVENT_LISTENER_CREATED); 379121d9b73aSPaolo Abeni } 3792f8c9dfbdSGeliang Tang 3793cf7da0d6SPeter Krystad unlock: 3794cfdcfeedSMenglong Dong release_sock(sk); 3795cf7da0d6SPeter Krystad return err; 3796cf7da0d6SPeter Krystad } 3797cf7da0d6SPeter Krystad 3798cf7da0d6SPeter Krystad static int mptcp_stream_accept(struct socket *sock, struct socket *newsock, 3799cf7da0d6SPeter Krystad int flags, bool kern) 3800cf7da0d6SPeter Krystad { 3801cf7da0d6SPeter Krystad struct mptcp_sock *msk = mptcp_sk(sock->sk); 38021f6610b9SPaolo Abeni struct sock *ssk, *newsk; 3803cf7da0d6SPeter Krystad int err; 3804cf7da0d6SPeter Krystad 3805cf7da0d6SPeter Krystad pr_debug("msk=%p", msk); 3806cf7da0d6SPeter Krystad 38075b825727SPaolo Abeni /* Buggy applications can call accept on socket states other then LISTEN 380861761231SPaolo Abeni * but no need to allocate the first subflow just to error out. 380961761231SPaolo Abeni */ 38101f6610b9SPaolo Abeni ssk = READ_ONCE(msk->first); 38111f6610b9SPaolo Abeni if (!ssk) 381271ba088cSPaolo Abeni return -EINVAL; 3813cf7da0d6SPeter Krystad 38148e2b8a9fSDavide Caratti pr_debug("ssk=%p, listener=%p", ssk, mptcp_subflow_ctx(ssk)); 38158e2b8a9fSDavide Caratti newsk = inet_csk_accept(ssk, flags, &err, kern); 3816e76c8ef5SPaolo Abeni if (!newsk) 3817e76c8ef5SPaolo Abeni return err; 3818e76c8ef5SPaolo Abeni 38198e2b8a9fSDavide Caratti pr_debug("newsk=%p, subflow is mptcp=%d", newsk, sk_is_mptcp(newsk)); 38208e2b8a9fSDavide Caratti if (sk_is_mptcp(newsk)) { 3821cf7da0d6SPeter Krystad struct mptcp_subflow_context *subflow; 38228e2b8a9fSDavide Caratti struct sock *new_mptcp_sock; 38238e2b8a9fSDavide Caratti 38248e2b8a9fSDavide Caratti subflow = mptcp_subflow_ctx(newsk); 38258e2b8a9fSDavide Caratti new_mptcp_sock = subflow->conn; 38268e2b8a9fSDavide Caratti 38278e2b8a9fSDavide Caratti /* is_mptcp should be false if subflow->conn is missing, see 38288e2b8a9fSDavide Caratti * subflow_syn_recv_sock() 38298e2b8a9fSDavide Caratti */ 38308e2b8a9fSDavide Caratti if (WARN_ON_ONCE(!new_mptcp_sock)) { 38318e2b8a9fSDavide Caratti tcp_sk(newsk)->is_mptcp = 0; 38328e2b8a9fSDavide Caratti goto tcpfallback; 38338e2b8a9fSDavide Caratti } 38348e2b8a9fSDavide Caratti 38358e2b8a9fSDavide Caratti newsk = new_mptcp_sock; 38368e2b8a9fSDavide Caratti MPTCP_INC_STATS(sock_net(ssk), MPTCP_MIB_MPCAPABLEPASSIVEACK); 38378e2b8a9fSDavide Caratti 38388e2b8a9fSDavide Caratti newsk->sk_kern_sock = kern; 38398e2b8a9fSDavide Caratti lock_sock(newsk); 38408e2b8a9fSDavide Caratti __inet_accept(sock, newsock, newsk); 38410397c6d8SPaolo Abeni 3842a5ef058dSPaolo Abeni set_bit(SOCK_CUSTOM_SOCKOPT, &newsock->flags); 38438e2b8a9fSDavide Caratti msk = mptcp_sk(newsk); 3844b6985b9bSPaolo Abeni msk->in_accept_queue = 0; 3845a5ef058dSPaolo Abeni 3846cf7da0d6SPeter Krystad /* set ssk->sk_socket of accept()ed flows to mptcp socket. 3847cf7da0d6SPeter Krystad * This is needed so NOSPACE flag can be set from tcp stack. 3848cf7da0d6SPeter Krystad */ 3849190f8b06SGeliang Tang mptcp_for_each_subflow(msk, subflow) { 3850cf7da0d6SPeter Krystad struct sock *ssk = mptcp_subflow_tcp_sock(subflow); 3851cf7da0d6SPeter Krystad 3852cf7da0d6SPeter Krystad if (!ssk->sk_socket) 3853cf7da0d6SPeter Krystad mptcp_sock_graft(ssk, newsock); 3854cf7da0d6SPeter Krystad } 385563740448SPaolo Abeni 385663740448SPaolo Abeni /* Do late cleanup for the first subflow as necessary. Also 385763740448SPaolo Abeni * deal with bad peers not doing a complete shutdown. 385863740448SPaolo Abeni */ 385939880bd8SPaolo Abeni if (unlikely(inet_sk_state_load(msk->first) == TCP_CLOSE)) { 386063740448SPaolo Abeni __mptcp_close_ssk(newsk, msk->first, 386163740448SPaolo Abeni mptcp_subflow_ctx(msk->first), 0); 386239880bd8SPaolo Abeni if (unlikely(list_is_singular(&msk->conn_list))) 3863c693a851SGeliang Tang mptcp_set_state(newsk, TCP_CLOSE); 386463740448SPaolo Abeni } 38658e2b8a9fSDavide Caratti } else { 38668e2b8a9fSDavide Caratti MPTCP_INC_STATS(sock_net(ssk), 38678e2b8a9fSDavide Caratti MPTCP_MIB_MPCAPABLEPASSIVEFALLBACK); 38688e2b8a9fSDavide Caratti tcpfallback: 38698e2b8a9fSDavide Caratti newsk->sk_kern_sock = kern; 38708e2b8a9fSDavide Caratti lock_sock(newsk); 38718e2b8a9fSDavide Caratti __inet_accept(sock, newsock, newsk); 38728e2b8a9fSDavide Caratti /* we are being invoked after accepting a non-mp-capable 38738e2b8a9fSDavide Caratti * flow: sk is a tcp_sk, not an mptcp one. 38748e2b8a9fSDavide Caratti * 38758e2b8a9fSDavide Caratti * Hand the socket over to tcp so all further socket ops 38768e2b8a9fSDavide Caratti * bypass mptcp. 38778e2b8a9fSDavide Caratti */ 38788e2b8a9fSDavide Caratti WRITE_ONCE(newsock->sk->sk_socket->ops, 38798e2b8a9fSDavide Caratti mptcp_fallback_tcp_ops(newsock->sk)); 3880cf7da0d6SPeter Krystad } 3881e76c8ef5SPaolo Abeni release_sock(newsk); 3882cf7da0d6SPeter Krystad 3883e76c8ef5SPaolo Abeni return 0; 3884cf7da0d6SPeter Krystad } 3885cf7da0d6SPeter Krystad 38868edf0864SFlorian Westphal static __poll_t mptcp_check_writeable(struct mptcp_sock *msk) 38878edf0864SFlorian Westphal { 38888edf0864SFlorian Westphal struct sock *sk = (struct sock *)msk; 38898edf0864SFlorian Westphal 38908edf0864SFlorian Westphal if (sk_stream_is_writeable(sk)) 38918edf0864SFlorian Westphal return EPOLLOUT | EPOLLWRNORM; 38928edf0864SFlorian Westphal 38935cf92bbaSPaolo Abeni mptcp_set_nospace(sk); 38946e628cd3SPaolo Abeni smp_mb__after_atomic(); /* msk->flags is changed by write_space cb */ 38956e628cd3SPaolo Abeni if (sk_stream_is_writeable(sk)) 38966e628cd3SPaolo Abeni return EPOLLOUT | EPOLLWRNORM; 38978edf0864SFlorian Westphal 38986e628cd3SPaolo Abeni return 0; 38998edf0864SFlorian Westphal } 39008edf0864SFlorian Westphal 39012303f994SPeter Krystad static __poll_t mptcp_poll(struct file *file, struct socket *sock, 39022303f994SPeter Krystad struct poll_table_struct *wait) 39032303f994SPeter Krystad { 39041891c4a0SFlorian Westphal struct sock *sk = sock->sk; 39058ab183deSPaolo Abeni struct mptcp_sock *msk; 39062303f994SPeter Krystad __poll_t mask = 0; 39076b9831bfSPaolo Abeni u8 shutdown; 39088a05661bSPaolo Abeni int state; 39092303f994SPeter Krystad 39101891c4a0SFlorian Westphal msk = mptcp_sk(sk); 39111891c4a0SFlorian Westphal sock_poll_wait(file, sock, wait); 39121891c4a0SFlorian Westphal 39138a05661bSPaolo Abeni state = inet_sk_state_load(sk); 39146719331cSPaolo Abeni pr_debug("msk=%p state=%d flags=%lx", msk, state, msk->flags); 391571ba088cSPaolo Abeni if (state == TCP_LISTEN) { 39165426a4efSPaolo Abeni struct sock *ssk = READ_ONCE(msk->first); 39175b825727SPaolo Abeni 39185426a4efSPaolo Abeni if (WARN_ON_ONCE(!ssk)) 391971ba088cSPaolo Abeni return 0; 392071ba088cSPaolo Abeni 39215426a4efSPaolo Abeni return inet_csk_listen_poll(ssk); 392271ba088cSPaolo Abeni } 39238a05661bSPaolo Abeni 39246b9831bfSPaolo Abeni shutdown = READ_ONCE(sk->sk_shutdown); 39256b9831bfSPaolo Abeni if (shutdown == SHUTDOWN_MASK || state == TCP_CLOSE) 39266b9831bfSPaolo Abeni mask |= EPOLLHUP; 39276b9831bfSPaolo Abeni if (shutdown & RCV_SHUTDOWN) 39286b9831bfSPaolo Abeni mask |= EPOLLIN | EPOLLRDNORM | EPOLLRDHUP; 39296b9831bfSPaolo Abeni 39308a05661bSPaolo Abeni if (state != TCP_SYN_SENT && state != TCP_SYN_RECV) { 39315684ab1aSPaolo Abeni mask |= mptcp_check_readable(sk); 39326b9831bfSPaolo Abeni if (shutdown & SEND_SHUTDOWN) 39336b9831bfSPaolo Abeni mask |= EPOLLOUT | EPOLLWRNORM; 39346b9831bfSPaolo Abeni else 39358edf0864SFlorian Westphal mask |= mptcp_check_writeable(msk); 393608e39c0dSEric Dumazet } else if (state == TCP_SYN_SENT && 393708e39c0dSEric Dumazet inet_test_bit(DEFER_CONNECT, sk)) { 3938a42cf9d1SBenjamin Hesmans /* cf tcp_poll() note about TFO */ 3939a42cf9d1SBenjamin Hesmans mask |= EPOLLOUT | EPOLLWRNORM; 39408a05661bSPaolo Abeni } 39411891c4a0SFlorian Westphal 3942d89e3ed7SPaolo Abeni /* This barrier is coupled with smp_wmb() in __mptcp_error_report() */ 394315cc1045SPaolo Abeni smp_rmb(); 39449ae8e5adSEric Dumazet if (READ_ONCE(sk->sk_err)) 394515cc1045SPaolo Abeni mask |= EPOLLERR; 394615cc1045SPaolo Abeni 39472303f994SPeter Krystad return mask; 39482303f994SPeter Krystad } 39492303f994SPeter Krystad 3950e42f1ac6SFlorian Westphal static const struct proto_ops mptcp_stream_ops = { 3951e42f1ac6SFlorian Westphal .family = PF_INET, 3952e42f1ac6SFlorian Westphal .owner = THIS_MODULE, 39530a3cc579SPaolo Abeni .release = inet_release, 3954e42f1ac6SFlorian Westphal .bind = mptcp_bind, 3955786fc124SPaolo Abeni .connect = inet_stream_connect, 3956e42f1ac6SFlorian Westphal .socketpair = sock_no_socketpair, 3957e42f1ac6SFlorian Westphal .accept = mptcp_stream_accept, 3958d2f77c53SPaolo Abeni .getname = inet_getname, 3959e42f1ac6SFlorian Westphal .poll = mptcp_poll, 3960e42f1ac6SFlorian Westphal .ioctl = inet_ioctl, 3961e42f1ac6SFlorian Westphal .gettstamp = sock_gettstamp, 3962e42f1ac6SFlorian Westphal .listen = mptcp_listen, 396376e2a55dSPaolo Abeni .shutdown = inet_shutdown, 3964e42f1ac6SFlorian Westphal .setsockopt = sock_common_setsockopt, 3965e42f1ac6SFlorian Westphal .getsockopt = sock_common_getsockopt, 3966e42f1ac6SFlorian Westphal .sendmsg = inet_sendmsg, 3967e42f1ac6SFlorian Westphal .recvmsg = inet_recvmsg, 3968e42f1ac6SFlorian Westphal .mmap = sock_no_mmap, 39695684ab1aSPaolo Abeni .set_rcvlowat = mptcp_set_rcvlowat, 3970e42f1ac6SFlorian Westphal }; 39712303f994SPeter Krystad 3972f870fa0bSMat Martineau static struct inet_protosw mptcp_protosw = { 3973f870fa0bSMat Martineau .type = SOCK_STREAM, 3974f870fa0bSMat Martineau .protocol = IPPROTO_MPTCP, 3975f870fa0bSMat Martineau .prot = &mptcp_prot, 39762303f994SPeter Krystad .ops = &mptcp_stream_ops, 39772303f994SPeter Krystad .flags = INET_PROTOSW_ICSK, 3978f870fa0bSMat Martineau }; 3979f870fa0bSMat Martineau 3980b19bc294SPaolo Abeni static int mptcp_napi_poll(struct napi_struct *napi, int budget) 3981b19bc294SPaolo Abeni { 3982b19bc294SPaolo Abeni struct mptcp_delegated_action *delegated; 3983b19bc294SPaolo Abeni struct mptcp_subflow_context *subflow; 3984b19bc294SPaolo Abeni int work_done = 0; 3985b19bc294SPaolo Abeni 3986b19bc294SPaolo Abeni delegated = container_of(napi, struct mptcp_delegated_action, napi); 3987b19bc294SPaolo Abeni while ((subflow = mptcp_subflow_delegated_next(delegated)) != NULL) { 3988b19bc294SPaolo Abeni struct sock *ssk = mptcp_subflow_tcp_sock(subflow); 3989b19bc294SPaolo Abeni 3990b19bc294SPaolo Abeni bh_lock_sock_nested(ssk); 3991a5efdbceSPaolo Abeni if (!sock_owned_by_user(ssk)) { 3992a5efdbceSPaolo Abeni mptcp_subflow_process_delegated(ssk, xchg(&subflow->delegated_status, 0)); 3993a5efdbceSPaolo Abeni } else { 3994a5efdbceSPaolo Abeni /* tcp_release_cb_override already processed 3995b19bc294SPaolo Abeni * the action or will do at next release_sock(). 3996b19bc294SPaolo Abeni * In both case must dequeue the subflow here - on the same 3997b19bc294SPaolo Abeni * CPU that scheduled it. 3998b19bc294SPaolo Abeni */ 3999a5efdbceSPaolo Abeni smp_wmb(); 4000a5efdbceSPaolo Abeni clear_bit(MPTCP_DELEGATE_SCHEDULED, &subflow->delegated_status); 4001a5efdbceSPaolo Abeni } 4002b19bc294SPaolo Abeni bh_unlock_sock(ssk); 4003b19bc294SPaolo Abeni sock_put(ssk); 4004b19bc294SPaolo Abeni 4005b19bc294SPaolo Abeni if (++work_done == budget) 4006b19bc294SPaolo Abeni return budget; 4007b19bc294SPaolo Abeni } 4008b19bc294SPaolo Abeni 4009b19bc294SPaolo Abeni /* always provide a 0 'work_done' argument, so that napi_complete_done 4010b19bc294SPaolo Abeni * will not try accessing the NULL napi->dev ptr 4011b19bc294SPaolo Abeni */ 4012b19bc294SPaolo Abeni napi_complete_done(napi, 0); 4013b19bc294SPaolo Abeni return work_done; 4014b19bc294SPaolo Abeni } 4015b19bc294SPaolo Abeni 4016d39dcecaSPaolo Abeni void __init mptcp_proto_init(void) 4017f870fa0bSMat Martineau { 4018b19bc294SPaolo Abeni struct mptcp_delegated_action *delegated; 4019b19bc294SPaolo Abeni int cpu; 4020b19bc294SPaolo Abeni 40212303f994SPeter Krystad mptcp_prot.h.hashinfo = tcp_prot.h.hashinfo; 40222303f994SPeter Krystad 4023d027236cSPaolo Abeni if (percpu_counter_init(&mptcp_sockets_allocated, 0, GFP_KERNEL)) 4024d027236cSPaolo Abeni panic("Failed to allocate MPTCP pcpu counter\n"); 4025d027236cSPaolo Abeni 4026b19bc294SPaolo Abeni init_dummy_netdev(&mptcp_napi_dev); 4027b19bc294SPaolo Abeni for_each_possible_cpu(cpu) { 4028b19bc294SPaolo Abeni delegated = per_cpu_ptr(&mptcp_delegated_actions, cpu); 4029b19bc294SPaolo Abeni INIT_LIST_HEAD(&delegated->head); 403016d083e2SJakub Kicinski netif_napi_add_tx(&mptcp_napi_dev, &delegated->napi, 403116d083e2SJakub Kicinski mptcp_napi_poll); 4032b19bc294SPaolo Abeni napi_enable(&delegated->napi); 4033b19bc294SPaolo Abeni } 4034b19bc294SPaolo Abeni 40352303f994SPeter Krystad mptcp_subflow_init(); 40361b1c7a0eSPeter Krystad mptcp_pm_init(); 4037ed1ad86bSGeliang Tang mptcp_sched_init(); 40382c5ebd00SPaolo Abeni mptcp_token_init(); 40392303f994SPeter Krystad 4040f870fa0bSMat Martineau if (proto_register(&mptcp_prot, 1) != 0) 4041f870fa0bSMat Martineau panic("Failed to register MPTCP proto.\n"); 4042f870fa0bSMat Martineau 4043f870fa0bSMat Martineau inet_register_protosw(&mptcp_protosw); 40446771bfd9SFlorian Westphal 40456771bfd9SFlorian Westphal BUILD_BUG_ON(sizeof(struct mptcp_skb_cb) > sizeof_field(struct sk_buff, cb)); 4046f870fa0bSMat Martineau } 4047f870fa0bSMat Martineau 4048f870fa0bSMat Martineau #if IS_ENABLED(CONFIG_MPTCP_IPV6) 4049e42f1ac6SFlorian Westphal static const struct proto_ops mptcp_v6_stream_ops = { 4050e42f1ac6SFlorian Westphal .family = PF_INET6, 4051e42f1ac6SFlorian Westphal .owner = THIS_MODULE, 40520a3cc579SPaolo Abeni .release = inet6_release, 4053e42f1ac6SFlorian Westphal .bind = mptcp_bind, 4054786fc124SPaolo Abeni .connect = inet_stream_connect, 4055e42f1ac6SFlorian Westphal .socketpair = sock_no_socketpair, 4056e42f1ac6SFlorian Westphal .accept = mptcp_stream_accept, 4057d2f77c53SPaolo Abeni .getname = inet6_getname, 4058e42f1ac6SFlorian Westphal .poll = mptcp_poll, 4059e42f1ac6SFlorian Westphal .ioctl = inet6_ioctl, 4060e42f1ac6SFlorian Westphal .gettstamp = sock_gettstamp, 4061e42f1ac6SFlorian Westphal .listen = mptcp_listen, 406276e2a55dSPaolo Abeni .shutdown = inet_shutdown, 4063e42f1ac6SFlorian Westphal .setsockopt = sock_common_setsockopt, 4064e42f1ac6SFlorian Westphal .getsockopt = sock_common_getsockopt, 4065e42f1ac6SFlorian Westphal .sendmsg = inet6_sendmsg, 4066e42f1ac6SFlorian Westphal .recvmsg = inet6_recvmsg, 4067e42f1ac6SFlorian Westphal .mmap = sock_no_mmap, 4068e42f1ac6SFlorian Westphal #ifdef CONFIG_COMPAT 40693986912fSChristoph Hellwig .compat_ioctl = inet6_compat_ioctl, 4070e42f1ac6SFlorian Westphal #endif 40715684ab1aSPaolo Abeni .set_rcvlowat = mptcp_set_rcvlowat, 4072e42f1ac6SFlorian Westphal }; 4073e42f1ac6SFlorian Westphal 4074f870fa0bSMat Martineau static struct proto mptcp_v6_prot; 4075f870fa0bSMat Martineau 4076f870fa0bSMat Martineau static struct inet_protosw mptcp_v6_protosw = { 4077f870fa0bSMat Martineau .type = SOCK_STREAM, 4078f870fa0bSMat Martineau .protocol = IPPROTO_MPTCP, 4079f870fa0bSMat Martineau .prot = &mptcp_v6_prot, 40802303f994SPeter Krystad .ops = &mptcp_v6_stream_ops, 4081f870fa0bSMat Martineau .flags = INET_PROTOSW_ICSK, 4082f870fa0bSMat Martineau }; 4083f870fa0bSMat Martineau 4084d39dcecaSPaolo Abeni int __init mptcp_proto_v6_init(void) 4085f870fa0bSMat Martineau { 4086f870fa0bSMat Martineau int err; 4087f870fa0bSMat Martineau 4088f870fa0bSMat Martineau mptcp_v6_prot = mptcp_prot; 4089f870fa0bSMat Martineau strcpy(mptcp_v6_prot.name, "MPTCPv6"); 4090f870fa0bSMat Martineau mptcp_v6_prot.slab = NULL; 4091b0519de8SFlorian Westphal mptcp_v6_prot.obj_size = sizeof(struct mptcp6_sock); 4092f5f80e32SEric Dumazet mptcp_v6_prot.ipv6_pinfo_offset = offsetof(struct mptcp6_sock, np); 4093f870fa0bSMat Martineau 4094f870fa0bSMat Martineau err = proto_register(&mptcp_v6_prot, 1); 4095f870fa0bSMat Martineau if (err) 4096f870fa0bSMat Martineau return err; 4097f870fa0bSMat Martineau 4098f870fa0bSMat Martineau err = inet6_register_protosw(&mptcp_v6_protosw); 4099f870fa0bSMat Martineau if (err) 4100f870fa0bSMat Martineau proto_unregister(&mptcp_v6_prot); 4101f870fa0bSMat Martineau 4102f870fa0bSMat Martineau return err; 4103f870fa0bSMat Martineau } 4104f870fa0bSMat Martineau #endif 4105