1f870fa0bSMat Martineau // SPDX-License-Identifier: GPL-2.0 2f870fa0bSMat Martineau /* Multipath TCP 3f870fa0bSMat Martineau * 4f870fa0bSMat Martineau * Copyright (c) 2017 - 2019, Intel Corporation. 5f870fa0bSMat Martineau */ 6f870fa0bSMat Martineau 7f870fa0bSMat Martineau #define pr_fmt(fmt) "MPTCP: " fmt 8f870fa0bSMat Martineau 9f870fa0bSMat Martineau #include <linux/kernel.h> 10f870fa0bSMat Martineau #include <linux/module.h> 11f870fa0bSMat Martineau #include <linux/netdevice.h> 127a6a6cbcSPaolo Abeni #include <linux/sched/signal.h> 137a6a6cbcSPaolo Abeni #include <linux/atomic.h> 14f870fa0bSMat Martineau #include <net/sock.h> 15f870fa0bSMat Martineau #include <net/inet_common.h> 16f870fa0bSMat Martineau #include <net/inet_hashtables.h> 17f870fa0bSMat Martineau #include <net/protocol.h> 18f870fa0bSMat Martineau #include <net/tcp.h> 193721b9b6SMat Martineau #include <net/tcp_states.h> 20cf7da0d6SPeter Krystad #if IS_ENABLED(CONFIG_MPTCP_IPV6) 21cf7da0d6SPeter Krystad #include <net/transp_v6.h> 22cf7da0d6SPeter Krystad #endif 23f870fa0bSMat Martineau #include <net/mptcp.h> 24e16163b6SPaolo Abeni #include <net/xfrm.h> 25644807e3SFlorian Westphal #include <asm/ioctls.h> 26f870fa0bSMat Martineau #include "protocol.h" 27fc518953SFlorian Westphal #include "mib.h" 28f870fa0bSMat Martineau 29e10a9892SGeliang Tang #define CREATE_TRACE_POINTS 30e10a9892SGeliang Tang #include <trace/events/mptcp.h> 31e10a9892SGeliang Tang 32b0519de8SFlorian Westphal #if IS_ENABLED(CONFIG_MPTCP_IPV6) 33b0519de8SFlorian Westphal struct mptcp6_sock { 34b0519de8SFlorian Westphal struct mptcp_sock msk; 35b0519de8SFlorian Westphal struct ipv6_pinfo np; 36b0519de8SFlorian Westphal }; 37b0519de8SFlorian Westphal #endif 38b0519de8SFlorian Westphal 396771bfd9SFlorian Westphal struct mptcp_skb_cb { 40ab174ad8SPaolo Abeni u64 map_seq; 41ab174ad8SPaolo Abeni u64 end_seq; 426771bfd9SFlorian Westphal u32 offset; 43b7f653b2SFlorian Westphal u8 has_rxtstamp:1; 446771bfd9SFlorian Westphal }; 456771bfd9SFlorian Westphal 466771bfd9SFlorian Westphal #define MPTCP_SKB_CB(__skb) ((struct mptcp_skb_cb *)&((__skb)->cb[0])) 476771bfd9SFlorian Westphal 48b7f653b2SFlorian Westphal enum { 49b7f653b2SFlorian Westphal MPTCP_CMSG_TS = BIT(0), 502c9e7765SFlorian Westphal MPTCP_CMSG_INQ = BIT(1), 51b7f653b2SFlorian Westphal }; 52b7f653b2SFlorian Westphal 5391b6d325SEric Dumazet static struct percpu_counter mptcp_sockets_allocated ____cacheline_aligned_in_smp; 54d027236cSPaolo Abeni 55e16163b6SPaolo Abeni static void __mptcp_destroy_sock(struct sock *sk); 56d9ca1de8SPaolo Abeni static void __mptcp_check_send_data_fin(struct sock *sk); 57e16163b6SPaolo Abeni 58b19bc294SPaolo Abeni DEFINE_PER_CPU(struct mptcp_delegated_action, mptcp_delegated_actions); 59b19bc294SPaolo Abeni static struct net_device mptcp_napi_dev; 60b19bc294SPaolo Abeni 612303f994SPeter Krystad /* If msk has an initial subflow socket, and the MP_CAPABLE handshake has not 622303f994SPeter Krystad * completed yet or has failed, return the subflow socket. 632303f994SPeter Krystad * Otherwise return NULL. 642303f994SPeter Krystad */ 651729cf18SGeliang Tang struct socket *__mptcp_nmpc_socket(const struct mptcp_sock *msk) 662303f994SPeter Krystad { 67d22f4988SChristoph Paasch if (!msk->subflow || READ_ONCE(msk->can_ack)) 682303f994SPeter Krystad return NULL; 692303f994SPeter Krystad 702303f994SPeter Krystad return msk->subflow; 712303f994SPeter Krystad } 722303f994SPeter Krystad 736f8a612aSFlorian Westphal /* Returns end sequence number of the receiver's advertised window */ 746f8a612aSFlorian Westphal static u64 mptcp_wnd_end(const struct mptcp_sock *msk) 756f8a612aSFlorian Westphal { 767439d687SPaolo Abeni return READ_ONCE(msk->wnd_end); 776f8a612aSFlorian Westphal } 786f8a612aSFlorian Westphal 79d2f77c53SPaolo Abeni static bool mptcp_is_tcpsk(struct sock *sk) 800b4f33deSFlorian Westphal { 810b4f33deSFlorian Westphal struct socket *sock = sk->sk_socket; 820b4f33deSFlorian Westphal 830b4f33deSFlorian Westphal if (unlikely(sk->sk_prot == &tcp_prot)) { 840b4f33deSFlorian Westphal /* we are being invoked after mptcp_accept() has 850b4f33deSFlorian Westphal * accepted a non-mp-capable flow: sk is a tcp_sk, 860b4f33deSFlorian Westphal * not an mptcp one. 870b4f33deSFlorian Westphal * 880b4f33deSFlorian Westphal * Hand the socket over to tcp so all further socket ops 890b4f33deSFlorian Westphal * bypass mptcp. 900b4f33deSFlorian Westphal */ 910b4f33deSFlorian Westphal sock->ops = &inet_stream_ops; 92d2f77c53SPaolo Abeni return true; 930b4f33deSFlorian Westphal #if IS_ENABLED(CONFIG_MPTCP_IPV6) 940b4f33deSFlorian Westphal } else if (unlikely(sk->sk_prot == &tcpv6_prot)) { 950b4f33deSFlorian Westphal sock->ops = &inet6_stream_ops; 96d2f77c53SPaolo Abeni return true; 970b4f33deSFlorian Westphal #endif 980b4f33deSFlorian Westphal } 990b4f33deSFlorian Westphal 100d2f77c53SPaolo Abeni return false; 1010b4f33deSFlorian Westphal } 1020b4f33deSFlorian Westphal 103fa68018dSPaolo Abeni static int __mptcp_socket_create(struct mptcp_sock *msk) 1042303f994SPeter Krystad { 1052303f994SPeter Krystad struct mptcp_subflow_context *subflow; 1062303f994SPeter Krystad struct sock *sk = (struct sock *)msk; 1072303f994SPeter Krystad struct socket *ssock; 1082303f994SPeter Krystad int err; 1092303f994SPeter Krystad 1102303f994SPeter Krystad err = mptcp_subflow_create_socket(sk, &ssock); 1112303f994SPeter Krystad if (err) 112fa68018dSPaolo Abeni return err; 1132303f994SPeter Krystad 1148ab183deSPaolo Abeni msk->first = ssock->sk; 1152303f994SPeter Krystad msk->subflow = ssock; 1162303f994SPeter Krystad subflow = mptcp_subflow_ctx(ssock->sk); 117cec37a6eSPeter Krystad list_add(&subflow->node, &msk->conn_list); 118e16163b6SPaolo Abeni sock_hold(ssock->sk); 1192303f994SPeter Krystad subflow->request_mptcp = 1; 1204cf86ae8SPaolo Abeni 1214cf86ae8SPaolo Abeni /* This is the first subflow, always with id 0 */ 1224cf86ae8SPaolo Abeni subflow->local_id_valid = 1; 123866f26f2SPaolo Abeni mptcp_sock_graft(msk->first, sk->sk_socket); 124e1ff9e82SDavide Caratti 125fa68018dSPaolo Abeni return 0; 1262303f994SPeter Krystad } 1272303f994SPeter Krystad 128ab174ad8SPaolo Abeni static void mptcp_drop(struct sock *sk, struct sk_buff *skb) 129ab174ad8SPaolo Abeni { 130ab174ad8SPaolo Abeni sk_drops_add(sk, skb); 131ab174ad8SPaolo Abeni __kfree_skb(skb); 132ab174ad8SPaolo Abeni } 133ab174ad8SPaolo Abeni 1346511882cSPaolo Abeni static void mptcp_rmem_charge(struct sock *sk, int size) 1356511882cSPaolo Abeni { 1366511882cSPaolo Abeni mptcp_sk(sk)->rmem_fwd_alloc -= size; 1376511882cSPaolo Abeni } 1386511882cSPaolo Abeni 1398268ed4cSPaolo Abeni static bool mptcp_try_coalesce(struct sock *sk, struct sk_buff *to, 1408268ed4cSPaolo Abeni struct sk_buff *from) 1418268ed4cSPaolo Abeni { 1428268ed4cSPaolo Abeni bool fragstolen; 1438268ed4cSPaolo Abeni int delta; 1448268ed4cSPaolo Abeni 1458268ed4cSPaolo Abeni if (MPTCP_SKB_CB(from)->offset || 1468268ed4cSPaolo Abeni !skb_try_coalesce(to, from, &fragstolen, &delta)) 1478268ed4cSPaolo Abeni return false; 1488268ed4cSPaolo Abeni 14906242e44SPaolo Abeni pr_debug("colesced seq %llx into %llx new len %d new end seq %llx", 15006242e44SPaolo Abeni MPTCP_SKB_CB(from)->map_seq, MPTCP_SKB_CB(to)->map_seq, 15106242e44SPaolo Abeni to->len, MPTCP_SKB_CB(from)->end_seq); 152ab174ad8SPaolo Abeni MPTCP_SKB_CB(to)->end_seq = MPTCP_SKB_CB(from)->end_seq; 1538268ed4cSPaolo Abeni kfree_skb_partial(from, fragstolen); 1548268ed4cSPaolo Abeni atomic_add(delta, &sk->sk_rmem_alloc); 1556511882cSPaolo Abeni mptcp_rmem_charge(sk, delta); 1568268ed4cSPaolo Abeni return true; 1578268ed4cSPaolo Abeni } 1588268ed4cSPaolo Abeni 159ab174ad8SPaolo Abeni static bool mptcp_ooo_try_coalesce(struct mptcp_sock *msk, struct sk_buff *to, 160ab174ad8SPaolo Abeni struct sk_buff *from) 1616771bfd9SFlorian Westphal { 162ab174ad8SPaolo Abeni if (MPTCP_SKB_CB(from)->map_seq != MPTCP_SKB_CB(to)->end_seq) 163ab174ad8SPaolo Abeni return false; 164ab174ad8SPaolo Abeni 165ab174ad8SPaolo Abeni return mptcp_try_coalesce((struct sock *)msk, to, from); 166ab174ad8SPaolo Abeni } 167ab174ad8SPaolo Abeni 1686511882cSPaolo Abeni static void __mptcp_rmem_reclaim(struct sock *sk, int amount) 1696511882cSPaolo Abeni { 1706511882cSPaolo Abeni amount >>= SK_MEM_QUANTUM_SHIFT; 1716511882cSPaolo Abeni mptcp_sk(sk)->rmem_fwd_alloc -= amount << SK_MEM_QUANTUM_SHIFT; 1726511882cSPaolo Abeni __sk_mem_reduce_allocated(sk, amount); 1736511882cSPaolo Abeni } 1746511882cSPaolo Abeni 1756511882cSPaolo Abeni static void mptcp_rmem_uncharge(struct sock *sk, int size) 1766511882cSPaolo Abeni { 1776511882cSPaolo Abeni struct mptcp_sock *msk = mptcp_sk(sk); 1786511882cSPaolo Abeni int reclaimable; 1796511882cSPaolo Abeni 1806511882cSPaolo Abeni msk->rmem_fwd_alloc += size; 1816511882cSPaolo Abeni reclaimable = msk->rmem_fwd_alloc - sk_unused_reserved_mem(sk); 1826511882cSPaolo Abeni 1836511882cSPaolo Abeni /* see sk_mem_uncharge() for the rationale behind the following schema */ 1846511882cSPaolo Abeni if (unlikely(reclaimable >= SK_RECLAIM_THRESHOLD)) 1856511882cSPaolo Abeni __mptcp_rmem_reclaim(sk, SK_RECLAIM_CHUNK); 1866511882cSPaolo Abeni } 1876511882cSPaolo Abeni 1886511882cSPaolo Abeni static void mptcp_rfree(struct sk_buff *skb) 1896511882cSPaolo Abeni { 1906511882cSPaolo Abeni unsigned int len = skb->truesize; 1916511882cSPaolo Abeni struct sock *sk = skb->sk; 1926511882cSPaolo Abeni 1936511882cSPaolo Abeni atomic_sub(len, &sk->sk_rmem_alloc); 1946511882cSPaolo Abeni mptcp_rmem_uncharge(sk, len); 1956511882cSPaolo Abeni } 1966511882cSPaolo Abeni 1976511882cSPaolo Abeni static void mptcp_set_owner_r(struct sk_buff *skb, struct sock *sk) 1986511882cSPaolo Abeni { 1996511882cSPaolo Abeni skb_orphan(skb); 2006511882cSPaolo Abeni skb->sk = sk; 2016511882cSPaolo Abeni skb->destructor = mptcp_rfree; 2026511882cSPaolo Abeni atomic_add(skb->truesize, &sk->sk_rmem_alloc); 2036511882cSPaolo Abeni mptcp_rmem_charge(sk, skb->truesize); 2046511882cSPaolo Abeni } 2056511882cSPaolo Abeni 206ab174ad8SPaolo Abeni /* "inspired" by tcp_data_queue_ofo(), main differences: 207ab174ad8SPaolo Abeni * - use mptcp seqs 208ab174ad8SPaolo Abeni * - don't cope with sacks 209ab174ad8SPaolo Abeni */ 210ab174ad8SPaolo Abeni static void mptcp_data_queue_ofo(struct mptcp_sock *msk, struct sk_buff *skb) 211ab174ad8SPaolo Abeni { 212ab174ad8SPaolo Abeni struct sock *sk = (struct sock *)msk; 213ab174ad8SPaolo Abeni struct rb_node **p, *parent; 214ab174ad8SPaolo Abeni u64 seq, end_seq, max_seq; 215ab174ad8SPaolo Abeni struct sk_buff *skb1; 216ab174ad8SPaolo Abeni 217ab174ad8SPaolo Abeni seq = MPTCP_SKB_CB(skb)->map_seq; 218ab174ad8SPaolo Abeni end_seq = MPTCP_SKB_CB(skb)->end_seq; 219fa3fe2b1SFlorian Westphal max_seq = READ_ONCE(msk->rcv_wnd_sent); 220ab174ad8SPaolo Abeni 22106242e44SPaolo Abeni pr_debug("msk=%p seq=%llx limit=%llx empty=%d", msk, seq, max_seq, 22206242e44SPaolo Abeni RB_EMPTY_ROOT(&msk->out_of_order_queue)); 223fa3fe2b1SFlorian Westphal if (after64(end_seq, max_seq)) { 224ab174ad8SPaolo Abeni /* out of window */ 225ab174ad8SPaolo Abeni mptcp_drop(sk, skb); 226fa3fe2b1SFlorian Westphal pr_debug("oow by %lld, rcv_wnd_sent %llu\n", 227fa3fe2b1SFlorian Westphal (unsigned long long)end_seq - (unsigned long)max_seq, 228fa3fe2b1SFlorian Westphal (unsigned long long)msk->rcv_wnd_sent); 22906242e44SPaolo Abeni MPTCP_INC_STATS(sock_net(sk), MPTCP_MIB_NODSSWINDOW); 230ab174ad8SPaolo Abeni return; 231ab174ad8SPaolo Abeni } 232ab174ad8SPaolo Abeni 233ab174ad8SPaolo Abeni p = &msk->out_of_order_queue.rb_node; 23406242e44SPaolo Abeni MPTCP_INC_STATS(sock_net(sk), MPTCP_MIB_OFOQUEUE); 235ab174ad8SPaolo Abeni if (RB_EMPTY_ROOT(&msk->out_of_order_queue)) { 236ab174ad8SPaolo Abeni rb_link_node(&skb->rbnode, NULL, p); 237ab174ad8SPaolo Abeni rb_insert_color(&skb->rbnode, &msk->out_of_order_queue); 238ab174ad8SPaolo Abeni msk->ooo_last_skb = skb; 239ab174ad8SPaolo Abeni goto end; 240ab174ad8SPaolo Abeni } 241ab174ad8SPaolo Abeni 242ab174ad8SPaolo Abeni /* with 2 subflows, adding at end of ooo queue is quite likely 243ab174ad8SPaolo Abeni * Use of ooo_last_skb avoids the O(Log(N)) rbtree lookup. 244ab174ad8SPaolo Abeni */ 24506242e44SPaolo Abeni if (mptcp_ooo_try_coalesce(msk, msk->ooo_last_skb, skb)) { 24606242e44SPaolo Abeni MPTCP_INC_STATS(sock_net(sk), MPTCP_MIB_OFOMERGE); 24706242e44SPaolo Abeni MPTCP_INC_STATS(sock_net(sk), MPTCP_MIB_OFOQUEUETAIL); 248ab174ad8SPaolo Abeni return; 24906242e44SPaolo Abeni } 250ab174ad8SPaolo Abeni 251ab174ad8SPaolo Abeni /* Can avoid an rbtree lookup if we are adding skb after ooo_last_skb */ 252ab174ad8SPaolo Abeni if (!before64(seq, MPTCP_SKB_CB(msk->ooo_last_skb)->end_seq)) { 25306242e44SPaolo Abeni MPTCP_INC_STATS(sock_net(sk), MPTCP_MIB_OFOQUEUETAIL); 254ab174ad8SPaolo Abeni parent = &msk->ooo_last_skb->rbnode; 255ab174ad8SPaolo Abeni p = &parent->rb_right; 256ab174ad8SPaolo Abeni goto insert; 257ab174ad8SPaolo Abeni } 258ab174ad8SPaolo Abeni 259ab174ad8SPaolo Abeni /* Find place to insert this segment. Handle overlaps on the way. */ 260ab174ad8SPaolo Abeni parent = NULL; 261ab174ad8SPaolo Abeni while (*p) { 262ab174ad8SPaolo Abeni parent = *p; 263ab174ad8SPaolo Abeni skb1 = rb_to_skb(parent); 264ab174ad8SPaolo Abeni if (before64(seq, MPTCP_SKB_CB(skb1)->map_seq)) { 265ab174ad8SPaolo Abeni p = &parent->rb_left; 266ab174ad8SPaolo Abeni continue; 267ab174ad8SPaolo Abeni } 268ab174ad8SPaolo Abeni if (before64(seq, MPTCP_SKB_CB(skb1)->end_seq)) { 269ab174ad8SPaolo Abeni if (!after64(end_seq, MPTCP_SKB_CB(skb1)->end_seq)) { 270ab174ad8SPaolo Abeni /* All the bits are present. Drop. */ 271ab174ad8SPaolo Abeni mptcp_drop(sk, skb); 27206242e44SPaolo Abeni MPTCP_INC_STATS(sock_net(sk), MPTCP_MIB_DUPDATA); 273ab174ad8SPaolo Abeni return; 274ab174ad8SPaolo Abeni } 275ab174ad8SPaolo Abeni if (after64(seq, MPTCP_SKB_CB(skb1)->map_seq)) { 276ab174ad8SPaolo Abeni /* partial overlap: 277ab174ad8SPaolo Abeni * | skb | 278ab174ad8SPaolo Abeni * | skb1 | 279ab174ad8SPaolo Abeni * continue traversing 280ab174ad8SPaolo Abeni */ 281ab174ad8SPaolo Abeni } else { 282ab174ad8SPaolo Abeni /* skb's seq == skb1's seq and skb covers skb1. 283ab174ad8SPaolo Abeni * Replace skb1 with skb. 284ab174ad8SPaolo Abeni */ 285ab174ad8SPaolo Abeni rb_replace_node(&skb1->rbnode, &skb->rbnode, 286ab174ad8SPaolo Abeni &msk->out_of_order_queue); 287ab174ad8SPaolo Abeni mptcp_drop(sk, skb1); 28806242e44SPaolo Abeni MPTCP_INC_STATS(sock_net(sk), MPTCP_MIB_DUPDATA); 289ab174ad8SPaolo Abeni goto merge_right; 290ab174ad8SPaolo Abeni } 291ab174ad8SPaolo Abeni } else if (mptcp_ooo_try_coalesce(msk, skb1, skb)) { 29206242e44SPaolo Abeni MPTCP_INC_STATS(sock_net(sk), MPTCP_MIB_OFOMERGE); 293ab174ad8SPaolo Abeni return; 294ab174ad8SPaolo Abeni } 295ab174ad8SPaolo Abeni p = &parent->rb_right; 296ab174ad8SPaolo Abeni } 29706242e44SPaolo Abeni 298ab174ad8SPaolo Abeni insert: 299ab174ad8SPaolo Abeni /* Insert segment into RB tree. */ 300ab174ad8SPaolo Abeni rb_link_node(&skb->rbnode, parent, p); 301ab174ad8SPaolo Abeni rb_insert_color(&skb->rbnode, &msk->out_of_order_queue); 302ab174ad8SPaolo Abeni 303ab174ad8SPaolo Abeni merge_right: 304ab174ad8SPaolo Abeni /* Remove other segments covered by skb. */ 305ab174ad8SPaolo Abeni while ((skb1 = skb_rb_next(skb)) != NULL) { 306ab174ad8SPaolo Abeni if (before64(end_seq, MPTCP_SKB_CB(skb1)->end_seq)) 307ab174ad8SPaolo Abeni break; 308ab174ad8SPaolo Abeni rb_erase(&skb1->rbnode, &msk->out_of_order_queue); 309ab174ad8SPaolo Abeni mptcp_drop(sk, skb1); 31006242e44SPaolo Abeni MPTCP_INC_STATS(sock_net(sk), MPTCP_MIB_DUPDATA); 311ab174ad8SPaolo Abeni } 312ab174ad8SPaolo Abeni /* If there is no skb after us, we are the last_skb ! */ 313ab174ad8SPaolo Abeni if (!skb1) 314ab174ad8SPaolo Abeni msk->ooo_last_skb = skb; 315ab174ad8SPaolo Abeni 316ab174ad8SPaolo Abeni end: 317ab174ad8SPaolo Abeni skb_condense(skb); 3186511882cSPaolo Abeni mptcp_set_owner_r(skb, sk); 3196511882cSPaolo Abeni } 3206511882cSPaolo Abeni 3216511882cSPaolo Abeni static bool mptcp_rmem_schedule(struct sock *sk, struct sock *ssk, int size) 3226511882cSPaolo Abeni { 3236511882cSPaolo Abeni struct mptcp_sock *msk = mptcp_sk(sk); 3246511882cSPaolo Abeni int amt, amount; 3256511882cSPaolo Abeni 3266511882cSPaolo Abeni if (size < msk->rmem_fwd_alloc) 3276511882cSPaolo Abeni return true; 3286511882cSPaolo Abeni 3296511882cSPaolo Abeni amt = sk_mem_pages(size); 3306511882cSPaolo Abeni amount = amt << SK_MEM_QUANTUM_SHIFT; 3316511882cSPaolo Abeni msk->rmem_fwd_alloc += amount; 3326511882cSPaolo Abeni if (!__sk_mem_raise_allocated(sk, size, amt, SK_MEM_RECV)) { 3336511882cSPaolo Abeni if (ssk->sk_forward_alloc < amount) { 3346511882cSPaolo Abeni msk->rmem_fwd_alloc -= amount; 3356511882cSPaolo Abeni return false; 3366511882cSPaolo Abeni } 3376511882cSPaolo Abeni 3386511882cSPaolo Abeni ssk->sk_forward_alloc -= amount; 3396511882cSPaolo Abeni } 3406511882cSPaolo Abeni return true; 341ab174ad8SPaolo Abeni } 342ab174ad8SPaolo Abeni 343ab174ad8SPaolo Abeni static bool __mptcp_move_skb(struct mptcp_sock *msk, struct sock *ssk, 344ab174ad8SPaolo Abeni struct sk_buff *skb, unsigned int offset, 345ab174ad8SPaolo Abeni size_t copy_len) 346ab174ad8SPaolo Abeni { 347ab174ad8SPaolo Abeni struct mptcp_subflow_context *subflow = mptcp_subflow_ctx(ssk); 3486771bfd9SFlorian Westphal struct sock *sk = (struct sock *)msk; 3494e637c70SFlorian Westphal struct sk_buff *tail; 350b7f653b2SFlorian Westphal bool has_rxtstamp; 3516771bfd9SFlorian Westphal 3526771bfd9SFlorian Westphal __skb_unlink(skb, &ssk->sk_receive_queue); 3534e637c70SFlorian Westphal 3544e637c70SFlorian Westphal skb_ext_reset(skb); 3554e637c70SFlorian Westphal skb_orphan(skb); 3564e637c70SFlorian Westphal 3579c3f94e1SPaolo Abeni /* try to fetch required memory from subflow */ 3586511882cSPaolo Abeni if (!mptcp_rmem_schedule(sk, ssk, skb->truesize)) 3599c3f94e1SPaolo Abeni goto drop; 36072f96132SPaolo Abeni 361b7f653b2SFlorian Westphal has_rxtstamp = TCP_SKB_CB(skb)->has_rxtstamp; 362b7f653b2SFlorian Westphal 363ab174ad8SPaolo Abeni /* the skb map_seq accounts for the skb offset: 364ab174ad8SPaolo Abeni * mptcp_subflow_get_mapped_dsn() is based on the current tp->copied_seq 365ab174ad8SPaolo Abeni * value 366ab174ad8SPaolo Abeni */ 367ab174ad8SPaolo Abeni MPTCP_SKB_CB(skb)->map_seq = mptcp_subflow_get_mapped_dsn(subflow); 368ab174ad8SPaolo Abeni MPTCP_SKB_CB(skb)->end_seq = MPTCP_SKB_CB(skb)->map_seq + copy_len; 369ab174ad8SPaolo Abeni MPTCP_SKB_CB(skb)->offset = offset; 370b7f653b2SFlorian Westphal MPTCP_SKB_CB(skb)->has_rxtstamp = has_rxtstamp; 371ab174ad8SPaolo Abeni 372ab174ad8SPaolo Abeni if (MPTCP_SKB_CB(skb)->map_seq == msk->ack_seq) { 373ab174ad8SPaolo Abeni /* in sequence */ 374917944daSMat Martineau WRITE_ONCE(msk->ack_seq, msk->ack_seq + copy_len); 3754e637c70SFlorian Westphal tail = skb_peek_tail(&sk->sk_receive_queue); 3768268ed4cSPaolo Abeni if (tail && mptcp_try_coalesce(sk, tail, skb)) 377ab174ad8SPaolo Abeni return true; 3784e637c70SFlorian Westphal 3796511882cSPaolo Abeni mptcp_set_owner_r(skb, sk); 3806771bfd9SFlorian Westphal __skb_queue_tail(&sk->sk_receive_queue, skb); 381ab174ad8SPaolo Abeni return true; 382ab174ad8SPaolo Abeni } else if (after64(MPTCP_SKB_CB(skb)->map_seq, msk->ack_seq)) { 383ab174ad8SPaolo Abeni mptcp_data_queue_ofo(msk, skb); 384ab174ad8SPaolo Abeni return false; 385ab174ad8SPaolo Abeni } 386ab174ad8SPaolo Abeni 387ab174ad8SPaolo Abeni /* old data, keep it simple and drop the whole pkt, sender 388ab174ad8SPaolo Abeni * will retransmit as needed, if needed. 389ab174ad8SPaolo Abeni */ 39006242e44SPaolo Abeni MPTCP_INC_STATS(sock_net(sk), MPTCP_MIB_DUPDATA); 3919c3f94e1SPaolo Abeni drop: 392ab174ad8SPaolo Abeni mptcp_drop(sk, skb); 393ab174ad8SPaolo Abeni return false; 3946771bfd9SFlorian Westphal } 3956771bfd9SFlorian Westphal 39616a9a9daSMat Martineau static void mptcp_stop_timer(struct sock *sk) 39716a9a9daSMat Martineau { 39816a9a9daSMat Martineau struct inet_connection_sock *icsk = inet_csk(sk); 39916a9a9daSMat Martineau 40016a9a9daSMat Martineau sk_stop_timer(sk, &icsk->icsk_retransmit_timer); 40116a9a9daSMat Martineau mptcp_sk(sk)->timer_ival = 0; 40216a9a9daSMat Martineau } 40316a9a9daSMat Martineau 404e16163b6SPaolo Abeni static void mptcp_close_wake_up(struct sock *sk) 405e16163b6SPaolo Abeni { 406e16163b6SPaolo Abeni if (sock_flag(sk, SOCK_DEAD)) 407e16163b6SPaolo Abeni return; 408e16163b6SPaolo Abeni 409e16163b6SPaolo Abeni sk->sk_state_change(sk); 410e16163b6SPaolo Abeni if (sk->sk_shutdown == SHUTDOWN_MASK || 411e16163b6SPaolo Abeni sk->sk_state == TCP_CLOSE) 412e16163b6SPaolo Abeni sk_wake_async(sk, SOCK_WAKE_WAITD, POLL_HUP); 413e16163b6SPaolo Abeni else 414e16163b6SPaolo Abeni sk_wake_async(sk, SOCK_WAKE_WAITD, POLL_IN); 415e16163b6SPaolo Abeni } 416e16163b6SPaolo Abeni 4176e628cd3SPaolo Abeni static bool mptcp_pending_data_fin_ack(struct sock *sk) 4186e628cd3SPaolo Abeni { 4196e628cd3SPaolo Abeni struct mptcp_sock *msk = mptcp_sk(sk); 4206e628cd3SPaolo Abeni 4216e628cd3SPaolo Abeni return !__mptcp_check_fallback(msk) && 4226e628cd3SPaolo Abeni ((1 << sk->sk_state) & 4236e628cd3SPaolo Abeni (TCPF_FIN_WAIT1 | TCPF_CLOSING | TCPF_LAST_ACK)) && 4246e628cd3SPaolo Abeni msk->write_seq == READ_ONCE(msk->snd_una); 4256e628cd3SPaolo Abeni } 4266e628cd3SPaolo Abeni 42716a9a9daSMat Martineau static void mptcp_check_data_fin_ack(struct sock *sk) 42816a9a9daSMat Martineau { 42916a9a9daSMat Martineau struct mptcp_sock *msk = mptcp_sk(sk); 43016a9a9daSMat Martineau 43116a9a9daSMat Martineau /* Look for an acknowledged DATA_FIN */ 4326e628cd3SPaolo Abeni if (mptcp_pending_data_fin_ack(sk)) { 43316a9a9daSMat Martineau WRITE_ONCE(msk->snd_data_fin_enable, 0); 43416a9a9daSMat Martineau 43516a9a9daSMat Martineau switch (sk->sk_state) { 43616a9a9daSMat Martineau case TCP_FIN_WAIT1: 43716a9a9daSMat Martineau inet_sk_state_store(sk, TCP_FIN_WAIT2); 43816a9a9daSMat Martineau break; 43916a9a9daSMat Martineau case TCP_CLOSING: 44016a9a9daSMat Martineau case TCP_LAST_ACK: 44116a9a9daSMat Martineau inet_sk_state_store(sk, TCP_CLOSE); 44216a9a9daSMat Martineau break; 44316a9a9daSMat Martineau } 44416a9a9daSMat Martineau 445e16163b6SPaolo Abeni mptcp_close_wake_up(sk); 44616a9a9daSMat Martineau } 44716a9a9daSMat Martineau } 44816a9a9daSMat Martineau 4493721b9b6SMat Martineau static bool mptcp_pending_data_fin(struct sock *sk, u64 *seq) 4503721b9b6SMat Martineau { 4513721b9b6SMat Martineau struct mptcp_sock *msk = mptcp_sk(sk); 4523721b9b6SMat Martineau 4533721b9b6SMat Martineau if (READ_ONCE(msk->rcv_data_fin) && 4543721b9b6SMat Martineau ((1 << sk->sk_state) & 4553721b9b6SMat Martineau (TCPF_ESTABLISHED | TCPF_FIN_WAIT1 | TCPF_FIN_WAIT2))) { 4563721b9b6SMat Martineau u64 rcv_data_fin_seq = READ_ONCE(msk->rcv_data_fin_seq); 4573721b9b6SMat Martineau 4583721b9b6SMat Martineau if (msk->ack_seq == rcv_data_fin_seq) { 4593721b9b6SMat Martineau if (seq) 4603721b9b6SMat Martineau *seq = rcv_data_fin_seq; 4613721b9b6SMat Martineau 4623721b9b6SMat Martineau return true; 4633721b9b6SMat Martineau } 4643721b9b6SMat Martineau } 4653721b9b6SMat Martineau 4663721b9b6SMat Martineau return false; 4673721b9b6SMat Martineau } 4683721b9b6SMat Martineau 4696477dd39SMat Martineau static void mptcp_set_datafin_timeout(const struct sock *sk) 4706477dd39SMat Martineau { 4716477dd39SMat Martineau struct inet_connection_sock *icsk = inet_csk(sk); 472877d11f0SMat Martineau u32 retransmits; 4736477dd39SMat Martineau 474877d11f0SMat Martineau retransmits = min_t(u32, icsk->icsk_retransmits, 475877d11f0SMat Martineau ilog2(TCP_RTO_MAX / TCP_RTO_MIN)); 476877d11f0SMat Martineau 477877d11f0SMat Martineau mptcp_sk(sk)->timer_ival = TCP_RTO_MIN << retransmits; 4786477dd39SMat Martineau } 4796477dd39SMat Martineau 48033d41c9cSPaolo Abeni static void __mptcp_set_timeout(struct sock *sk, long tout) 4813721b9b6SMat Martineau { 4823721b9b6SMat Martineau mptcp_sk(sk)->timer_ival = tout > 0 ? tout : TCP_RTO_MIN; 4833721b9b6SMat Martineau } 4843721b9b6SMat Martineau 48533d41c9cSPaolo Abeni static long mptcp_timeout_from_subflow(const struct mptcp_subflow_context *subflow) 48633d41c9cSPaolo Abeni { 48733d41c9cSPaolo Abeni const struct sock *ssk = mptcp_subflow_tcp_sock(subflow); 48833d41c9cSPaolo Abeni 48971b7dec2SPaolo Abeni return inet_csk(ssk)->icsk_pending && !subflow->stale_count ? 49071b7dec2SPaolo Abeni inet_csk(ssk)->icsk_timeout - jiffies : 0; 49133d41c9cSPaolo Abeni } 49233d41c9cSPaolo Abeni 49333d41c9cSPaolo Abeni static void mptcp_set_timeout(struct sock *sk) 49433d41c9cSPaolo Abeni { 49533d41c9cSPaolo Abeni struct mptcp_subflow_context *subflow; 49633d41c9cSPaolo Abeni long tout = 0; 49733d41c9cSPaolo Abeni 49833d41c9cSPaolo Abeni mptcp_for_each_subflow(mptcp_sk(sk), subflow) 49933d41c9cSPaolo Abeni tout = max(tout, mptcp_timeout_from_subflow(subflow)); 50033d41c9cSPaolo Abeni __mptcp_set_timeout(sk, tout); 50133d41c9cSPaolo Abeni } 50233d41c9cSPaolo Abeni 503fd897679SPaolo Abeni static bool tcp_can_send_ack(const struct sock *ssk) 504fd897679SPaolo Abeni { 505fd897679SPaolo Abeni return !((1 << inet_sk_state_load(ssk)) & 50620bc80b6SPaolo Abeni (TCPF_SYN_SENT | TCPF_SYN_RECV | TCPF_TIME_WAIT | TCPF_CLOSE | TCPF_LISTEN)); 507fd897679SPaolo Abeni } 508fd897679SPaolo Abeni 509340fa666SMat Martineau void mptcp_subflow_send_ack(struct sock *ssk) 5107ed90803SPaolo Abeni { 51175e908c3SPaolo Abeni bool slow; 5127ed90803SPaolo Abeni 51375e908c3SPaolo Abeni slow = lock_sock_fast(ssk); 514fd897679SPaolo Abeni if (tcp_can_send_ack(ssk)) 5157ed90803SPaolo Abeni tcp_send_ack(ssk); 51675e908c3SPaolo Abeni unlock_sock_fast(ssk, slow); 517fd897679SPaolo Abeni } 518340fa666SMat Martineau 519340fa666SMat Martineau static void mptcp_send_ack(struct mptcp_sock *msk) 520340fa666SMat Martineau { 521340fa666SMat Martineau struct mptcp_subflow_context *subflow; 522340fa666SMat Martineau 523340fa666SMat Martineau mptcp_for_each_subflow(msk, subflow) 524340fa666SMat Martineau mptcp_subflow_send_ack(mptcp_subflow_tcp_sock(subflow)); 525ea4ca586SPaolo Abeni } 526ea4ca586SPaolo Abeni 527fde56eeaSPaolo Abeni static void mptcp_subflow_cleanup_rbuf(struct sock *ssk) 528fd897679SPaolo Abeni { 52975e908c3SPaolo Abeni bool slow; 530fd897679SPaolo Abeni 53175e908c3SPaolo Abeni slow = lock_sock_fast(ssk); 532fde56eeaSPaolo Abeni if (tcp_can_send_ack(ssk)) 533fd897679SPaolo Abeni tcp_cleanup_rbuf(ssk, 1); 53475e908c3SPaolo Abeni unlock_sock_fast(ssk, slow); 535fde56eeaSPaolo Abeni } 536fde56eeaSPaolo Abeni 537fde56eeaSPaolo Abeni static bool mptcp_subflow_could_cleanup(const struct sock *ssk, bool rx_empty) 538fde56eeaSPaolo Abeni { 539fde56eeaSPaolo Abeni const struct inet_connection_sock *icsk = inet_csk(ssk); 540c4512c63SMatthieu Baerts u8 ack_pending = READ_ONCE(icsk->icsk_ack.pending); 541fde56eeaSPaolo Abeni const struct tcp_sock *tp = tcp_sk(ssk); 542fde56eeaSPaolo Abeni 543fde56eeaSPaolo Abeni return (ack_pending & ICSK_ACK_SCHED) && 544fde56eeaSPaolo Abeni ((READ_ONCE(tp->rcv_nxt) - READ_ONCE(tp->rcv_wup) > 545fde56eeaSPaolo Abeni READ_ONCE(icsk->icsk_ack.rcv_mss)) || 546fde56eeaSPaolo Abeni (rx_empty && ack_pending & 547fde56eeaSPaolo Abeni (ICSK_ACK_PUSHED2 | ICSK_ACK_PUSHED))); 548ea4ca586SPaolo Abeni } 549fd897679SPaolo Abeni 550fd897679SPaolo Abeni static void mptcp_cleanup_rbuf(struct mptcp_sock *msk) 551fd897679SPaolo Abeni { 552e3859603SPaolo Abeni int old_space = READ_ONCE(msk->old_wspace); 553fd897679SPaolo Abeni struct mptcp_subflow_context *subflow; 554e3859603SPaolo Abeni struct sock *sk = (struct sock *)msk; 555fde56eeaSPaolo Abeni int space = __mptcp_space(sk); 556fde56eeaSPaolo Abeni bool cleanup, rx_empty; 557e3859603SPaolo Abeni 558fde56eeaSPaolo Abeni cleanup = (space > 0) && (space >= (old_space << 1)); 559ce599c51SPaolo Abeni rx_empty = !__mptcp_rmem(sk); 560fd897679SPaolo Abeni 561fd897679SPaolo Abeni mptcp_for_each_subflow(msk, subflow) { 562fd897679SPaolo Abeni struct sock *ssk = mptcp_subflow_tcp_sock(subflow); 563fd897679SPaolo Abeni 564fde56eeaSPaolo Abeni if (cleanup || mptcp_subflow_could_cleanup(ssk, rx_empty)) 565fde56eeaSPaolo Abeni mptcp_subflow_cleanup_rbuf(ssk); 5667ed90803SPaolo Abeni } 5677ed90803SPaolo Abeni } 5687ed90803SPaolo Abeni 5697ed90803SPaolo Abeni static bool mptcp_check_data_fin(struct sock *sk) 5703721b9b6SMat Martineau { 5713721b9b6SMat Martineau struct mptcp_sock *msk = mptcp_sk(sk); 5723721b9b6SMat Martineau u64 rcv_data_fin_seq; 5737ed90803SPaolo Abeni bool ret = false; 5743721b9b6SMat Martineau 575781bf13dSPaolo Abeni if (__mptcp_check_fallback(msk)) 5767ed90803SPaolo Abeni return ret; 5773721b9b6SMat Martineau 5783721b9b6SMat Martineau /* Need to ack a DATA_FIN received from a peer while this side 5793721b9b6SMat Martineau * of the connection is in ESTABLISHED, FIN_WAIT1, or FIN_WAIT2. 5803721b9b6SMat Martineau * msk->rcv_data_fin was set when parsing the incoming options 5813721b9b6SMat Martineau * at the subflow level and the msk lock was not held, so this 5823721b9b6SMat Martineau * is the first opportunity to act on the DATA_FIN and change 5833721b9b6SMat Martineau * the msk state. 5843721b9b6SMat Martineau * 5853721b9b6SMat Martineau * If we are caught up to the sequence number of the incoming 5863721b9b6SMat Martineau * DATA_FIN, send the DATA_ACK now and do state transition. If 5873721b9b6SMat Martineau * not caught up, do nothing and let the recv code send DATA_ACK 5883721b9b6SMat Martineau * when catching up. 5893721b9b6SMat Martineau */ 5903721b9b6SMat Martineau 5913721b9b6SMat Martineau if (mptcp_pending_data_fin(sk, &rcv_data_fin_seq)) { 592917944daSMat Martineau WRITE_ONCE(msk->ack_seq, msk->ack_seq + 1); 5933721b9b6SMat Martineau WRITE_ONCE(msk->rcv_data_fin, 0); 5943721b9b6SMat Martineau 5953721b9b6SMat Martineau sk->sk_shutdown |= RCV_SHUTDOWN; 59616a9a9daSMat Martineau smp_mb__before_atomic(); /* SHUTDOWN must be visible first */ 5973721b9b6SMat Martineau 5983721b9b6SMat Martineau switch (sk->sk_state) { 5993721b9b6SMat Martineau case TCP_ESTABLISHED: 6003721b9b6SMat Martineau inet_sk_state_store(sk, TCP_CLOSE_WAIT); 6013721b9b6SMat Martineau break; 6023721b9b6SMat Martineau case TCP_FIN_WAIT1: 6033721b9b6SMat Martineau inet_sk_state_store(sk, TCP_CLOSING); 6043721b9b6SMat Martineau break; 6053721b9b6SMat Martineau case TCP_FIN_WAIT2: 6063721b9b6SMat Martineau inet_sk_state_store(sk, TCP_CLOSE); 6073721b9b6SMat Martineau break; 6083721b9b6SMat Martineau default: 6093721b9b6SMat Martineau /* Other states not expected */ 6103721b9b6SMat Martineau WARN_ON_ONCE(1); 6113721b9b6SMat Martineau break; 6123721b9b6SMat Martineau } 6133721b9b6SMat Martineau 6147ed90803SPaolo Abeni ret = true; 615fd897679SPaolo Abeni mptcp_send_ack(msk); 616e16163b6SPaolo Abeni mptcp_close_wake_up(sk); 6173721b9b6SMat Martineau } 6187ed90803SPaolo Abeni return ret; 6193721b9b6SMat Martineau } 6203721b9b6SMat Martineau 6216771bfd9SFlorian Westphal static bool __mptcp_move_skbs_from_subflow(struct mptcp_sock *msk, 6226771bfd9SFlorian Westphal struct sock *ssk, 6236771bfd9SFlorian Westphal unsigned int *bytes) 6246771bfd9SFlorian Westphal { 6256771bfd9SFlorian Westphal struct mptcp_subflow_context *subflow = mptcp_subflow_ctx(ssk); 626600911ffSFlorian Westphal struct sock *sk = (struct sock *)msk; 6276771bfd9SFlorian Westphal unsigned int moved = 0; 6286771bfd9SFlorian Westphal bool more_data_avail; 6296771bfd9SFlorian Westphal struct tcp_sock *tp; 6306771bfd9SFlorian Westphal bool done = false; 63113c7ba0cSFlorian Westphal int sk_rbuf; 63213c7ba0cSFlorian Westphal 63313c7ba0cSFlorian Westphal sk_rbuf = READ_ONCE(sk->sk_rcvbuf); 63413c7ba0cSFlorian Westphal 63513c7ba0cSFlorian Westphal if (!(sk->sk_userlocks & SOCK_RCVBUF_LOCK)) { 63613c7ba0cSFlorian Westphal int ssk_rbuf = READ_ONCE(ssk->sk_rcvbuf); 63713c7ba0cSFlorian Westphal 63813c7ba0cSFlorian Westphal if (unlikely(ssk_rbuf > sk_rbuf)) { 63913c7ba0cSFlorian Westphal WRITE_ONCE(sk->sk_rcvbuf, ssk_rbuf); 64013c7ba0cSFlorian Westphal sk_rbuf = ssk_rbuf; 64113c7ba0cSFlorian Westphal } 64213c7ba0cSFlorian Westphal } 643600911ffSFlorian Westphal 644ab174ad8SPaolo Abeni pr_debug("msk=%p ssk=%p", msk, ssk); 6456771bfd9SFlorian Westphal tp = tcp_sk(ssk); 6466771bfd9SFlorian Westphal do { 6476771bfd9SFlorian Westphal u32 map_remaining, offset; 6486771bfd9SFlorian Westphal u32 seq = tp->copied_seq; 6496771bfd9SFlorian Westphal struct sk_buff *skb; 6506771bfd9SFlorian Westphal bool fin; 6516771bfd9SFlorian Westphal 6526771bfd9SFlorian Westphal /* try to move as much data as available */ 6536771bfd9SFlorian Westphal map_remaining = subflow->map_data_len - 6546771bfd9SFlorian Westphal mptcp_subflow_get_map_offset(subflow); 6556771bfd9SFlorian Westphal 6566771bfd9SFlorian Westphal skb = skb_peek(&ssk->sk_receive_queue); 657d9fb8c50SPaolo Abeni if (!skb) { 658d9fb8c50SPaolo Abeni /* if no data is found, a racing workqueue/recvmsg 659d9fb8c50SPaolo Abeni * already processed the new data, stop here or we 660d9fb8c50SPaolo Abeni * can enter an infinite loop 661d9fb8c50SPaolo Abeni */ 662d9fb8c50SPaolo Abeni if (!moved) 663d9fb8c50SPaolo Abeni done = true; 6646771bfd9SFlorian Westphal break; 665d9fb8c50SPaolo Abeni } 6666771bfd9SFlorian Westphal 667e1ff9e82SDavide Caratti if (__mptcp_check_fallback(msk)) { 668e1ff9e82SDavide Caratti /* if we are running under the workqueue, TCP could have 669e1ff9e82SDavide Caratti * collapsed skbs between dummy map creation and now 670e1ff9e82SDavide Caratti * be sure to adjust the size 671e1ff9e82SDavide Caratti */ 672e1ff9e82SDavide Caratti map_remaining = skb->len; 673e1ff9e82SDavide Caratti subflow->map_data_len = skb->len; 674e1ff9e82SDavide Caratti } 675e1ff9e82SDavide Caratti 6766771bfd9SFlorian Westphal offset = seq - TCP_SKB_CB(skb)->seq; 6776771bfd9SFlorian Westphal fin = TCP_SKB_CB(skb)->tcp_flags & TCPHDR_FIN; 6786771bfd9SFlorian Westphal if (fin) { 6796771bfd9SFlorian Westphal done = true; 6806771bfd9SFlorian Westphal seq++; 6816771bfd9SFlorian Westphal } 6826771bfd9SFlorian Westphal 6836771bfd9SFlorian Westphal if (offset < skb->len) { 6846771bfd9SFlorian Westphal size_t len = skb->len - offset; 6856771bfd9SFlorian Westphal 6866771bfd9SFlorian Westphal if (tp->urg_data) 6876771bfd9SFlorian Westphal done = true; 6886771bfd9SFlorian Westphal 689ab174ad8SPaolo Abeni if (__mptcp_move_skb(msk, ssk, skb, offset, len)) 6906771bfd9SFlorian Westphal moved += len; 691ab174ad8SPaolo Abeni seq += len; 6926771bfd9SFlorian Westphal 6936771bfd9SFlorian Westphal if (WARN_ON_ONCE(map_remaining < len)) 6946771bfd9SFlorian Westphal break; 6956771bfd9SFlorian Westphal } else { 6966771bfd9SFlorian Westphal WARN_ON_ONCE(!fin); 6976771bfd9SFlorian Westphal sk_eat_skb(ssk, skb); 6986771bfd9SFlorian Westphal done = true; 6996771bfd9SFlorian Westphal } 7006771bfd9SFlorian Westphal 7016771bfd9SFlorian Westphal WRITE_ONCE(tp->copied_seq, seq); 7026771bfd9SFlorian Westphal more_data_avail = mptcp_subflow_data_available(ssk); 703600911ffSFlorian Westphal 70413c7ba0cSFlorian Westphal if (atomic_read(&sk->sk_rmem_alloc) > sk_rbuf) { 705600911ffSFlorian Westphal done = true; 706600911ffSFlorian Westphal break; 707600911ffSFlorian Westphal } 7086771bfd9SFlorian Westphal } while (more_data_avail); 7096771bfd9SFlorian Westphal 7106719331cSPaolo Abeni *bytes += moved; 7116771bfd9SFlorian Westphal return done; 7126771bfd9SFlorian Westphal } 7136771bfd9SFlorian Westphal 71487952603SPaolo Abeni static bool __mptcp_ofo_queue(struct mptcp_sock *msk) 715ab174ad8SPaolo Abeni { 716ab174ad8SPaolo Abeni struct sock *sk = (struct sock *)msk; 717ab174ad8SPaolo Abeni struct sk_buff *skb, *tail; 718ab174ad8SPaolo Abeni bool moved = false; 719ab174ad8SPaolo Abeni struct rb_node *p; 720ab174ad8SPaolo Abeni u64 end_seq; 721ab174ad8SPaolo Abeni 722ab174ad8SPaolo Abeni p = rb_first(&msk->out_of_order_queue); 72306242e44SPaolo Abeni pr_debug("msk=%p empty=%d", msk, RB_EMPTY_ROOT(&msk->out_of_order_queue)); 724ab174ad8SPaolo Abeni while (p) { 725ab174ad8SPaolo Abeni skb = rb_to_skb(p); 726ab174ad8SPaolo Abeni if (after64(MPTCP_SKB_CB(skb)->map_seq, msk->ack_seq)) 727ab174ad8SPaolo Abeni break; 728ab174ad8SPaolo Abeni 729ab174ad8SPaolo Abeni p = rb_next(p); 730ab174ad8SPaolo Abeni rb_erase(&skb->rbnode, &msk->out_of_order_queue); 731ab174ad8SPaolo Abeni 732ab174ad8SPaolo Abeni if (unlikely(!after64(MPTCP_SKB_CB(skb)->end_seq, 733ab174ad8SPaolo Abeni msk->ack_seq))) { 734ab174ad8SPaolo Abeni mptcp_drop(sk, skb); 73506242e44SPaolo Abeni MPTCP_INC_STATS(sock_net(sk), MPTCP_MIB_DUPDATA); 736ab174ad8SPaolo Abeni continue; 737ab174ad8SPaolo Abeni } 738ab174ad8SPaolo Abeni 739ab174ad8SPaolo Abeni end_seq = MPTCP_SKB_CB(skb)->end_seq; 740ab174ad8SPaolo Abeni tail = skb_peek_tail(&sk->sk_receive_queue); 741ab174ad8SPaolo Abeni if (!tail || !mptcp_ooo_try_coalesce(msk, tail, skb)) { 742ab174ad8SPaolo Abeni int delta = msk->ack_seq - MPTCP_SKB_CB(skb)->map_seq; 743ab174ad8SPaolo Abeni 744ab174ad8SPaolo Abeni /* skip overlapping data, if any */ 74506242e44SPaolo Abeni pr_debug("uncoalesced seq=%llx ack seq=%llx delta=%d", 74606242e44SPaolo Abeni MPTCP_SKB_CB(skb)->map_seq, msk->ack_seq, 74706242e44SPaolo Abeni delta); 748ab174ad8SPaolo Abeni MPTCP_SKB_CB(skb)->offset += delta; 7492c9e7765SFlorian Westphal MPTCP_SKB_CB(skb)->map_seq += delta; 750ab174ad8SPaolo Abeni __skb_queue_tail(&sk->sk_receive_queue, skb); 751ab174ad8SPaolo Abeni } 752ab174ad8SPaolo Abeni msk->ack_seq = end_seq; 753ab174ad8SPaolo Abeni moved = true; 754ab174ad8SPaolo Abeni } 755ab174ad8SPaolo Abeni return moved; 756ab174ad8SPaolo Abeni } 757ab174ad8SPaolo Abeni 7582e52213cSFlorian Westphal /* In most cases we will be able to lock the mptcp socket. If its already 7592e52213cSFlorian Westphal * owned, we need to defer to the work queue to avoid ABBA deadlock. 7602e52213cSFlorian Westphal */ 76199d1055cSPaolo Abeni static bool move_skbs_to_msk(struct mptcp_sock *msk, struct sock *ssk) 7622e52213cSFlorian Westphal { 7632e52213cSFlorian Westphal struct sock *sk = (struct sock *)msk; 7642e52213cSFlorian Westphal unsigned int moved = 0; 7652e52213cSFlorian Westphal 7662e52213cSFlorian Westphal __mptcp_move_skbs_from_subflow(msk, ssk, &moved); 76787952603SPaolo Abeni __mptcp_ofo_queue(msk); 768499ada50SPaolo Abeni if (unlikely(ssk->sk_err)) { 769499ada50SPaolo Abeni if (!sock_owned_by_user(sk)) 770499ada50SPaolo Abeni __mptcp_error_report(sk); 771499ada50SPaolo Abeni else 772e9d09bacSPaolo Abeni __set_bit(MPTCP_ERROR_REPORT, &msk->cb_flags); 773499ada50SPaolo Abeni } 774ab174ad8SPaolo Abeni 775ab174ad8SPaolo Abeni /* If the moves have caught up with the DATA_FIN sequence number 776ab174ad8SPaolo Abeni * it's time to ack the DATA_FIN and change socket state, but 777ab174ad8SPaolo Abeni * this is not a good place to change state. Let the workqueue 778ab174ad8SPaolo Abeni * do it. 779ab174ad8SPaolo Abeni */ 780ba8f48f7SPaolo Abeni if (mptcp_pending_data_fin(sk, NULL)) 781ba8f48f7SPaolo Abeni mptcp_schedule_work(sk); 78299d1055cSPaolo Abeni return moved > 0; 7832e52213cSFlorian Westphal } 7842e52213cSFlorian Westphal 7852e52213cSFlorian Westphal void mptcp_data_ready(struct sock *sk, struct sock *ssk) 786101f6f85SFlorian Westphal { 7876719331cSPaolo Abeni struct mptcp_subflow_context *subflow = mptcp_subflow_ctx(ssk); 788101f6f85SFlorian Westphal struct mptcp_sock *msk = mptcp_sk(sk); 78913c7ba0cSFlorian Westphal int sk_rbuf, ssk_rbuf; 790101f6f85SFlorian Westphal 791d7b1bfd0SPaolo Abeni /* The peer can send data while we are shutting down this 792d7b1bfd0SPaolo Abeni * subflow at msk destruction time, but we must avoid enqueuing 793d7b1bfd0SPaolo Abeni * more data to the msk receive queue 794d7b1bfd0SPaolo Abeni */ 795d7b1bfd0SPaolo Abeni if (unlikely(subflow->disposable)) 796d7b1bfd0SPaolo Abeni return; 797d7b1bfd0SPaolo Abeni 79813c7ba0cSFlorian Westphal ssk_rbuf = READ_ONCE(ssk->sk_rcvbuf); 79913c7ba0cSFlorian Westphal sk_rbuf = READ_ONCE(sk->sk_rcvbuf); 80013c7ba0cSFlorian Westphal if (unlikely(ssk_rbuf > sk_rbuf)) 80113c7ba0cSFlorian Westphal sk_rbuf = ssk_rbuf; 80213c7ba0cSFlorian Westphal 80399d1055cSPaolo Abeni /* over limit? can't append more skbs to msk, Also, no need to wake-up*/ 804ce599c51SPaolo Abeni if (__mptcp_rmem(sk) > sk_rbuf) { 805ce599c51SPaolo Abeni MPTCP_INC_STATS(sock_net(sk), MPTCP_MIB_RCVPRUNED); 80699d1055cSPaolo Abeni return; 807ce599c51SPaolo Abeni } 8082e52213cSFlorian Westphal 80999d1055cSPaolo Abeni /* Wake-up the reader only for in-sequence data */ 81099d1055cSPaolo Abeni mptcp_data_lock(sk); 811612f71d7SPaolo Abeni if (move_skbs_to_msk(msk, ssk)) 812101f6f85SFlorian Westphal sk->sk_data_ready(sk); 813612f71d7SPaolo Abeni 81499d1055cSPaolo Abeni mptcp_data_unlock(sk); 81599d1055cSPaolo Abeni } 816101f6f85SFlorian Westphal 8173e501490SPaolo Abeni static bool __mptcp_finish_join(struct mptcp_sock *msk, struct sock *ssk) 818ec3edaa7SPeter Krystad { 8193e501490SPaolo Abeni struct sock *sk = (struct sock *)msk; 8205cf92bbaSPaolo Abeni 8213e501490SPaolo Abeni if (sk->sk_state != TCP_ESTABLISHED) 82278962489SFlorian Westphal return false; 823ec3edaa7SPeter Krystad 8243e501490SPaolo Abeni /* attach to msk socket only after we are sure we will deal with it 8253e501490SPaolo Abeni * at close time 8263e501490SPaolo Abeni */ 8273e501490SPaolo Abeni if (sk->sk_socket && !ssk->sk_socket) 8283e501490SPaolo Abeni mptcp_sock_graft(ssk, sk->sk_socket); 82978962489SFlorian Westphal 8303e501490SPaolo Abeni mptcp_propagate_sndbuf((struct sock *)msk, ssk); 8313e501490SPaolo Abeni mptcp_sockopt_sync_locked(msk, ssk); 8323e501490SPaolo Abeni return true; 83378962489SFlorian Westphal } 83478962489SFlorian Westphal 8353e501490SPaolo Abeni static void __mptcp_flush_join_list(struct sock *sk) 83678962489SFlorian Westphal { 8373e501490SPaolo Abeni struct mptcp_subflow_context *tmp, *subflow; 8383e501490SPaolo Abeni struct mptcp_sock *msk = mptcp_sk(sk); 83978962489SFlorian Westphal 8403e501490SPaolo Abeni list_for_each_entry_safe(subflow, tmp, &msk->join_list, node) { 8413e501490SPaolo Abeni struct sock *ssk = mptcp_subflow_tcp_sock(subflow); 8423e501490SPaolo Abeni bool slow = lock_sock_fast(ssk); 8433e501490SPaolo Abeni 8443e501490SPaolo Abeni list_move_tail(&subflow->node, &msk->conn_list); 8453e501490SPaolo Abeni if (!__mptcp_finish_join(msk, ssk)) 8463e501490SPaolo Abeni mptcp_subflow_reset(ssk); 8473e501490SPaolo Abeni unlock_sock_fast(ssk, slow); 84878962489SFlorian Westphal } 849ec3edaa7SPeter Krystad } 850ec3edaa7SPeter Krystad 851b51f9b80SPaolo Abeni static bool mptcp_timer_pending(struct sock *sk) 852b51f9b80SPaolo Abeni { 853b51f9b80SPaolo Abeni return timer_pending(&inet_csk(sk)->icsk_retransmit_timer); 854b51f9b80SPaolo Abeni } 855b51f9b80SPaolo Abeni 856b51f9b80SPaolo Abeni static void mptcp_reset_timer(struct sock *sk) 857b51f9b80SPaolo Abeni { 858b51f9b80SPaolo Abeni struct inet_connection_sock *icsk = inet_csk(sk); 859b51f9b80SPaolo Abeni unsigned long tout; 860b51f9b80SPaolo Abeni 861e16163b6SPaolo Abeni /* prevent rescheduling on close */ 862e16163b6SPaolo Abeni if (unlikely(inet_sk_state_load(sk) == TCP_CLOSE)) 863e16163b6SPaolo Abeni return; 864e16163b6SPaolo Abeni 86533d41c9cSPaolo Abeni tout = mptcp_sk(sk)->timer_ival; 866b51f9b80SPaolo Abeni sk_reset_timer(sk, &icsk->icsk_retransmit_timer, jiffies + tout); 867b51f9b80SPaolo Abeni } 868b51f9b80SPaolo Abeni 869ba8f48f7SPaolo Abeni bool mptcp_schedule_work(struct sock *sk) 870ba8f48f7SPaolo Abeni { 871ba8f48f7SPaolo Abeni if (inet_sk_state_load(sk) != TCP_CLOSE && 872ba8f48f7SPaolo Abeni schedule_work(&mptcp_sk(sk)->work)) { 873ba8f48f7SPaolo Abeni /* each subflow already holds a reference to the sk, and the 874ba8f48f7SPaolo Abeni * workqueue is invoked by a subflow, so sk can't go away here. 875ba8f48f7SPaolo Abeni */ 876ba8f48f7SPaolo Abeni sock_hold(sk); 877ba8f48f7SPaolo Abeni return true; 878ba8f48f7SPaolo Abeni } 879ba8f48f7SPaolo Abeni return false; 880ba8f48f7SPaolo Abeni } 881ba8f48f7SPaolo Abeni 88259832e24SFlorian Westphal void mptcp_subflow_eof(struct sock *sk) 88359832e24SFlorian Westphal { 884ba8f48f7SPaolo Abeni if (!test_and_set_bit(MPTCP_WORK_EOF, &mptcp_sk(sk)->flags)) 885ba8f48f7SPaolo Abeni mptcp_schedule_work(sk); 88659832e24SFlorian Westphal } 88759832e24SFlorian Westphal 8885969856aSPaolo Abeni static void mptcp_check_for_eof(struct mptcp_sock *msk) 8895969856aSPaolo Abeni { 8905969856aSPaolo Abeni struct mptcp_subflow_context *subflow; 8915969856aSPaolo Abeni struct sock *sk = (struct sock *)msk; 8925969856aSPaolo Abeni int receivers = 0; 8935969856aSPaolo Abeni 8945969856aSPaolo Abeni mptcp_for_each_subflow(msk, subflow) 8955969856aSPaolo Abeni receivers += !subflow->rx_eof; 896e16163b6SPaolo Abeni if (receivers) 897e16163b6SPaolo Abeni return; 8985969856aSPaolo Abeni 899e16163b6SPaolo Abeni if (!(sk->sk_shutdown & RCV_SHUTDOWN)) { 9005969856aSPaolo Abeni /* hopefully temporary hack: propagate shutdown status 9015969856aSPaolo Abeni * to msk, when all subflows agree on it 9025969856aSPaolo Abeni */ 9035969856aSPaolo Abeni sk->sk_shutdown |= RCV_SHUTDOWN; 9045969856aSPaolo Abeni 9055969856aSPaolo Abeni smp_mb__before_atomic(); /* SHUTDOWN must be visible first */ 9065969856aSPaolo Abeni sk->sk_data_ready(sk); 9075969856aSPaolo Abeni } 908e16163b6SPaolo Abeni 909e16163b6SPaolo Abeni switch (sk->sk_state) { 910e16163b6SPaolo Abeni case TCP_ESTABLISHED: 911e16163b6SPaolo Abeni inet_sk_state_store(sk, TCP_CLOSE_WAIT); 912e16163b6SPaolo Abeni break; 913e16163b6SPaolo Abeni case TCP_FIN_WAIT1: 91426aa2314SPaolo Abeni inet_sk_state_store(sk, TCP_CLOSING); 91526aa2314SPaolo Abeni break; 91626aa2314SPaolo Abeni case TCP_FIN_WAIT2: 917e16163b6SPaolo Abeni inet_sk_state_store(sk, TCP_CLOSE); 918e16163b6SPaolo Abeni break; 919e16163b6SPaolo Abeni default: 920e16163b6SPaolo Abeni return; 921e16163b6SPaolo Abeni } 922e16163b6SPaolo Abeni mptcp_close_wake_up(sk); 9235969856aSPaolo Abeni } 9245969856aSPaolo Abeni 9257a6a6cbcSPaolo Abeni static struct sock *mptcp_subflow_recv_lookup(const struct mptcp_sock *msk) 9267a6a6cbcSPaolo Abeni { 9277a6a6cbcSPaolo Abeni struct mptcp_subflow_context *subflow; 9287a6a6cbcSPaolo Abeni struct sock *sk = (struct sock *)msk; 9297a6a6cbcSPaolo Abeni 9307a6a6cbcSPaolo Abeni sock_owned_by_me(sk); 9317a6a6cbcSPaolo Abeni 9327a6a6cbcSPaolo Abeni mptcp_for_each_subflow(msk, subflow) { 93399d1055cSPaolo Abeni if (READ_ONCE(subflow->data_avail)) 9347a6a6cbcSPaolo Abeni return mptcp_subflow_tcp_sock(subflow); 9357a6a6cbcSPaolo Abeni } 9367a6a6cbcSPaolo Abeni 9377a6a6cbcSPaolo Abeni return NULL; 9387a6a6cbcSPaolo Abeni } 9397a6a6cbcSPaolo Abeni 9403f8e0aaeSPaolo Abeni static bool mptcp_skb_can_collapse_to(u64 write_seq, 94157040755SPaolo Abeni const struct sk_buff *skb, 94257040755SPaolo Abeni const struct mptcp_ext *mpext) 9436d0060f6SMat Martineau { 94457040755SPaolo Abeni if (!tcp_skb_can_collapse_to(skb)) 94557040755SPaolo Abeni return false; 94657040755SPaolo Abeni 9475a369ca6SPaolo Abeni /* can collapse only if MPTCP level sequence is in order and this 9485a369ca6SPaolo Abeni * mapping has not been xmitted yet 9495a369ca6SPaolo Abeni */ 9505a369ca6SPaolo Abeni return mpext && mpext->data_seq + mpext->data_len == write_seq && 9515a369ca6SPaolo Abeni !mpext->frozen; 95257040755SPaolo Abeni } 95357040755SPaolo Abeni 95429249eacSPaolo Abeni /* we can append data to the given data frag if: 95529249eacSPaolo Abeni * - there is space available in the backing page_frag 95629249eacSPaolo Abeni * - the data frag tail matches the current page_frag free offset 95729249eacSPaolo Abeni * - the data frag end sequence number matches the current write seq 95829249eacSPaolo Abeni */ 95918b683bfSPaolo Abeni static bool mptcp_frag_can_collapse_to(const struct mptcp_sock *msk, 96018b683bfSPaolo Abeni const struct page_frag *pfrag, 96118b683bfSPaolo Abeni const struct mptcp_data_frag *df) 96218b683bfSPaolo Abeni { 96318b683bfSPaolo Abeni return df && pfrag->page == df->page && 964d9ca1de8SPaolo Abeni pfrag->size - pfrag->offset > 0 && 96529249eacSPaolo Abeni pfrag->offset == (df->offset + df->data_len) && 96618b683bfSPaolo Abeni df->data_seq + df->data_len == msk->write_seq; 96718b683bfSPaolo Abeni } 96818b683bfSPaolo Abeni 9691094c6feSPaolo Abeni static void __mptcp_mem_reclaim_partial(struct sock *sk) 9701094c6feSPaolo Abeni { 9716511882cSPaolo Abeni int reclaimable = mptcp_sk(sk)->rmem_fwd_alloc - sk_unused_reserved_mem(sk); 9726511882cSPaolo Abeni 9731094c6feSPaolo Abeni lockdep_assert_held_once(&sk->sk_lock.slock); 9746511882cSPaolo Abeni 975269bda9eSMat Martineau if (reclaimable > SK_MEM_QUANTUM) 9766511882cSPaolo Abeni __mptcp_rmem_reclaim(sk, reclaimable - 1); 977269bda9eSMat Martineau 9781094c6feSPaolo Abeni sk_mem_reclaim_partial(sk); 9791094c6feSPaolo Abeni } 9801094c6feSPaolo Abeni 981724cfd2eSPaolo Abeni static void mptcp_mem_reclaim_partial(struct sock *sk) 982724cfd2eSPaolo Abeni { 983724cfd2eSPaolo Abeni mptcp_data_lock(sk); 9846511882cSPaolo Abeni __mptcp_mem_reclaim_partial(sk); 985724cfd2eSPaolo Abeni mptcp_data_unlock(sk); 986724cfd2eSPaolo Abeni } 987724cfd2eSPaolo Abeni 988d027236cSPaolo Abeni static void dfrag_uncharge(struct sock *sk, int len) 98918b683bfSPaolo Abeni { 990d027236cSPaolo Abeni sk_mem_uncharge(sk, len); 9917948f6ccSFlorian Westphal sk_wmem_queued_add(sk, -len); 992d027236cSPaolo Abeni } 993d027236cSPaolo Abeni 994d027236cSPaolo Abeni static void dfrag_clear(struct sock *sk, struct mptcp_data_frag *dfrag) 995d027236cSPaolo Abeni { 996d027236cSPaolo Abeni int len = dfrag->data_len + dfrag->overhead; 997d027236cSPaolo Abeni 99818b683bfSPaolo Abeni list_del(&dfrag->list); 999d027236cSPaolo Abeni dfrag_uncharge(sk, len); 100018b683bfSPaolo Abeni put_page(dfrag->page); 100118b683bfSPaolo Abeni } 100218b683bfSPaolo Abeni 10036e628cd3SPaolo Abeni static void __mptcp_clean_una(struct sock *sk) 100418b683bfSPaolo Abeni { 100518b683bfSPaolo Abeni struct mptcp_sock *msk = mptcp_sk(sk); 100618b683bfSPaolo Abeni struct mptcp_data_frag *dtmp, *dfrag; 1007d027236cSPaolo Abeni bool cleaned = false; 1008e1ff9e82SDavide Caratti u64 snd_una; 1009e1ff9e82SDavide Caratti 1010e1ff9e82SDavide Caratti /* on fallback we just need to ignore snd_una, as this is really 1011e1ff9e82SDavide Caratti * plain TCP 1012e1ff9e82SDavide Caratti */ 1013e1ff9e82SDavide Caratti if (__mptcp_check_fallback(msk)) 10147439d687SPaolo Abeni msk->snd_una = READ_ONCE(msk->snd_nxt); 10157439d687SPaolo Abeni 10167439d687SPaolo Abeni snd_una = msk->snd_una; 101718b683bfSPaolo Abeni list_for_each_entry_safe(dfrag, dtmp, &msk->rtx_queue, list) { 101818b683bfSPaolo Abeni if (after64(dfrag->data_seq + dfrag->data_len, snd_una)) 101918b683bfSPaolo Abeni break; 102018b683bfSPaolo Abeni 10211e1d9d6fSPaolo Abeni if (unlikely(dfrag == msk->first_pending)) { 10221e1d9d6fSPaolo Abeni /* in recovery mode can see ack after the current snd head */ 10231e1d9d6fSPaolo Abeni if (WARN_ON_ONCE(!msk->recovery)) 1024d9ca1de8SPaolo Abeni break; 10251e1d9d6fSPaolo Abeni 10261e1d9d6fSPaolo Abeni WRITE_ONCE(msk->first_pending, mptcp_send_next(sk)); 10271e1d9d6fSPaolo Abeni } 10281e1d9d6fSPaolo Abeni 1029d027236cSPaolo Abeni dfrag_clear(sk, dfrag); 1030d027236cSPaolo Abeni cleaned = true; 1031d027236cSPaolo Abeni } 1032d027236cSPaolo Abeni 10337948f6ccSFlorian Westphal dfrag = mptcp_rtx_head(sk); 10347948f6ccSFlorian Westphal if (dfrag && after64(snd_una, dfrag->data_seq)) { 103553eb4c38SPaolo Abeni u64 delta = snd_una - dfrag->data_seq; 103653eb4c38SPaolo Abeni 10371e1d9d6fSPaolo Abeni /* prevent wrap around in recovery mode */ 10381e1d9d6fSPaolo Abeni if (unlikely(delta > dfrag->already_sent)) { 10391e1d9d6fSPaolo Abeni if (WARN_ON_ONCE(!msk->recovery)) 104053eb4c38SPaolo Abeni goto out; 10411e1d9d6fSPaolo Abeni if (WARN_ON_ONCE(delta > dfrag->data_len)) 10421e1d9d6fSPaolo Abeni goto out; 10431e1d9d6fSPaolo Abeni dfrag->already_sent += delta - dfrag->already_sent; 10441e1d9d6fSPaolo Abeni } 10457948f6ccSFlorian Westphal 10467948f6ccSFlorian Westphal dfrag->data_seq += delta; 104753eb4c38SPaolo Abeni dfrag->offset += delta; 10487948f6ccSFlorian Westphal dfrag->data_len -= delta; 1049d9ca1de8SPaolo Abeni dfrag->already_sent -= delta; 10507948f6ccSFlorian Westphal 10517948f6ccSFlorian Westphal dfrag_uncharge(sk, delta); 10527948f6ccSFlorian Westphal cleaned = true; 10537948f6ccSFlorian Westphal } 10547948f6ccSFlorian Westphal 10551e1d9d6fSPaolo Abeni /* all retransmitted data acked, recovery completed */ 10561e1d9d6fSPaolo Abeni if (unlikely(msk->recovery) && after64(msk->snd_una, msk->recovery_snd_nxt)) 10571e1d9d6fSPaolo Abeni msk->recovery = false; 10581e1d9d6fSPaolo Abeni 105953eb4c38SPaolo Abeni out: 10601094c6feSPaolo Abeni if (cleaned && tcp_under_memory_pressure(sk)) 10611094c6feSPaolo Abeni __mptcp_mem_reclaim_partial(sk); 106218b683bfSPaolo Abeni 10633241a9c0SFlorian Westphal if (snd_una == READ_ONCE(msk->snd_nxt) && 10643241a9c0SFlorian Westphal snd_una == READ_ONCE(msk->write_seq)) { 106533d41c9cSPaolo Abeni if (mptcp_timer_pending(sk) && !mptcp_data_fin_enabled(msk)) 10666e628cd3SPaolo Abeni mptcp_stop_timer(sk); 10676e628cd3SPaolo Abeni } else { 10686e628cd3SPaolo Abeni mptcp_reset_timer(sk); 10696e628cd3SPaolo Abeni } 10706e628cd3SPaolo Abeni } 10716e628cd3SPaolo Abeni 1072417789dfSPaolo Abeni static void __mptcp_clean_una_wakeup(struct sock *sk) 1073417789dfSPaolo Abeni { 1074765ff425SPaolo Abeni lockdep_assert_held_once(&sk->sk_lock.slock); 1075765ff425SPaolo Abeni 1076417789dfSPaolo Abeni __mptcp_clean_una(sk); 1077417789dfSPaolo Abeni mptcp_write_space(sk); 1078417789dfSPaolo Abeni } 1079417789dfSPaolo Abeni 1080b5941f06SPaolo Abeni static void mptcp_clean_una_wakeup(struct sock *sk) 1081b5941f06SPaolo Abeni { 1082b5941f06SPaolo Abeni mptcp_data_lock(sk); 1083b5941f06SPaolo Abeni __mptcp_clean_una_wakeup(sk); 1084b5941f06SPaolo Abeni mptcp_data_unlock(sk); 1085b5941f06SPaolo Abeni } 1086b5941f06SPaolo Abeni 1087724cfd2eSPaolo Abeni static void mptcp_enter_memory_pressure(struct sock *sk) 108818b683bfSPaolo Abeni { 1089d9ca1de8SPaolo Abeni struct mptcp_subflow_context *subflow; 1090d9ca1de8SPaolo Abeni struct mptcp_sock *msk = mptcp_sk(sk); 1091d9ca1de8SPaolo Abeni bool first = true; 1092d9ca1de8SPaolo Abeni 109318b683bfSPaolo Abeni sk_stream_moderate_sndbuf(sk); 1094d9ca1de8SPaolo Abeni mptcp_for_each_subflow(msk, subflow) { 1095d9ca1de8SPaolo Abeni struct sock *ssk = mptcp_subflow_tcp_sock(subflow); 1096d9ca1de8SPaolo Abeni 1097d9ca1de8SPaolo Abeni if (first) 1098d9ca1de8SPaolo Abeni tcp_enter_memory_pressure(ssk); 1099d9ca1de8SPaolo Abeni sk_stream_moderate_sndbuf(ssk); 1100d9ca1de8SPaolo Abeni first = false; 1101d9ca1de8SPaolo Abeni } 1102724cfd2eSPaolo Abeni } 1103724cfd2eSPaolo Abeni 1104724cfd2eSPaolo Abeni /* ensure we get enough memory for the frag hdr, beyond some minimal amount of 1105724cfd2eSPaolo Abeni * data 1106724cfd2eSPaolo Abeni */ 1107724cfd2eSPaolo Abeni static bool mptcp_page_frag_refill(struct sock *sk, struct page_frag *pfrag) 1108724cfd2eSPaolo Abeni { 1109724cfd2eSPaolo Abeni if (likely(skb_page_frag_refill(32U + sizeof(struct mptcp_data_frag), 1110724cfd2eSPaolo Abeni pfrag, sk->sk_allocation))) 1111724cfd2eSPaolo Abeni return true; 1112724cfd2eSPaolo Abeni 1113724cfd2eSPaolo Abeni mptcp_enter_memory_pressure(sk); 111418b683bfSPaolo Abeni return false; 111518b683bfSPaolo Abeni } 111618b683bfSPaolo Abeni 111718b683bfSPaolo Abeni static struct mptcp_data_frag * 111818b683bfSPaolo Abeni mptcp_carve_data_frag(const struct mptcp_sock *msk, struct page_frag *pfrag, 111918b683bfSPaolo Abeni int orig_offset) 112018b683bfSPaolo Abeni { 112118b683bfSPaolo Abeni int offset = ALIGN(orig_offset, sizeof(long)); 112218b683bfSPaolo Abeni struct mptcp_data_frag *dfrag; 112318b683bfSPaolo Abeni 112418b683bfSPaolo Abeni dfrag = (struct mptcp_data_frag *)(page_to_virt(pfrag->page) + offset); 112518b683bfSPaolo Abeni dfrag->data_len = 0; 112618b683bfSPaolo Abeni dfrag->data_seq = msk->write_seq; 112718b683bfSPaolo Abeni dfrag->overhead = offset - orig_offset + sizeof(struct mptcp_data_frag); 112818b683bfSPaolo Abeni dfrag->offset = offset + sizeof(struct mptcp_data_frag); 1129d9ca1de8SPaolo Abeni dfrag->already_sent = 0; 113018b683bfSPaolo Abeni dfrag->page = pfrag->page; 113118b683bfSPaolo Abeni 113218b683bfSPaolo Abeni return dfrag; 113318b683bfSPaolo Abeni } 113418b683bfSPaolo Abeni 1135caf971dfSPaolo Abeni struct mptcp_sendmsg_info { 1136caf971dfSPaolo Abeni int mss_now; 1137caf971dfSPaolo Abeni int size_goal; 1138d9ca1de8SPaolo Abeni u16 limit; 1139d9ca1de8SPaolo Abeni u16 sent; 1140d9ca1de8SPaolo Abeni unsigned int flags; 11411094c6feSPaolo Abeni bool data_lock_held; 1142caf971dfSPaolo Abeni }; 1143caf971dfSPaolo Abeni 11446f8a612aSFlorian Westphal static int mptcp_check_allowed_size(struct mptcp_sock *msk, u64 data_seq, 11456f8a612aSFlorian Westphal int avail_size) 11466f8a612aSFlorian Westphal { 11476f8a612aSFlorian Westphal u64 window_end = mptcp_wnd_end(msk); 11486f8a612aSFlorian Westphal 11496f8a612aSFlorian Westphal if (__mptcp_check_fallback(msk)) 11506f8a612aSFlorian Westphal return avail_size; 11516f8a612aSFlorian Westphal 11526f8a612aSFlorian Westphal if (!before64(data_seq + avail_size, window_end)) { 11536f8a612aSFlorian Westphal u64 allowed_size = window_end - data_seq; 11546f8a612aSFlorian Westphal 11556f8a612aSFlorian Westphal return min_t(unsigned int, allowed_size, avail_size); 11566f8a612aSFlorian Westphal } 11576f8a612aSFlorian Westphal 11586f8a612aSFlorian Westphal return avail_size; 11596f8a612aSFlorian Westphal } 11606f8a612aSFlorian Westphal 1161724cfd2eSPaolo Abeni static bool __mptcp_add_ext(struct sk_buff *skb, gfp_t gfp) 1162724cfd2eSPaolo Abeni { 1163724cfd2eSPaolo Abeni struct skb_ext *mpext = __skb_ext_alloc(gfp); 1164724cfd2eSPaolo Abeni 1165724cfd2eSPaolo Abeni if (!mpext) 1166724cfd2eSPaolo Abeni return false; 1167724cfd2eSPaolo Abeni __skb_ext_set(skb, SKB_EXT_MPTCP, mpext); 1168724cfd2eSPaolo Abeni return true; 1169724cfd2eSPaolo Abeni } 1170724cfd2eSPaolo Abeni 11716e628cd3SPaolo Abeni static struct sk_buff *__mptcp_do_alloc_tx_skb(struct sock *sk, gfp_t gfp) 1172724cfd2eSPaolo Abeni { 1173724cfd2eSPaolo Abeni struct sk_buff *skb; 1174724cfd2eSPaolo Abeni 11756e628cd3SPaolo Abeni skb = alloc_skb_fclone(MAX_TCP_HEADER, gfp); 1176724cfd2eSPaolo Abeni if (likely(skb)) { 11776e628cd3SPaolo Abeni if (likely(__mptcp_add_ext(skb, gfp))) { 1178724cfd2eSPaolo Abeni skb_reserve(skb, MAX_TCP_HEADER); 1179a52fe46eSEric Dumazet skb->ip_summed = CHECKSUM_PARTIAL; 1180f70cad10SPaolo Abeni INIT_LIST_HEAD(&skb->tcp_tsorted_anchor); 1181724cfd2eSPaolo Abeni return skb; 1182724cfd2eSPaolo Abeni } 1183724cfd2eSPaolo Abeni __kfree_skb(skb); 1184724cfd2eSPaolo Abeni } else { 1185724cfd2eSPaolo Abeni mptcp_enter_memory_pressure(sk); 1186724cfd2eSPaolo Abeni } 1187724cfd2eSPaolo Abeni return NULL; 1188724cfd2eSPaolo Abeni } 1189724cfd2eSPaolo Abeni 1190f70cad10SPaolo Abeni static struct sk_buff *__mptcp_alloc_tx_skb(struct sock *sk, struct sock *ssk, gfp_t gfp) 1191724cfd2eSPaolo Abeni { 1192724cfd2eSPaolo Abeni struct sk_buff *skb; 1193724cfd2eSPaolo Abeni 11946e628cd3SPaolo Abeni skb = __mptcp_do_alloc_tx_skb(sk, gfp); 1195724cfd2eSPaolo Abeni if (!skb) 1196f70cad10SPaolo Abeni return NULL; 1197724cfd2eSPaolo Abeni 1198724cfd2eSPaolo Abeni if (likely(sk_wmem_schedule(ssk, skb->truesize))) { 1199f70cad10SPaolo Abeni tcp_skb_entail(ssk, skb); 1200f70cad10SPaolo Abeni return skb; 1201724cfd2eSPaolo Abeni } 12023ef3905aSYonglong Li tcp_skb_tsorted_anchor_cleanup(skb); 1203724cfd2eSPaolo Abeni kfree_skb(skb); 1204f70cad10SPaolo Abeni return NULL; 1205724cfd2eSPaolo Abeni } 1206724cfd2eSPaolo Abeni 1207f70cad10SPaolo Abeni static struct sk_buff *mptcp_alloc_tx_skb(struct sock *sk, struct sock *ssk, bool data_lock_held) 1208724cfd2eSPaolo Abeni { 12091094c6feSPaolo Abeni gfp_t gfp = data_lock_held ? GFP_ATOMIC : sk->sk_allocation; 1210724cfd2eSPaolo Abeni 12111094c6feSPaolo Abeni if (unlikely(tcp_under_memory_pressure(sk))) { 12121094c6feSPaolo Abeni if (data_lock_held) 12131094c6feSPaolo Abeni __mptcp_mem_reclaim_partial(sk); 12141094c6feSPaolo Abeni else 1215724cfd2eSPaolo Abeni mptcp_mem_reclaim_partial(sk); 12161094c6feSPaolo Abeni } 12171094c6feSPaolo Abeni return __mptcp_alloc_tx_skb(sk, ssk, gfp); 1218724cfd2eSPaolo Abeni } 1219724cfd2eSPaolo Abeni 1220d0cc2987SGeliang Tang /* note: this always recompute the csum on the whole skb, even 1221d0cc2987SGeliang Tang * if we just appended a single frag. More status info needed 1222d0cc2987SGeliang Tang */ 1223d0cc2987SGeliang Tang static void mptcp_update_data_checksum(struct sk_buff *skb, int added) 1224d0cc2987SGeliang Tang { 1225d0cc2987SGeliang Tang struct mptcp_ext *mpext = mptcp_get_ext(skb); 1226d0cc2987SGeliang Tang __wsum csum = ~csum_unfold(mpext->csum); 1227d0cc2987SGeliang Tang int offset = skb->len - added; 1228d0cc2987SGeliang Tang 1229d0cc2987SGeliang Tang mpext->csum = csum_fold(csum_block_add(csum, skb_checksum(skb, offset, added, 0), offset)); 1230d0cc2987SGeliang Tang } 1231d0cc2987SGeliang Tang 1232*1e39e5a3SGeliang Tang static void mptcp_update_infinite_map(struct mptcp_sock *msk, 1233*1e39e5a3SGeliang Tang struct sock *ssk, 1234*1e39e5a3SGeliang Tang struct mptcp_ext *mpext) 1235*1e39e5a3SGeliang Tang { 1236*1e39e5a3SGeliang Tang if (!mpext) 1237*1e39e5a3SGeliang Tang return; 1238*1e39e5a3SGeliang Tang 1239*1e39e5a3SGeliang Tang mpext->infinite_map = 1; 1240*1e39e5a3SGeliang Tang mpext->data_len = 0; 1241*1e39e5a3SGeliang Tang 1242*1e39e5a3SGeliang Tang mptcp_subflow_ctx(ssk)->send_infinite_map = 0; 1243*1e39e5a3SGeliang Tang pr_fallback(msk); 1244*1e39e5a3SGeliang Tang __mptcp_do_fallback(msk); 1245*1e39e5a3SGeliang Tang } 1246*1e39e5a3SGeliang Tang 124757040755SPaolo Abeni static int mptcp_sendmsg_frag(struct sock *sk, struct sock *ssk, 1248d9ca1de8SPaolo Abeni struct mptcp_data_frag *dfrag, 1249caf971dfSPaolo Abeni struct mptcp_sendmsg_info *info) 125057040755SPaolo Abeni { 1251d9ca1de8SPaolo Abeni u64 data_seq = dfrag->data_seq + info->sent; 1252f70cad10SPaolo Abeni int offset = dfrag->offset + info->sent; 12536d0060f6SMat Martineau struct mptcp_sock *msk = mptcp_sk(sk); 12546f8a612aSFlorian Westphal bool zero_window_probe = false; 12556d0060f6SMat Martineau struct mptcp_ext *mpext = NULL; 1256f70cad10SPaolo Abeni bool can_coalesce = false; 1257f70cad10SPaolo Abeni bool reuse_skb = true; 1258f70cad10SPaolo Abeni struct sk_buff *skb; 1259f70cad10SPaolo Abeni size_t copy; 1260f70cad10SPaolo Abeni int i; 12616d0060f6SMat Martineau 1262e4b61351SGeliang Tang pr_debug("msk=%p ssk=%p sending dfrag at seq=%llu len=%u already sent=%u", 1263d9ca1de8SPaolo Abeni msk, ssk, dfrag->data_seq, dfrag->data_len, info->sent); 12646d0060f6SMat Martineau 1265f70cad10SPaolo Abeni if (WARN_ON_ONCE(info->sent > info->limit || 1266f70cad10SPaolo Abeni info->limit > dfrag->data_len)) 1267f70cad10SPaolo Abeni return 0; 1268f70cad10SPaolo Abeni 1269d9ca1de8SPaolo Abeni /* compute send limit */ 1270d9ca1de8SPaolo Abeni info->mss_now = tcp_send_mss(ssk, &info->size_goal, info->flags); 1271f70cad10SPaolo Abeni copy = info->size_goal; 1272f70cad10SPaolo Abeni 127357040755SPaolo Abeni skb = tcp_write_queue_tail(ssk); 1274f70cad10SPaolo Abeni if (skb && copy > skb->len) { 127557040755SPaolo Abeni /* Limit the write to the size available in the 127657040755SPaolo Abeni * current skb, if any, so that we create at most a new skb. 127757040755SPaolo Abeni * Explicitly tells TCP internals to avoid collapsing on later 127857040755SPaolo Abeni * queue management operation, to avoid breaking the ext <-> 127957040755SPaolo Abeni * SSN association set here 128057040755SPaolo Abeni */ 1281d9ca1de8SPaolo Abeni mpext = skb_ext_find(skb, SKB_EXT_MPTCP); 12821094c6feSPaolo Abeni if (!mptcp_skb_can_collapse_to(data_seq, skb, mpext)) { 128357040755SPaolo Abeni TCP_SKB_CB(skb)->eor = 1; 12841094c6feSPaolo Abeni goto alloc_skb; 12851094c6feSPaolo Abeni } 12861094c6feSPaolo Abeni 1287f70cad10SPaolo Abeni i = skb_shinfo(skb)->nr_frags; 1288f70cad10SPaolo Abeni can_coalesce = skb_can_coalesce(skb, i, dfrag->page, offset); 1289f70cad10SPaolo Abeni if (!can_coalesce && i >= sysctl_max_skb_frags) { 1290f70cad10SPaolo Abeni tcp_mark_push(tcp_sk(ssk), skb); 1291f70cad10SPaolo Abeni goto alloc_skb; 129215e6ca97SPaolo Abeni } 129318b683bfSPaolo Abeni 1294f70cad10SPaolo Abeni copy -= skb->len; 1295f70cad10SPaolo Abeni } else { 12961094c6feSPaolo Abeni alloc_skb: 1297f70cad10SPaolo Abeni skb = mptcp_alloc_tx_skb(sk, ssk, info->data_lock_held); 1298f70cad10SPaolo Abeni if (!skb) 1299f70cad10SPaolo Abeni return -ENOMEM; 1300f70cad10SPaolo Abeni 1301f70cad10SPaolo Abeni i = skb_shinfo(skb)->nr_frags; 1302f70cad10SPaolo Abeni reuse_skb = false; 1303f70cad10SPaolo Abeni mpext = skb_ext_find(skb, SKB_EXT_MPTCP); 1304f70cad10SPaolo Abeni } 13051094c6feSPaolo Abeni 13066f8a612aSFlorian Westphal /* Zero window and all data acked? Probe. */ 1307f70cad10SPaolo Abeni copy = mptcp_check_allowed_size(msk, data_seq, copy); 1308f70cad10SPaolo Abeni if (copy == 0) { 13097439d687SPaolo Abeni u64 snd_una = READ_ONCE(msk->snd_una); 13107439d687SPaolo Abeni 1311f70cad10SPaolo Abeni if (snd_una != msk->snd_nxt) { 131227728ba8SEric Dumazet tcp_remove_empty_skb(ssk); 13136f8a612aSFlorian Westphal return 0; 13146f8a612aSFlorian Westphal } 13156f8a612aSFlorian Westphal 1316f70cad10SPaolo Abeni zero_window_probe = true; 1317f70cad10SPaolo Abeni data_seq = snd_una - 1; 1318f70cad10SPaolo Abeni copy = 1; 131957040755SPaolo Abeni 1320f70cad10SPaolo Abeni /* all mptcp-level data is acked, no skbs should be present into the 1321f70cad10SPaolo Abeni * ssk write queue 1322f70cad10SPaolo Abeni */ 1323f70cad10SPaolo Abeni WARN_ON_ONCE(reuse_skb); 1324f70cad10SPaolo Abeni } 1325f70cad10SPaolo Abeni 1326f70cad10SPaolo Abeni copy = min_t(size_t, copy, info->limit - info->sent); 1327f70cad10SPaolo Abeni if (!sk_wmem_schedule(ssk, copy)) { 132827728ba8SEric Dumazet tcp_remove_empty_skb(ssk); 1329e2223995SPaolo Abeni return -ENOMEM; 133035759383SFlorian Westphal } 133118b683bfSPaolo Abeni 1332f70cad10SPaolo Abeni if (can_coalesce) { 1333f70cad10SPaolo Abeni skb_frag_size_add(&skb_shinfo(skb)->frags[i - 1], copy); 1334f70cad10SPaolo Abeni } else { 1335f70cad10SPaolo Abeni get_page(dfrag->page); 1336f70cad10SPaolo Abeni skb_fill_page_desc(skb, i, dfrag->page, offset, copy); 133757040755SPaolo Abeni } 133857040755SPaolo Abeni 1339f70cad10SPaolo Abeni skb->len += copy; 1340f70cad10SPaolo Abeni skb->data_len += copy; 1341f70cad10SPaolo Abeni skb->truesize += copy; 1342f70cad10SPaolo Abeni sk_wmem_queued_add(ssk, copy); 1343f70cad10SPaolo Abeni sk_mem_charge(ssk, copy); 1344f70cad10SPaolo Abeni WRITE_ONCE(tcp_sk(ssk)->write_seq, tcp_sk(ssk)->write_seq + copy); 1345f70cad10SPaolo Abeni TCP_SKB_CB(skb)->end_seq += copy; 1346f70cad10SPaolo Abeni tcp_skb_pcount_set(skb, 0); 1347f70cad10SPaolo Abeni 1348f70cad10SPaolo Abeni /* on skb reuse we just need to update the DSS len */ 1349f70cad10SPaolo Abeni if (reuse_skb) { 1350f70cad10SPaolo Abeni TCP_SKB_CB(skb)->tcp_flags &= ~TCPHDR_PSH; 1351f70cad10SPaolo Abeni mpext->data_len += copy; 1352f70cad10SPaolo Abeni WARN_ON_ONCE(zero_window_probe); 1353f70cad10SPaolo Abeni goto out; 1354724cfd2eSPaolo Abeni } 13556d0060f6SMat Martineau 13566d0060f6SMat Martineau memset(mpext, 0, sizeof(*mpext)); 1357d9ca1de8SPaolo Abeni mpext->data_seq = data_seq; 13586d0060f6SMat Martineau mpext->subflow_seq = mptcp_subflow_ctx(ssk)->rel_write_seq; 1359f70cad10SPaolo Abeni mpext->data_len = copy; 13606d0060f6SMat Martineau mpext->use_map = 1; 13616d0060f6SMat Martineau mpext->dsn64 = 1; 13626d0060f6SMat Martineau 13636d0060f6SMat Martineau pr_debug("data_seq=%llu subflow_seq=%u data_len=%u dsn64=%d", 13646d0060f6SMat Martineau mpext->data_seq, mpext->subflow_seq, mpext->data_len, 13656d0060f6SMat Martineau mpext->dsn64); 13666d0060f6SMat Martineau 13676f8a612aSFlorian Westphal if (zero_window_probe) { 1368f70cad10SPaolo Abeni mptcp_subflow_ctx(ssk)->rel_write_seq += copy; 13696f8a612aSFlorian Westphal mpext->frozen = 1; 1370d0cc2987SGeliang Tang if (READ_ONCE(msk->csum_enabled)) 1371f70cad10SPaolo Abeni mptcp_update_data_checksum(skb, copy); 13726f8a612aSFlorian Westphal tcp_push_pending_frames(ssk); 1373d0cc2987SGeliang Tang return 0; 13746f8a612aSFlorian Westphal } 137557040755SPaolo Abeni out: 1376d0cc2987SGeliang Tang if (READ_ONCE(msk->csum_enabled)) 1377f70cad10SPaolo Abeni mptcp_update_data_checksum(skb, copy); 1378*1e39e5a3SGeliang Tang if (mptcp_subflow_ctx(ssk)->send_infinite_map) 1379*1e39e5a3SGeliang Tang mptcp_update_infinite_map(msk, ssk, mpext); 13800eb4e7eeSGeliang Tang trace_mptcp_sendmsg_frag(mpext); 1381f70cad10SPaolo Abeni mptcp_subflow_ctx(ssk)->rel_write_seq += copy; 1382f70cad10SPaolo Abeni return copy; 13836d0060f6SMat Martineau } 13846d0060f6SMat Martineau 1385d5f49190SPaolo Abeni #define MPTCP_SEND_BURST_SIZE ((1 << 16) - \ 1386d5f49190SPaolo Abeni sizeof(struct tcphdr) - \ 1387d5f49190SPaolo Abeni MAX_TCP_OPTION_SPACE - \ 1388d5f49190SPaolo Abeni sizeof(struct ipv6hdr) - \ 1389d5f49190SPaolo Abeni sizeof(struct frag_hdr)) 1390d5f49190SPaolo Abeni 1391d5f49190SPaolo Abeni struct subflow_send_info { 1392d5f49190SPaolo Abeni struct sock *ssk; 13933ce0852cSPaolo Abeni u64 linger_time; 1394d5f49190SPaolo Abeni }; 1395d5f49190SPaolo Abeni 1396ff5a0b42SPaolo Abeni void mptcp_subflow_set_active(struct mptcp_subflow_context *subflow) 1397ff5a0b42SPaolo Abeni { 1398ff5a0b42SPaolo Abeni if (!subflow->stale) 1399ff5a0b42SPaolo Abeni return; 1400ff5a0b42SPaolo Abeni 1401ff5a0b42SPaolo Abeni subflow->stale = 0; 1402fc1b4e3bSPaolo Abeni MPTCP_INC_STATS(sock_net(mptcp_subflow_tcp_sock(subflow)), MPTCP_MIB_SUBFLOWRECOVER); 1403ff5a0b42SPaolo Abeni } 1404ff5a0b42SPaolo Abeni 1405ff5a0b42SPaolo Abeni bool mptcp_subflow_active(struct mptcp_subflow_context *subflow) 1406ff5a0b42SPaolo Abeni { 1407ff5a0b42SPaolo Abeni if (unlikely(subflow->stale)) { 1408ff5a0b42SPaolo Abeni u32 rcv_tstamp = READ_ONCE(tcp_sk(mptcp_subflow_tcp_sock(subflow))->rcv_tstamp); 1409ff5a0b42SPaolo Abeni 1410ff5a0b42SPaolo Abeni if (subflow->stale_rcv_tstamp == rcv_tstamp) 1411ff5a0b42SPaolo Abeni return false; 1412ff5a0b42SPaolo Abeni 1413ff5a0b42SPaolo Abeni mptcp_subflow_set_active(subflow); 1414ff5a0b42SPaolo Abeni } 1415ff5a0b42SPaolo Abeni return __mptcp_subflow_active(subflow); 1416ff5a0b42SPaolo Abeni } 1417ff5a0b42SPaolo Abeni 14183ce0852cSPaolo Abeni #define SSK_MODE_ACTIVE 0 14193ce0852cSPaolo Abeni #define SSK_MODE_BACKUP 1 14203ce0852cSPaolo Abeni #define SSK_MODE_MAX 2 14213ce0852cSPaolo Abeni 142233d41c9cSPaolo Abeni /* implement the mptcp packet scheduler; 142333d41c9cSPaolo Abeni * returns the subflow that will transmit the next DSS 142433d41c9cSPaolo Abeni * additionally updates the rtx timeout 142533d41c9cSPaolo Abeni */ 14265cf92bbaSPaolo Abeni static struct sock *mptcp_subflow_get_send(struct mptcp_sock *msk) 1427f296234cSPeter Krystad { 14283ce0852cSPaolo Abeni struct subflow_send_info send_info[SSK_MODE_MAX]; 1429f296234cSPeter Krystad struct mptcp_subflow_context *subflow; 143033d41c9cSPaolo Abeni struct sock *sk = (struct sock *)msk; 14313ce0852cSPaolo Abeni u32 pace, burst, wmem; 1432d5f49190SPaolo Abeni int i, nr_active = 0; 1433d5f49190SPaolo Abeni struct sock *ssk; 14343ce0852cSPaolo Abeni u64 linger_time; 143533d41c9cSPaolo Abeni long tout = 0; 1436f296234cSPeter Krystad 143733d41c9cSPaolo Abeni sock_owned_by_me(sk); 1438f296234cSPeter Krystad 1439d5f49190SPaolo Abeni if (__mptcp_check_fallback(msk)) { 1440d5f49190SPaolo Abeni if (!msk->first) 1441f296234cSPeter Krystad return NULL; 1442d5f49190SPaolo Abeni return sk_stream_memory_free(msk->first) ? msk->first : NULL; 1443f296234cSPeter Krystad } 1444f296234cSPeter Krystad 1445d5f49190SPaolo Abeni /* re-use last subflow, if the burst allow that */ 1446d5f49190SPaolo Abeni if (msk->last_snd && msk->snd_burst > 0 && 1447d5f49190SPaolo Abeni sk_stream_memory_free(msk->last_snd) && 144833d41c9cSPaolo Abeni mptcp_subflow_active(mptcp_subflow_ctx(msk->last_snd))) { 144933d41c9cSPaolo Abeni mptcp_set_timeout(sk); 1450d5f49190SPaolo Abeni return msk->last_snd; 145133d41c9cSPaolo Abeni } 1452f296234cSPeter Krystad 1453d5f49190SPaolo Abeni /* pick the subflow with the lower wmem/wspace ratio */ 14543ce0852cSPaolo Abeni for (i = 0; i < SSK_MODE_MAX; ++i) { 1455d5f49190SPaolo Abeni send_info[i].ssk = NULL; 14563ce0852cSPaolo Abeni send_info[i].linger_time = -1; 1457d5f49190SPaolo Abeni } 14583ce0852cSPaolo Abeni 1459d5f49190SPaolo Abeni mptcp_for_each_subflow(msk, subflow) { 1460e10a9892SGeliang Tang trace_mptcp_subflow_get_send(subflow); 1461d5f49190SPaolo Abeni ssk = mptcp_subflow_tcp_sock(subflow); 1462d5f49190SPaolo Abeni if (!mptcp_subflow_active(subflow)) 1463f296234cSPeter Krystad continue; 1464d5f49190SPaolo Abeni 146533d41c9cSPaolo Abeni tout = max(tout, mptcp_timeout_from_subflow(subflow)); 1466d5f49190SPaolo Abeni nr_active += !subflow->backup; 14673ce0852cSPaolo Abeni pace = subflow->avg_pacing_rate; 14683ce0852cSPaolo Abeni if (unlikely(!pace)) { 14693ce0852cSPaolo Abeni /* init pacing rate from socket */ 14703ce0852cSPaolo Abeni subflow->avg_pacing_rate = READ_ONCE(ssk->sk_pacing_rate); 14713ce0852cSPaolo Abeni pace = subflow->avg_pacing_rate; 1472d5f49190SPaolo Abeni if (!pace) 1473d5f49190SPaolo Abeni continue; 14743ce0852cSPaolo Abeni } 1475d5f49190SPaolo Abeni 14763ce0852cSPaolo Abeni linger_time = div_u64((u64)READ_ONCE(ssk->sk_wmem_queued) << 32, pace); 14773ce0852cSPaolo Abeni if (linger_time < send_info[subflow->backup].linger_time) { 1478d5f49190SPaolo Abeni send_info[subflow->backup].ssk = ssk; 14793ce0852cSPaolo Abeni send_info[subflow->backup].linger_time = linger_time; 1480d5f49190SPaolo Abeni } 1481f296234cSPeter Krystad } 148233d41c9cSPaolo Abeni __mptcp_set_timeout(sk, tout); 1483f296234cSPeter Krystad 1484d5f49190SPaolo Abeni /* pick the best backup if no other subflow is active */ 1485d5f49190SPaolo Abeni if (!nr_active) 14863ce0852cSPaolo Abeni send_info[SSK_MODE_ACTIVE].ssk = send_info[SSK_MODE_BACKUP].ssk; 1487d5f49190SPaolo Abeni 14883ce0852cSPaolo Abeni /* According to the blest algorithm, to avoid HoL blocking for the 14893ce0852cSPaolo Abeni * faster flow, we need to: 14903ce0852cSPaolo Abeni * - estimate the faster flow linger time 14913ce0852cSPaolo Abeni * - use the above to estimate the amount of byte transferred 14923ce0852cSPaolo Abeni * by the faster flow 14933ce0852cSPaolo Abeni * - check that the amount of queued data is greter than the above, 14943ce0852cSPaolo Abeni * otherwise do not use the picked, slower, subflow 14953ce0852cSPaolo Abeni * We select the subflow with the shorter estimated time to flush 14963ce0852cSPaolo Abeni * the queued mem, which basically ensure the above. We just need 14973ce0852cSPaolo Abeni * to check that subflow has a non empty cwin. 14983ce0852cSPaolo Abeni */ 14993ce0852cSPaolo Abeni ssk = send_info[SSK_MODE_ACTIVE].ssk; 15003ce0852cSPaolo Abeni if (!ssk || !sk_stream_memory_free(ssk) || !tcp_sk(ssk)->snd_wnd) 1501d5f49190SPaolo Abeni return NULL; 15023ce0852cSPaolo Abeni 15033ce0852cSPaolo Abeni burst = min_t(int, MPTCP_SEND_BURST_SIZE, tcp_sk(ssk)->snd_wnd); 15043ce0852cSPaolo Abeni wmem = READ_ONCE(ssk->sk_wmem_queued); 15053ce0852cSPaolo Abeni subflow = mptcp_subflow_ctx(ssk); 15063ce0852cSPaolo Abeni subflow->avg_pacing_rate = div_u64((u64)subflow->avg_pacing_rate * wmem + 15073ce0852cSPaolo Abeni READ_ONCE(ssk->sk_pacing_rate) * burst, 15083ce0852cSPaolo Abeni burst + wmem); 15093ce0852cSPaolo Abeni msk->last_snd = ssk; 15103ce0852cSPaolo Abeni msk->snd_burst = burst; 15113ce0852cSPaolo Abeni return ssk; 1512f296234cSPeter Krystad } 1513f296234cSPeter Krystad 1514b8e0def3SGeliang Tang static void mptcp_push_release(struct sock *ssk, struct mptcp_sendmsg_info *info) 1515d9ca1de8SPaolo Abeni { 1516d9ca1de8SPaolo Abeni tcp_push(ssk, 0, info->mss_now, tcp_sk(ssk)->nonagle, info->size_goal); 1517d9ca1de8SPaolo Abeni release_sock(ssk); 1518d9ca1de8SPaolo Abeni } 1519d9ca1de8SPaolo Abeni 15200d199e43SFlorian Westphal static void mptcp_update_post_push(struct mptcp_sock *msk, 15210d199e43SFlorian Westphal struct mptcp_data_frag *dfrag, 15220d199e43SFlorian Westphal u32 sent) 15230d199e43SFlorian Westphal { 15240d199e43SFlorian Westphal u64 snd_nxt_new = dfrag->data_seq; 15250d199e43SFlorian Westphal 15260d199e43SFlorian Westphal dfrag->already_sent += sent; 15270d199e43SFlorian Westphal 15280d199e43SFlorian Westphal msk->snd_burst -= sent; 15290d199e43SFlorian Westphal 15300d199e43SFlorian Westphal snd_nxt_new += dfrag->already_sent; 15310d199e43SFlorian Westphal 15320d199e43SFlorian Westphal /* snd_nxt_new can be smaller than snd_nxt in case mptcp 15330d199e43SFlorian Westphal * is recovering after a failover. In that event, this re-sends 15340d199e43SFlorian Westphal * old segments. 15350d199e43SFlorian Westphal * 15360d199e43SFlorian Westphal * Thus compute snd_nxt_new candidate based on 15370d199e43SFlorian Westphal * the dfrag->data_seq that was sent and the data 15380d199e43SFlorian Westphal * that has been handed to the subflow for transmission 15390d199e43SFlorian Westphal * and skip update in case it was old dfrag. 15400d199e43SFlorian Westphal */ 15410d199e43SFlorian Westphal if (likely(after64(snd_nxt_new, msk->snd_nxt))) 15420d199e43SFlorian Westphal msk->snd_nxt = snd_nxt_new; 15430d199e43SFlorian Westphal } 15440d199e43SFlorian Westphal 15458b38217aSMaxim Galaganov void mptcp_check_and_set_pending(struct sock *sk) 15463241a9c0SFlorian Westphal { 1547e9d09bacSPaolo Abeni if (mptcp_send_head(sk)) 1548e9d09bacSPaolo Abeni mptcp_sk(sk)->push_pending |= BIT(MPTCP_PUSH_PENDING); 15493241a9c0SFlorian Westphal } 15503241a9c0SFlorian Westphal 1551ff5a0b42SPaolo Abeni void __mptcp_push_pending(struct sock *sk, unsigned int flags) 1552d9ca1de8SPaolo Abeni { 1553d9ca1de8SPaolo Abeni struct sock *prev_ssk = NULL, *ssk = NULL; 1554d9ca1de8SPaolo Abeni struct mptcp_sock *msk = mptcp_sk(sk); 1555d9ca1de8SPaolo Abeni struct mptcp_sendmsg_info info = { 1556d9ca1de8SPaolo Abeni .flags = flags, 1557d9ca1de8SPaolo Abeni }; 1558d9ca1de8SPaolo Abeni struct mptcp_data_frag *dfrag; 1559d9ca1de8SPaolo Abeni int len, copied = 0; 1560d9ca1de8SPaolo Abeni 1561d9ca1de8SPaolo Abeni while ((dfrag = mptcp_send_head(sk))) { 1562d9ca1de8SPaolo Abeni info.sent = dfrag->already_sent; 1563d9ca1de8SPaolo Abeni info.limit = dfrag->data_len; 1564d9ca1de8SPaolo Abeni len = dfrag->data_len - dfrag->already_sent; 1565d9ca1de8SPaolo Abeni while (len > 0) { 1566d9ca1de8SPaolo Abeni int ret = 0; 1567d9ca1de8SPaolo Abeni 1568d9ca1de8SPaolo Abeni prev_ssk = ssk; 15695cf92bbaSPaolo Abeni ssk = mptcp_subflow_get_send(msk); 1570d9ca1de8SPaolo Abeni 15719758f40eSPaolo Abeni /* First check. If the ssk has changed since 15729758f40eSPaolo Abeni * the last round, release prev_ssk 1573d9ca1de8SPaolo Abeni */ 1574d9ca1de8SPaolo Abeni if (ssk != prev_ssk && prev_ssk) 1575b8e0def3SGeliang Tang mptcp_push_release(prev_ssk, &info); 1576d9ca1de8SPaolo Abeni if (!ssk) 1577d9ca1de8SPaolo Abeni goto out; 1578d9ca1de8SPaolo Abeni 15799758f40eSPaolo Abeni /* Need to lock the new subflow only if different 15809758f40eSPaolo Abeni * from the previous one, otherwise we are still 15819758f40eSPaolo Abeni * helding the relevant lock 15829758f40eSPaolo Abeni */ 15839758f40eSPaolo Abeni if (ssk != prev_ssk) 1584d9ca1de8SPaolo Abeni lock_sock(ssk); 1585d9ca1de8SPaolo Abeni 1586d9ca1de8SPaolo Abeni ret = mptcp_sendmsg_frag(sk, ssk, dfrag, &info); 1587d9ca1de8SPaolo Abeni if (ret <= 0) { 1588b8e0def3SGeliang Tang mptcp_push_release(ssk, &info); 1589d9ca1de8SPaolo Abeni goto out; 1590d9ca1de8SPaolo Abeni } 1591d9ca1de8SPaolo Abeni 1592d9ca1de8SPaolo Abeni info.sent += ret; 1593d9ca1de8SPaolo Abeni copied += ret; 1594d9ca1de8SPaolo Abeni len -= ret; 15950d199e43SFlorian Westphal 15960d199e43SFlorian Westphal mptcp_update_post_push(msk, dfrag, ret); 1597d9ca1de8SPaolo Abeni } 1598d9ca1de8SPaolo Abeni WRITE_ONCE(msk->first_pending, mptcp_send_next(sk)); 1599d9ca1de8SPaolo Abeni } 1600d9ca1de8SPaolo Abeni 1601d9ca1de8SPaolo Abeni /* at this point we held the socket lock for the last subflow we used */ 1602d9ca1de8SPaolo Abeni if (ssk) 1603b8e0def3SGeliang Tang mptcp_push_release(ssk, &info); 1604d9ca1de8SPaolo Abeni 1605d9ca1de8SPaolo Abeni out: 160633d41c9cSPaolo Abeni /* ensure the rtx timer is running */ 1607d9ca1de8SPaolo Abeni if (!mptcp_timer_pending(sk)) 1608d9ca1de8SPaolo Abeni mptcp_reset_timer(sk); 160933d41c9cSPaolo Abeni if (copied) 1610d9ca1de8SPaolo Abeni __mptcp_check_send_data_fin(sk); 1611d9ca1de8SPaolo Abeni } 1612d9ca1de8SPaolo Abeni 16136e628cd3SPaolo Abeni static void __mptcp_subflow_push_pending(struct sock *sk, struct sock *ssk) 16146e628cd3SPaolo Abeni { 16156e628cd3SPaolo Abeni struct mptcp_sock *msk = mptcp_sk(sk); 16161094c6feSPaolo Abeni struct mptcp_sendmsg_info info = { 16171094c6feSPaolo Abeni .data_lock_held = true, 16181094c6feSPaolo Abeni }; 16196e628cd3SPaolo Abeni struct mptcp_data_frag *dfrag; 1620b19bc294SPaolo Abeni struct sock *xmit_ssk; 16216e628cd3SPaolo Abeni int len, copied = 0; 1622b19bc294SPaolo Abeni bool first = true; 16236e628cd3SPaolo Abeni 16246e628cd3SPaolo Abeni info.flags = 0; 16256e628cd3SPaolo Abeni while ((dfrag = mptcp_send_head(sk))) { 16266e628cd3SPaolo Abeni info.sent = dfrag->already_sent; 16276e628cd3SPaolo Abeni info.limit = dfrag->data_len; 16286e628cd3SPaolo Abeni len = dfrag->data_len - dfrag->already_sent; 16296e628cd3SPaolo Abeni while (len > 0) { 16306e628cd3SPaolo Abeni int ret = 0; 16316e628cd3SPaolo Abeni 1632b19bc294SPaolo Abeni /* the caller already invoked the packet scheduler, 1633b19bc294SPaolo Abeni * check for a different subflow usage only after 1634b19bc294SPaolo Abeni * spooling the first chunk of data 1635b19bc294SPaolo Abeni */ 1636b19bc294SPaolo Abeni xmit_ssk = first ? ssk : mptcp_subflow_get_send(mptcp_sk(sk)); 1637b19bc294SPaolo Abeni if (!xmit_ssk) 1638b19bc294SPaolo Abeni goto out; 1639b19bc294SPaolo Abeni if (xmit_ssk != ssk) { 1640bcd97734SPaolo Abeni mptcp_subflow_delegate(mptcp_subflow_ctx(xmit_ssk), 1641bcd97734SPaolo Abeni MPTCP_DELEGATE_SEND); 1642b19bc294SPaolo Abeni goto out; 1643b19bc294SPaolo Abeni } 1644b19bc294SPaolo Abeni 16456e628cd3SPaolo Abeni ret = mptcp_sendmsg_frag(sk, ssk, dfrag, &info); 16466e628cd3SPaolo Abeni if (ret <= 0) 16476e628cd3SPaolo Abeni goto out; 16486e628cd3SPaolo Abeni 16496e628cd3SPaolo Abeni info.sent += ret; 16506e628cd3SPaolo Abeni copied += ret; 16516e628cd3SPaolo Abeni len -= ret; 1652b19bc294SPaolo Abeni first = false; 16530d199e43SFlorian Westphal 16540d199e43SFlorian Westphal mptcp_update_post_push(msk, dfrag, ret); 16556e628cd3SPaolo Abeni } 16566e628cd3SPaolo Abeni WRITE_ONCE(msk->first_pending, mptcp_send_next(sk)); 16576e628cd3SPaolo Abeni } 16586e628cd3SPaolo Abeni 16596e628cd3SPaolo Abeni out: 16606e628cd3SPaolo Abeni /* __mptcp_alloc_tx_skb could have released some wmem and we are 16616e628cd3SPaolo Abeni * not going to flush it via release_sock() 16626e628cd3SPaolo Abeni */ 16636e628cd3SPaolo Abeni if (copied) { 16646e628cd3SPaolo Abeni tcp_push(ssk, 0, info.mss_now, tcp_sk(ssk)->nonagle, 16656e628cd3SPaolo Abeni info.size_goal); 1666d09d818eSPaolo Abeni if (!mptcp_timer_pending(sk)) 1667d09d818eSPaolo Abeni mptcp_reset_timer(sk); 1668d09d818eSPaolo Abeni 16696e628cd3SPaolo Abeni if (msk->snd_data_fin_enable && 16706e628cd3SPaolo Abeni msk->snd_nxt + 1 == msk->write_seq) 16716e628cd3SPaolo Abeni mptcp_schedule_work(sk); 16726e628cd3SPaolo Abeni } 16736e628cd3SPaolo Abeni } 16746e628cd3SPaolo Abeni 16755cf92bbaSPaolo Abeni static void mptcp_set_nospace(struct sock *sk) 16765cf92bbaSPaolo Abeni { 16775cf92bbaSPaolo Abeni /* enable autotune */ 16785cf92bbaSPaolo Abeni set_bit(SOCK_NOSPACE, &sk->sk_socket->flags); 16795cf92bbaSPaolo Abeni 16805cf92bbaSPaolo Abeni /* will be cleared on avail space */ 16815cf92bbaSPaolo Abeni set_bit(MPTCP_NOSPACE, &mptcp_sk(sk)->flags); 16825cf92bbaSPaolo Abeni } 16835cf92bbaSPaolo Abeni 1684f870fa0bSMat Martineau static int mptcp_sendmsg(struct sock *sk, struct msghdr *msg, size_t len) 1685f870fa0bSMat Martineau { 1686f870fa0bSMat Martineau struct mptcp_sock *msk = mptcp_sk(sk); 168717091708SFlorian Westphal struct page_frag *pfrag; 16886d0060f6SMat Martineau size_t copied = 0; 1689caf971dfSPaolo Abeni int ret = 0; 16906d0060f6SMat Martineau long timeo; 1691f870fa0bSMat Martineau 1692987858e5SPaolo Abeni /* we don't support FASTOPEN yet */ 1693987858e5SPaolo Abeni if (msg->msg_flags & MSG_FASTOPEN) 1694f870fa0bSMat Martineau return -EOPNOTSUPP; 1695f870fa0bSMat Martineau 1696987858e5SPaolo Abeni /* silently ignore everything else */ 1697987858e5SPaolo Abeni msg->msg_flags &= MSG_MORE | MSG_DONTWAIT | MSG_NOSIGNAL; 1698987858e5SPaolo Abeni 16996511882cSPaolo Abeni lock_sock(sk); 17001954b860SMat Martineau 17011954b860SMat Martineau timeo = sock_sndtimeo(sk, msg->msg_flags & MSG_DONTWAIT); 17021954b860SMat Martineau 17031954b860SMat Martineau if ((1 << sk->sk_state) & ~(TCPF_ESTABLISHED | TCPF_CLOSE_WAIT)) { 17041954b860SMat Martineau ret = sk_stream_wait_connect(sk, &timeo); 17051954b860SMat Martineau if (ret) 17061954b860SMat Martineau goto out; 17071954b860SMat Martineau } 17081954b860SMat Martineau 170917091708SFlorian Westphal pfrag = sk_page_frag(sk); 171018b683bfSPaolo Abeni 1711d9ca1de8SPaolo Abeni while (msg_data_left(msg)) { 1712724cfd2eSPaolo Abeni int total_ts, frag_truesize = 0; 1713d9ca1de8SPaolo Abeni struct mptcp_data_frag *dfrag; 1714d9ca1de8SPaolo Abeni bool dfrag_collapsed; 1715d9ca1de8SPaolo Abeni size_t psize, offset; 1716d9ca1de8SPaolo Abeni 171757baaf28SMat Martineau if (sk->sk_err || (sk->sk_shutdown & SEND_SHUTDOWN)) { 171857baaf28SMat Martineau ret = -EPIPE; 171957baaf28SMat Martineau goto out; 172057baaf28SMat Martineau } 172157baaf28SMat Martineau 1722d9ca1de8SPaolo Abeni /* reuse tail pfrag, if possible, or carve a new one from the 1723d9ca1de8SPaolo Abeni * page allocator 1724fb529e62SFlorian Westphal */ 1725d9ca1de8SPaolo Abeni dfrag = mptcp_pending_tail(sk); 1726d9ca1de8SPaolo Abeni dfrag_collapsed = mptcp_frag_can_collapse_to(msk, pfrag, dfrag); 1727d9ca1de8SPaolo Abeni if (!dfrag_collapsed) { 1728d9ca1de8SPaolo Abeni if (!sk_stream_memory_free(sk)) 1729d9ca1de8SPaolo Abeni goto wait_for_memory; 17306e628cd3SPaolo Abeni 1731d9ca1de8SPaolo Abeni if (!mptcp_page_frag_refill(sk, pfrag)) 1732d9ca1de8SPaolo Abeni goto wait_for_memory; 1733d9ca1de8SPaolo Abeni 1734d9ca1de8SPaolo Abeni dfrag = mptcp_carve_data_frag(msk, pfrag, pfrag->offset); 1735d9ca1de8SPaolo Abeni frag_truesize = dfrag->overhead; 1736fb529e62SFlorian Westphal } 1737fb529e62SFlorian Westphal 1738d9ca1de8SPaolo Abeni /* we do not bound vs wspace, to allow a single packet. 1739d9ca1de8SPaolo Abeni * memory accounting will prevent execessive memory usage 1740d9ca1de8SPaolo Abeni * anyway 1741d9ca1de8SPaolo Abeni */ 1742d9ca1de8SPaolo Abeni offset = dfrag->offset + dfrag->data_len; 1743d9ca1de8SPaolo Abeni psize = pfrag->size - offset; 1744d9ca1de8SPaolo Abeni psize = min_t(size_t, psize, msg_data_left(msg)); 1745724cfd2eSPaolo Abeni total_ts = psize + frag_truesize; 17468ce568edSPaolo Abeni 17476511882cSPaolo Abeni if (!sk_wmem_schedule(sk, total_ts)) 1748d9ca1de8SPaolo Abeni goto wait_for_memory; 1749d9ca1de8SPaolo Abeni 1750d9ca1de8SPaolo Abeni if (copy_page_from_iter(dfrag->page, offset, psize, 1751d9ca1de8SPaolo Abeni &msg->msg_iter) != psize) { 1752d9ca1de8SPaolo Abeni ret = -EFAULT; 1753d9ca1de8SPaolo Abeni goto out; 1754d9ca1de8SPaolo Abeni } 1755d9ca1de8SPaolo Abeni 1756d9ca1de8SPaolo Abeni /* data successfully copied into the write queue */ 17576511882cSPaolo Abeni sk->sk_forward_alloc -= total_ts; 1758d9ca1de8SPaolo Abeni copied += psize; 1759d9ca1de8SPaolo Abeni dfrag->data_len += psize; 1760d9ca1de8SPaolo Abeni frag_truesize += psize; 1761d9ca1de8SPaolo Abeni pfrag->offset += frag_truesize; 1762d9ca1de8SPaolo Abeni WRITE_ONCE(msk->write_seq, msk->write_seq + psize); 1763d9ca1de8SPaolo Abeni 1764d9ca1de8SPaolo Abeni /* charge data on mptcp pending queue to the msk socket 1765d9ca1de8SPaolo Abeni * Note: we charge such data both to sk and ssk 1766d9ca1de8SPaolo Abeni */ 1767d9ca1de8SPaolo Abeni sk_wmem_queued_add(sk, frag_truesize); 1768d9ca1de8SPaolo Abeni if (!dfrag_collapsed) { 1769d9ca1de8SPaolo Abeni get_page(dfrag->page); 1770d9ca1de8SPaolo Abeni list_add_tail(&dfrag->list, &msk->rtx_queue); 1771d9ca1de8SPaolo Abeni if (!msk->first_pending) 1772d9ca1de8SPaolo Abeni WRITE_ONCE(msk->first_pending, dfrag); 1773d9ca1de8SPaolo Abeni } 1774e4b61351SGeliang Tang pr_debug("msk=%p dfrag at seq=%llu len=%u sent=%u new=%d", msk, 1775d9ca1de8SPaolo Abeni dfrag->data_seq, dfrag->data_len, dfrag->already_sent, 1776d9ca1de8SPaolo Abeni !dfrag_collapsed); 1777d9ca1de8SPaolo Abeni 1778d9ca1de8SPaolo Abeni continue; 1779d9ca1de8SPaolo Abeni 1780d9ca1de8SPaolo Abeni wait_for_memory: 17815cf92bbaSPaolo Abeni mptcp_set_nospace(sk); 1782c2e6048fSPaolo Abeni __mptcp_push_pending(sk, msg->msg_flags); 1783f296234cSPeter Krystad ret = sk_stream_wait_memory(sk, &timeo); 1784f296234cSPeter Krystad if (ret) 1785f296234cSPeter Krystad goto out; 1786cec37a6eSPeter Krystad } 1787cec37a6eSPeter Krystad 178813e16037SPaolo Abeni if (copied) 1789c2e6048fSPaolo Abeni __mptcp_push_pending(sk, msg->msg_flags); 1790da51aef5SPaolo Abeni 17911954b860SMat Martineau out: 1792cec37a6eSPeter Krystad release_sock(sk); 17938555c6bfSPaolo Abeni return copied ? : ret; 1794f870fa0bSMat Martineau } 1795f870fa0bSMat Martineau 17966771bfd9SFlorian Westphal static int __mptcp_recvmsg_mskq(struct mptcp_sock *msk, 17976771bfd9SFlorian Westphal struct msghdr *msg, 1798b7f653b2SFlorian Westphal size_t len, int flags, 1799b7f653b2SFlorian Westphal struct scm_timestamping_internal *tss, 1800b7f653b2SFlorian Westphal int *cmsg_flags) 18016771bfd9SFlorian Westphal { 1802ca4fb892SYonglong Li struct sk_buff *skb, *tmp; 18036771bfd9SFlorian Westphal int copied = 0; 18046771bfd9SFlorian Westphal 1805ca4fb892SYonglong Li skb_queue_walk_safe(&msk->receive_queue, skb, tmp) { 18066771bfd9SFlorian Westphal u32 offset = MPTCP_SKB_CB(skb)->offset; 18076771bfd9SFlorian Westphal u32 data_len = skb->len - offset; 18086771bfd9SFlorian Westphal u32 count = min_t(size_t, len - copied, data_len); 18096771bfd9SFlorian Westphal int err; 18106771bfd9SFlorian Westphal 1811d976092cSPaolo Abeni if (!(flags & MSG_TRUNC)) { 18126771bfd9SFlorian Westphal err = skb_copy_datagram_msg(skb, offset, msg, count); 18136771bfd9SFlorian Westphal if (unlikely(err < 0)) { 18146771bfd9SFlorian Westphal if (!copied) 18156771bfd9SFlorian Westphal return err; 18166771bfd9SFlorian Westphal break; 18176771bfd9SFlorian Westphal } 1818d976092cSPaolo Abeni } 18196771bfd9SFlorian Westphal 1820b7f653b2SFlorian Westphal if (MPTCP_SKB_CB(skb)->has_rxtstamp) { 1821b7f653b2SFlorian Westphal tcp_update_recv_tstamps(skb, tss); 1822b7f653b2SFlorian Westphal *cmsg_flags |= MPTCP_CMSG_TS; 1823b7f653b2SFlorian Westphal } 1824b7f653b2SFlorian Westphal 18256771bfd9SFlorian Westphal copied += count; 18266771bfd9SFlorian Westphal 18276771bfd9SFlorian Westphal if (count < data_len) { 18282c9e7765SFlorian Westphal if (!(flags & MSG_PEEK)) { 18296771bfd9SFlorian Westphal MPTCP_SKB_CB(skb)->offset += count; 18302c9e7765SFlorian Westphal MPTCP_SKB_CB(skb)->map_seq += count; 18312c9e7765SFlorian Westphal } 18326771bfd9SFlorian Westphal break; 18336771bfd9SFlorian Westphal } 18346771bfd9SFlorian Westphal 1835ca4fb892SYonglong Li if (!(flags & MSG_PEEK)) { 183687952603SPaolo Abeni /* we will bulk release the skb memory later */ 183787952603SPaolo Abeni skb->destructor = NULL; 1838ce599c51SPaolo Abeni WRITE_ONCE(msk->rmem_released, msk->rmem_released + skb->truesize); 183987952603SPaolo Abeni __skb_unlink(skb, &msk->receive_queue); 18406771bfd9SFlorian Westphal __kfree_skb(skb); 1841ca4fb892SYonglong Li } 18426771bfd9SFlorian Westphal 18436771bfd9SFlorian Westphal if (copied >= len) 18446771bfd9SFlorian Westphal break; 18456771bfd9SFlorian Westphal } 18466771bfd9SFlorian Westphal 18476771bfd9SFlorian Westphal return copied; 18486771bfd9SFlorian Westphal } 18496771bfd9SFlorian Westphal 1850a6b118feSFlorian Westphal /* receive buffer autotuning. See tcp_rcv_space_adjust for more information. 1851a6b118feSFlorian Westphal * 1852a6b118feSFlorian Westphal * Only difference: Use highest rtt estimate of the subflows in use. 1853a6b118feSFlorian Westphal */ 1854a6b118feSFlorian Westphal static void mptcp_rcv_space_adjust(struct mptcp_sock *msk, int copied) 1855a6b118feSFlorian Westphal { 1856a6b118feSFlorian Westphal struct mptcp_subflow_context *subflow; 1857a6b118feSFlorian Westphal struct sock *sk = (struct sock *)msk; 1858a6b118feSFlorian Westphal u32 time, advmss = 1; 1859a6b118feSFlorian Westphal u64 rtt_us, mstamp; 1860a6b118feSFlorian Westphal 1861a6b118feSFlorian Westphal sock_owned_by_me(sk); 1862a6b118feSFlorian Westphal 1863a6b118feSFlorian Westphal if (copied <= 0) 1864a6b118feSFlorian Westphal return; 1865a6b118feSFlorian Westphal 1866a6b118feSFlorian Westphal msk->rcvq_space.copied += copied; 1867a6b118feSFlorian Westphal 1868a6b118feSFlorian Westphal mstamp = div_u64(tcp_clock_ns(), NSEC_PER_USEC); 1869a6b118feSFlorian Westphal time = tcp_stamp_us_delta(mstamp, msk->rcvq_space.time); 1870a6b118feSFlorian Westphal 1871a6b118feSFlorian Westphal rtt_us = msk->rcvq_space.rtt_us; 1872a6b118feSFlorian Westphal if (rtt_us && time < (rtt_us >> 3)) 1873a6b118feSFlorian Westphal return; 1874a6b118feSFlorian Westphal 1875a6b118feSFlorian Westphal rtt_us = 0; 1876a6b118feSFlorian Westphal mptcp_for_each_subflow(msk, subflow) { 1877a6b118feSFlorian Westphal const struct tcp_sock *tp; 1878a6b118feSFlorian Westphal u64 sf_rtt_us; 1879a6b118feSFlorian Westphal u32 sf_advmss; 1880a6b118feSFlorian Westphal 1881a6b118feSFlorian Westphal tp = tcp_sk(mptcp_subflow_tcp_sock(subflow)); 1882a6b118feSFlorian Westphal 1883a6b118feSFlorian Westphal sf_rtt_us = READ_ONCE(tp->rcv_rtt_est.rtt_us); 1884a6b118feSFlorian Westphal sf_advmss = READ_ONCE(tp->advmss); 1885a6b118feSFlorian Westphal 1886a6b118feSFlorian Westphal rtt_us = max(sf_rtt_us, rtt_us); 1887a6b118feSFlorian Westphal advmss = max(sf_advmss, advmss); 1888a6b118feSFlorian Westphal } 1889a6b118feSFlorian Westphal 1890a6b118feSFlorian Westphal msk->rcvq_space.rtt_us = rtt_us; 1891a6b118feSFlorian Westphal if (time < (rtt_us >> 3) || rtt_us == 0) 1892a6b118feSFlorian Westphal return; 1893a6b118feSFlorian Westphal 1894a6b118feSFlorian Westphal if (msk->rcvq_space.copied <= msk->rcvq_space.space) 1895a6b118feSFlorian Westphal goto new_measure; 1896a6b118feSFlorian Westphal 1897a6b118feSFlorian Westphal if (sock_net(sk)->ipv4.sysctl_tcp_moderate_rcvbuf && 1898a6b118feSFlorian Westphal !(sk->sk_userlocks & SOCK_RCVBUF_LOCK)) { 1899a6b118feSFlorian Westphal int rcvmem, rcvbuf; 1900a6b118feSFlorian Westphal u64 rcvwin, grow; 1901a6b118feSFlorian Westphal 1902a6b118feSFlorian Westphal rcvwin = ((u64)msk->rcvq_space.copied << 1) + 16 * advmss; 1903a6b118feSFlorian Westphal 1904a6b118feSFlorian Westphal grow = rcvwin * (msk->rcvq_space.copied - msk->rcvq_space.space); 1905a6b118feSFlorian Westphal 1906a6b118feSFlorian Westphal do_div(grow, msk->rcvq_space.space); 1907a6b118feSFlorian Westphal rcvwin += (grow << 1); 1908a6b118feSFlorian Westphal 1909a6b118feSFlorian Westphal rcvmem = SKB_TRUESIZE(advmss + MAX_TCP_HEADER); 1910a6b118feSFlorian Westphal while (tcp_win_from_space(sk, rcvmem) < advmss) 1911a6b118feSFlorian Westphal rcvmem += 128; 1912a6b118feSFlorian Westphal 1913a6b118feSFlorian Westphal do_div(rcvwin, advmss); 1914a6b118feSFlorian Westphal rcvbuf = min_t(u64, rcvwin * rcvmem, 1915a6b118feSFlorian Westphal sock_net(sk)->ipv4.sysctl_tcp_rmem[2]); 1916a6b118feSFlorian Westphal 1917a6b118feSFlorian Westphal if (rcvbuf > sk->sk_rcvbuf) { 1918a6b118feSFlorian Westphal u32 window_clamp; 1919a6b118feSFlorian Westphal 1920a6b118feSFlorian Westphal window_clamp = tcp_win_from_space(sk, rcvbuf); 1921a6b118feSFlorian Westphal WRITE_ONCE(sk->sk_rcvbuf, rcvbuf); 1922a6b118feSFlorian Westphal 1923a6b118feSFlorian Westphal /* Make subflows follow along. If we do not do this, we 1924a6b118feSFlorian Westphal * get drops at subflow level if skbs can't be moved to 1925a6b118feSFlorian Westphal * the mptcp rx queue fast enough (announced rcv_win can 1926a6b118feSFlorian Westphal * exceed ssk->sk_rcvbuf). 1927a6b118feSFlorian Westphal */ 1928a6b118feSFlorian Westphal mptcp_for_each_subflow(msk, subflow) { 1929a6b118feSFlorian Westphal struct sock *ssk; 1930c76c6956SPaolo Abeni bool slow; 1931a6b118feSFlorian Westphal 1932a6b118feSFlorian Westphal ssk = mptcp_subflow_tcp_sock(subflow); 1933c76c6956SPaolo Abeni slow = lock_sock_fast(ssk); 1934a6b118feSFlorian Westphal WRITE_ONCE(ssk->sk_rcvbuf, rcvbuf); 1935a6b118feSFlorian Westphal tcp_sk(ssk)->window_clamp = window_clamp; 1936c76c6956SPaolo Abeni tcp_cleanup_rbuf(ssk, 1); 1937c76c6956SPaolo Abeni unlock_sock_fast(ssk, slow); 1938a6b118feSFlorian Westphal } 1939a6b118feSFlorian Westphal } 1940a6b118feSFlorian Westphal } 1941a6b118feSFlorian Westphal 1942a6b118feSFlorian Westphal msk->rcvq_space.space = msk->rcvq_space.copied; 1943a6b118feSFlorian Westphal new_measure: 1944a6b118feSFlorian Westphal msk->rcvq_space.copied = 0; 1945a6b118feSFlorian Westphal msk->rcvq_space.time = mstamp; 1946a6b118feSFlorian Westphal } 1947a6b118feSFlorian Westphal 194887952603SPaolo Abeni static void __mptcp_update_rmem(struct sock *sk) 194987952603SPaolo Abeni { 195087952603SPaolo Abeni struct mptcp_sock *msk = mptcp_sk(sk); 195187952603SPaolo Abeni 195287952603SPaolo Abeni if (!msk->rmem_released) 195387952603SPaolo Abeni return; 195487952603SPaolo Abeni 195587952603SPaolo Abeni atomic_sub(msk->rmem_released, &sk->sk_rmem_alloc); 19566511882cSPaolo Abeni mptcp_rmem_uncharge(sk, msk->rmem_released); 1957ce599c51SPaolo Abeni WRITE_ONCE(msk->rmem_released, 0); 195887952603SPaolo Abeni } 195987952603SPaolo Abeni 196087952603SPaolo Abeni static void __mptcp_splice_receive_queue(struct sock *sk) 196187952603SPaolo Abeni { 196287952603SPaolo Abeni struct mptcp_sock *msk = mptcp_sk(sk); 196387952603SPaolo Abeni 196487952603SPaolo Abeni skb_queue_splice_tail_init(&sk->sk_receive_queue, &msk->receive_queue); 196587952603SPaolo Abeni } 196687952603SPaolo Abeni 1967e3859603SPaolo Abeni static bool __mptcp_move_skbs(struct mptcp_sock *msk) 19686771bfd9SFlorian Westphal { 196987952603SPaolo Abeni struct sock *sk = (struct sock *)msk; 19706771bfd9SFlorian Westphal unsigned int moved = 0; 197187952603SPaolo Abeni bool ret, done; 1972d5f49190SPaolo Abeni 19736771bfd9SFlorian Westphal do { 19746771bfd9SFlorian Westphal struct sock *ssk = mptcp_subflow_recv_lookup(msk); 197565f49fe7SFlorian Westphal bool slowpath; 19766771bfd9SFlorian Westphal 197787952603SPaolo Abeni /* we can have data pending in the subflows only if the msk 197887952603SPaolo Abeni * receive buffer was full at subflow_data_ready() time, 197987952603SPaolo Abeni * that is an unlikely slow path. 198087952603SPaolo Abeni */ 198187952603SPaolo Abeni if (likely(!ssk)) 19826771bfd9SFlorian Westphal break; 19836771bfd9SFlorian Westphal 198465f49fe7SFlorian Westphal slowpath = lock_sock_fast(ssk); 198587952603SPaolo Abeni mptcp_data_lock(sk); 1986e3859603SPaolo Abeni __mptcp_update_rmem(sk); 19876771bfd9SFlorian Westphal done = __mptcp_move_skbs_from_subflow(msk, ssk, &moved); 198887952603SPaolo Abeni mptcp_data_unlock(sk); 1989499ada50SPaolo Abeni 1990499ada50SPaolo Abeni if (unlikely(ssk->sk_err)) 1991499ada50SPaolo Abeni __mptcp_error_report(sk); 199265f49fe7SFlorian Westphal unlock_sock_fast(ssk, slowpath); 19936771bfd9SFlorian Westphal } while (!done); 19946771bfd9SFlorian Westphal 199587952603SPaolo Abeni /* acquire the data lock only if some input data is pending */ 199687952603SPaolo Abeni ret = moved > 0; 199787952603SPaolo Abeni if (!RB_EMPTY_ROOT(&msk->out_of_order_queue) || 199887952603SPaolo Abeni !skb_queue_empty_lockless(&sk->sk_receive_queue)) { 199987952603SPaolo Abeni mptcp_data_lock(sk); 200087952603SPaolo Abeni __mptcp_update_rmem(sk); 200187952603SPaolo Abeni ret |= __mptcp_ofo_queue(msk); 200287952603SPaolo Abeni __mptcp_splice_receive_queue(sk); 200387952603SPaolo Abeni mptcp_data_unlock(sk); 2004ab174ad8SPaolo Abeni } 200587952603SPaolo Abeni if (ret) 200687952603SPaolo Abeni mptcp_check_data_fin((struct sock *)msk); 200787952603SPaolo Abeni return !skb_queue_empty(&msk->receive_queue); 20086771bfd9SFlorian Westphal } 20096771bfd9SFlorian Westphal 20102c9e7765SFlorian Westphal static unsigned int mptcp_inq_hint(const struct sock *sk) 20112c9e7765SFlorian Westphal { 20122c9e7765SFlorian Westphal const struct mptcp_sock *msk = mptcp_sk(sk); 20132c9e7765SFlorian Westphal const struct sk_buff *skb; 20142c9e7765SFlorian Westphal 20152c9e7765SFlorian Westphal skb = skb_peek(&msk->receive_queue); 20162c9e7765SFlorian Westphal if (skb) { 20172c9e7765SFlorian Westphal u64 hint_val = msk->ack_seq - MPTCP_SKB_CB(skb)->map_seq; 20182c9e7765SFlorian Westphal 20192c9e7765SFlorian Westphal if (hint_val >= INT_MAX) 20202c9e7765SFlorian Westphal return INT_MAX; 20212c9e7765SFlorian Westphal 20222c9e7765SFlorian Westphal return (unsigned int)hint_val; 20232c9e7765SFlorian Westphal } 20242c9e7765SFlorian Westphal 20252c9e7765SFlorian Westphal if (sk->sk_state == TCP_CLOSE || (sk->sk_shutdown & RCV_SHUTDOWN)) 20262c9e7765SFlorian Westphal return 1; 20272c9e7765SFlorian Westphal 20282c9e7765SFlorian Westphal return 0; 20292c9e7765SFlorian Westphal } 20302c9e7765SFlorian Westphal 2031f870fa0bSMat Martineau static int mptcp_recvmsg(struct sock *sk, struct msghdr *msg, size_t len, 2032ec095263SOliver Hartkopp int flags, int *addr_len) 2033f870fa0bSMat Martineau { 2034f870fa0bSMat Martineau struct mptcp_sock *msk = mptcp_sk(sk); 2035b7f653b2SFlorian Westphal struct scm_timestamping_internal tss; 2036b7f653b2SFlorian Westphal int copied = 0, cmsg_flags = 0; 20377a6a6cbcSPaolo Abeni int target; 20387a6a6cbcSPaolo Abeni long timeo; 2039f870fa0bSMat Martineau 2040cb9d80f4SPaolo Abeni /* MSG_ERRQUEUE is really a no-op till we support IP_RECVERR */ 2041cb9d80f4SPaolo Abeni if (unlikely(flags & MSG_ERRQUEUE)) 2042cb9d80f4SPaolo Abeni return inet_recv_error(sk, msg, len, addr_len); 2043cb9d80f4SPaolo Abeni 20446511882cSPaolo Abeni lock_sock(sk); 2045fd897679SPaolo Abeni if (unlikely(sk->sk_state == TCP_LISTEN)) { 2046fd897679SPaolo Abeni copied = -ENOTCONN; 2047fd897679SPaolo Abeni goto out_err; 2048fd897679SPaolo Abeni } 2049fd897679SPaolo Abeni 2050ec095263SOliver Hartkopp timeo = sock_rcvtimeo(sk, flags & MSG_DONTWAIT); 20517a6a6cbcSPaolo Abeni 20527a6a6cbcSPaolo Abeni len = min_t(size_t, len, INT_MAX); 20537a6a6cbcSPaolo Abeni target = sock_rcvlowat(sk, flags & MSG_WAITALL, len); 20547a6a6cbcSPaolo Abeni 20552c9e7765SFlorian Westphal if (unlikely(msk->recvmsg_inq)) 20562c9e7765SFlorian Westphal cmsg_flags = MPTCP_CMSG_INQ; 20572c9e7765SFlorian Westphal 205805e3eceaSEric Dumazet while (copied < len) { 2059e3859603SPaolo Abeni int bytes_read; 20607a6a6cbcSPaolo Abeni 2061b7f653b2SFlorian Westphal bytes_read = __mptcp_recvmsg_mskq(msk, msg, len - copied, flags, &tss, &cmsg_flags); 20626771bfd9SFlorian Westphal if (unlikely(bytes_read < 0)) { 20637a6a6cbcSPaolo Abeni if (!copied) 20647a6a6cbcSPaolo Abeni copied = bytes_read; 20656771bfd9SFlorian Westphal goto out_err; 2066cec37a6eSPeter Krystad } 2067cec37a6eSPeter Krystad 20687a6a6cbcSPaolo Abeni copied += bytes_read; 20697a6a6cbcSPaolo Abeni 2070ea4ca586SPaolo Abeni /* be sure to advertise window change */ 2071fd897679SPaolo Abeni mptcp_cleanup_rbuf(msk); 2072ea4ca586SPaolo Abeni 2073e3859603SPaolo Abeni if (skb_queue_empty(&msk->receive_queue) && __mptcp_move_skbs(msk)) 2074e3859603SPaolo Abeni continue; 2075e3859603SPaolo Abeni 20767a6a6cbcSPaolo Abeni /* only the master socket status is relevant here. The exit 20777a6a6cbcSPaolo Abeni * conditions mirror closely tcp_recvmsg() 20787a6a6cbcSPaolo Abeni */ 20797a6a6cbcSPaolo Abeni if (copied >= target) 20807a6a6cbcSPaolo Abeni break; 20817a6a6cbcSPaolo Abeni 20827a6a6cbcSPaolo Abeni if (copied) { 20837a6a6cbcSPaolo Abeni if (sk->sk_err || 20847a6a6cbcSPaolo Abeni sk->sk_state == TCP_CLOSE || 20857a6a6cbcSPaolo Abeni (sk->sk_shutdown & RCV_SHUTDOWN) || 20867a6a6cbcSPaolo Abeni !timeo || 20877a6a6cbcSPaolo Abeni signal_pending(current)) 20887a6a6cbcSPaolo Abeni break; 20897a6a6cbcSPaolo Abeni } else { 20907a6a6cbcSPaolo Abeni if (sk->sk_err) { 20917a6a6cbcSPaolo Abeni copied = sock_error(sk); 20927a6a6cbcSPaolo Abeni break; 20937a6a6cbcSPaolo Abeni } 20947a6a6cbcSPaolo Abeni 20955969856aSPaolo Abeni if (test_and_clear_bit(MPTCP_WORK_EOF, &msk->flags)) 20965969856aSPaolo Abeni mptcp_check_for_eof(msk); 20975969856aSPaolo Abeni 209887952603SPaolo Abeni if (sk->sk_shutdown & RCV_SHUTDOWN) { 209987952603SPaolo Abeni /* race breaker: the shutdown could be after the 210087952603SPaolo Abeni * previous receive queue check 210187952603SPaolo Abeni */ 2102e3859603SPaolo Abeni if (__mptcp_move_skbs(msk)) 210387952603SPaolo Abeni continue; 21047a6a6cbcSPaolo Abeni break; 210587952603SPaolo Abeni } 21067a6a6cbcSPaolo Abeni 21077a6a6cbcSPaolo Abeni if (sk->sk_state == TCP_CLOSE) { 21087a6a6cbcSPaolo Abeni copied = -ENOTCONN; 21097a6a6cbcSPaolo Abeni break; 21107a6a6cbcSPaolo Abeni } 21117a6a6cbcSPaolo Abeni 21127a6a6cbcSPaolo Abeni if (!timeo) { 21137a6a6cbcSPaolo Abeni copied = -EAGAIN; 21147a6a6cbcSPaolo Abeni break; 21157a6a6cbcSPaolo Abeni } 21167a6a6cbcSPaolo Abeni 21177a6a6cbcSPaolo Abeni if (signal_pending(current)) { 21187a6a6cbcSPaolo Abeni copied = sock_intr_errno(timeo); 21197a6a6cbcSPaolo Abeni break; 21207a6a6cbcSPaolo Abeni } 21217a6a6cbcSPaolo Abeni } 21227a6a6cbcSPaolo Abeni 21237a6a6cbcSPaolo Abeni pr_debug("block timeout %ld", timeo); 2124612f71d7SPaolo Abeni sk_wait_data(sk, &timeo, NULL); 21257a6a6cbcSPaolo Abeni } 21263c90e377SPaolo Abeni 21276771bfd9SFlorian Westphal out_err: 2128b7f653b2SFlorian Westphal if (cmsg_flags && copied >= 0) { 2129b7f653b2SFlorian Westphal if (cmsg_flags & MPTCP_CMSG_TS) 2130b7f653b2SFlorian Westphal tcp_recv_timestamp(msg, sk, &tss); 21312c9e7765SFlorian Westphal 21322c9e7765SFlorian Westphal if (cmsg_flags & MPTCP_CMSG_INQ) { 21332c9e7765SFlorian Westphal unsigned int inq = mptcp_inq_hint(sk); 21342c9e7765SFlorian Westphal 21352c9e7765SFlorian Westphal put_cmsg(msg, SOL_TCP, TCP_CM_INQ, sizeof(inq), &inq); 21362c9e7765SFlorian Westphal } 2137b7f653b2SFlorian Westphal } 2138b7f653b2SFlorian Westphal 2139612f71d7SPaolo Abeni pr_debug("msk=%p rx queue empty=%d:%d copied=%d", 2140612f71d7SPaolo Abeni msk, skb_queue_empty_lockless(&sk->sk_receive_queue), 2141612f71d7SPaolo Abeni skb_queue_empty(&msk->receive_queue), copied); 2142ca4fb892SYonglong Li if (!(flags & MSG_PEEK)) 2143a6b118feSFlorian Westphal mptcp_rcv_space_adjust(msk, copied); 2144a6b118feSFlorian Westphal 2145cec37a6eSPeter Krystad release_sock(sk); 2146cec37a6eSPeter Krystad return copied; 2147cec37a6eSPeter Krystad } 2148cec37a6eSPeter Krystad 2149b51f9b80SPaolo Abeni static void mptcp_retransmit_timer(struct timer_list *t) 2150b51f9b80SPaolo Abeni { 2151b51f9b80SPaolo Abeni struct inet_connection_sock *icsk = from_timer(icsk, t, 2152b51f9b80SPaolo Abeni icsk_retransmit_timer); 2153b51f9b80SPaolo Abeni struct sock *sk = &icsk->icsk_inet.sk; 21542d6f5a2bSPaolo Abeni struct mptcp_sock *msk = mptcp_sk(sk); 2155b51f9b80SPaolo Abeni 2156b51f9b80SPaolo Abeni bh_lock_sock(sk); 2157b51f9b80SPaolo Abeni if (!sock_owned_by_user(sk)) { 21582d6f5a2bSPaolo Abeni /* we need a process context to retransmit */ 21592d6f5a2bSPaolo Abeni if (!test_and_set_bit(MPTCP_WORK_RTX, &msk->flags)) 21602d6f5a2bSPaolo Abeni mptcp_schedule_work(sk); 2161b51f9b80SPaolo Abeni } else { 2162b51f9b80SPaolo Abeni /* delegate our work to tcp_release_cb() */ 2163e9d09bacSPaolo Abeni __set_bit(MPTCP_RETRANSMIT, &msk->cb_flags); 2164b51f9b80SPaolo Abeni } 2165b51f9b80SPaolo Abeni bh_unlock_sock(sk); 2166b51f9b80SPaolo Abeni sock_put(sk); 2167b51f9b80SPaolo Abeni } 2168b51f9b80SPaolo Abeni 2169e16163b6SPaolo Abeni static void mptcp_timeout_timer(struct timer_list *t) 2170e16163b6SPaolo Abeni { 2171e16163b6SPaolo Abeni struct sock *sk = from_timer(sk, t, sk_timer); 2172e16163b6SPaolo Abeni 2173e16163b6SPaolo Abeni mptcp_schedule_work(sk); 2174b6d69fc8SFlorian Westphal sock_put(sk); 2175e16163b6SPaolo Abeni } 2176e16163b6SPaolo Abeni 21773b1d6210SPaolo Abeni /* Find an idle subflow. Return NULL if there is unacked data at tcp 21783b1d6210SPaolo Abeni * level. 21793b1d6210SPaolo Abeni * 21803b1d6210SPaolo Abeni * A backup subflow is returned only if that is the only kind available. 21813b1d6210SPaolo Abeni */ 2182ff5a0b42SPaolo Abeni static struct sock *mptcp_subflow_get_retrans(struct mptcp_sock *msk) 21833b1d6210SPaolo Abeni { 218471b7dec2SPaolo Abeni struct sock *backup = NULL, *pick = NULL; 21853b1d6210SPaolo Abeni struct mptcp_subflow_context *subflow; 218671b7dec2SPaolo Abeni int min_stale_count = INT_MAX; 21873b1d6210SPaolo Abeni 21883b1d6210SPaolo Abeni sock_owned_by_me((const struct sock *)msk); 21893b1d6210SPaolo Abeni 2190d5f49190SPaolo Abeni if (__mptcp_check_fallback(msk)) 2191d9ca1de8SPaolo Abeni return NULL; 2192d5f49190SPaolo Abeni 21933b1d6210SPaolo Abeni mptcp_for_each_subflow(msk, subflow) { 21943b1d6210SPaolo Abeni struct sock *ssk = mptcp_subflow_tcp_sock(subflow); 21953b1d6210SPaolo Abeni 2196ff5a0b42SPaolo Abeni if (!__mptcp_subflow_active(subflow)) 2197d5f49190SPaolo Abeni continue; 2198d5f49190SPaolo Abeni 219971b7dec2SPaolo Abeni /* still data outstanding at TCP level? skip this */ 220071b7dec2SPaolo Abeni if (!tcp_rtx_and_write_queues_empty(ssk)) { 220171b7dec2SPaolo Abeni mptcp_pm_subflow_chk_stale(msk, ssk); 220271b7dec2SPaolo Abeni min_stale_count = min_t(int, min_stale_count, subflow->stale_count); 2203860975c6SFlorian Westphal continue; 2204860975c6SFlorian Westphal } 22053b1d6210SPaolo Abeni 22063b1d6210SPaolo Abeni if (subflow->backup) { 22073b1d6210SPaolo Abeni if (!backup) 22083b1d6210SPaolo Abeni backup = ssk; 22093b1d6210SPaolo Abeni continue; 22103b1d6210SPaolo Abeni } 22113b1d6210SPaolo Abeni 221271b7dec2SPaolo Abeni if (!pick) 221371b7dec2SPaolo Abeni pick = ssk; 22143b1d6210SPaolo Abeni } 22153b1d6210SPaolo Abeni 221671b7dec2SPaolo Abeni if (pick) 221771b7dec2SPaolo Abeni return pick; 221871b7dec2SPaolo Abeni 221971b7dec2SPaolo Abeni /* use backup only if there are no progresses anywhere */ 222071b7dec2SPaolo Abeni return min_stale_count > 1 ? backup : NULL; 22213b1d6210SPaolo Abeni } 22223b1d6210SPaolo Abeni 222317aee05dSFlorian Westphal static void mptcp_dispose_initial_subflow(struct mptcp_sock *msk) 222417aee05dSFlorian Westphal { 222517aee05dSFlorian Westphal if (msk->subflow) { 222617aee05dSFlorian Westphal iput(SOCK_INODE(msk->subflow)); 222717aee05dSFlorian Westphal msk->subflow = NULL; 222817aee05dSFlorian Westphal } 222917aee05dSFlorian Westphal } 223017aee05dSFlorian Westphal 22311e1d9d6fSPaolo Abeni bool __mptcp_retransmit_pending_data(struct sock *sk) 22321e1d9d6fSPaolo Abeni { 22331e1d9d6fSPaolo Abeni struct mptcp_data_frag *cur, *rtx_head; 22341e1d9d6fSPaolo Abeni struct mptcp_sock *msk = mptcp_sk(sk); 22351e1d9d6fSPaolo Abeni 22361e1d9d6fSPaolo Abeni if (__mptcp_check_fallback(mptcp_sk(sk))) 22371e1d9d6fSPaolo Abeni return false; 22381e1d9d6fSPaolo Abeni 22391e1d9d6fSPaolo Abeni if (tcp_rtx_and_write_queues_empty(sk)) 22401e1d9d6fSPaolo Abeni return false; 22411e1d9d6fSPaolo Abeni 22421e1d9d6fSPaolo Abeni /* the closing socket has some data untransmitted and/or unacked: 22431e1d9d6fSPaolo Abeni * some data in the mptcp rtx queue has not really xmitted yet. 22441e1d9d6fSPaolo Abeni * keep it simple and re-inject the whole mptcp level rtx queue 22451e1d9d6fSPaolo Abeni */ 22461e1d9d6fSPaolo Abeni mptcp_data_lock(sk); 22471e1d9d6fSPaolo Abeni __mptcp_clean_una_wakeup(sk); 22481e1d9d6fSPaolo Abeni rtx_head = mptcp_rtx_head(sk); 22491e1d9d6fSPaolo Abeni if (!rtx_head) { 22501e1d9d6fSPaolo Abeni mptcp_data_unlock(sk); 22511e1d9d6fSPaolo Abeni return false; 22521e1d9d6fSPaolo Abeni } 22531e1d9d6fSPaolo Abeni 22541e1d9d6fSPaolo Abeni msk->recovery_snd_nxt = msk->snd_nxt; 22551e1d9d6fSPaolo Abeni msk->recovery = true; 22561e1d9d6fSPaolo Abeni mptcp_data_unlock(sk); 22571e1d9d6fSPaolo Abeni 22581e1d9d6fSPaolo Abeni msk->first_pending = rtx_head; 22591e1d9d6fSPaolo Abeni msk->snd_burst = 0; 22601e1d9d6fSPaolo Abeni 22611e1d9d6fSPaolo Abeni /* be sure to clear the "sent status" on all re-injected fragments */ 22621e1d9d6fSPaolo Abeni list_for_each_entry(cur, &msk->rtx_queue, list) { 22631e1d9d6fSPaolo Abeni if (!cur->already_sent) 22641e1d9d6fSPaolo Abeni break; 22651e1d9d6fSPaolo Abeni cur->already_sent = 0; 22661e1d9d6fSPaolo Abeni } 22671e1d9d6fSPaolo Abeni 22681e1d9d6fSPaolo Abeni return true; 22691e1d9d6fSPaolo Abeni } 22701e1d9d6fSPaolo Abeni 2271b29fcfb5SPaolo Abeni /* flags for __mptcp_close_ssk() */ 2272b29fcfb5SPaolo Abeni #define MPTCP_CF_PUSH BIT(1) 2273b29fcfb5SPaolo Abeni #define MPTCP_CF_FASTCLOSE BIT(2) 2274b29fcfb5SPaolo Abeni 2275cec37a6eSPeter Krystad /* subflow sockets can be either outgoing (connect) or incoming 2276cec37a6eSPeter Krystad * (accept). 2277cec37a6eSPeter Krystad * 2278cec37a6eSPeter Krystad * Outgoing subflows use in-kernel sockets. 2279cec37a6eSPeter Krystad * Incoming subflows do not have their own 'struct socket' allocated, 2280cec37a6eSPeter Krystad * so we need to use tcp_close() after detaching them from the mptcp 2281cec37a6eSPeter Krystad * parent socket. 2282cec37a6eSPeter Krystad */ 2283a141e02eSFlorian Westphal static void __mptcp_close_ssk(struct sock *sk, struct sock *ssk, 2284b29fcfb5SPaolo Abeni struct mptcp_subflow_context *subflow, 2285b29fcfb5SPaolo Abeni unsigned int flags) 2286cec37a6eSPeter Krystad { 2287e0be4931SFlorian Westphal struct mptcp_sock *msk = mptcp_sk(sk); 2288b29fcfb5SPaolo Abeni bool need_push, dispose_it; 2289e0be4931SFlorian Westphal 2290b29fcfb5SPaolo Abeni dispose_it = !msk->subflow || ssk != msk->subflow->sk; 2291b29fcfb5SPaolo Abeni if (dispose_it) 2292cec37a6eSPeter Krystad list_del(&subflow->node); 2293cec37a6eSPeter Krystad 22943f8b2667SPaolo Abeni lock_sock_nested(ssk, SINGLE_DEPTH_NESTING); 2295e16163b6SPaolo Abeni 2296b29fcfb5SPaolo Abeni if (flags & MPTCP_CF_FASTCLOSE) 2297b29fcfb5SPaolo Abeni subflow->send_fastclose = 1; 2298b29fcfb5SPaolo Abeni 2299b29fcfb5SPaolo Abeni need_push = (flags & MPTCP_CF_PUSH) && __mptcp_retransmit_pending_data(sk); 2300b29fcfb5SPaolo Abeni if (!dispose_it) { 2301b29fcfb5SPaolo Abeni tcp_disconnect(ssk, 0); 2302b29fcfb5SPaolo Abeni msk->subflow->state = SS_UNCONNECTED; 2303b29fcfb5SPaolo Abeni mptcp_subflow_ctx_reset(subflow); 2304b29fcfb5SPaolo Abeni release_sock(ssk); 2305b29fcfb5SPaolo Abeni 2306b29fcfb5SPaolo Abeni goto out; 2307b29fcfb5SPaolo Abeni } 2308b29fcfb5SPaolo Abeni 2309e16163b6SPaolo Abeni /* if we are invoked by the msk cleanup code, the subflow is 2310e16163b6SPaolo Abeni * already orphaned 2311e16163b6SPaolo Abeni */ 2312866f26f2SPaolo Abeni if (ssk->sk_socket) 2313e16163b6SPaolo Abeni sock_orphan(ssk); 2314e16163b6SPaolo Abeni 2315d7b1bfd0SPaolo Abeni subflow->disposable = 1; 2316d7b1bfd0SPaolo Abeni 2317e16163b6SPaolo Abeni /* if ssk hit tcp_done(), tcp_cleanup_ulp() cleared the related ops 2318e16163b6SPaolo Abeni * the ssk has been already destroyed, we just need to release the 2319e16163b6SPaolo Abeni * reference owned by msk; 2320e16163b6SPaolo Abeni */ 2321e16163b6SPaolo Abeni if (!inet_csk(ssk)->icsk_ulp_ops) { 2322e16163b6SPaolo Abeni kfree_rcu(subflow, rcu); 2323e16163b6SPaolo Abeni } else { 2324d7b1bfd0SPaolo Abeni /* otherwise tcp will dispose of the ssk and subflow ctx */ 2325e16163b6SPaolo Abeni __tcp_close(ssk, 0); 2326e16163b6SPaolo Abeni 2327e16163b6SPaolo Abeni /* close acquired an extra ref */ 2328e16163b6SPaolo Abeni __sock_put(ssk); 2329e16163b6SPaolo Abeni } 2330e16163b6SPaolo Abeni release_sock(ssk); 2331e16163b6SPaolo Abeni 2332e16163b6SPaolo Abeni sock_put(ssk); 2333e0be4931SFlorian Westphal 2334c8fe62f0SFlorian Westphal if (ssk == msk->first) 2335c8fe62f0SFlorian Westphal msk->first = NULL; 2336c8fe62f0SFlorian Westphal 2337b29fcfb5SPaolo Abeni out: 2338b29fcfb5SPaolo Abeni if (ssk == msk->last_snd) 2339b29fcfb5SPaolo Abeni msk->last_snd = NULL; 23401e1d9d6fSPaolo Abeni 23411e1d9d6fSPaolo Abeni if (need_push) 23421e1d9d6fSPaolo Abeni __mptcp_push_pending(sk, 0); 2343f870fa0bSMat Martineau } 2344f870fa0bSMat Martineau 2345a141e02eSFlorian Westphal void mptcp_close_ssk(struct sock *sk, struct sock *ssk, 2346a141e02eSFlorian Westphal struct mptcp_subflow_context *subflow) 2347a141e02eSFlorian Westphal { 2348b911c97cSFlorian Westphal if (sk->sk_state == TCP_ESTABLISHED) 2349b911c97cSFlorian Westphal mptcp_event(MPTCP_EVENT_SUB_CLOSED, mptcp_sk(sk), ssk, GFP_KERNEL); 2350a88c9e49SPaolo Abeni 2351a88c9e49SPaolo Abeni /* subflow aborted before reaching the fully_established status 2352a88c9e49SPaolo Abeni * attempt the creation of the next subflow 2353a88c9e49SPaolo Abeni */ 2354a88c9e49SPaolo Abeni mptcp_pm_subflow_check_next(mptcp_sk(sk), ssk, subflow); 2355a88c9e49SPaolo Abeni 2356b29fcfb5SPaolo Abeni __mptcp_close_ssk(sk, ssk, subflow, MPTCP_CF_PUSH); 2357a141e02eSFlorian Westphal } 2358a141e02eSFlorian Westphal 2359dc24f8b4SPaolo Abeni static unsigned int mptcp_sync_mss(struct sock *sk, u32 pmtu) 2360dc24f8b4SPaolo Abeni { 2361dc24f8b4SPaolo Abeni return 0; 2362dc24f8b4SPaolo Abeni } 2363dc24f8b4SPaolo Abeni 23640e4f35d7SPaolo Abeni static void __mptcp_close_subflow(struct mptcp_sock *msk) 23650e4f35d7SPaolo Abeni { 23660e4f35d7SPaolo Abeni struct mptcp_subflow_context *subflow, *tmp; 23670e4f35d7SPaolo Abeni 23683abc05d9SFlorian Westphal might_sleep(); 23693abc05d9SFlorian Westphal 23700e4f35d7SPaolo Abeni list_for_each_entry_safe(subflow, tmp, &msk->conn_list, node) { 23710e4f35d7SPaolo Abeni struct sock *ssk = mptcp_subflow_tcp_sock(subflow); 23720e4f35d7SPaolo Abeni 23730e4f35d7SPaolo Abeni if (inet_sk_state_load(ssk) != TCP_CLOSE) 23740e4f35d7SPaolo Abeni continue; 23750e4f35d7SPaolo Abeni 237640947e13SFlorian Westphal /* 'subflow_data_ready' will re-sched once rx queue is empty */ 237740947e13SFlorian Westphal if (!skb_queue_empty_lockless(&ssk->sk_receive_queue)) 237840947e13SFlorian Westphal continue; 237940947e13SFlorian Westphal 2380a141e02eSFlorian Westphal mptcp_close_ssk((struct sock *)msk, ssk, subflow); 23810e4f35d7SPaolo Abeni } 23820e4f35d7SPaolo Abeni } 23830e4f35d7SPaolo Abeni 2384e16163b6SPaolo Abeni static bool mptcp_check_close_timeout(const struct sock *sk) 2385e16163b6SPaolo Abeni { 2386e16163b6SPaolo Abeni s32 delta = tcp_jiffies32 - inet_csk(sk)->icsk_mtup.probe_timestamp; 2387e16163b6SPaolo Abeni struct mptcp_subflow_context *subflow; 2388e16163b6SPaolo Abeni 2389e16163b6SPaolo Abeni if (delta >= TCP_TIMEWAIT_LEN) 2390e16163b6SPaolo Abeni return true; 2391e16163b6SPaolo Abeni 2392e16163b6SPaolo Abeni /* if all subflows are in closed status don't bother with additional 2393e16163b6SPaolo Abeni * timeout 2394e16163b6SPaolo Abeni */ 2395e16163b6SPaolo Abeni mptcp_for_each_subflow(mptcp_sk(sk), subflow) { 2396e16163b6SPaolo Abeni if (inet_sk_state_load(mptcp_subflow_tcp_sock(subflow)) != 2397e16163b6SPaolo Abeni TCP_CLOSE) 2398e16163b6SPaolo Abeni return false; 2399e16163b6SPaolo Abeni } 2400e16163b6SPaolo Abeni return true; 2401e16163b6SPaolo Abeni } 2402e16163b6SPaolo Abeni 240350c504a2SFlorian Westphal static void mptcp_check_fastclose(struct mptcp_sock *msk) 240450c504a2SFlorian Westphal { 240550c504a2SFlorian Westphal struct mptcp_subflow_context *subflow, *tmp; 240650c504a2SFlorian Westphal struct sock *sk = &msk->sk.icsk_inet.sk; 240750c504a2SFlorian Westphal 240850c504a2SFlorian Westphal if (likely(!READ_ONCE(msk->rcv_fastclose))) 240950c504a2SFlorian Westphal return; 241050c504a2SFlorian Westphal 241150c504a2SFlorian Westphal mptcp_token_destroy(msk); 241250c504a2SFlorian Westphal 241350c504a2SFlorian Westphal list_for_each_entry_safe(subflow, tmp, &msk->conn_list, node) { 241450c504a2SFlorian Westphal struct sock *tcp_sk = mptcp_subflow_tcp_sock(subflow); 241575e908c3SPaolo Abeni bool slow; 241650c504a2SFlorian Westphal 241775e908c3SPaolo Abeni slow = lock_sock_fast(tcp_sk); 241850c504a2SFlorian Westphal if (tcp_sk->sk_state != TCP_CLOSE) { 241950c504a2SFlorian Westphal tcp_send_active_reset(tcp_sk, GFP_ATOMIC); 242050c504a2SFlorian Westphal tcp_set_state(tcp_sk, TCP_CLOSE); 242150c504a2SFlorian Westphal } 242275e908c3SPaolo Abeni unlock_sock_fast(tcp_sk, slow); 242350c504a2SFlorian Westphal } 242450c504a2SFlorian Westphal 242550c504a2SFlorian Westphal inet_sk_state_store(sk, TCP_CLOSE); 242650c504a2SFlorian Westphal sk->sk_shutdown = SHUTDOWN_MASK; 242750c504a2SFlorian Westphal smp_mb__before_atomic(); /* SHUTDOWN must be visible first */ 242850c504a2SFlorian Westphal set_bit(MPTCP_WORK_CLOSE_SUBFLOW, &msk->flags); 242950c504a2SFlorian Westphal 243050c504a2SFlorian Westphal mptcp_close_wake_up(sk); 243150c504a2SFlorian Westphal } 243250c504a2SFlorian Westphal 24332948d0a1SPaolo Abeni static void __mptcp_retrans(struct sock *sk) 243480992017SPaolo Abeni { 24352948d0a1SPaolo Abeni struct mptcp_sock *msk = mptcp_sk(sk); 2436caf971dfSPaolo Abeni struct mptcp_sendmsg_info info = {}; 24373b1d6210SPaolo Abeni struct mptcp_data_frag *dfrag; 24383b1d6210SPaolo Abeni size_t copied = 0; 24392948d0a1SPaolo Abeni struct sock *ssk; 24402948d0a1SPaolo Abeni int ret; 24412948d0a1SPaolo Abeni 2442b5941f06SPaolo Abeni mptcp_clean_una_wakeup(sk); 24433241a9c0SFlorian Westphal 24443241a9c0SFlorian Westphal /* first check ssk: need to kick "stale" logic */ 24453241a9c0SFlorian Westphal ssk = mptcp_subflow_get_retrans(msk); 24462948d0a1SPaolo Abeni dfrag = mptcp_rtx_head(sk); 24476477dd39SMat Martineau if (!dfrag) { 24486477dd39SMat Martineau if (mptcp_data_fin_enabled(msk)) { 24496477dd39SMat Martineau struct inet_connection_sock *icsk = inet_csk(sk); 24506477dd39SMat Martineau 24516477dd39SMat Martineau icsk->icsk_retransmits++; 24526477dd39SMat Martineau mptcp_set_datafin_timeout(sk); 24536477dd39SMat Martineau mptcp_send_ack(msk); 24546477dd39SMat Martineau 24556477dd39SMat Martineau goto reset_timer; 24566477dd39SMat Martineau } 24576477dd39SMat Martineau 24583241a9c0SFlorian Westphal if (!mptcp_send_head(sk)) 24592948d0a1SPaolo Abeni return; 24603241a9c0SFlorian Westphal 24613241a9c0SFlorian Westphal goto reset_timer; 24626477dd39SMat Martineau } 24632948d0a1SPaolo Abeni 24642948d0a1SPaolo Abeni if (!ssk) 24652948d0a1SPaolo Abeni goto reset_timer; 24662948d0a1SPaolo Abeni 24672948d0a1SPaolo Abeni lock_sock(ssk); 24682948d0a1SPaolo Abeni 24692948d0a1SPaolo Abeni /* limit retransmission to the bytes already sent on some subflows */ 24702948d0a1SPaolo Abeni info.sent = 0; 24714e14867dSPaolo Abeni info.limit = READ_ONCE(msk->csum_enabled) ? dfrag->data_len : dfrag->already_sent; 24724e14867dSPaolo Abeni while (info.sent < info.limit) { 24732948d0a1SPaolo Abeni ret = mptcp_sendmsg_frag(sk, ssk, dfrag, &info); 24742948d0a1SPaolo Abeni if (ret <= 0) 24752948d0a1SPaolo Abeni break; 24762948d0a1SPaolo Abeni 24772948d0a1SPaolo Abeni MPTCP_INC_STATS(sock_net(sk), MPTCP_MIB_RETRANSSEGS); 24782948d0a1SPaolo Abeni copied += ret; 24792948d0a1SPaolo Abeni info.sent += ret; 24802948d0a1SPaolo Abeni } 24814e14867dSPaolo Abeni if (copied) { 24824e14867dSPaolo Abeni dfrag->already_sent = max(dfrag->already_sent, info.sent); 24832948d0a1SPaolo Abeni tcp_push(ssk, 0, info.mss_now, tcp_sk(ssk)->nonagle, 24842948d0a1SPaolo Abeni info.size_goal); 24850530020aSGeliang Tang WRITE_ONCE(msk->allow_infinite_fallback, false); 24864e14867dSPaolo Abeni } 24872948d0a1SPaolo Abeni 24882948d0a1SPaolo Abeni release_sock(ssk); 24892948d0a1SPaolo Abeni 24902948d0a1SPaolo Abeni reset_timer: 24913241a9c0SFlorian Westphal mptcp_check_and_set_pending(sk); 24923241a9c0SFlorian Westphal 24932948d0a1SPaolo Abeni if (!mptcp_timer_pending(sk)) 24942948d0a1SPaolo Abeni mptcp_reset_timer(sk); 24952948d0a1SPaolo Abeni } 24962948d0a1SPaolo Abeni 24972948d0a1SPaolo Abeni static void mptcp_worker(struct work_struct *work) 24982948d0a1SPaolo Abeni { 24992948d0a1SPaolo Abeni struct mptcp_sock *msk = container_of(work, struct mptcp_sock, work); 25002948d0a1SPaolo Abeni struct sock *sk = &msk->sk.icsk_inet.sk; 25012948d0a1SPaolo Abeni int state; 250280992017SPaolo Abeni 250380992017SPaolo Abeni lock_sock(sk); 2504e16163b6SPaolo Abeni state = sk->sk_state; 2505e16163b6SPaolo Abeni if (unlikely(state == TCP_CLOSE)) 2506e16163b6SPaolo Abeni goto unlock; 2507e16163b6SPaolo Abeni 250843b54c6eSMat Martineau mptcp_check_data_fin_ack(sk); 250950c504a2SFlorian Westphal 251050c504a2SFlorian Westphal mptcp_check_fastclose(msk); 251150c504a2SFlorian Westphal 2512e9801430SFlorian Westphal mptcp_pm_nl_work(msk); 2513b416268bSFlorian Westphal 251459832e24SFlorian Westphal if (test_and_clear_bit(MPTCP_WORK_EOF, &msk->flags)) 251559832e24SFlorian Westphal mptcp_check_for_eof(msk); 251659832e24SFlorian Westphal 25176e628cd3SPaolo Abeni __mptcp_check_send_data_fin(sk); 251843b54c6eSMat Martineau mptcp_check_data_fin(sk); 251943b54c6eSMat Martineau 2520341c6524SPaolo Abeni /* There is no point in keeping around an orphaned sk timedout or 2521341c6524SPaolo Abeni * closed, but we need the msk around to reply to incoming DATA_FIN, 2522341c6524SPaolo Abeni * even if it is orphaned and in FIN_WAIT2 state 2523e16163b6SPaolo Abeni */ 2524e16163b6SPaolo Abeni if (sock_flag(sk, SOCK_DEAD) && 2525341c6524SPaolo Abeni (mptcp_check_close_timeout(sk) || sk->sk_state == TCP_CLOSE)) { 2526e16163b6SPaolo Abeni inet_sk_state_store(sk, TCP_CLOSE); 2527e16163b6SPaolo Abeni __mptcp_destroy_sock(sk); 2528e16163b6SPaolo Abeni goto unlock; 2529e16163b6SPaolo Abeni } 2530e16163b6SPaolo Abeni 2531b263b0d7SFlorian Westphal if (test_and_clear_bit(MPTCP_WORK_CLOSE_SUBFLOW, &msk->flags)) 2532b263b0d7SFlorian Westphal __mptcp_close_subflow(msk); 2533b263b0d7SFlorian Westphal 25342948d0a1SPaolo Abeni if (test_and_clear_bit(MPTCP_WORK_RTX, &msk->flags)) 25352948d0a1SPaolo Abeni __mptcp_retrans(sk); 25363b1d6210SPaolo Abeni 25373b1d6210SPaolo Abeni unlock: 253880992017SPaolo Abeni release_sock(sk); 253980992017SPaolo Abeni sock_put(sk); 254080992017SPaolo Abeni } 254180992017SPaolo Abeni 2542784325e9SMatthieu Baerts static int __mptcp_init_sock(struct sock *sk) 2543f870fa0bSMat Martineau { 2544cec37a6eSPeter Krystad struct mptcp_sock *msk = mptcp_sk(sk); 2545cec37a6eSPeter Krystad 2546cec37a6eSPeter Krystad INIT_LIST_HEAD(&msk->conn_list); 2547ec3edaa7SPeter Krystad INIT_LIST_HEAD(&msk->join_list); 254818b683bfSPaolo Abeni INIT_LIST_HEAD(&msk->rtx_queue); 254980992017SPaolo Abeni INIT_WORK(&msk->work, mptcp_worker); 255087952603SPaolo Abeni __skb_queue_head_init(&msk->receive_queue); 2551ab174ad8SPaolo Abeni msk->out_of_order_queue = RB_ROOT; 2552f0e6a4cfSPaolo Abeni msk->first_pending = NULL; 25536511882cSPaolo Abeni msk->rmem_fwd_alloc = 0; 2554ce599c51SPaolo Abeni WRITE_ONCE(msk->rmem_released, 0); 255533d41c9cSPaolo Abeni msk->timer_ival = TCP_RTO_MIN; 2556cec37a6eSPeter Krystad 25578ab183deSPaolo Abeni msk->first = NULL; 2558dc24f8b4SPaolo Abeni inet_csk(sk)->icsk_sync_mss = mptcp_sync_mss; 2559752e9067SGeliang Tang WRITE_ONCE(msk->csum_enabled, mptcp_is_checksum_enabled(sock_net(sk))); 25600530020aSGeliang Tang WRITE_ONCE(msk->allow_infinite_fallback, true); 25611e1d9d6fSPaolo Abeni msk->recovery = false; 25628ab183deSPaolo Abeni 25631b1c7a0eSPeter Krystad mptcp_pm_data_init(msk); 25641b1c7a0eSPeter Krystad 2565b51f9b80SPaolo Abeni /* re-use the csk retrans timer for MPTCP-level retrans */ 2566b51f9b80SPaolo Abeni timer_setup(&msk->sk.icsk_retransmit_timer, mptcp_retransmit_timer, 0); 2567e16163b6SPaolo Abeni timer_setup(&sk->sk_timer, mptcp_timeout_timer, 0); 2568aa1fbd94SFlorian Westphal 2569f870fa0bSMat Martineau return 0; 2570f870fa0bSMat Martineau } 2571f870fa0bSMat Martineau 2572b29fcfb5SPaolo Abeni static void mptcp_ca_reset(struct sock *sk) 2573784325e9SMatthieu Baerts { 257420b5759fSPaolo Abeni struct inet_connection_sock *icsk = inet_csk(sk); 2575b29fcfb5SPaolo Abeni 2576b29fcfb5SPaolo Abeni tcp_assign_congestion_control(sk); 2577b29fcfb5SPaolo Abeni strcpy(mptcp_sk(sk)->ca_name, icsk->icsk_ca_ops->name); 2578b29fcfb5SPaolo Abeni 2579b29fcfb5SPaolo Abeni /* no need to keep a reference to the ops, the name will suffice */ 2580b29fcfb5SPaolo Abeni tcp_cleanup_congestion_control(sk); 2581b29fcfb5SPaolo Abeni icsk->icsk_ca_ops = NULL; 2582b29fcfb5SPaolo Abeni } 2583b29fcfb5SPaolo Abeni 2584b29fcfb5SPaolo Abeni static int mptcp_init_sock(struct sock *sk) 2585b29fcfb5SPaolo Abeni { 2586fc518953SFlorian Westphal struct net *net = sock_net(sk); 2587fc518953SFlorian Westphal int ret; 258818b683bfSPaolo Abeni 2589b6c08380SGeliang Tang ret = __mptcp_init_sock(sk); 2590b6c08380SGeliang Tang if (ret) 2591b6c08380SGeliang Tang return ret; 2592b6c08380SGeliang Tang 2593fc518953SFlorian Westphal if (!mptcp_is_enabled(net)) 2594fc518953SFlorian Westphal return -ENOPROTOOPT; 2595fc518953SFlorian Westphal 2596fc518953SFlorian Westphal if (unlikely(!net->mib.mptcp_statistics) && !mptcp_mib_alloc(net)) 2597fc518953SFlorian Westphal return -ENOMEM; 2598fc518953SFlorian Westphal 2599fa68018dSPaolo Abeni ret = __mptcp_socket_create(mptcp_sk(sk)); 2600fa68018dSPaolo Abeni if (ret) 2601fa68018dSPaolo Abeni return ret; 2602fa68018dSPaolo Abeni 260320b5759fSPaolo Abeni /* fetch the ca name; do it outside __mptcp_init_sock(), so that clone will 260420b5759fSPaolo Abeni * propagate the correct value 260520b5759fSPaolo Abeni */ 2606b29fcfb5SPaolo Abeni mptcp_ca_reset(sk); 260720b5759fSPaolo Abeni 2608d027236cSPaolo Abeni sk_sockets_allocated_inc(sk); 2609a6b118feSFlorian Westphal sk->sk_rcvbuf = sock_net(sk)->ipv4.sysctl_tcp_rmem[1]; 2610da51aef5SPaolo Abeni sk->sk_sndbuf = sock_net(sk)->ipv4.sysctl_tcp_wmem[1]; 2611d027236cSPaolo Abeni 261218b683bfSPaolo Abeni return 0; 261318b683bfSPaolo Abeni } 261418b683bfSPaolo Abeni 261518b683bfSPaolo Abeni static void __mptcp_clear_xmit(struct sock *sk) 261618b683bfSPaolo Abeni { 261718b683bfSPaolo Abeni struct mptcp_sock *msk = mptcp_sk(sk); 261818b683bfSPaolo Abeni struct mptcp_data_frag *dtmp, *dfrag; 261918b683bfSPaolo Abeni 2620d9ca1de8SPaolo Abeni WRITE_ONCE(msk->first_pending, NULL); 262118b683bfSPaolo Abeni list_for_each_entry_safe(dfrag, dtmp, &msk->rtx_queue, list) 2622d027236cSPaolo Abeni dfrag_clear(sk, dfrag); 2623784325e9SMatthieu Baerts } 2624784325e9SMatthieu Baerts 262580992017SPaolo Abeni static void mptcp_cancel_work(struct sock *sk) 262680992017SPaolo Abeni { 262780992017SPaolo Abeni struct mptcp_sock *msk = mptcp_sk(sk); 262880992017SPaolo Abeni 2629b2771d24SPaolo Abeni if (cancel_work_sync(&msk->work)) 2630e16163b6SPaolo Abeni __sock_put(sk); 263180992017SPaolo Abeni } 263280992017SPaolo Abeni 2633d0876b22SGeliang Tang void mptcp_subflow_shutdown(struct sock *sk, struct sock *ssk, int how) 263421498490SPeter Krystad { 263521498490SPeter Krystad lock_sock(ssk); 263621498490SPeter Krystad 263721498490SPeter Krystad switch (ssk->sk_state) { 263821498490SPeter Krystad case TCP_LISTEN: 263921498490SPeter Krystad if (!(how & RCV_SHUTDOWN)) 264021498490SPeter Krystad break; 2641df561f66SGustavo A. R. Silva fallthrough; 264221498490SPeter Krystad case TCP_SYN_SENT: 264321498490SPeter Krystad tcp_disconnect(ssk, O_NONBLOCK); 264421498490SPeter Krystad break; 264521498490SPeter Krystad default: 264643b54c6eSMat Martineau if (__mptcp_check_fallback(mptcp_sk(sk))) { 264743b54c6eSMat Martineau pr_debug("Fallback"); 264821498490SPeter Krystad ssk->sk_shutdown |= how; 264921498490SPeter Krystad tcp_shutdown(ssk, how); 265043b54c6eSMat Martineau } else { 265143b54c6eSMat Martineau pr_debug("Sending DATA_FIN on subflow %p", ssk); 265243b54c6eSMat Martineau tcp_send_ack(ssk); 26536477dd39SMat Martineau if (!mptcp_timer_pending(sk)) 26546477dd39SMat Martineau mptcp_reset_timer(sk); 265543b54c6eSMat Martineau } 265621498490SPeter Krystad break; 265721498490SPeter Krystad } 265821498490SPeter Krystad 265921498490SPeter Krystad release_sock(ssk); 266021498490SPeter Krystad } 266121498490SPeter Krystad 26626920b851SMat Martineau static const unsigned char new_state[16] = { 26636920b851SMat Martineau /* current state: new state: action: */ 26646920b851SMat Martineau [0 /* (Invalid) */] = TCP_CLOSE, 26656920b851SMat Martineau [TCP_ESTABLISHED] = TCP_FIN_WAIT1 | TCP_ACTION_FIN, 26666920b851SMat Martineau [TCP_SYN_SENT] = TCP_CLOSE, 26676920b851SMat Martineau [TCP_SYN_RECV] = TCP_FIN_WAIT1 | TCP_ACTION_FIN, 26686920b851SMat Martineau [TCP_FIN_WAIT1] = TCP_FIN_WAIT1, 26696920b851SMat Martineau [TCP_FIN_WAIT2] = TCP_FIN_WAIT2, 26706920b851SMat Martineau [TCP_TIME_WAIT] = TCP_CLOSE, /* should not happen ! */ 26716920b851SMat Martineau [TCP_CLOSE] = TCP_CLOSE, 26726920b851SMat Martineau [TCP_CLOSE_WAIT] = TCP_LAST_ACK | TCP_ACTION_FIN, 26736920b851SMat Martineau [TCP_LAST_ACK] = TCP_LAST_ACK, 26746920b851SMat Martineau [TCP_LISTEN] = TCP_CLOSE, 26756920b851SMat Martineau [TCP_CLOSING] = TCP_CLOSING, 26766920b851SMat Martineau [TCP_NEW_SYN_RECV] = TCP_CLOSE, /* should not happen ! */ 26776920b851SMat Martineau }; 26786920b851SMat Martineau 26796920b851SMat Martineau static int mptcp_close_state(struct sock *sk) 26806920b851SMat Martineau { 26816920b851SMat Martineau int next = (int)new_state[sk->sk_state]; 26826920b851SMat Martineau int ns = next & TCP_STATE_MASK; 26836920b851SMat Martineau 26846920b851SMat Martineau inet_sk_state_store(sk, ns); 26856920b851SMat Martineau 26866920b851SMat Martineau return next & TCP_ACTION_FIN; 26876920b851SMat Martineau } 26886920b851SMat Martineau 2689e16163b6SPaolo Abeni static void __mptcp_check_send_data_fin(struct sock *sk) 2690e16163b6SPaolo Abeni { 2691e16163b6SPaolo Abeni struct mptcp_subflow_context *subflow; 2692e16163b6SPaolo Abeni struct mptcp_sock *msk = mptcp_sk(sk); 2693e16163b6SPaolo Abeni 2694e16163b6SPaolo Abeni pr_debug("msk=%p snd_data_fin_enable=%d pending=%d snd_nxt=%llu write_seq=%llu", 2695e16163b6SPaolo Abeni msk, msk->snd_data_fin_enable, !!mptcp_send_head(sk), 2696e16163b6SPaolo Abeni msk->snd_nxt, msk->write_seq); 2697e16163b6SPaolo Abeni 2698e16163b6SPaolo Abeni /* we still need to enqueue subflows or not really shutting down, 2699e16163b6SPaolo Abeni * skip this 2700e16163b6SPaolo Abeni */ 2701e16163b6SPaolo Abeni if (!msk->snd_data_fin_enable || msk->snd_nxt + 1 != msk->write_seq || 2702e16163b6SPaolo Abeni mptcp_send_head(sk)) 2703e16163b6SPaolo Abeni return; 2704e16163b6SPaolo Abeni 2705e16163b6SPaolo Abeni WRITE_ONCE(msk->snd_nxt, msk->write_seq); 2706e16163b6SPaolo Abeni 270726aa2314SPaolo Abeni /* fallback socket will not get data_fin/ack, can move to the next 270826aa2314SPaolo Abeni * state now 270926aa2314SPaolo Abeni */ 271026aa2314SPaolo Abeni if (__mptcp_check_fallback(msk)) { 271158cd405bSPaolo Abeni WRITE_ONCE(msk->snd_una, msk->write_seq); 271226aa2314SPaolo Abeni if ((1 << sk->sk_state) & (TCPF_CLOSING | TCPF_LAST_ACK)) { 2713e16163b6SPaolo Abeni inet_sk_state_store(sk, TCP_CLOSE); 2714e16163b6SPaolo Abeni mptcp_close_wake_up(sk); 271526aa2314SPaolo Abeni } else if (sk->sk_state == TCP_FIN_WAIT1) { 271626aa2314SPaolo Abeni inet_sk_state_store(sk, TCP_FIN_WAIT2); 271726aa2314SPaolo Abeni } 2718e16163b6SPaolo Abeni } 2719e16163b6SPaolo Abeni 2720e16163b6SPaolo Abeni mptcp_for_each_subflow(msk, subflow) { 2721e16163b6SPaolo Abeni struct sock *tcp_sk = mptcp_subflow_tcp_sock(subflow); 2722e16163b6SPaolo Abeni 2723e16163b6SPaolo Abeni mptcp_subflow_shutdown(sk, tcp_sk, SEND_SHUTDOWN); 2724e16163b6SPaolo Abeni } 2725e16163b6SPaolo Abeni } 2726e16163b6SPaolo Abeni 2727e16163b6SPaolo Abeni static void __mptcp_wr_shutdown(struct sock *sk) 2728e16163b6SPaolo Abeni { 2729e16163b6SPaolo Abeni struct mptcp_sock *msk = mptcp_sk(sk); 2730e16163b6SPaolo Abeni 2731e16163b6SPaolo Abeni pr_debug("msk=%p snd_data_fin_enable=%d shutdown=%x state=%d pending=%d", 2732e16163b6SPaolo Abeni msk, msk->snd_data_fin_enable, sk->sk_shutdown, sk->sk_state, 2733e16163b6SPaolo Abeni !!mptcp_send_head(sk)); 2734e16163b6SPaolo Abeni 2735e16163b6SPaolo Abeni /* will be ignored by fallback sockets */ 2736e16163b6SPaolo Abeni WRITE_ONCE(msk->write_seq, msk->write_seq + 1); 2737e16163b6SPaolo Abeni WRITE_ONCE(msk->snd_data_fin_enable, 1); 2738e16163b6SPaolo Abeni 2739e16163b6SPaolo Abeni __mptcp_check_send_data_fin(sk); 2740e16163b6SPaolo Abeni } 2741e16163b6SPaolo Abeni 2742e16163b6SPaolo Abeni static void __mptcp_destroy_sock(struct sock *sk) 2743f870fa0bSMat Martineau { 2744cec37a6eSPeter Krystad struct mptcp_subflow_context *subflow, *tmp; 2745f870fa0bSMat Martineau struct mptcp_sock *msk = mptcp_sk(sk); 2746b2c5b614SFlorian Westphal LIST_HEAD(conn_list); 2747f870fa0bSMat Martineau 2748e16163b6SPaolo Abeni pr_debug("msk=%p", msk); 27492c22c06cSFlorian Westphal 27503abc05d9SFlorian Westphal might_sleep(); 27513abc05d9SFlorian Westphal 27523e501490SPaolo Abeni /* join list will be eventually flushed (with rst) at sock lock release time*/ 2753b2c5b614SFlorian Westphal list_splice_init(&msk->conn_list, &conn_list); 2754b2c5b614SFlorian Westphal 27556e628cd3SPaolo Abeni sk_stop_timer(sk, &msk->sk.icsk_retransmit_timer); 2756e16163b6SPaolo Abeni sk_stop_timer(sk, &sk->sk_timer); 2757e16163b6SPaolo Abeni msk->pm.status = 0; 2758b2c5b614SFlorian Westphal 2759b29fcfb5SPaolo Abeni /* clears msk->subflow, allowing the following loop to close 2760b29fcfb5SPaolo Abeni * even the initial subflow 2761b29fcfb5SPaolo Abeni */ 2762b29fcfb5SPaolo Abeni mptcp_dispose_initial_subflow(msk); 2763b2c5b614SFlorian Westphal list_for_each_entry_safe(subflow, tmp, &conn_list, node) { 2764cec37a6eSPeter Krystad struct sock *ssk = mptcp_subflow_tcp_sock(subflow); 2765b29fcfb5SPaolo Abeni __mptcp_close_ssk(sk, ssk, subflow, 0); 2766f870fa0bSMat Martineau } 2767f870fa0bSMat Martineau 2768e16163b6SPaolo Abeni sk->sk_prot->destroy(sk); 2769e16163b6SPaolo Abeni 27706511882cSPaolo Abeni WARN_ON_ONCE(msk->rmem_fwd_alloc); 277187952603SPaolo Abeni WARN_ON_ONCE(msk->rmem_released); 2772e16163b6SPaolo Abeni sk_stream_kill_queues(sk); 2773e16163b6SPaolo Abeni xfrm_sk_free_policy(sk); 2774aa1fbd94SFlorian Westphal 2775e16163b6SPaolo Abeni sk_refcnt_debug_release(sk); 2776e16163b6SPaolo Abeni sock_put(sk); 2777e16163b6SPaolo Abeni } 2778e16163b6SPaolo Abeni 2779e16163b6SPaolo Abeni static void mptcp_close(struct sock *sk, long timeout) 2780e16163b6SPaolo Abeni { 2781e16163b6SPaolo Abeni struct mptcp_subflow_context *subflow; 2782e16163b6SPaolo Abeni bool do_cancel_work = false; 2783e16163b6SPaolo Abeni 2784e16163b6SPaolo Abeni lock_sock(sk); 2785e16163b6SPaolo Abeni sk->sk_shutdown = SHUTDOWN_MASK; 2786e16163b6SPaolo Abeni 2787e16163b6SPaolo Abeni if ((1 << sk->sk_state) & (TCPF_LISTEN | TCPF_CLOSE)) { 2788e16163b6SPaolo Abeni inet_sk_state_store(sk, TCP_CLOSE); 2789e16163b6SPaolo Abeni goto cleanup; 2790e16163b6SPaolo Abeni } 2791e16163b6SPaolo Abeni 2792e16163b6SPaolo Abeni if (mptcp_close_state(sk)) 2793e16163b6SPaolo Abeni __mptcp_wr_shutdown(sk); 2794e16163b6SPaolo Abeni 2795e16163b6SPaolo Abeni sk_stream_wait_close(sk, timeout); 2796e16163b6SPaolo Abeni 2797e16163b6SPaolo Abeni cleanup: 2798e16163b6SPaolo Abeni /* orphan all the subflows */ 2799e16163b6SPaolo Abeni inet_csk(sk)->icsk_mtup.probe_timestamp = tcp_jiffies32; 280044227915SGeliang Tang mptcp_for_each_subflow(mptcp_sk(sk), subflow) { 2801e16163b6SPaolo Abeni struct sock *ssk = mptcp_subflow_tcp_sock(subflow); 280249054556SPaolo Abeni bool slow = lock_sock_fast_nested(ssk); 2803e16163b6SPaolo Abeni 2804e16163b6SPaolo Abeni sock_orphan(ssk); 2805e16163b6SPaolo Abeni unlock_sock_fast(ssk, slow); 2806e16163b6SPaolo Abeni } 2807e16163b6SPaolo Abeni sock_orphan(sk); 2808e16163b6SPaolo Abeni 2809e16163b6SPaolo Abeni sock_hold(sk); 2810e16163b6SPaolo Abeni pr_debug("msk=%p state=%d", sk, sk->sk_state); 2811b29fcfb5SPaolo Abeni if (mptcp_sk(sk)->token) 2812b29fcfb5SPaolo Abeni mptcp_event(MPTCP_EVENT_CLOSED, mptcp_sk(sk), NULL, GFP_KERNEL); 2813b29fcfb5SPaolo Abeni 2814e16163b6SPaolo Abeni if (sk->sk_state == TCP_CLOSE) { 2815e16163b6SPaolo Abeni __mptcp_destroy_sock(sk); 2816e16163b6SPaolo Abeni do_cancel_work = true; 2817e16163b6SPaolo Abeni } else { 2818e16163b6SPaolo Abeni sk_reset_timer(sk, &sk->sk_timer, jiffies + TCP_TIMEWAIT_LEN); 2819e16163b6SPaolo Abeni } 2820e16163b6SPaolo Abeni release_sock(sk); 2821e16163b6SPaolo Abeni if (do_cancel_work) 282280992017SPaolo Abeni mptcp_cancel_work(sk); 2823b911c97cSFlorian Westphal 2824e16163b6SPaolo Abeni sock_put(sk); 2825f870fa0bSMat Martineau } 2826f870fa0bSMat Martineau 2827cf7da0d6SPeter Krystad static void mptcp_copy_inaddrs(struct sock *msk, const struct sock *ssk) 2828cf7da0d6SPeter Krystad { 2829cf7da0d6SPeter Krystad #if IS_ENABLED(CONFIG_MPTCP_IPV6) 2830cf7da0d6SPeter Krystad const struct ipv6_pinfo *ssk6 = inet6_sk(ssk); 2831cf7da0d6SPeter Krystad struct ipv6_pinfo *msk6 = inet6_sk(msk); 2832cf7da0d6SPeter Krystad 2833cf7da0d6SPeter Krystad msk->sk_v6_daddr = ssk->sk_v6_daddr; 2834cf7da0d6SPeter Krystad msk->sk_v6_rcv_saddr = ssk->sk_v6_rcv_saddr; 2835cf7da0d6SPeter Krystad 2836cf7da0d6SPeter Krystad if (msk6 && ssk6) { 2837cf7da0d6SPeter Krystad msk6->saddr = ssk6->saddr; 2838cf7da0d6SPeter Krystad msk6->flow_label = ssk6->flow_label; 2839cf7da0d6SPeter Krystad } 2840cf7da0d6SPeter Krystad #endif 2841cf7da0d6SPeter Krystad 2842cf7da0d6SPeter Krystad inet_sk(msk)->inet_num = inet_sk(ssk)->inet_num; 2843cf7da0d6SPeter Krystad inet_sk(msk)->inet_dport = inet_sk(ssk)->inet_dport; 2844cf7da0d6SPeter Krystad inet_sk(msk)->inet_sport = inet_sk(ssk)->inet_sport; 2845cf7da0d6SPeter Krystad inet_sk(msk)->inet_daddr = inet_sk(ssk)->inet_daddr; 2846cf7da0d6SPeter Krystad inet_sk(msk)->inet_saddr = inet_sk(ssk)->inet_saddr; 2847cf7da0d6SPeter Krystad inet_sk(msk)->inet_rcv_saddr = inet_sk(ssk)->inet_rcv_saddr; 2848cf7da0d6SPeter Krystad } 2849cf7da0d6SPeter Krystad 285018b683bfSPaolo Abeni static int mptcp_disconnect(struct sock *sk, int flags) 285118b683bfSPaolo Abeni { 285276e2a55dSPaolo Abeni struct mptcp_subflow_context *subflow; 285376e2a55dSPaolo Abeni struct mptcp_sock *msk = mptcp_sk(sk); 285476e2a55dSPaolo Abeni 2855b29fcfb5SPaolo Abeni inet_sk_state_store(sk, TCP_CLOSE); 2856b29fcfb5SPaolo Abeni 285713a9499eSPaolo Abeni mptcp_for_each_subflow(msk, subflow) { 285813a9499eSPaolo Abeni struct sock *ssk = mptcp_subflow_tcp_sock(subflow); 285913a9499eSPaolo Abeni 2860b29fcfb5SPaolo Abeni __mptcp_close_ssk(sk, ssk, subflow, MPTCP_CF_FASTCLOSE); 286113a9499eSPaolo Abeni } 2862b29fcfb5SPaolo Abeni 2863b29fcfb5SPaolo Abeni sk_stop_timer(sk, &msk->sk.icsk_retransmit_timer); 2864b29fcfb5SPaolo Abeni sk_stop_timer(sk, &sk->sk_timer); 2865b29fcfb5SPaolo Abeni 2866b29fcfb5SPaolo Abeni if (mptcp_sk(sk)->token) 2867b29fcfb5SPaolo Abeni mptcp_event(MPTCP_EVENT_CLOSED, mptcp_sk(sk), NULL, GFP_KERNEL); 2868b29fcfb5SPaolo Abeni 2869b29fcfb5SPaolo Abeni mptcp_destroy_common(msk); 2870b29fcfb5SPaolo Abeni msk->last_snd = NULL; 2871e9d09bacSPaolo Abeni WRITE_ONCE(msk->flags, 0); 2872e9d09bacSPaolo Abeni msk->cb_flags = 0; 2873e9d09bacSPaolo Abeni msk->push_pending = 0; 2874b29fcfb5SPaolo Abeni msk->recovery = false; 2875b29fcfb5SPaolo Abeni msk->can_ack = false; 2876b29fcfb5SPaolo Abeni msk->fully_established = false; 2877b29fcfb5SPaolo Abeni msk->rcv_data_fin = false; 2878b29fcfb5SPaolo Abeni msk->snd_data_fin_enable = false; 2879b29fcfb5SPaolo Abeni msk->rcv_fastclose = false; 2880b29fcfb5SPaolo Abeni msk->use_64bit_ack = false; 2881b29fcfb5SPaolo Abeni WRITE_ONCE(msk->csum_enabled, mptcp_is_checksum_enabled(sock_net(sk))); 2882b29fcfb5SPaolo Abeni mptcp_pm_data_reset(msk); 2883b29fcfb5SPaolo Abeni mptcp_ca_reset(sk); 2884b29fcfb5SPaolo Abeni 2885b29fcfb5SPaolo Abeni sk->sk_shutdown = 0; 2886b29fcfb5SPaolo Abeni sk_error_report(sk); 288742c556feSFlorian Westphal return 0; 288818b683bfSPaolo Abeni } 288918b683bfSPaolo Abeni 2890b0519de8SFlorian Westphal #if IS_ENABLED(CONFIG_MPTCP_IPV6) 2891b0519de8SFlorian Westphal static struct ipv6_pinfo *mptcp_inet6_sk(const struct sock *sk) 2892b0519de8SFlorian Westphal { 2893b0519de8SFlorian Westphal unsigned int offset = sizeof(struct mptcp6_sock) - sizeof(struct ipv6_pinfo); 2894b0519de8SFlorian Westphal 2895b0519de8SFlorian Westphal return (struct ipv6_pinfo *)(((u8 *)sk) + offset); 2896b0519de8SFlorian Westphal } 2897b0519de8SFlorian Westphal #endif 2898b0519de8SFlorian Westphal 2899fca5c82cSPaolo Abeni struct sock *mptcp_sk_clone(const struct sock *sk, 2900cfde141eSPaolo Abeni const struct mptcp_options_received *mp_opt, 2901fca5c82cSPaolo Abeni struct request_sock *req) 2902b0519de8SFlorian Westphal { 290358b09919SPaolo Abeni struct mptcp_subflow_request_sock *subflow_req = mptcp_subflow_rsk(req); 2904b0519de8SFlorian Westphal struct sock *nsk = sk_clone_lock(sk, GFP_ATOMIC); 290558b09919SPaolo Abeni struct mptcp_sock *msk; 290658b09919SPaolo Abeni u64 ack_seq; 2907b0519de8SFlorian Westphal 2908b0519de8SFlorian Westphal if (!nsk) 2909b0519de8SFlorian Westphal return NULL; 2910b0519de8SFlorian Westphal 2911b0519de8SFlorian Westphal #if IS_ENABLED(CONFIG_MPTCP_IPV6) 2912b0519de8SFlorian Westphal if (nsk->sk_family == AF_INET6) 2913b0519de8SFlorian Westphal inet_sk(nsk)->pinet6 = mptcp_inet6_sk(nsk); 2914b0519de8SFlorian Westphal #endif 2915b0519de8SFlorian Westphal 291658b09919SPaolo Abeni __mptcp_init_sock(nsk); 291758b09919SPaolo Abeni 291858b09919SPaolo Abeni msk = mptcp_sk(nsk); 291958b09919SPaolo Abeni msk->local_key = subflow_req->local_key; 292058b09919SPaolo Abeni msk->token = subflow_req->token; 292158b09919SPaolo Abeni msk->subflow = NULL; 2922b93df08cSPaolo Abeni WRITE_ONCE(msk->fully_established, false); 292374c7dfbeSPaolo Abeni if (mp_opt->suboptions & OPTION_MPTCP_CSUMREQD) 292406251181SGeliang Tang WRITE_ONCE(msk->csum_enabled, true); 292558b09919SPaolo Abeni 292658b09919SPaolo Abeni msk->write_seq = subflow_req->idsn + 1; 2927eaa2ffabSPaolo Abeni msk->snd_nxt = msk->write_seq; 29287439d687SPaolo Abeni msk->snd_una = msk->write_seq; 29297439d687SPaolo Abeni msk->wnd_end = msk->snd_nxt + req->rsk_rcv_wnd; 2930df00b087SFlorian Westphal msk->setsockopt_seq = mptcp_sk(sk)->setsockopt_seq; 29316f8a612aSFlorian Westphal 293274c7dfbeSPaolo Abeni if (mp_opt->suboptions & OPTIONS_MPTCP_MPC) { 293358b09919SPaolo Abeni msk->can_ack = true; 2934cfde141eSPaolo Abeni msk->remote_key = mp_opt->sndr_key; 293558b09919SPaolo Abeni mptcp_crypto_key_sha(msk->remote_key, NULL, &ack_seq); 293658b09919SPaolo Abeni ack_seq++; 2937917944daSMat Martineau WRITE_ONCE(msk->ack_seq, ack_seq); 2938fa3fe2b1SFlorian Westphal WRITE_ONCE(msk->rcv_wnd_sent, ack_seq); 293958b09919SPaolo Abeni } 29407f20d5fcSPaolo Abeni 29415e20087dSFlorian Westphal sock_reset_flag(nsk, SOCK_RCU_FREE); 29427f20d5fcSPaolo Abeni /* will be fully established after successful MPC subflow creation */ 29437f20d5fcSPaolo Abeni inet_sk_state_store(nsk, TCP_SYN_RECV); 29440c148460SPaolo Abeni 29450c148460SPaolo Abeni security_inet_csk_clone(nsk, req); 294658b09919SPaolo Abeni bh_unlock_sock(nsk); 294758b09919SPaolo Abeni 294858b09919SPaolo Abeni /* keep a single reference */ 294958b09919SPaolo Abeni __sock_put(nsk); 2950b0519de8SFlorian Westphal return nsk; 2951b0519de8SFlorian Westphal } 2952b0519de8SFlorian Westphal 2953a6b118feSFlorian Westphal void mptcp_rcv_space_init(struct mptcp_sock *msk, const struct sock *ssk) 2954a6b118feSFlorian Westphal { 2955a6b118feSFlorian Westphal const struct tcp_sock *tp = tcp_sk(ssk); 2956a6b118feSFlorian Westphal 2957a6b118feSFlorian Westphal msk->rcvq_space.copied = 0; 2958a6b118feSFlorian Westphal msk->rcvq_space.rtt_us = 0; 2959a6b118feSFlorian Westphal 2960a6b118feSFlorian Westphal msk->rcvq_space.time = tp->tcp_mstamp; 2961a6b118feSFlorian Westphal 2962a6b118feSFlorian Westphal /* initial rcv_space offering made to peer */ 2963a6b118feSFlorian Westphal msk->rcvq_space.space = min_t(u32, tp->rcv_wnd, 2964a6b118feSFlorian Westphal TCP_INIT_CWND * tp->advmss); 2965a6b118feSFlorian Westphal if (msk->rcvq_space.space == 0) 2966a6b118feSFlorian Westphal msk->rcvq_space.space = TCP_INIT_CWND * TCP_MSS_DEFAULT; 29676f8a612aSFlorian Westphal 29687439d687SPaolo Abeni WRITE_ONCE(msk->wnd_end, msk->snd_nxt + tcp_sk(ssk)->snd_wnd); 2969a6b118feSFlorian Westphal } 2970a6b118feSFlorian Westphal 2971cf7da0d6SPeter Krystad static struct sock *mptcp_accept(struct sock *sk, int flags, int *err, 2972cf7da0d6SPeter Krystad bool kern) 2973cf7da0d6SPeter Krystad { 2974cf7da0d6SPeter Krystad struct mptcp_sock *msk = mptcp_sk(sk); 2975cf7da0d6SPeter Krystad struct socket *listener; 2976cf7da0d6SPeter Krystad struct sock *newsk; 2977cf7da0d6SPeter Krystad 2978cf7da0d6SPeter Krystad listener = __mptcp_nmpc_socket(msk); 2979cf7da0d6SPeter Krystad if (WARN_ON_ONCE(!listener)) { 2980cf7da0d6SPeter Krystad *err = -EINVAL; 2981cf7da0d6SPeter Krystad return NULL; 2982cf7da0d6SPeter Krystad } 2983cf7da0d6SPeter Krystad 2984cf7da0d6SPeter Krystad pr_debug("msk=%p, listener=%p", msk, mptcp_subflow_ctx(listener->sk)); 2985cf7da0d6SPeter Krystad newsk = inet_csk_accept(listener->sk, flags, err, kern); 2986cf7da0d6SPeter Krystad if (!newsk) 2987cf7da0d6SPeter Krystad return NULL; 2988cf7da0d6SPeter Krystad 2989cf7da0d6SPeter Krystad pr_debug("msk=%p, subflow is mptcp=%d", msk, sk_is_mptcp(newsk)); 2990cf7da0d6SPeter Krystad if (sk_is_mptcp(newsk)) { 2991cf7da0d6SPeter Krystad struct mptcp_subflow_context *subflow; 2992cf7da0d6SPeter Krystad struct sock *new_mptcp_sock; 2993cf7da0d6SPeter Krystad 2994cf7da0d6SPeter Krystad subflow = mptcp_subflow_ctx(newsk); 299558b09919SPaolo Abeni new_mptcp_sock = subflow->conn; 299658b09919SPaolo Abeni 299758b09919SPaolo Abeni /* is_mptcp should be false if subflow->conn is missing, see 299858b09919SPaolo Abeni * subflow_syn_recv_sock() 299958b09919SPaolo Abeni */ 300058b09919SPaolo Abeni if (WARN_ON_ONCE(!new_mptcp_sock)) { 300158b09919SPaolo Abeni tcp_sk(newsk)->is_mptcp = 0; 3002d6692b3bSFlorian Westphal goto out; 300358b09919SPaolo Abeni } 300458b09919SPaolo Abeni 300558b09919SPaolo Abeni /* acquire the 2nd reference for the owning socket */ 300658b09919SPaolo Abeni sock_hold(new_mptcp_sock); 3007cf7da0d6SPeter Krystad newsk = new_mptcp_sock; 30080397c6d8SPaolo Abeni MPTCP_INC_STATS(sock_net(sk), MPTCP_MIB_MPCAPABLEPASSIVEACK); 3009fc518953SFlorian Westphal } else { 3010fc518953SFlorian Westphal MPTCP_INC_STATS(sock_net(sk), 3011fc518953SFlorian Westphal MPTCP_MIB_MPCAPABLEPASSIVEFALLBACK); 3012cf7da0d6SPeter Krystad } 3013cf7da0d6SPeter Krystad 3014d6692b3bSFlorian Westphal out: 3015d6692b3bSFlorian Westphal newsk->sk_kern_sock = kern; 3016cf7da0d6SPeter Krystad return newsk; 3017cf7da0d6SPeter Krystad } 3018cf7da0d6SPeter Krystad 30195c8c1640SGeliang Tang void mptcp_destroy_common(struct mptcp_sock *msk) 30205c8c1640SGeliang Tang { 302187952603SPaolo Abeni struct sock *sk = (struct sock *)msk; 302287952603SPaolo Abeni 30236e628cd3SPaolo Abeni __mptcp_clear_xmit(sk); 30246e628cd3SPaolo Abeni 302587952603SPaolo Abeni /* move to sk_receive_queue, sk_stream_kill_queues will purge it */ 3026b29fcfb5SPaolo Abeni mptcp_data_lock(sk); 302787952603SPaolo Abeni skb_queue_splice_tail_init(&msk->receive_queue, &sk->sk_receive_queue); 30286511882cSPaolo Abeni __skb_queue_purge(&sk->sk_receive_queue); 30295c8c1640SGeliang Tang skb_rbtree_purge(&msk->out_of_order_queue); 3030b29fcfb5SPaolo Abeni mptcp_data_unlock(sk); 30316511882cSPaolo Abeni 30326511882cSPaolo Abeni /* move all the rx fwd alloc into the sk_mem_reclaim_final in 30336511882cSPaolo Abeni * inet_sock_destruct() will dispose it 30346511882cSPaolo Abeni */ 30356511882cSPaolo Abeni sk->sk_forward_alloc += msk->rmem_fwd_alloc; 30366511882cSPaolo Abeni msk->rmem_fwd_alloc = 0; 30375c8c1640SGeliang Tang mptcp_token_destroy(msk); 30385c8c1640SGeliang Tang mptcp_pm_free_anno_list(msk); 30395c8c1640SGeliang Tang } 30405c8c1640SGeliang Tang 304179c0949eSPeter Krystad static void mptcp_destroy(struct sock *sk) 304279c0949eSPeter Krystad { 3043c9fd9c5fSFlorian Westphal struct mptcp_sock *msk = mptcp_sk(sk); 3044c9fd9c5fSFlorian Westphal 30455c8c1640SGeliang Tang mptcp_destroy_common(msk); 3046d027236cSPaolo Abeni sk_sockets_allocated_dec(sk); 304779c0949eSPeter Krystad } 304879c0949eSPeter Krystad 30496e628cd3SPaolo Abeni void __mptcp_data_acked(struct sock *sk) 30506e628cd3SPaolo Abeni { 30516e628cd3SPaolo Abeni if (!sock_owned_by_user(sk)) 30526e628cd3SPaolo Abeni __mptcp_clean_una(sk); 30536e628cd3SPaolo Abeni else 3054e9d09bacSPaolo Abeni __set_bit(MPTCP_CLEAN_UNA, &mptcp_sk(sk)->cb_flags); 30556e628cd3SPaolo Abeni 30566e628cd3SPaolo Abeni if (mptcp_pending_data_fin_ack(sk)) 30576e628cd3SPaolo Abeni mptcp_schedule_work(sk); 30586e628cd3SPaolo Abeni } 30596e628cd3SPaolo Abeni 3060219d0499SPaolo Abeni void __mptcp_check_push(struct sock *sk, struct sock *ssk) 30616e628cd3SPaolo Abeni { 30626e628cd3SPaolo Abeni if (!mptcp_send_head(sk)) 30636e628cd3SPaolo Abeni return; 30646e628cd3SPaolo Abeni 306540dc9416SPaolo Abeni if (!sock_owned_by_user(sk)) { 3066b19bc294SPaolo Abeni struct sock *xmit_ssk = mptcp_subflow_get_send(mptcp_sk(sk)); 3067b19bc294SPaolo Abeni 3068b19bc294SPaolo Abeni if (xmit_ssk == ssk) 30696e628cd3SPaolo Abeni __mptcp_subflow_push_pending(sk, ssk); 3070b19bc294SPaolo Abeni else if (xmit_ssk) 3071bcd97734SPaolo Abeni mptcp_subflow_delegate(mptcp_subflow_ctx(xmit_ssk), MPTCP_DELEGATE_SEND); 307240dc9416SPaolo Abeni } else { 3073e9d09bacSPaolo Abeni __set_bit(MPTCP_PUSH_PENDING, &mptcp_sk(sk)->cb_flags); 30746e628cd3SPaolo Abeni } 307540dc9416SPaolo Abeni } 30766e628cd3SPaolo Abeni 3077e9d09bacSPaolo Abeni #define MPTCP_FLAGS_PROCESS_CTX_NEED (BIT(MPTCP_PUSH_PENDING) | \ 3078e9d09bacSPaolo Abeni BIT(MPTCP_RETRANSMIT) | \ 3079e9d09bacSPaolo Abeni BIT(MPTCP_FLUSH_JOIN_LIST)) 3080e9d09bacSPaolo Abeni 3081e93da928SPaolo Abeni /* processes deferred events and flush wmem */ 308214c441b5SPaolo Abeni static void mptcp_release_cb(struct sock *sk) 3083e9d09bacSPaolo Abeni __must_hold(&sk->sk_lock.slock) 308414c441b5SPaolo Abeni { 3085e9d09bacSPaolo Abeni struct mptcp_sock *msk = mptcp_sk(sk); 30862d6f5a2bSPaolo Abeni 3087e9d09bacSPaolo Abeni for (;;) { 3088e9d09bacSPaolo Abeni unsigned long flags = (msk->cb_flags & MPTCP_FLAGS_PROCESS_CTX_NEED) | 3089e9d09bacSPaolo Abeni msk->push_pending; 3090c2e6048fSPaolo Abeni if (!flags) 3091c2e6048fSPaolo Abeni break; 3092c2e6048fSPaolo Abeni 3093c2e6048fSPaolo Abeni /* the following actions acquire the subflow socket lock 30946e628cd3SPaolo Abeni * 30956e628cd3SPaolo Abeni * 1) can't be invoked in atomic scope 30966e628cd3SPaolo Abeni * 2) must avoid ABBA deadlock with msk socket spinlock: the RX 30976e628cd3SPaolo Abeni * datapath acquires the msk socket spinlock while helding 30986e628cd3SPaolo Abeni * the subflow socket lock 30996e628cd3SPaolo Abeni */ 3100e9d09bacSPaolo Abeni msk->push_pending = 0; 3101e9d09bacSPaolo Abeni msk->cb_flags &= ~flags; 31026e628cd3SPaolo Abeni spin_unlock_bh(&sk->sk_lock.slock); 31033e501490SPaolo Abeni if (flags & BIT(MPTCP_FLUSH_JOIN_LIST)) 31043e501490SPaolo Abeni __mptcp_flush_join_list(sk); 31052e5de7e0SDan Carpenter if (flags & BIT(MPTCP_PUSH_PENDING)) 3106c2e6048fSPaolo Abeni __mptcp_push_pending(sk, 0); 31072d6f5a2bSPaolo Abeni if (flags & BIT(MPTCP_RETRANSMIT)) 31082d6f5a2bSPaolo Abeni __mptcp_retrans(sk); 3109c2e6048fSPaolo Abeni 3110c2e6048fSPaolo Abeni cond_resched(); 31116e628cd3SPaolo Abeni spin_lock_bh(&sk->sk_lock.slock); 31126e628cd3SPaolo Abeni } 3113c2e6048fSPaolo Abeni 311465a569b0SPaolo Abeni if (__test_and_clear_bit(MPTCP_CLEAN_UNA, &msk->cb_flags)) 311565a569b0SPaolo Abeni __mptcp_clean_una_wakeup(sk); 311665a569b0SPaolo Abeni if (unlikely(&msk->cb_flags)) { 3117490274b4SPaolo Abeni /* be sure to set the current sk state before tacking actions 311865a569b0SPaolo Abeni * depending on sk_state, that is processing MPTCP_ERROR_REPORT 3119490274b4SPaolo Abeni */ 3120e9d09bacSPaolo Abeni if (__test_and_clear_bit(MPTCP_CONNECTED, &msk->cb_flags)) 3121490274b4SPaolo Abeni __mptcp_set_connected(sk); 3122e9d09bacSPaolo Abeni if (__test_and_clear_bit(MPTCP_ERROR_REPORT, &msk->cb_flags)) 312315cc1045SPaolo Abeni __mptcp_error_report(sk); 312443f5b111SPaolo Abeni if (__test_and_clear_bit(MPTCP_RESET_SCHEDULER, &msk->cb_flags)) 312543f5b111SPaolo Abeni msk->last_snd = NULL; 312665a569b0SPaolo Abeni } 31276e628cd3SPaolo Abeni 312887952603SPaolo Abeni __mptcp_update_rmem(sk); 312914c441b5SPaolo Abeni } 313014c441b5SPaolo Abeni 3131bcd97734SPaolo Abeni /* MP_JOIN client subflow must wait for 4th ack before sending any data: 3132bcd97734SPaolo Abeni * TCP can't schedule delack timer before the subflow is fully established. 3133bcd97734SPaolo Abeni * MPTCP uses the delack timer to do 3rd ack retransmissions 3134bcd97734SPaolo Abeni */ 3135bcd97734SPaolo Abeni static void schedule_3rdack_retransmission(struct sock *ssk) 3136bcd97734SPaolo Abeni { 3137bcd97734SPaolo Abeni struct inet_connection_sock *icsk = inet_csk(ssk); 3138bcd97734SPaolo Abeni struct tcp_sock *tp = tcp_sk(ssk); 3139bcd97734SPaolo Abeni unsigned long timeout; 3140bcd97734SPaolo Abeni 3141bcd97734SPaolo Abeni if (mptcp_subflow_ctx(ssk)->fully_established) 3142bcd97734SPaolo Abeni return; 3143bcd97734SPaolo Abeni 3144bcd97734SPaolo Abeni /* reschedule with a timeout above RTT, as we must look only for drop */ 3145bcd97734SPaolo Abeni if (tp->srtt_us) 3146bcd97734SPaolo Abeni timeout = usecs_to_jiffies(tp->srtt_us >> (3 - 1)); 3147bcd97734SPaolo Abeni else 3148bcd97734SPaolo Abeni timeout = TCP_TIMEOUT_INIT; 3149bcd97734SPaolo Abeni timeout += jiffies; 3150bcd97734SPaolo Abeni 3151bcd97734SPaolo Abeni WARN_ON_ONCE(icsk->icsk_ack.pending & ICSK_ACK_TIMER); 3152bcd97734SPaolo Abeni icsk->icsk_ack.pending |= ICSK_ACK_SCHED | ICSK_ACK_TIMER; 3153bcd97734SPaolo Abeni icsk->icsk_ack.timeout = timeout; 3154bcd97734SPaolo Abeni sk_reset_timer(ssk, &icsk->icsk_delack_timer, timeout); 3155bcd97734SPaolo Abeni } 3156bcd97734SPaolo Abeni 3157b19bc294SPaolo Abeni void mptcp_subflow_process_delegated(struct sock *ssk) 3158b19bc294SPaolo Abeni { 3159b19bc294SPaolo Abeni struct mptcp_subflow_context *subflow = mptcp_subflow_ctx(ssk); 3160b19bc294SPaolo Abeni struct sock *sk = subflow->conn; 3161b19bc294SPaolo Abeni 3162bcd97734SPaolo Abeni if (test_bit(MPTCP_DELEGATE_SEND, &subflow->delegated_status)) { 3163b19bc294SPaolo Abeni mptcp_data_lock(sk); 3164b19bc294SPaolo Abeni if (!sock_owned_by_user(sk)) 3165b19bc294SPaolo Abeni __mptcp_subflow_push_pending(sk, ssk); 3166b19bc294SPaolo Abeni else 3167e9d09bacSPaolo Abeni __set_bit(MPTCP_PUSH_PENDING, &mptcp_sk(sk)->cb_flags); 3168b19bc294SPaolo Abeni mptcp_data_unlock(sk); 3169bcd97734SPaolo Abeni mptcp_subflow_delegated_done(subflow, MPTCP_DELEGATE_SEND); 3170bcd97734SPaolo Abeni } 3171bcd97734SPaolo Abeni if (test_bit(MPTCP_DELEGATE_ACK, &subflow->delegated_status)) { 3172bcd97734SPaolo Abeni schedule_3rdack_retransmission(ssk); 3173bcd97734SPaolo Abeni mptcp_subflow_delegated_done(subflow, MPTCP_DELEGATE_ACK); 3174bcd97734SPaolo Abeni } 3175b19bc294SPaolo Abeni } 3176b19bc294SPaolo Abeni 31772c5ebd00SPaolo Abeni static int mptcp_hash(struct sock *sk) 31782c5ebd00SPaolo Abeni { 31792c5ebd00SPaolo Abeni /* should never be called, 31802c5ebd00SPaolo Abeni * we hash the TCP subflows not the master socket 31812c5ebd00SPaolo Abeni */ 31822c5ebd00SPaolo Abeni WARN_ON_ONCE(1); 31832c5ebd00SPaolo Abeni return 0; 31842c5ebd00SPaolo Abeni } 31852c5ebd00SPaolo Abeni 31862c5ebd00SPaolo Abeni static void mptcp_unhash(struct sock *sk) 31872c5ebd00SPaolo Abeni { 31882c5ebd00SPaolo Abeni /* called from sk_common_release(), but nothing to do here */ 31892c5ebd00SPaolo Abeni } 31902c5ebd00SPaolo Abeni 3191cec37a6eSPeter Krystad static int mptcp_get_port(struct sock *sk, unsigned short snum) 3192f870fa0bSMat Martineau { 3193f870fa0bSMat Martineau struct mptcp_sock *msk = mptcp_sk(sk); 3194cec37a6eSPeter Krystad struct socket *ssock; 3195f870fa0bSMat Martineau 3196cec37a6eSPeter Krystad ssock = __mptcp_nmpc_socket(msk); 3197cec37a6eSPeter Krystad pr_debug("msk=%p, subflow=%p", msk, ssock); 3198cec37a6eSPeter Krystad if (WARN_ON_ONCE(!ssock)) 3199cec37a6eSPeter Krystad return -EINVAL; 3200f870fa0bSMat Martineau 3201cec37a6eSPeter Krystad return inet_csk_get_port(ssock->sk, snum); 3202cec37a6eSPeter Krystad } 3203f870fa0bSMat Martineau 3204cec37a6eSPeter Krystad void mptcp_finish_connect(struct sock *ssk) 3205cec37a6eSPeter Krystad { 3206cec37a6eSPeter Krystad struct mptcp_subflow_context *subflow; 3207cec37a6eSPeter Krystad struct mptcp_sock *msk; 3208cec37a6eSPeter Krystad struct sock *sk; 32096d0060f6SMat Martineau u64 ack_seq; 3210f870fa0bSMat Martineau 3211cec37a6eSPeter Krystad subflow = mptcp_subflow_ctx(ssk); 3212cec37a6eSPeter Krystad sk = subflow->conn; 3213cec37a6eSPeter Krystad msk = mptcp_sk(sk); 3214cec37a6eSPeter Krystad 3215648ef4b8SMat Martineau pr_debug("msk=%p, token=%u", sk, subflow->token); 3216648ef4b8SMat Martineau 32176d0060f6SMat Martineau mptcp_crypto_key_sha(subflow->remote_key, NULL, &ack_seq); 32186d0060f6SMat Martineau ack_seq++; 3219648ef4b8SMat Martineau subflow->map_seq = ack_seq; 3220648ef4b8SMat Martineau subflow->map_subflow_seq = 1; 32216d0060f6SMat Martineau 3222cec37a6eSPeter Krystad /* the socket is not connected yet, no msk/subflow ops can access/race 3223cec37a6eSPeter Krystad * accessing the field below 3224cec37a6eSPeter Krystad */ 3225cec37a6eSPeter Krystad WRITE_ONCE(msk->remote_key, subflow->remote_key); 3226cec37a6eSPeter Krystad WRITE_ONCE(msk->local_key, subflow->local_key); 32276d0060f6SMat Martineau WRITE_ONCE(msk->write_seq, subflow->idsn + 1); 3228eaa2ffabSPaolo Abeni WRITE_ONCE(msk->snd_nxt, msk->write_seq); 32296d0060f6SMat Martineau WRITE_ONCE(msk->ack_seq, ack_seq); 3230fa3fe2b1SFlorian Westphal WRITE_ONCE(msk->rcv_wnd_sent, ack_seq); 3231d22f4988SChristoph Paasch WRITE_ONCE(msk->can_ack, 1); 32327439d687SPaolo Abeni WRITE_ONCE(msk->snd_una, msk->write_seq); 32331b1c7a0eSPeter Krystad 32346c714f1bSFlorian Westphal mptcp_pm_new_connection(msk, ssk, 0); 3235a6b118feSFlorian Westphal 3236a6b118feSFlorian Westphal mptcp_rcv_space_init(msk, ssk); 3237f870fa0bSMat Martineau } 3238f870fa0bSMat Martineau 3239866f26f2SPaolo Abeni void mptcp_sock_graft(struct sock *sk, struct socket *parent) 3240cf7da0d6SPeter Krystad { 3241cf7da0d6SPeter Krystad write_lock_bh(&sk->sk_callback_lock); 3242cf7da0d6SPeter Krystad rcu_assign_pointer(sk->sk_wq, &parent->wq); 3243cf7da0d6SPeter Krystad sk_set_socket(sk, parent); 3244cf7da0d6SPeter Krystad sk->sk_uid = SOCK_INODE(parent)->i_uid; 3245cf7da0d6SPeter Krystad write_unlock_bh(&sk->sk_callback_lock); 3246cf7da0d6SPeter Krystad } 3247cf7da0d6SPeter Krystad 3248e16163b6SPaolo Abeni bool mptcp_finish_join(struct sock *ssk) 3249f296234cSPeter Krystad { 3250e16163b6SPaolo Abeni struct mptcp_subflow_context *subflow = mptcp_subflow_ctx(ssk); 3251f296234cSPeter Krystad struct mptcp_sock *msk = mptcp_sk(subflow->conn); 3252f296234cSPeter Krystad struct sock *parent = (void *)msk; 32533e501490SPaolo Abeni bool ret = true; 3254f296234cSPeter Krystad 3255f296234cSPeter Krystad pr_debug("msk=%p, subflow=%p", msk, subflow); 3256f296234cSPeter Krystad 3257f296234cSPeter Krystad /* mptcp socket already closing? */ 3258dc87efdbSFlorian Westphal if (!mptcp_is_fully_established(parent)) { 3259dc87efdbSFlorian Westphal subflow->reset_reason = MPTCP_RST_EMPTCP; 3260f296234cSPeter Krystad return false; 3261dc87efdbSFlorian Westphal } 3262f296234cSPeter Krystad 3263f296234cSPeter Krystad if (!msk->pm.server_side) 3264b911c97cSFlorian Westphal goto out; 3265f296234cSPeter Krystad 32663e501490SPaolo Abeni if (!mptcp_pm_allow_new_subflow(msk)) 32673e501490SPaolo Abeni goto err_prohibited; 326810f6d46cSPaolo Abeni 32693e501490SPaolo Abeni if (WARN_ON_ONCE(!list_empty(&subflow->node))) 32703e501490SPaolo Abeni goto err_prohibited; 32713e501490SPaolo Abeni 32723e501490SPaolo Abeni /* active connections are already on conn_list. 32733e501490SPaolo Abeni * If we can't acquire msk socket lock here, let the release callback 32743e501490SPaolo Abeni * handle it 327510f6d46cSPaolo Abeni */ 32763e501490SPaolo Abeni mptcp_data_lock(parent); 32773e501490SPaolo Abeni if (!sock_owned_by_user(parent)) { 32783e501490SPaolo Abeni ret = __mptcp_finish_join(msk, ssk); 32793e501490SPaolo Abeni if (ret) { 3280e16163b6SPaolo Abeni sock_hold(ssk); 32813e501490SPaolo Abeni list_add_tail(&subflow->node, &msk->conn_list); 3282e16163b6SPaolo Abeni } 32833e501490SPaolo Abeni } else { 32843e501490SPaolo Abeni sock_hold(ssk); 32853e501490SPaolo Abeni list_add_tail(&subflow->node, &msk->join_list); 3286e9d09bacSPaolo Abeni __set_bit(MPTCP_FLUSH_JOIN_LIST, &msk->cb_flags); 32873e501490SPaolo Abeni } 32883e501490SPaolo Abeni mptcp_data_unlock(parent); 32893e501490SPaolo Abeni 3290dc87efdbSFlorian Westphal if (!ret) { 32913e501490SPaolo Abeni err_prohibited: 3292dc87efdbSFlorian Westphal subflow->reset_reason = MPTCP_RST_EPROHIBIT; 329310f6d46cSPaolo Abeni return false; 3294dc87efdbSFlorian Westphal } 329510f6d46cSPaolo Abeni 3296917944daSMat Martineau subflow->map_seq = READ_ONCE(msk->ack_seq); 32970530020aSGeliang Tang WRITE_ONCE(msk->allow_infinite_fallback, false); 32983e501490SPaolo Abeni 3299b911c97cSFlorian Westphal out: 3300b911c97cSFlorian Westphal mptcp_event(MPTCP_EVENT_SUB_ESTABLISHED, msk, ssk, GFP_ATOMIC); 330110f6d46cSPaolo Abeni return true; 3302f296234cSPeter Krystad } 3303f296234cSPeter Krystad 330476e2a55dSPaolo Abeni static void mptcp_shutdown(struct sock *sk, int how) 330576e2a55dSPaolo Abeni { 330676e2a55dSPaolo Abeni pr_debug("sk=%p, how=%d", sk, how); 330776e2a55dSPaolo Abeni 330876e2a55dSPaolo Abeni if ((how & SEND_SHUTDOWN) && mptcp_close_state(sk)) 330976e2a55dSPaolo Abeni __mptcp_wr_shutdown(sk); 331076e2a55dSPaolo Abeni } 331176e2a55dSPaolo Abeni 33126511882cSPaolo Abeni static int mptcp_forward_alloc_get(const struct sock *sk) 33136511882cSPaolo Abeni { 33146511882cSPaolo Abeni return sk->sk_forward_alloc + mptcp_sk(sk)->rmem_fwd_alloc; 33156511882cSPaolo Abeni } 33166511882cSPaolo Abeni 3317644807e3SFlorian Westphal static int mptcp_ioctl_outq(const struct mptcp_sock *msk, u64 v) 3318644807e3SFlorian Westphal { 3319644807e3SFlorian Westphal const struct sock *sk = (void *)msk; 3320644807e3SFlorian Westphal u64 delta; 3321644807e3SFlorian Westphal 3322644807e3SFlorian Westphal if (sk->sk_state == TCP_LISTEN) 3323644807e3SFlorian Westphal return -EINVAL; 3324644807e3SFlorian Westphal 3325644807e3SFlorian Westphal if ((1 << sk->sk_state) & (TCPF_SYN_SENT | TCPF_SYN_RECV)) 3326644807e3SFlorian Westphal return 0; 3327644807e3SFlorian Westphal 3328644807e3SFlorian Westphal delta = msk->write_seq - v; 332907c2c7a3SPaolo Abeni if (__mptcp_check_fallback(msk) && msk->first) { 333007c2c7a3SPaolo Abeni struct tcp_sock *tp = tcp_sk(msk->first); 333107c2c7a3SPaolo Abeni 333207c2c7a3SPaolo Abeni /* the first subflow is disconnected after close - see 333307c2c7a3SPaolo Abeni * __mptcp_close_ssk(). tcp_disconnect() moves the write_seq 333407c2c7a3SPaolo Abeni * so ignore that status, too. 333507c2c7a3SPaolo Abeni */ 333607c2c7a3SPaolo Abeni if (!((1 << msk->first->sk_state) & 333707c2c7a3SPaolo Abeni (TCPF_SYN_SENT | TCPF_SYN_RECV | TCPF_CLOSE))) 333807c2c7a3SPaolo Abeni delta += READ_ONCE(tp->write_seq) - tp->snd_una; 333907c2c7a3SPaolo Abeni } 3340644807e3SFlorian Westphal if (delta > INT_MAX) 3341644807e3SFlorian Westphal delta = INT_MAX; 3342644807e3SFlorian Westphal 3343644807e3SFlorian Westphal return (int)delta; 3344644807e3SFlorian Westphal } 3345644807e3SFlorian Westphal 3346644807e3SFlorian Westphal static int mptcp_ioctl(struct sock *sk, int cmd, unsigned long arg) 3347644807e3SFlorian Westphal { 3348644807e3SFlorian Westphal struct mptcp_sock *msk = mptcp_sk(sk); 3349644807e3SFlorian Westphal bool slow; 3350644807e3SFlorian Westphal int answ; 3351644807e3SFlorian Westphal 3352644807e3SFlorian Westphal switch (cmd) { 3353644807e3SFlorian Westphal case SIOCINQ: 3354644807e3SFlorian Westphal if (sk->sk_state == TCP_LISTEN) 3355644807e3SFlorian Westphal return -EINVAL; 3356644807e3SFlorian Westphal 3357644807e3SFlorian Westphal lock_sock(sk); 3358644807e3SFlorian Westphal __mptcp_move_skbs(msk); 3359644807e3SFlorian Westphal answ = mptcp_inq_hint(sk); 3360644807e3SFlorian Westphal release_sock(sk); 3361644807e3SFlorian Westphal break; 3362644807e3SFlorian Westphal case SIOCOUTQ: 3363644807e3SFlorian Westphal slow = lock_sock_fast(sk); 3364644807e3SFlorian Westphal answ = mptcp_ioctl_outq(msk, READ_ONCE(msk->snd_una)); 3365644807e3SFlorian Westphal unlock_sock_fast(sk, slow); 3366644807e3SFlorian Westphal break; 3367644807e3SFlorian Westphal case SIOCOUTQNSD: 3368644807e3SFlorian Westphal slow = lock_sock_fast(sk); 3369644807e3SFlorian Westphal answ = mptcp_ioctl_outq(msk, msk->snd_nxt); 3370644807e3SFlorian Westphal unlock_sock_fast(sk, slow); 3371644807e3SFlorian Westphal break; 3372644807e3SFlorian Westphal default: 3373644807e3SFlorian Westphal return -ENOIOCTLCMD; 3374644807e3SFlorian Westphal } 3375644807e3SFlorian Westphal 3376644807e3SFlorian Westphal return put_user(answ, (int __user *)arg); 3377644807e3SFlorian Westphal } 3378644807e3SFlorian Westphal 3379f870fa0bSMat Martineau static struct proto mptcp_prot = { 3380f870fa0bSMat Martineau .name = "MPTCP", 3381f870fa0bSMat Martineau .owner = THIS_MODULE, 3382f870fa0bSMat Martineau .init = mptcp_init_sock, 338318b683bfSPaolo Abeni .disconnect = mptcp_disconnect, 3384f870fa0bSMat Martineau .close = mptcp_close, 3385cf7da0d6SPeter Krystad .accept = mptcp_accept, 3386717e79c8SPeter Krystad .setsockopt = mptcp_setsockopt, 3387717e79c8SPeter Krystad .getsockopt = mptcp_getsockopt, 338876e2a55dSPaolo Abeni .shutdown = mptcp_shutdown, 338979c0949eSPeter Krystad .destroy = mptcp_destroy, 3390f870fa0bSMat Martineau .sendmsg = mptcp_sendmsg, 3391644807e3SFlorian Westphal .ioctl = mptcp_ioctl, 3392f870fa0bSMat Martineau .recvmsg = mptcp_recvmsg, 339314c441b5SPaolo Abeni .release_cb = mptcp_release_cb, 33942c5ebd00SPaolo Abeni .hash = mptcp_hash, 33952c5ebd00SPaolo Abeni .unhash = mptcp_unhash, 3396cec37a6eSPeter Krystad .get_port = mptcp_get_port, 33976511882cSPaolo Abeni .forward_alloc_get = mptcp_forward_alloc_get, 3398d027236cSPaolo Abeni .sockets_allocated = &mptcp_sockets_allocated, 3399d027236cSPaolo Abeni .memory_allocated = &tcp_memory_allocated, 3400d027236cSPaolo Abeni .memory_pressure = &tcp_memory_pressure, 3401d027236cSPaolo Abeni .sysctl_wmem_offset = offsetof(struct net, ipv4.sysctl_tcp_wmem), 3402989ef49bSPaolo Abeni .sysctl_rmem_offset = offsetof(struct net, ipv4.sysctl_tcp_rmem), 3403d027236cSPaolo Abeni .sysctl_mem = sysctl_tcp_mem, 3404f870fa0bSMat Martineau .obj_size = sizeof(struct mptcp_sock), 34052c5ebd00SPaolo Abeni .slab_flags = SLAB_TYPESAFE_BY_RCU, 3406f870fa0bSMat Martineau .no_autobind = true, 3407f870fa0bSMat Martineau }; 3408f870fa0bSMat Martineau 34092303f994SPeter Krystad static int mptcp_bind(struct socket *sock, struct sockaddr *uaddr, int addr_len) 34102303f994SPeter Krystad { 34112303f994SPeter Krystad struct mptcp_sock *msk = mptcp_sk(sock->sk); 34122303f994SPeter Krystad struct socket *ssock; 3413cf7da0d6SPeter Krystad int err; 34142303f994SPeter Krystad 34152303f994SPeter Krystad lock_sock(sock->sk); 3416fa68018dSPaolo Abeni ssock = __mptcp_nmpc_socket(msk); 3417fa68018dSPaolo Abeni if (!ssock) { 3418fa68018dSPaolo Abeni err = -EINVAL; 34192303f994SPeter Krystad goto unlock; 34202303f994SPeter Krystad } 34212303f994SPeter Krystad 34222303f994SPeter Krystad err = ssock->ops->bind(ssock, uaddr, addr_len); 3423cf7da0d6SPeter Krystad if (!err) 3424cf7da0d6SPeter Krystad mptcp_copy_inaddrs(sock->sk, ssock->sk); 34252303f994SPeter Krystad 34262303f994SPeter Krystad unlock: 34272303f994SPeter Krystad release_sock(sock->sk); 34282303f994SPeter Krystad return err; 34292303f994SPeter Krystad } 34302303f994SPeter Krystad 34310235d075SPaolo Abeni static void mptcp_subflow_early_fallback(struct mptcp_sock *msk, 34320235d075SPaolo Abeni struct mptcp_subflow_context *subflow) 34330235d075SPaolo Abeni { 34340235d075SPaolo Abeni subflow->request_mptcp = 0; 34350235d075SPaolo Abeni __mptcp_do_fallback(msk); 34360235d075SPaolo Abeni } 34370235d075SPaolo Abeni 34382303f994SPeter Krystad static int mptcp_stream_connect(struct socket *sock, struct sockaddr *uaddr, 34392303f994SPeter Krystad int addr_len, int flags) 34402303f994SPeter Krystad { 34412303f994SPeter Krystad struct mptcp_sock *msk = mptcp_sk(sock->sk); 34422c5ebd00SPaolo Abeni struct mptcp_subflow_context *subflow; 34432303f994SPeter Krystad struct socket *ssock; 34443d1d6d66SPaolo Abeni int err = -EINVAL; 34452303f994SPeter Krystad 34462303f994SPeter Krystad lock_sock(sock->sk); 34473d1d6d66SPaolo Abeni if (uaddr) { 34483d1d6d66SPaolo Abeni if (addr_len < sizeof(uaddr->sa_family)) 34493d1d6d66SPaolo Abeni goto unlock; 34503d1d6d66SPaolo Abeni 34513d1d6d66SPaolo Abeni if (uaddr->sa_family == AF_UNSPEC) { 34523d1d6d66SPaolo Abeni err = mptcp_disconnect(sock->sk, flags); 34533d1d6d66SPaolo Abeni sock->state = err ? SS_DISCONNECTING : SS_UNCONNECTED; 34543d1d6d66SPaolo Abeni goto unlock; 34553d1d6d66SPaolo Abeni } 34563d1d6d66SPaolo Abeni } 34573d1d6d66SPaolo Abeni 345841be81a8SPaolo Abeni if (sock->state != SS_UNCONNECTED && msk->subflow) { 345941be81a8SPaolo Abeni /* pending connection or invalid state, let existing subflow 346041be81a8SPaolo Abeni * cope with that 346141be81a8SPaolo Abeni */ 346241be81a8SPaolo Abeni ssock = msk->subflow; 346341be81a8SPaolo Abeni goto do_connect; 346441be81a8SPaolo Abeni } 346541be81a8SPaolo Abeni 3466fa68018dSPaolo Abeni ssock = __mptcp_nmpc_socket(msk); 34673d1d6d66SPaolo Abeni if (!ssock) 34682303f994SPeter Krystad goto unlock; 34692303f994SPeter Krystad 3470fa68018dSPaolo Abeni mptcp_token_destroy(msk); 3471fa68018dSPaolo Abeni inet_sk_state_store(sock->sk, TCP_SYN_SENT); 34722c5ebd00SPaolo Abeni subflow = mptcp_subflow_ctx(ssock->sk); 3473cf7da0d6SPeter Krystad #ifdef CONFIG_TCP_MD5SIG 3474cf7da0d6SPeter Krystad /* no MPTCP if MD5SIG is enabled on this socket or we may run out of 3475cf7da0d6SPeter Krystad * TCP option space. 3476cf7da0d6SPeter Krystad */ 3477cf7da0d6SPeter Krystad if (rcu_access_pointer(tcp_sk(ssock->sk)->md5sig_info)) 34780235d075SPaolo Abeni mptcp_subflow_early_fallback(msk, subflow); 3479cf7da0d6SPeter Krystad #endif 3480a16195e3SPaolo Abeni if (subflow->request_mptcp && mptcp_token_new_connect(ssock->sk)) { 3481a16195e3SPaolo Abeni MPTCP_INC_STATS(sock_net(ssock->sk), MPTCP_MIB_TOKENFALLBACKINIT); 34820235d075SPaolo Abeni mptcp_subflow_early_fallback(msk, subflow); 3483a16195e3SPaolo Abeni } 34845695eb88SPaolo Abeni if (likely(!__mptcp_check_fallback(msk))) 34855695eb88SPaolo Abeni MPTCP_INC_STATS(sock_net(sock->sk), MPTCP_MIB_MPCAPABLEACTIVE); 3486cf7da0d6SPeter Krystad 348741be81a8SPaolo Abeni do_connect: 34882303f994SPeter Krystad err = ssock->ops->connect(ssock, uaddr, addr_len, flags); 348941be81a8SPaolo Abeni sock->state = ssock->state; 349041be81a8SPaolo Abeni 349141be81a8SPaolo Abeni /* on successful connect, the msk state will be moved to established by 349241be81a8SPaolo Abeni * subflow_finish_connect() 349341be81a8SPaolo Abeni */ 3494367fe04eSMatthieu Baerts if (!err || err == -EINPROGRESS) 3495cf7da0d6SPeter Krystad mptcp_copy_inaddrs(sock->sk, ssock->sk); 349641be81a8SPaolo Abeni else 349741be81a8SPaolo Abeni inet_sk_state_store(sock->sk, inet_sk_state_load(ssock->sk)); 34982303f994SPeter Krystad 34992303f994SPeter Krystad unlock: 35002303f994SPeter Krystad release_sock(sock->sk); 35012303f994SPeter Krystad return err; 35022303f994SPeter Krystad } 35032303f994SPeter Krystad 3504cf7da0d6SPeter Krystad static int mptcp_listen(struct socket *sock, int backlog) 3505cf7da0d6SPeter Krystad { 3506cf7da0d6SPeter Krystad struct mptcp_sock *msk = mptcp_sk(sock->sk); 3507cf7da0d6SPeter Krystad struct socket *ssock; 3508cf7da0d6SPeter Krystad int err; 3509cf7da0d6SPeter Krystad 3510cf7da0d6SPeter Krystad pr_debug("msk=%p", msk); 3511cf7da0d6SPeter Krystad 3512cf7da0d6SPeter Krystad lock_sock(sock->sk); 3513fa68018dSPaolo Abeni ssock = __mptcp_nmpc_socket(msk); 3514fa68018dSPaolo Abeni if (!ssock) { 3515fa68018dSPaolo Abeni err = -EINVAL; 3516cf7da0d6SPeter Krystad goto unlock; 3517cf7da0d6SPeter Krystad } 3518cf7da0d6SPeter Krystad 3519fa68018dSPaolo Abeni mptcp_token_destroy(msk); 3520fa68018dSPaolo Abeni inet_sk_state_store(sock->sk, TCP_LISTEN); 35215e20087dSFlorian Westphal sock_set_flag(sock->sk, SOCK_RCU_FREE); 35225e20087dSFlorian Westphal 3523cf7da0d6SPeter Krystad err = ssock->ops->listen(ssock, backlog); 3524cf7da0d6SPeter Krystad inet_sk_state_store(sock->sk, inet_sk_state_load(ssock->sk)); 3525cf7da0d6SPeter Krystad if (!err) 3526cf7da0d6SPeter Krystad mptcp_copy_inaddrs(sock->sk, ssock->sk); 3527cf7da0d6SPeter Krystad 3528cf7da0d6SPeter Krystad unlock: 3529cf7da0d6SPeter Krystad release_sock(sock->sk); 3530cf7da0d6SPeter Krystad return err; 3531cf7da0d6SPeter Krystad } 3532cf7da0d6SPeter Krystad 3533cf7da0d6SPeter Krystad static int mptcp_stream_accept(struct socket *sock, struct socket *newsock, 3534cf7da0d6SPeter Krystad int flags, bool kern) 3535cf7da0d6SPeter Krystad { 3536cf7da0d6SPeter Krystad struct mptcp_sock *msk = mptcp_sk(sock->sk); 3537cf7da0d6SPeter Krystad struct socket *ssock; 3538cf7da0d6SPeter Krystad int err; 3539cf7da0d6SPeter Krystad 3540cf7da0d6SPeter Krystad pr_debug("msk=%p", msk); 3541cf7da0d6SPeter Krystad 3542cf7da0d6SPeter Krystad ssock = __mptcp_nmpc_socket(msk); 3543cf7da0d6SPeter Krystad if (!ssock) 354471ba088cSPaolo Abeni return -EINVAL; 3545cf7da0d6SPeter Krystad 3546cf7da0d6SPeter Krystad err = ssock->ops->accept(sock, newsock, flags, kern); 3547d2f77c53SPaolo Abeni if (err == 0 && !mptcp_is_tcpsk(newsock->sk)) { 3548cf7da0d6SPeter Krystad struct mptcp_sock *msk = mptcp_sk(newsock->sk); 3549cf7da0d6SPeter Krystad struct mptcp_subflow_context *subflow; 35500397c6d8SPaolo Abeni struct sock *newsk = newsock->sk; 35510397c6d8SPaolo Abeni 35524d54cc32SFlorian Westphal lock_sock(newsk); 35535b950ff4SPaolo Abeni 35545b950ff4SPaolo Abeni /* PM/worker can now acquire the first subflow socket 35555b950ff4SPaolo Abeni * lock without racing with listener queue cleanup, 35565b950ff4SPaolo Abeni * we can notify it, if needed. 3557c8fe62f0SFlorian Westphal * 3558c8fe62f0SFlorian Westphal * Even if remote has reset the initial subflow by now 3559c8fe62f0SFlorian Westphal * the refcnt is still at least one. 35605b950ff4SPaolo Abeni */ 35615b950ff4SPaolo Abeni subflow = mptcp_subflow_ctx(msk->first); 35625b950ff4SPaolo Abeni list_add(&subflow->node, &msk->conn_list); 35635b950ff4SPaolo Abeni sock_hold(msk->first); 35645b950ff4SPaolo Abeni if (mptcp_is_fully_established(newsk)) 35656c714f1bSFlorian Westphal mptcp_pm_fully_established(msk, msk->first, GFP_KERNEL); 35665b950ff4SPaolo Abeni 35670397c6d8SPaolo Abeni mptcp_copy_inaddrs(newsk, msk->first); 35680397c6d8SPaolo Abeni mptcp_rcv_space_init(msk, msk->first); 35695cf92bbaSPaolo Abeni mptcp_propagate_sndbuf(newsk, msk->first); 3570cf7da0d6SPeter Krystad 3571cf7da0d6SPeter Krystad /* set ssk->sk_socket of accept()ed flows to mptcp socket. 3572cf7da0d6SPeter Krystad * This is needed so NOSPACE flag can be set from tcp stack. 3573cf7da0d6SPeter Krystad */ 3574190f8b06SGeliang Tang mptcp_for_each_subflow(msk, subflow) { 3575cf7da0d6SPeter Krystad struct sock *ssk = mptcp_subflow_tcp_sock(subflow); 3576cf7da0d6SPeter Krystad 3577cf7da0d6SPeter Krystad if (!ssk->sk_socket) 3578cf7da0d6SPeter Krystad mptcp_sock_graft(ssk, newsock); 3579cf7da0d6SPeter Krystad } 35804d54cc32SFlorian Westphal release_sock(newsk); 3581cf7da0d6SPeter Krystad } 3582cf7da0d6SPeter Krystad 3583cf7da0d6SPeter Krystad return err; 3584cf7da0d6SPeter Krystad } 3585cf7da0d6SPeter Krystad 35868a05661bSPaolo Abeni static __poll_t mptcp_check_readable(struct mptcp_sock *msk) 35878a05661bSPaolo Abeni { 3588612f71d7SPaolo Abeni /* Concurrent splices from sk_receive_queue into receive_queue will 3589612f71d7SPaolo Abeni * always show at least one non-empty queue when checked in this order. 3590612f71d7SPaolo Abeni */ 3591612f71d7SPaolo Abeni if (skb_queue_empty_lockless(&((struct sock *)msk)->sk_receive_queue) && 3592612f71d7SPaolo Abeni skb_queue_empty_lockless(&msk->receive_queue)) 3593612f71d7SPaolo Abeni return 0; 3594612f71d7SPaolo Abeni 3595612f71d7SPaolo Abeni return EPOLLIN | EPOLLRDNORM; 35968a05661bSPaolo Abeni } 35978a05661bSPaolo Abeni 35988edf0864SFlorian Westphal static __poll_t mptcp_check_writeable(struct mptcp_sock *msk) 35998edf0864SFlorian Westphal { 36008edf0864SFlorian Westphal struct sock *sk = (struct sock *)msk; 36018edf0864SFlorian Westphal 36028edf0864SFlorian Westphal if (unlikely(sk->sk_shutdown & SEND_SHUTDOWN)) 3603dd913410SPaolo Abeni return EPOLLOUT | EPOLLWRNORM; 36048edf0864SFlorian Westphal 36058edf0864SFlorian Westphal if (sk_stream_is_writeable(sk)) 36068edf0864SFlorian Westphal return EPOLLOUT | EPOLLWRNORM; 36078edf0864SFlorian Westphal 36085cf92bbaSPaolo Abeni mptcp_set_nospace(sk); 36096e628cd3SPaolo Abeni smp_mb__after_atomic(); /* msk->flags is changed by write_space cb */ 36106e628cd3SPaolo Abeni if (sk_stream_is_writeable(sk)) 36116e628cd3SPaolo Abeni return EPOLLOUT | EPOLLWRNORM; 36128edf0864SFlorian Westphal 36136e628cd3SPaolo Abeni return 0; 36148edf0864SFlorian Westphal } 36158edf0864SFlorian Westphal 36162303f994SPeter Krystad static __poll_t mptcp_poll(struct file *file, struct socket *sock, 36172303f994SPeter Krystad struct poll_table_struct *wait) 36182303f994SPeter Krystad { 36191891c4a0SFlorian Westphal struct sock *sk = sock->sk; 36208ab183deSPaolo Abeni struct mptcp_sock *msk; 36212303f994SPeter Krystad __poll_t mask = 0; 36228a05661bSPaolo Abeni int state; 36232303f994SPeter Krystad 36241891c4a0SFlorian Westphal msk = mptcp_sk(sk); 36251891c4a0SFlorian Westphal sock_poll_wait(file, sock, wait); 36261891c4a0SFlorian Westphal 36278a05661bSPaolo Abeni state = inet_sk_state_load(sk); 36286719331cSPaolo Abeni pr_debug("msk=%p state=%d flags=%lx", msk, state, msk->flags); 362971ba088cSPaolo Abeni if (state == TCP_LISTEN) { 363071ba088cSPaolo Abeni if (WARN_ON_ONCE(!msk->subflow || !msk->subflow->sk)) 363171ba088cSPaolo Abeni return 0; 363271ba088cSPaolo Abeni 363371ba088cSPaolo Abeni return inet_csk_listen_poll(msk->subflow->sk); 363471ba088cSPaolo Abeni } 36358a05661bSPaolo Abeni 36368a05661bSPaolo Abeni if (state != TCP_SYN_SENT && state != TCP_SYN_RECV) { 36378a05661bSPaolo Abeni mask |= mptcp_check_readable(msk); 36388edf0864SFlorian Westphal mask |= mptcp_check_writeable(msk); 36398a05661bSPaolo Abeni } 3640dd913410SPaolo Abeni if (sk->sk_shutdown == SHUTDOWN_MASK || state == TCP_CLOSE) 3641dd913410SPaolo Abeni mask |= EPOLLHUP; 36421891c4a0SFlorian Westphal if (sk->sk_shutdown & RCV_SHUTDOWN) 36431891c4a0SFlorian Westphal mask |= EPOLLIN | EPOLLRDNORM | EPOLLRDHUP; 36441891c4a0SFlorian Westphal 364515cc1045SPaolo Abeni /* This barrier is coupled with smp_wmb() in tcp_reset() */ 364615cc1045SPaolo Abeni smp_rmb(); 364715cc1045SPaolo Abeni if (sk->sk_err) 364815cc1045SPaolo Abeni mask |= EPOLLERR; 364915cc1045SPaolo Abeni 36502303f994SPeter Krystad return mask; 36512303f994SPeter Krystad } 36522303f994SPeter Krystad 3653e42f1ac6SFlorian Westphal static const struct proto_ops mptcp_stream_ops = { 3654e42f1ac6SFlorian Westphal .family = PF_INET, 3655e42f1ac6SFlorian Westphal .owner = THIS_MODULE, 36560a3cc579SPaolo Abeni .release = inet_release, 3657e42f1ac6SFlorian Westphal .bind = mptcp_bind, 3658e42f1ac6SFlorian Westphal .connect = mptcp_stream_connect, 3659e42f1ac6SFlorian Westphal .socketpair = sock_no_socketpair, 3660e42f1ac6SFlorian Westphal .accept = mptcp_stream_accept, 3661d2f77c53SPaolo Abeni .getname = inet_getname, 3662e42f1ac6SFlorian Westphal .poll = mptcp_poll, 3663e42f1ac6SFlorian Westphal .ioctl = inet_ioctl, 3664e42f1ac6SFlorian Westphal .gettstamp = sock_gettstamp, 3665e42f1ac6SFlorian Westphal .listen = mptcp_listen, 366676e2a55dSPaolo Abeni .shutdown = inet_shutdown, 3667e42f1ac6SFlorian Westphal .setsockopt = sock_common_setsockopt, 3668e42f1ac6SFlorian Westphal .getsockopt = sock_common_getsockopt, 3669e42f1ac6SFlorian Westphal .sendmsg = inet_sendmsg, 3670e42f1ac6SFlorian Westphal .recvmsg = inet_recvmsg, 3671e42f1ac6SFlorian Westphal .mmap = sock_no_mmap, 3672e42f1ac6SFlorian Westphal .sendpage = inet_sendpage, 3673e42f1ac6SFlorian Westphal }; 36742303f994SPeter Krystad 3675f870fa0bSMat Martineau static struct inet_protosw mptcp_protosw = { 3676f870fa0bSMat Martineau .type = SOCK_STREAM, 3677f870fa0bSMat Martineau .protocol = IPPROTO_MPTCP, 3678f870fa0bSMat Martineau .prot = &mptcp_prot, 36792303f994SPeter Krystad .ops = &mptcp_stream_ops, 36802303f994SPeter Krystad .flags = INET_PROTOSW_ICSK, 3681f870fa0bSMat Martineau }; 3682f870fa0bSMat Martineau 3683b19bc294SPaolo Abeni static int mptcp_napi_poll(struct napi_struct *napi, int budget) 3684b19bc294SPaolo Abeni { 3685b19bc294SPaolo Abeni struct mptcp_delegated_action *delegated; 3686b19bc294SPaolo Abeni struct mptcp_subflow_context *subflow; 3687b19bc294SPaolo Abeni int work_done = 0; 3688b19bc294SPaolo Abeni 3689b19bc294SPaolo Abeni delegated = container_of(napi, struct mptcp_delegated_action, napi); 3690b19bc294SPaolo Abeni while ((subflow = mptcp_subflow_delegated_next(delegated)) != NULL) { 3691b19bc294SPaolo Abeni struct sock *ssk = mptcp_subflow_tcp_sock(subflow); 3692b19bc294SPaolo Abeni 3693b19bc294SPaolo Abeni bh_lock_sock_nested(ssk); 3694b19bc294SPaolo Abeni if (!sock_owned_by_user(ssk) && 3695b19bc294SPaolo Abeni mptcp_subflow_has_delegated_action(subflow)) 3696b19bc294SPaolo Abeni mptcp_subflow_process_delegated(ssk); 3697b19bc294SPaolo Abeni /* ... elsewhere tcp_release_cb_override already processed 3698b19bc294SPaolo Abeni * the action or will do at next release_sock(). 3699b19bc294SPaolo Abeni * In both case must dequeue the subflow here - on the same 3700b19bc294SPaolo Abeni * CPU that scheduled it. 3701b19bc294SPaolo Abeni */ 3702b19bc294SPaolo Abeni bh_unlock_sock(ssk); 3703b19bc294SPaolo Abeni sock_put(ssk); 3704b19bc294SPaolo Abeni 3705b19bc294SPaolo Abeni if (++work_done == budget) 3706b19bc294SPaolo Abeni return budget; 3707b19bc294SPaolo Abeni } 3708b19bc294SPaolo Abeni 3709b19bc294SPaolo Abeni /* always provide a 0 'work_done' argument, so that napi_complete_done 3710b19bc294SPaolo Abeni * will not try accessing the NULL napi->dev ptr 3711b19bc294SPaolo Abeni */ 3712b19bc294SPaolo Abeni napi_complete_done(napi, 0); 3713b19bc294SPaolo Abeni return work_done; 3714b19bc294SPaolo Abeni } 3715b19bc294SPaolo Abeni 3716d39dcecaSPaolo Abeni void __init mptcp_proto_init(void) 3717f870fa0bSMat Martineau { 3718b19bc294SPaolo Abeni struct mptcp_delegated_action *delegated; 3719b19bc294SPaolo Abeni int cpu; 3720b19bc294SPaolo Abeni 37212303f994SPeter Krystad mptcp_prot.h.hashinfo = tcp_prot.h.hashinfo; 37222303f994SPeter Krystad 3723d027236cSPaolo Abeni if (percpu_counter_init(&mptcp_sockets_allocated, 0, GFP_KERNEL)) 3724d027236cSPaolo Abeni panic("Failed to allocate MPTCP pcpu counter\n"); 3725d027236cSPaolo Abeni 3726b19bc294SPaolo Abeni init_dummy_netdev(&mptcp_napi_dev); 3727b19bc294SPaolo Abeni for_each_possible_cpu(cpu) { 3728b19bc294SPaolo Abeni delegated = per_cpu_ptr(&mptcp_delegated_actions, cpu); 3729b19bc294SPaolo Abeni INIT_LIST_HEAD(&delegated->head); 3730b19bc294SPaolo Abeni netif_tx_napi_add(&mptcp_napi_dev, &delegated->napi, mptcp_napi_poll, 3731b19bc294SPaolo Abeni NAPI_POLL_WEIGHT); 3732b19bc294SPaolo Abeni napi_enable(&delegated->napi); 3733b19bc294SPaolo Abeni } 3734b19bc294SPaolo Abeni 37352303f994SPeter Krystad mptcp_subflow_init(); 37361b1c7a0eSPeter Krystad mptcp_pm_init(); 37372c5ebd00SPaolo Abeni mptcp_token_init(); 37382303f994SPeter Krystad 3739f870fa0bSMat Martineau if (proto_register(&mptcp_prot, 1) != 0) 3740f870fa0bSMat Martineau panic("Failed to register MPTCP proto.\n"); 3741f870fa0bSMat Martineau 3742f870fa0bSMat Martineau inet_register_protosw(&mptcp_protosw); 37436771bfd9SFlorian Westphal 37446771bfd9SFlorian Westphal BUILD_BUG_ON(sizeof(struct mptcp_skb_cb) > sizeof_field(struct sk_buff, cb)); 3745f870fa0bSMat Martineau } 3746f870fa0bSMat Martineau 3747f870fa0bSMat Martineau #if IS_ENABLED(CONFIG_MPTCP_IPV6) 3748e42f1ac6SFlorian Westphal static const struct proto_ops mptcp_v6_stream_ops = { 3749e42f1ac6SFlorian Westphal .family = PF_INET6, 3750e42f1ac6SFlorian Westphal .owner = THIS_MODULE, 37510a3cc579SPaolo Abeni .release = inet6_release, 3752e42f1ac6SFlorian Westphal .bind = mptcp_bind, 3753e42f1ac6SFlorian Westphal .connect = mptcp_stream_connect, 3754e42f1ac6SFlorian Westphal .socketpair = sock_no_socketpair, 3755e42f1ac6SFlorian Westphal .accept = mptcp_stream_accept, 3756d2f77c53SPaolo Abeni .getname = inet6_getname, 3757e42f1ac6SFlorian Westphal .poll = mptcp_poll, 3758e42f1ac6SFlorian Westphal .ioctl = inet6_ioctl, 3759e42f1ac6SFlorian Westphal .gettstamp = sock_gettstamp, 3760e42f1ac6SFlorian Westphal .listen = mptcp_listen, 376176e2a55dSPaolo Abeni .shutdown = inet_shutdown, 3762e42f1ac6SFlorian Westphal .setsockopt = sock_common_setsockopt, 3763e42f1ac6SFlorian Westphal .getsockopt = sock_common_getsockopt, 3764e42f1ac6SFlorian Westphal .sendmsg = inet6_sendmsg, 3765e42f1ac6SFlorian Westphal .recvmsg = inet6_recvmsg, 3766e42f1ac6SFlorian Westphal .mmap = sock_no_mmap, 3767e42f1ac6SFlorian Westphal .sendpage = inet_sendpage, 3768e42f1ac6SFlorian Westphal #ifdef CONFIG_COMPAT 37693986912fSChristoph Hellwig .compat_ioctl = inet6_compat_ioctl, 3770e42f1ac6SFlorian Westphal #endif 3771e42f1ac6SFlorian Westphal }; 3772e42f1ac6SFlorian Westphal 3773f870fa0bSMat Martineau static struct proto mptcp_v6_prot; 3774f870fa0bSMat Martineau 377579c0949eSPeter Krystad static void mptcp_v6_destroy(struct sock *sk) 377679c0949eSPeter Krystad { 377779c0949eSPeter Krystad mptcp_destroy(sk); 377879c0949eSPeter Krystad inet6_destroy_sock(sk); 377979c0949eSPeter Krystad } 378079c0949eSPeter Krystad 3781f870fa0bSMat Martineau static struct inet_protosw mptcp_v6_protosw = { 3782f870fa0bSMat Martineau .type = SOCK_STREAM, 3783f870fa0bSMat Martineau .protocol = IPPROTO_MPTCP, 3784f870fa0bSMat Martineau .prot = &mptcp_v6_prot, 37852303f994SPeter Krystad .ops = &mptcp_v6_stream_ops, 3786f870fa0bSMat Martineau .flags = INET_PROTOSW_ICSK, 3787f870fa0bSMat Martineau }; 3788f870fa0bSMat Martineau 3789d39dcecaSPaolo Abeni int __init mptcp_proto_v6_init(void) 3790f870fa0bSMat Martineau { 3791f870fa0bSMat Martineau int err; 3792f870fa0bSMat Martineau 3793f870fa0bSMat Martineau mptcp_v6_prot = mptcp_prot; 3794f870fa0bSMat Martineau strcpy(mptcp_v6_prot.name, "MPTCPv6"); 3795f870fa0bSMat Martineau mptcp_v6_prot.slab = NULL; 379679c0949eSPeter Krystad mptcp_v6_prot.destroy = mptcp_v6_destroy; 3797b0519de8SFlorian Westphal mptcp_v6_prot.obj_size = sizeof(struct mptcp6_sock); 3798f870fa0bSMat Martineau 3799f870fa0bSMat Martineau err = proto_register(&mptcp_v6_prot, 1); 3800f870fa0bSMat Martineau if (err) 3801f870fa0bSMat Martineau return err; 3802f870fa0bSMat Martineau 3803f870fa0bSMat Martineau err = inet6_register_protosw(&mptcp_v6_protosw); 3804f870fa0bSMat Martineau if (err) 3805f870fa0bSMat Martineau proto_unregister(&mptcp_v6_prot); 3806f870fa0bSMat Martineau 3807f870fa0bSMat Martineau return err; 3808f870fa0bSMat Martineau } 3809f870fa0bSMat Martineau #endif 3810