1f870fa0bSMat Martineau // SPDX-License-Identifier: GPL-2.0
2f870fa0bSMat Martineau /* Multipath TCP
3f870fa0bSMat Martineau *
4f870fa0bSMat Martineau * Copyright (c) 2017 - 2019, Intel Corporation.
5f870fa0bSMat Martineau */
6f870fa0bSMat Martineau
7f870fa0bSMat Martineau #define pr_fmt(fmt) "MPTCP: " fmt
8f870fa0bSMat Martineau
9f870fa0bSMat Martineau #include <linux/kernel.h>
10f870fa0bSMat Martineau #include <linux/module.h>
11f870fa0bSMat Martineau #include <linux/netdevice.h>
127a6a6cbcSPaolo Abeni #include <linux/sched/signal.h>
137a6a6cbcSPaolo Abeni #include <linux/atomic.h>
14f870fa0bSMat Martineau #include <net/sock.h>
15f870fa0bSMat Martineau #include <net/inet_common.h>
16f870fa0bSMat Martineau #include <net/inet_hashtables.h>
17f870fa0bSMat Martineau #include <net/protocol.h>
183721b9b6SMat Martineau #include <net/tcp_states.h>
19cf7da0d6SPeter Krystad #if IS_ENABLED(CONFIG_MPTCP_IPV6)
20cf7da0d6SPeter Krystad #include <net/transp_v6.h>
21cf7da0d6SPeter Krystad #endif
22f870fa0bSMat Martineau #include <net/mptcp.h>
23a86a0661SEric Dumazet #include <net/hotdata.h>
24e16163b6SPaolo Abeni #include <net/xfrm.h>
25644807e3SFlorian Westphal #include <asm/ioctls.h>
26f870fa0bSMat Martineau #include "protocol.h"
27fc518953SFlorian Westphal #include "mib.h"
28f870fa0bSMat Martineau
29e10a9892SGeliang Tang #define CREATE_TRACE_POINTS
30e10a9892SGeliang Tang #include <trace/events/mptcp.h>
31e10a9892SGeliang Tang
32b0519de8SFlorian Westphal #if IS_ENABLED(CONFIG_MPTCP_IPV6)
33b0519de8SFlorian Westphal struct mptcp6_sock {
34b0519de8SFlorian Westphal struct mptcp_sock msk;
35b0519de8SFlorian Westphal struct ipv6_pinfo np;
36b0519de8SFlorian Westphal };
37b0519de8SFlorian Westphal #endif
38b0519de8SFlorian Westphal
39b7f653b2SFlorian Westphal enum {
40b7f653b2SFlorian Westphal MPTCP_CMSG_TS = BIT(0),
412c9e7765SFlorian Westphal MPTCP_CMSG_INQ = BIT(1),
42b7f653b2SFlorian Westphal };
43b7f653b2SFlorian Westphal
4491b6d325SEric Dumazet static struct percpu_counter mptcp_sockets_allocated ____cacheline_aligned_in_smp;
45d027236cSPaolo Abeni
46e16163b6SPaolo Abeni static void __mptcp_destroy_sock(struct sock *sk);
4781c1d029SPaolo Abeni static void mptcp_check_send_data_fin(struct sock *sk);
48e16163b6SPaolo Abeni
49b19bc294SPaolo Abeni DEFINE_PER_CPU(struct mptcp_delegated_action, mptcp_delegated_actions);
50b19bc294SPaolo Abeni static struct net_device mptcp_napi_dev;
51b19bc294SPaolo Abeni
526f8a612aSFlorian Westphal /* Returns end sequence number of the receiver's advertised window */
mptcp_wnd_end(const struct mptcp_sock * msk)536f8a612aSFlorian Westphal static u64 mptcp_wnd_end(const struct mptcp_sock *msk)
546f8a612aSFlorian Westphal {
557439d687SPaolo Abeni return READ_ONCE(msk->wnd_end);
566f8a612aSFlorian Westphal }
576f8a612aSFlorian Westphal
mptcp_fallback_tcp_ops(const struct sock * sk)588e2b8a9fSDavide Caratti static const struct proto_ops *mptcp_fallback_tcp_ops(const struct sock *sk)
590b4f33deSFlorian Westphal {
600b4f33deSFlorian Westphal #if IS_ENABLED(CONFIG_MPTCP_IPV6)
618e2b8a9fSDavide Caratti if (sk->sk_prot == &tcpv6_prot)
628e2b8a9fSDavide Caratti return &inet6_stream_ops;
630b4f33deSFlorian Westphal #endif
648e2b8a9fSDavide Caratti WARN_ON_ONCE(sk->sk_prot != &tcp_prot);
658e2b8a9fSDavide Caratti return &inet_stream_ops;
660b4f33deSFlorian Westphal }
670b4f33deSFlorian Westphal
__mptcp_socket_create(struct mptcp_sock * msk)68fa68018dSPaolo Abeni static int __mptcp_socket_create(struct mptcp_sock *msk)
692303f994SPeter Krystad {
702303f994SPeter Krystad struct mptcp_subflow_context *subflow;
712303f994SPeter Krystad struct sock *sk = (struct sock *)msk;
722303f994SPeter Krystad struct socket *ssock;
732303f994SPeter Krystad int err;
742303f994SPeter Krystad
756bc1fe7dSPaolo Abeni err = mptcp_subflow_create_socket(sk, sk->sk_family, &ssock);
762303f994SPeter Krystad if (err)
77fa68018dSPaolo Abeni return err;
782303f994SPeter Krystad
79b8dc6d6cSPaolo Abeni msk->scaling_ratio = tcp_sk(ssock->sk)->scaling_ratio;
801b1b43eeSPaolo Abeni WRITE_ONCE(msk->first, ssock->sk);
812303f994SPeter Krystad subflow = mptcp_subflow_ctx(ssock->sk);
82cec37a6eSPeter Krystad list_add(&subflow->node, &msk->conn_list);
83e16163b6SPaolo Abeni sock_hold(ssock->sk);
842303f994SPeter Krystad subflow->request_mptcp = 1;
856f06b4d4SPaolo Abeni subflow->subflow_id = msk->subflow_id++;
864cf86ae8SPaolo Abeni
874cf86ae8SPaolo Abeni /* This is the first subflow, always with id 0 */
88a7cfe776SPaolo Abeni WRITE_ONCE(subflow->local_id, 0);
89866f26f2SPaolo Abeni mptcp_sock_graft(msk->first, sk->sk_socket);
9039880bd8SPaolo Abeni iput(SOCK_INODE(ssock));
91e1ff9e82SDavide Caratti
92fa68018dSPaolo Abeni return 0;
932303f994SPeter Krystad }
942303f994SPeter Krystad
95ddb1a072SPaolo Abeni /* If the MPC handshake is not started, returns the first subflow,
96ddb1a072SPaolo Abeni * eventually allocating it.
97ddb1a072SPaolo Abeni */
__mptcp_nmpc_sk(struct mptcp_sock * msk)983f326a82SPaolo Abeni struct sock *__mptcp_nmpc_sk(struct mptcp_sock *msk)
99ddb1a072SPaolo Abeni {
100ddb1a072SPaolo Abeni struct sock *sk = (struct sock *)msk;
101ddb1a072SPaolo Abeni int ret;
102ddb1a072SPaolo Abeni
103ddb1a072SPaolo Abeni if (!((1 << sk->sk_state) & (TCPF_CLOSE | TCPF_LISTEN)))
104ddb1a072SPaolo Abeni return ERR_PTR(-EINVAL);
105ddb1a072SPaolo Abeni
1063f326a82SPaolo Abeni if (!msk->first) {
107ddb1a072SPaolo Abeni ret = __mptcp_socket_create(msk);
108ddb1a072SPaolo Abeni if (ret)
109ddb1a072SPaolo Abeni return ERR_PTR(ret);
110ddb1a072SPaolo Abeni }
111ddb1a072SPaolo Abeni
1123f326a82SPaolo Abeni return msk->first;
113ddb1a072SPaolo Abeni }
114ddb1a072SPaolo Abeni
mptcp_drop(struct sock * sk,struct sk_buff * skb)115ab174ad8SPaolo Abeni static void mptcp_drop(struct sock *sk, struct sk_buff *skb)
116ab174ad8SPaolo Abeni {
117ab174ad8SPaolo Abeni sk_drops_add(sk, skb);
118ab174ad8SPaolo Abeni __kfree_skb(skb);
119ab174ad8SPaolo Abeni }
120ab174ad8SPaolo Abeni
mptcp_rmem_fwd_alloc_add(struct sock * sk,int size)1219531e4a8SEric Dumazet static void mptcp_rmem_fwd_alloc_add(struct sock *sk, int size)
1229531e4a8SEric Dumazet {
1239531e4a8SEric Dumazet WRITE_ONCE(mptcp_sk(sk)->rmem_fwd_alloc,
1249531e4a8SEric Dumazet mptcp_sk(sk)->rmem_fwd_alloc + size);
1259531e4a8SEric Dumazet }
1269531e4a8SEric Dumazet
mptcp_rmem_charge(struct sock * sk,int size)1276511882cSPaolo Abeni static void mptcp_rmem_charge(struct sock *sk, int size)
1286511882cSPaolo Abeni {
1299531e4a8SEric Dumazet mptcp_rmem_fwd_alloc_add(sk, -size);
1306511882cSPaolo Abeni }
1316511882cSPaolo Abeni
mptcp_try_coalesce(struct sock * sk,struct sk_buff * to,struct sk_buff * from)1328268ed4cSPaolo Abeni static bool mptcp_try_coalesce(struct sock *sk, struct sk_buff *to,
1338268ed4cSPaolo Abeni struct sk_buff *from)
1348268ed4cSPaolo Abeni {
1358268ed4cSPaolo Abeni bool fragstolen;
1368268ed4cSPaolo Abeni int delta;
1378268ed4cSPaolo Abeni
1388268ed4cSPaolo Abeni if (MPTCP_SKB_CB(from)->offset ||
1398268ed4cSPaolo Abeni !skb_try_coalesce(to, from, &fragstolen, &delta))
1408268ed4cSPaolo Abeni return false;
1418268ed4cSPaolo Abeni
14206242e44SPaolo Abeni pr_debug("colesced seq %llx into %llx new len %d new end seq %llx",
14306242e44SPaolo Abeni MPTCP_SKB_CB(from)->map_seq, MPTCP_SKB_CB(to)->map_seq,
14406242e44SPaolo Abeni to->len, MPTCP_SKB_CB(from)->end_seq);
145ab174ad8SPaolo Abeni MPTCP_SKB_CB(to)->end_seq = MPTCP_SKB_CB(from)->end_seq;
1467288ff6eSPaolo Abeni
1477288ff6eSPaolo Abeni /* note the fwd memory can reach a negative value after accounting
1487288ff6eSPaolo Abeni * for the delta, but the later skb free will restore a non
1497288ff6eSPaolo Abeni * negative one
1507288ff6eSPaolo Abeni */
1518268ed4cSPaolo Abeni atomic_add(delta, &sk->sk_rmem_alloc);
1526511882cSPaolo Abeni mptcp_rmem_charge(sk, delta);
1537288ff6eSPaolo Abeni kfree_skb_partial(from, fragstolen);
1547288ff6eSPaolo Abeni
1558268ed4cSPaolo Abeni return true;
1568268ed4cSPaolo Abeni }
1578268ed4cSPaolo Abeni
mptcp_ooo_try_coalesce(struct mptcp_sock * msk,struct sk_buff * to,struct sk_buff * from)158ab174ad8SPaolo Abeni static bool mptcp_ooo_try_coalesce(struct mptcp_sock *msk, struct sk_buff *to,
159ab174ad8SPaolo Abeni struct sk_buff *from)
1606771bfd9SFlorian Westphal {
161ab174ad8SPaolo Abeni if (MPTCP_SKB_CB(from)->map_seq != MPTCP_SKB_CB(to)->end_seq)
162ab174ad8SPaolo Abeni return false;
163ab174ad8SPaolo Abeni
164ab174ad8SPaolo Abeni return mptcp_try_coalesce((struct sock *)msk, to, from);
165ab174ad8SPaolo Abeni }
166ab174ad8SPaolo Abeni
__mptcp_rmem_reclaim(struct sock * sk,int amount)1676511882cSPaolo Abeni static void __mptcp_rmem_reclaim(struct sock *sk, int amount)
1686511882cSPaolo Abeni {
169100fdd1fSEric Dumazet amount >>= PAGE_SHIFT;
1709531e4a8SEric Dumazet mptcp_rmem_charge(sk, amount << PAGE_SHIFT);
1716511882cSPaolo Abeni __sk_mem_reduce_allocated(sk, amount);
1726511882cSPaolo Abeni }
1736511882cSPaolo Abeni
mptcp_rmem_uncharge(struct sock * sk,int size)1746511882cSPaolo Abeni static void mptcp_rmem_uncharge(struct sock *sk, int size)
1756511882cSPaolo Abeni {
1766511882cSPaolo Abeni struct mptcp_sock *msk = mptcp_sk(sk);
1776511882cSPaolo Abeni int reclaimable;
1786511882cSPaolo Abeni
1799531e4a8SEric Dumazet mptcp_rmem_fwd_alloc_add(sk, size);
1806511882cSPaolo Abeni reclaimable = msk->rmem_fwd_alloc - sk_unused_reserved_mem(sk);
1816511882cSPaolo Abeni
1826511882cSPaolo Abeni /* see sk_mem_uncharge() for the rationale behind the following schema */
183d24141feSPaolo Abeni if (unlikely(reclaimable >= PAGE_SIZE))
184d24141feSPaolo Abeni __mptcp_rmem_reclaim(sk, reclaimable);
1856511882cSPaolo Abeni }
1866511882cSPaolo Abeni
mptcp_rfree(struct sk_buff * skb)1876511882cSPaolo Abeni static void mptcp_rfree(struct sk_buff *skb)
1886511882cSPaolo Abeni {
1896511882cSPaolo Abeni unsigned int len = skb->truesize;
1906511882cSPaolo Abeni struct sock *sk = skb->sk;
1916511882cSPaolo Abeni
1926511882cSPaolo Abeni atomic_sub(len, &sk->sk_rmem_alloc);
1936511882cSPaolo Abeni mptcp_rmem_uncharge(sk, len);
1946511882cSPaolo Abeni }
1956511882cSPaolo Abeni
mptcp_set_owner_r(struct sk_buff * skb,struct sock * sk)19636b122baSDmytro Shytyi void mptcp_set_owner_r(struct sk_buff *skb, struct sock *sk)
1976511882cSPaolo Abeni {
1986511882cSPaolo Abeni skb_orphan(skb);
1996511882cSPaolo Abeni skb->sk = sk;
2006511882cSPaolo Abeni skb->destructor = mptcp_rfree;
2016511882cSPaolo Abeni atomic_add(skb->truesize, &sk->sk_rmem_alloc);
2026511882cSPaolo Abeni mptcp_rmem_charge(sk, skb->truesize);
2036511882cSPaolo Abeni }
2046511882cSPaolo Abeni
205ab174ad8SPaolo Abeni /* "inspired" by tcp_data_queue_ofo(), main differences:
206ab174ad8SPaolo Abeni * - use mptcp seqs
207ab174ad8SPaolo Abeni * - don't cope with sacks
208ab174ad8SPaolo Abeni */
mptcp_data_queue_ofo(struct mptcp_sock * msk,struct sk_buff * skb)209ab174ad8SPaolo Abeni static void mptcp_data_queue_ofo(struct mptcp_sock *msk, struct sk_buff *skb)
210ab174ad8SPaolo Abeni {
211ab174ad8SPaolo Abeni struct sock *sk = (struct sock *)msk;
212ab174ad8SPaolo Abeni struct rb_node **p, *parent;
213ab174ad8SPaolo Abeni u64 seq, end_seq, max_seq;
214ab174ad8SPaolo Abeni struct sk_buff *skb1;
215ab174ad8SPaolo Abeni
216ab174ad8SPaolo Abeni seq = MPTCP_SKB_CB(skb)->map_seq;
217ab174ad8SPaolo Abeni end_seq = MPTCP_SKB_CB(skb)->end_seq;
218f3589be0SPaolo Abeni max_seq = atomic64_read(&msk->rcv_wnd_sent);
219ab174ad8SPaolo Abeni
22006242e44SPaolo Abeni pr_debug("msk=%p seq=%llx limit=%llx empty=%d", msk, seq, max_seq,
22106242e44SPaolo Abeni RB_EMPTY_ROOT(&msk->out_of_order_queue));
222fa3fe2b1SFlorian Westphal if (after64(end_seq, max_seq)) {
223ab174ad8SPaolo Abeni /* out of window */
224ab174ad8SPaolo Abeni mptcp_drop(sk, skb);
225fa3fe2b1SFlorian Westphal pr_debug("oow by %lld, rcv_wnd_sent %llu\n",
226fa3fe2b1SFlorian Westphal (unsigned long long)end_seq - (unsigned long)max_seq,
227f3589be0SPaolo Abeni (unsigned long long)atomic64_read(&msk->rcv_wnd_sent));
22806242e44SPaolo Abeni MPTCP_INC_STATS(sock_net(sk), MPTCP_MIB_NODSSWINDOW);
229ab174ad8SPaolo Abeni return;
230ab174ad8SPaolo Abeni }
231ab174ad8SPaolo Abeni
232ab174ad8SPaolo Abeni p = &msk->out_of_order_queue.rb_node;
23306242e44SPaolo Abeni MPTCP_INC_STATS(sock_net(sk), MPTCP_MIB_OFOQUEUE);
234ab174ad8SPaolo Abeni if (RB_EMPTY_ROOT(&msk->out_of_order_queue)) {
235ab174ad8SPaolo Abeni rb_link_node(&skb->rbnode, NULL, p);
236ab174ad8SPaolo Abeni rb_insert_color(&skb->rbnode, &msk->out_of_order_queue);
237ab174ad8SPaolo Abeni msk->ooo_last_skb = skb;
238ab174ad8SPaolo Abeni goto end;
239ab174ad8SPaolo Abeni }
240ab174ad8SPaolo Abeni
241ab174ad8SPaolo Abeni /* with 2 subflows, adding at end of ooo queue is quite likely
242ab174ad8SPaolo Abeni * Use of ooo_last_skb avoids the O(Log(N)) rbtree lookup.
243ab174ad8SPaolo Abeni */
24406242e44SPaolo Abeni if (mptcp_ooo_try_coalesce(msk, msk->ooo_last_skb, skb)) {
24506242e44SPaolo Abeni MPTCP_INC_STATS(sock_net(sk), MPTCP_MIB_OFOMERGE);
24606242e44SPaolo Abeni MPTCP_INC_STATS(sock_net(sk), MPTCP_MIB_OFOQUEUETAIL);
247ab174ad8SPaolo Abeni return;
24806242e44SPaolo Abeni }
249ab174ad8SPaolo Abeni
250ab174ad8SPaolo Abeni /* Can avoid an rbtree lookup if we are adding skb after ooo_last_skb */
251ab174ad8SPaolo Abeni if (!before64(seq, MPTCP_SKB_CB(msk->ooo_last_skb)->end_seq)) {
25206242e44SPaolo Abeni MPTCP_INC_STATS(sock_net(sk), MPTCP_MIB_OFOQUEUETAIL);
253ab174ad8SPaolo Abeni parent = &msk->ooo_last_skb->rbnode;
254ab174ad8SPaolo Abeni p = &parent->rb_right;
255ab174ad8SPaolo Abeni goto insert;
256ab174ad8SPaolo Abeni }
257ab174ad8SPaolo Abeni
258ab174ad8SPaolo Abeni /* Find place to insert this segment. Handle overlaps on the way. */
259ab174ad8SPaolo Abeni parent = NULL;
260ab174ad8SPaolo Abeni while (*p) {
261ab174ad8SPaolo Abeni parent = *p;
262ab174ad8SPaolo Abeni skb1 = rb_to_skb(parent);
263ab174ad8SPaolo Abeni if (before64(seq, MPTCP_SKB_CB(skb1)->map_seq)) {
264ab174ad8SPaolo Abeni p = &parent->rb_left;
265ab174ad8SPaolo Abeni continue;
266ab174ad8SPaolo Abeni }
267ab174ad8SPaolo Abeni if (before64(seq, MPTCP_SKB_CB(skb1)->end_seq)) {
268ab174ad8SPaolo Abeni if (!after64(end_seq, MPTCP_SKB_CB(skb1)->end_seq)) {
269ab174ad8SPaolo Abeni /* All the bits are present. Drop. */
270ab174ad8SPaolo Abeni mptcp_drop(sk, skb);
27106242e44SPaolo Abeni MPTCP_INC_STATS(sock_net(sk), MPTCP_MIB_DUPDATA);
272ab174ad8SPaolo Abeni return;
273ab174ad8SPaolo Abeni }
274ab174ad8SPaolo Abeni if (after64(seq, MPTCP_SKB_CB(skb1)->map_seq)) {
275ab174ad8SPaolo Abeni /* partial overlap:
276ab174ad8SPaolo Abeni * | skb |
277ab174ad8SPaolo Abeni * | skb1 |
278ab174ad8SPaolo Abeni * continue traversing
279ab174ad8SPaolo Abeni */
280ab174ad8SPaolo Abeni } else {
281ab174ad8SPaolo Abeni /* skb's seq == skb1's seq and skb covers skb1.
282ab174ad8SPaolo Abeni * Replace skb1 with skb.
283ab174ad8SPaolo Abeni */
284ab174ad8SPaolo Abeni rb_replace_node(&skb1->rbnode, &skb->rbnode,
285ab174ad8SPaolo Abeni &msk->out_of_order_queue);
286ab174ad8SPaolo Abeni mptcp_drop(sk, skb1);
28706242e44SPaolo Abeni MPTCP_INC_STATS(sock_net(sk), MPTCP_MIB_DUPDATA);
288ab174ad8SPaolo Abeni goto merge_right;
289ab174ad8SPaolo Abeni }
290ab174ad8SPaolo Abeni } else if (mptcp_ooo_try_coalesce(msk, skb1, skb)) {
29106242e44SPaolo Abeni MPTCP_INC_STATS(sock_net(sk), MPTCP_MIB_OFOMERGE);
292ab174ad8SPaolo Abeni return;
293ab174ad8SPaolo Abeni }
294ab174ad8SPaolo Abeni p = &parent->rb_right;
295ab174ad8SPaolo Abeni }
29606242e44SPaolo Abeni
297ab174ad8SPaolo Abeni insert:
298ab174ad8SPaolo Abeni /* Insert segment into RB tree. */
299ab174ad8SPaolo Abeni rb_link_node(&skb->rbnode, parent, p);
300ab174ad8SPaolo Abeni rb_insert_color(&skb->rbnode, &msk->out_of_order_queue);
301ab174ad8SPaolo Abeni
302ab174ad8SPaolo Abeni merge_right:
303ab174ad8SPaolo Abeni /* Remove other segments covered by skb. */
304ab174ad8SPaolo Abeni while ((skb1 = skb_rb_next(skb)) != NULL) {
305ab174ad8SPaolo Abeni if (before64(end_seq, MPTCP_SKB_CB(skb1)->end_seq))
306ab174ad8SPaolo Abeni break;
307ab174ad8SPaolo Abeni rb_erase(&skb1->rbnode, &msk->out_of_order_queue);
308ab174ad8SPaolo Abeni mptcp_drop(sk, skb1);
30906242e44SPaolo Abeni MPTCP_INC_STATS(sock_net(sk), MPTCP_MIB_DUPDATA);
310ab174ad8SPaolo Abeni }
311ab174ad8SPaolo Abeni /* If there is no skb after us, we are the last_skb ! */
312ab174ad8SPaolo Abeni if (!skb1)
313ab174ad8SPaolo Abeni msk->ooo_last_skb = skb;
314ab174ad8SPaolo Abeni
315ab174ad8SPaolo Abeni end:
316ab174ad8SPaolo Abeni skb_condense(skb);
3176511882cSPaolo Abeni mptcp_set_owner_r(skb, sk);
3186511882cSPaolo Abeni }
3196511882cSPaolo Abeni
mptcp_rmem_schedule(struct sock * sk,struct sock * ssk,int size)3206511882cSPaolo Abeni static bool mptcp_rmem_schedule(struct sock *sk, struct sock *ssk, int size)
3216511882cSPaolo Abeni {
3226511882cSPaolo Abeni struct mptcp_sock *msk = mptcp_sk(sk);
3236511882cSPaolo Abeni int amt, amount;
3246511882cSPaolo Abeni
32569d93daeSPaolo Abeni if (size <= msk->rmem_fwd_alloc)
3266511882cSPaolo Abeni return true;
3276511882cSPaolo Abeni
32869d93daeSPaolo Abeni size -= msk->rmem_fwd_alloc;
3296511882cSPaolo Abeni amt = sk_mem_pages(size);
330100fdd1fSEric Dumazet amount = amt << PAGE_SHIFT;
3314aaa1685SPaolo Abeni if (!__sk_mem_raise_allocated(sk, size, amt, SK_MEM_RECV))
3326511882cSPaolo Abeni return false;
3336511882cSPaolo Abeni
3349531e4a8SEric Dumazet mptcp_rmem_fwd_alloc_add(sk, amount);
3356511882cSPaolo Abeni return true;
336ab174ad8SPaolo Abeni }
337ab174ad8SPaolo Abeni
__mptcp_move_skb(struct mptcp_sock * msk,struct sock * ssk,struct sk_buff * skb,unsigned int offset,size_t copy_len)338ab174ad8SPaolo Abeni static bool __mptcp_move_skb(struct mptcp_sock *msk, struct sock *ssk,
339ab174ad8SPaolo Abeni struct sk_buff *skb, unsigned int offset,
340ab174ad8SPaolo Abeni size_t copy_len)
341ab174ad8SPaolo Abeni {
342ab174ad8SPaolo Abeni struct mptcp_subflow_context *subflow = mptcp_subflow_ctx(ssk);
3436771bfd9SFlorian Westphal struct sock *sk = (struct sock *)msk;
3444e637c70SFlorian Westphal struct sk_buff *tail;
345b7f653b2SFlorian Westphal bool has_rxtstamp;
3466771bfd9SFlorian Westphal
3476771bfd9SFlorian Westphal __skb_unlink(skb, &ssk->sk_receive_queue);
3484e637c70SFlorian Westphal
3494e637c70SFlorian Westphal skb_ext_reset(skb);
3504e637c70SFlorian Westphal skb_orphan(skb);
3514e637c70SFlorian Westphal
3529c3f94e1SPaolo Abeni /* try to fetch required memory from subflow */
3536511882cSPaolo Abeni if (!mptcp_rmem_schedule(sk, ssk, skb->truesize))
3549c3f94e1SPaolo Abeni goto drop;
35572f96132SPaolo Abeni
356b7f653b2SFlorian Westphal has_rxtstamp = TCP_SKB_CB(skb)->has_rxtstamp;
357b7f653b2SFlorian Westphal
358ab174ad8SPaolo Abeni /* the skb map_seq accounts for the skb offset:
359ab174ad8SPaolo Abeni * mptcp_subflow_get_mapped_dsn() is based on the current tp->copied_seq
360ab174ad8SPaolo Abeni * value
361ab174ad8SPaolo Abeni */
362ab174ad8SPaolo Abeni MPTCP_SKB_CB(skb)->map_seq = mptcp_subflow_get_mapped_dsn(subflow);
363ab174ad8SPaolo Abeni MPTCP_SKB_CB(skb)->end_seq = MPTCP_SKB_CB(skb)->map_seq + copy_len;
364ab174ad8SPaolo Abeni MPTCP_SKB_CB(skb)->offset = offset;
365b7f653b2SFlorian Westphal MPTCP_SKB_CB(skb)->has_rxtstamp = has_rxtstamp;
366ab174ad8SPaolo Abeni
367ab174ad8SPaolo Abeni if (MPTCP_SKB_CB(skb)->map_seq == msk->ack_seq) {
368ab174ad8SPaolo Abeni /* in sequence */
36938967f42SPaolo Abeni msk->bytes_received += copy_len;
370917944daSMat Martineau WRITE_ONCE(msk->ack_seq, msk->ack_seq + copy_len);
3714e637c70SFlorian Westphal tail = skb_peek_tail(&sk->sk_receive_queue);
3728268ed4cSPaolo Abeni if (tail && mptcp_try_coalesce(sk, tail, skb))
373ab174ad8SPaolo Abeni return true;
3744e637c70SFlorian Westphal
3756511882cSPaolo Abeni mptcp_set_owner_r(skb, sk);
3766771bfd9SFlorian Westphal __skb_queue_tail(&sk->sk_receive_queue, skb);
377ab174ad8SPaolo Abeni return true;
378ab174ad8SPaolo Abeni } else if (after64(MPTCP_SKB_CB(skb)->map_seq, msk->ack_seq)) {
379ab174ad8SPaolo Abeni mptcp_data_queue_ofo(msk, skb);
380ab174ad8SPaolo Abeni return false;
381ab174ad8SPaolo Abeni }
382ab174ad8SPaolo Abeni
383ab174ad8SPaolo Abeni /* old data, keep it simple and drop the whole pkt, sender
384ab174ad8SPaolo Abeni * will retransmit as needed, if needed.
385ab174ad8SPaolo Abeni */
38606242e44SPaolo Abeni MPTCP_INC_STATS(sock_net(sk), MPTCP_MIB_DUPDATA);
3879c3f94e1SPaolo Abeni drop:
388ab174ad8SPaolo Abeni mptcp_drop(sk, skb);
389ab174ad8SPaolo Abeni return false;
3906771bfd9SFlorian Westphal }
3916771bfd9SFlorian Westphal
mptcp_stop_rtx_timer(struct sock * sk)392f6909dc1SPaolo Abeni static void mptcp_stop_rtx_timer(struct sock *sk)
39316a9a9daSMat Martineau {
39416a9a9daSMat Martineau struct inet_connection_sock *icsk = inet_csk(sk);
39516a9a9daSMat Martineau
39616a9a9daSMat Martineau sk_stop_timer(sk, &icsk->icsk_retransmit_timer);
39716a9a9daSMat Martineau mptcp_sk(sk)->timer_ival = 0;
39816a9a9daSMat Martineau }
39916a9a9daSMat Martineau
mptcp_close_wake_up(struct sock * sk)400e16163b6SPaolo Abeni static void mptcp_close_wake_up(struct sock *sk)
401e16163b6SPaolo Abeni {
402e16163b6SPaolo Abeni if (sock_flag(sk, SOCK_DEAD))
403e16163b6SPaolo Abeni return;
404e16163b6SPaolo Abeni
405e16163b6SPaolo Abeni sk->sk_state_change(sk);
406e16163b6SPaolo Abeni if (sk->sk_shutdown == SHUTDOWN_MASK ||
407e16163b6SPaolo Abeni sk->sk_state == TCP_CLOSE)
408e16163b6SPaolo Abeni sk_wake_async(sk, SOCK_WAKE_WAITD, POLL_HUP);
409e16163b6SPaolo Abeni else
410e16163b6SPaolo Abeni sk_wake_async(sk, SOCK_WAKE_WAITD, POLL_IN);
411e16163b6SPaolo Abeni }
412e16163b6SPaolo Abeni
4139426ce47SPaolo Abeni /* called under the msk socket lock */
mptcp_pending_data_fin_ack(struct sock * sk)4146e628cd3SPaolo Abeni static bool mptcp_pending_data_fin_ack(struct sock *sk)
4156e628cd3SPaolo Abeni {
4166e628cd3SPaolo Abeni struct mptcp_sock *msk = mptcp_sk(sk);
4176e628cd3SPaolo Abeni
41881c1d029SPaolo Abeni return ((1 << sk->sk_state) &
4196e628cd3SPaolo Abeni (TCPF_FIN_WAIT1 | TCPF_CLOSING | TCPF_LAST_ACK)) &&
4206e628cd3SPaolo Abeni msk->write_seq == READ_ONCE(msk->snd_una);
4216e628cd3SPaolo Abeni }
4226e628cd3SPaolo Abeni
mptcp_check_data_fin_ack(struct sock * sk)42316a9a9daSMat Martineau static void mptcp_check_data_fin_ack(struct sock *sk)
42416a9a9daSMat Martineau {
42516a9a9daSMat Martineau struct mptcp_sock *msk = mptcp_sk(sk);
42616a9a9daSMat Martineau
42716a9a9daSMat Martineau /* Look for an acknowledged DATA_FIN */
4286e628cd3SPaolo Abeni if (mptcp_pending_data_fin_ack(sk)) {
42916a9a9daSMat Martineau WRITE_ONCE(msk->snd_data_fin_enable, 0);
43016a9a9daSMat Martineau
43116a9a9daSMat Martineau switch (sk->sk_state) {
43216a9a9daSMat Martineau case TCP_FIN_WAIT1:
433c693a851SGeliang Tang mptcp_set_state(sk, TCP_FIN_WAIT2);
43416a9a9daSMat Martineau break;
43516a9a9daSMat Martineau case TCP_CLOSING:
43616a9a9daSMat Martineau case TCP_LAST_ACK:
437c693a851SGeliang Tang mptcp_set_state(sk, TCP_CLOSE);
43816a9a9daSMat Martineau break;
43916a9a9daSMat Martineau }
44016a9a9daSMat Martineau
441e16163b6SPaolo Abeni mptcp_close_wake_up(sk);
44216a9a9daSMat Martineau }
44316a9a9daSMat Martineau }
44416a9a9daSMat Martineau
4459426ce47SPaolo Abeni /* can be called with no lock acquired */
mptcp_pending_data_fin(struct sock * sk,u64 * seq)4463721b9b6SMat Martineau static bool mptcp_pending_data_fin(struct sock *sk, u64 *seq)
4473721b9b6SMat Martineau {
4483721b9b6SMat Martineau struct mptcp_sock *msk = mptcp_sk(sk);
4493721b9b6SMat Martineau
4503721b9b6SMat Martineau if (READ_ONCE(msk->rcv_data_fin) &&
4519426ce47SPaolo Abeni ((1 << inet_sk_state_load(sk)) &
4523721b9b6SMat Martineau (TCPF_ESTABLISHED | TCPF_FIN_WAIT1 | TCPF_FIN_WAIT2))) {
4533721b9b6SMat Martineau u64 rcv_data_fin_seq = READ_ONCE(msk->rcv_data_fin_seq);
4543721b9b6SMat Martineau
4559426ce47SPaolo Abeni if (READ_ONCE(msk->ack_seq) == rcv_data_fin_seq) {
4563721b9b6SMat Martineau if (seq)
4573721b9b6SMat Martineau *seq = rcv_data_fin_seq;
4583721b9b6SMat Martineau
4593721b9b6SMat Martineau return true;
4603721b9b6SMat Martineau }
4613721b9b6SMat Martineau }
4623721b9b6SMat Martineau
4633721b9b6SMat Martineau return false;
4643721b9b6SMat Martineau }
4653721b9b6SMat Martineau
mptcp_set_datafin_timeout(struct sock * sk)466403a40f2SEric Dumazet static void mptcp_set_datafin_timeout(struct sock *sk)
4676477dd39SMat Martineau {
4686477dd39SMat Martineau struct inet_connection_sock *icsk = inet_csk(sk);
469877d11f0SMat Martineau u32 retransmits;
4706477dd39SMat Martineau
471877d11f0SMat Martineau retransmits = min_t(u32, icsk->icsk_retransmits,
472877d11f0SMat Martineau ilog2(TCP_RTO_MAX / TCP_RTO_MIN));
473877d11f0SMat Martineau
474877d11f0SMat Martineau mptcp_sk(sk)->timer_ival = TCP_RTO_MIN << retransmits;
4756477dd39SMat Martineau }
4766477dd39SMat Martineau
__mptcp_set_timeout(struct sock * sk,long tout)47733d41c9cSPaolo Abeni static void __mptcp_set_timeout(struct sock *sk, long tout)
4783721b9b6SMat Martineau {
4793721b9b6SMat Martineau mptcp_sk(sk)->timer_ival = tout > 0 ? tout : TCP_RTO_MIN;
4803721b9b6SMat Martineau }
4813721b9b6SMat Martineau
mptcp_timeout_from_subflow(const struct mptcp_subflow_context * subflow)48233d41c9cSPaolo Abeni static long mptcp_timeout_from_subflow(const struct mptcp_subflow_context *subflow)
48333d41c9cSPaolo Abeni {
48433d41c9cSPaolo Abeni const struct sock *ssk = mptcp_subflow_tcp_sock(subflow);
48533d41c9cSPaolo Abeni
48671b7dec2SPaolo Abeni return inet_csk(ssk)->icsk_pending && !subflow->stale_count ?
48771b7dec2SPaolo Abeni inet_csk(ssk)->icsk_timeout - jiffies : 0;
48833d41c9cSPaolo Abeni }
48933d41c9cSPaolo Abeni
mptcp_set_timeout(struct sock * sk)49033d41c9cSPaolo Abeni static void mptcp_set_timeout(struct sock *sk)
49133d41c9cSPaolo Abeni {
49233d41c9cSPaolo Abeni struct mptcp_subflow_context *subflow;
49333d41c9cSPaolo Abeni long tout = 0;
49433d41c9cSPaolo Abeni
49533d41c9cSPaolo Abeni mptcp_for_each_subflow(mptcp_sk(sk), subflow)
49633d41c9cSPaolo Abeni tout = max(tout, mptcp_timeout_from_subflow(subflow));
49733d41c9cSPaolo Abeni __mptcp_set_timeout(sk, tout);
49833d41c9cSPaolo Abeni }
49933d41c9cSPaolo Abeni
tcp_can_send_ack(const struct sock * ssk)50076a13b31SGeliang Tang static inline bool tcp_can_send_ack(const struct sock *ssk)
501fd897679SPaolo Abeni {
502fd897679SPaolo Abeni return !((1 << inet_sk_state_load(ssk)) &
50320bc80b6SPaolo Abeni (TCPF_SYN_SENT | TCPF_SYN_RECV | TCPF_TIME_WAIT | TCPF_CLOSE | TCPF_LISTEN));
504fd897679SPaolo Abeni }
505fd897679SPaolo Abeni
__mptcp_subflow_send_ack(struct sock * ssk)506a6574302SMat Martineau void __mptcp_subflow_send_ack(struct sock *ssk)
507a6574302SMat Martineau {
508a6574302SMat Martineau if (tcp_can_send_ack(ssk))
509a6574302SMat Martineau tcp_send_ack(ssk);
510a6574302SMat Martineau }
511a6574302SMat Martineau
mptcp_subflow_send_ack(struct sock * ssk)512f5360e9bSPaolo Abeni static void mptcp_subflow_send_ack(struct sock *ssk)
5137ed90803SPaolo Abeni {
51475e908c3SPaolo Abeni bool slow;
5157ed90803SPaolo Abeni
51675e908c3SPaolo Abeni slow = lock_sock_fast(ssk);
517a6574302SMat Martineau __mptcp_subflow_send_ack(ssk);
51875e908c3SPaolo Abeni unlock_sock_fast(ssk, slow);
519fd897679SPaolo Abeni }
520340fa666SMat Martineau
mptcp_send_ack(struct mptcp_sock * msk)521340fa666SMat Martineau static void mptcp_send_ack(struct mptcp_sock *msk)
522340fa666SMat Martineau {
523340fa666SMat Martineau struct mptcp_subflow_context *subflow;
524340fa666SMat Martineau
525340fa666SMat Martineau mptcp_for_each_subflow(msk, subflow)
526340fa666SMat Martineau mptcp_subflow_send_ack(mptcp_subflow_tcp_sock(subflow));
527ea4ca586SPaolo Abeni }
528ea4ca586SPaolo Abeni
mptcp_subflow_cleanup_rbuf(struct sock * ssk)529fde56eeaSPaolo Abeni static void mptcp_subflow_cleanup_rbuf(struct sock *ssk)
530fd897679SPaolo Abeni {
53175e908c3SPaolo Abeni bool slow;
532fd897679SPaolo Abeni
53375e908c3SPaolo Abeni slow = lock_sock_fast(ssk);
534fde56eeaSPaolo Abeni if (tcp_can_send_ack(ssk))
535fd897679SPaolo Abeni tcp_cleanup_rbuf(ssk, 1);
53675e908c3SPaolo Abeni unlock_sock_fast(ssk, slow);
537fde56eeaSPaolo Abeni }
538fde56eeaSPaolo Abeni
mptcp_subflow_could_cleanup(const struct sock * ssk,bool rx_empty)539fde56eeaSPaolo Abeni static bool mptcp_subflow_could_cleanup(const struct sock *ssk, bool rx_empty)
540fde56eeaSPaolo Abeni {
541fde56eeaSPaolo Abeni const struct inet_connection_sock *icsk = inet_csk(ssk);
542c4512c63SMatthieu Baerts u8 ack_pending = READ_ONCE(icsk->icsk_ack.pending);
543fde56eeaSPaolo Abeni const struct tcp_sock *tp = tcp_sk(ssk);
544fde56eeaSPaolo Abeni
545fde56eeaSPaolo Abeni return (ack_pending & ICSK_ACK_SCHED) &&
546fde56eeaSPaolo Abeni ((READ_ONCE(tp->rcv_nxt) - READ_ONCE(tp->rcv_wup) >
547fde56eeaSPaolo Abeni READ_ONCE(icsk->icsk_ack.rcv_mss)) ||
548fde56eeaSPaolo Abeni (rx_empty && ack_pending &
549fde56eeaSPaolo Abeni (ICSK_ACK_PUSHED2 | ICSK_ACK_PUSHED)));
550ea4ca586SPaolo Abeni }
551fd897679SPaolo Abeni
mptcp_cleanup_rbuf(struct mptcp_sock * msk)552fd897679SPaolo Abeni static void mptcp_cleanup_rbuf(struct mptcp_sock *msk)
553fd897679SPaolo Abeni {
554e3859603SPaolo Abeni int old_space = READ_ONCE(msk->old_wspace);
555fd897679SPaolo Abeni struct mptcp_subflow_context *subflow;
556e3859603SPaolo Abeni struct sock *sk = (struct sock *)msk;
557fde56eeaSPaolo Abeni int space = __mptcp_space(sk);
558fde56eeaSPaolo Abeni bool cleanup, rx_empty;
559e3859603SPaolo Abeni
560fde56eeaSPaolo Abeni cleanup = (space > 0) && (space >= (old_space << 1));
561ce599c51SPaolo Abeni rx_empty = !__mptcp_rmem(sk);
562fd897679SPaolo Abeni
563fd897679SPaolo Abeni mptcp_for_each_subflow(msk, subflow) {
564fd897679SPaolo Abeni struct sock *ssk = mptcp_subflow_tcp_sock(subflow);
565fd897679SPaolo Abeni
566fde56eeaSPaolo Abeni if (cleanup || mptcp_subflow_could_cleanup(ssk, rx_empty))
567fde56eeaSPaolo Abeni mptcp_subflow_cleanup_rbuf(ssk);
5687ed90803SPaolo Abeni }
5697ed90803SPaolo Abeni }
5707ed90803SPaolo Abeni
mptcp_check_data_fin(struct sock * sk)5717ed90803SPaolo Abeni static bool mptcp_check_data_fin(struct sock *sk)
5723721b9b6SMat Martineau {
5733721b9b6SMat Martineau struct mptcp_sock *msk = mptcp_sk(sk);
5743721b9b6SMat Martineau u64 rcv_data_fin_seq;
5757ed90803SPaolo Abeni bool ret = false;
5763721b9b6SMat Martineau
5773721b9b6SMat Martineau /* Need to ack a DATA_FIN received from a peer while this side
5783721b9b6SMat Martineau * of the connection is in ESTABLISHED, FIN_WAIT1, or FIN_WAIT2.
5793721b9b6SMat Martineau * msk->rcv_data_fin was set when parsing the incoming options
5803721b9b6SMat Martineau * at the subflow level and the msk lock was not held, so this
5813721b9b6SMat Martineau * is the first opportunity to act on the DATA_FIN and change
5823721b9b6SMat Martineau * the msk state.
5833721b9b6SMat Martineau *
5843721b9b6SMat Martineau * If we are caught up to the sequence number of the incoming
5853721b9b6SMat Martineau * DATA_FIN, send the DATA_ACK now and do state transition. If
5863721b9b6SMat Martineau * not caught up, do nothing and let the recv code send DATA_ACK
5873721b9b6SMat Martineau * when catching up.
5883721b9b6SMat Martineau */
5893721b9b6SMat Martineau
5903721b9b6SMat Martineau if (mptcp_pending_data_fin(sk, &rcv_data_fin_seq)) {
591917944daSMat Martineau WRITE_ONCE(msk->ack_seq, msk->ack_seq + 1);
5923721b9b6SMat Martineau WRITE_ONCE(msk->rcv_data_fin, 0);
5933721b9b6SMat Martineau
5946b9831bfSPaolo Abeni WRITE_ONCE(sk->sk_shutdown, sk->sk_shutdown | RCV_SHUTDOWN);
59516a9a9daSMat Martineau smp_mb__before_atomic(); /* SHUTDOWN must be visible first */
5963721b9b6SMat Martineau
5973721b9b6SMat Martineau switch (sk->sk_state) {
5983721b9b6SMat Martineau case TCP_ESTABLISHED:
599c693a851SGeliang Tang mptcp_set_state(sk, TCP_CLOSE_WAIT);
6003721b9b6SMat Martineau break;
6013721b9b6SMat Martineau case TCP_FIN_WAIT1:
602c693a851SGeliang Tang mptcp_set_state(sk, TCP_CLOSING);
6033721b9b6SMat Martineau break;
6043721b9b6SMat Martineau case TCP_FIN_WAIT2:
605c693a851SGeliang Tang mptcp_set_state(sk, TCP_CLOSE);
6063721b9b6SMat Martineau break;
6073721b9b6SMat Martineau default:
6083721b9b6SMat Martineau /* Other states not expected */
6093721b9b6SMat Martineau WARN_ON_ONCE(1);
6103721b9b6SMat Martineau break;
6113721b9b6SMat Martineau }
6123721b9b6SMat Martineau
6137ed90803SPaolo Abeni ret = true;
61481c1d029SPaolo Abeni if (!__mptcp_check_fallback(msk))
615fd897679SPaolo Abeni mptcp_send_ack(msk);
616e16163b6SPaolo Abeni mptcp_close_wake_up(sk);
6173721b9b6SMat Martineau }
6187ed90803SPaolo Abeni return ret;
6193721b9b6SMat Martineau }
6203721b9b6SMat Martineau
__mptcp_move_skbs_from_subflow(struct mptcp_sock * msk,struct sock * ssk,unsigned int * bytes)6216771bfd9SFlorian Westphal static bool __mptcp_move_skbs_from_subflow(struct mptcp_sock *msk,
6226771bfd9SFlorian Westphal struct sock *ssk,
6236771bfd9SFlorian Westphal unsigned int *bytes)
6246771bfd9SFlorian Westphal {
6256771bfd9SFlorian Westphal struct mptcp_subflow_context *subflow = mptcp_subflow_ctx(ssk);
626600911ffSFlorian Westphal struct sock *sk = (struct sock *)msk;
6276771bfd9SFlorian Westphal unsigned int moved = 0;
6286771bfd9SFlorian Westphal bool more_data_avail;
6296771bfd9SFlorian Westphal struct tcp_sock *tp;
6306771bfd9SFlorian Westphal bool done = false;
63113c7ba0cSFlorian Westphal int sk_rbuf;
63213c7ba0cSFlorian Westphal
63313c7ba0cSFlorian Westphal sk_rbuf = READ_ONCE(sk->sk_rcvbuf);
63413c7ba0cSFlorian Westphal
63513c7ba0cSFlorian Westphal if (!(sk->sk_userlocks & SOCK_RCVBUF_LOCK)) {
63613c7ba0cSFlorian Westphal int ssk_rbuf = READ_ONCE(ssk->sk_rcvbuf);
63713c7ba0cSFlorian Westphal
63813c7ba0cSFlorian Westphal if (unlikely(ssk_rbuf > sk_rbuf)) {
63913c7ba0cSFlorian Westphal WRITE_ONCE(sk->sk_rcvbuf, ssk_rbuf);
64013c7ba0cSFlorian Westphal sk_rbuf = ssk_rbuf;
64113c7ba0cSFlorian Westphal }
64213c7ba0cSFlorian Westphal }
643600911ffSFlorian Westphal
644ab174ad8SPaolo Abeni pr_debug("msk=%p ssk=%p", msk, ssk);
6456771bfd9SFlorian Westphal tp = tcp_sk(ssk);
6466771bfd9SFlorian Westphal do {
6476771bfd9SFlorian Westphal u32 map_remaining, offset;
6486771bfd9SFlorian Westphal u32 seq = tp->copied_seq;
6496771bfd9SFlorian Westphal struct sk_buff *skb;
6506771bfd9SFlorian Westphal bool fin;
6516771bfd9SFlorian Westphal
6526771bfd9SFlorian Westphal /* try to move as much data as available */
6536771bfd9SFlorian Westphal map_remaining = subflow->map_data_len -
6546771bfd9SFlorian Westphal mptcp_subflow_get_map_offset(subflow);
6556771bfd9SFlorian Westphal
6566771bfd9SFlorian Westphal skb = skb_peek(&ssk->sk_receive_queue);
657d9fb8c50SPaolo Abeni if (!skb) {
658d89e3ed7SPaolo Abeni /* With racing move_skbs_to_msk() and __mptcp_move_skbs(),
659d89e3ed7SPaolo Abeni * a different CPU can have already processed the pending
660d89e3ed7SPaolo Abeni * data, stop here or we can enter an infinite loop
661d9fb8c50SPaolo Abeni */
662d9fb8c50SPaolo Abeni if (!moved)
663d9fb8c50SPaolo Abeni done = true;
6646771bfd9SFlorian Westphal break;
665d9fb8c50SPaolo Abeni }
6666771bfd9SFlorian Westphal
667e1ff9e82SDavide Caratti if (__mptcp_check_fallback(msk)) {
668d89e3ed7SPaolo Abeni /* Under fallback skbs have no MPTCP extension and TCP could
669d89e3ed7SPaolo Abeni * collapse them between the dummy map creation and the
670d89e3ed7SPaolo Abeni * current dequeue. Be sure to adjust the map size.
671e1ff9e82SDavide Caratti */
672e1ff9e82SDavide Caratti map_remaining = skb->len;
673e1ff9e82SDavide Caratti subflow->map_data_len = skb->len;
674e1ff9e82SDavide Caratti }
675e1ff9e82SDavide Caratti
6766771bfd9SFlorian Westphal offset = seq - TCP_SKB_CB(skb)->seq;
6776771bfd9SFlorian Westphal fin = TCP_SKB_CB(skb)->tcp_flags & TCPHDR_FIN;
6786771bfd9SFlorian Westphal if (fin) {
6796771bfd9SFlorian Westphal done = true;
6806771bfd9SFlorian Westphal seq++;
6816771bfd9SFlorian Westphal }
6826771bfd9SFlorian Westphal
6836771bfd9SFlorian Westphal if (offset < skb->len) {
6846771bfd9SFlorian Westphal size_t len = skb->len - offset;
6856771bfd9SFlorian Westphal
6866771bfd9SFlorian Westphal if (tp->urg_data)
6876771bfd9SFlorian Westphal done = true;
6886771bfd9SFlorian Westphal
689ab174ad8SPaolo Abeni if (__mptcp_move_skb(msk, ssk, skb, offset, len))
6906771bfd9SFlorian Westphal moved += len;
691ab174ad8SPaolo Abeni seq += len;
6926771bfd9SFlorian Westphal
6936771bfd9SFlorian Westphal if (WARN_ON_ONCE(map_remaining < len))
6946771bfd9SFlorian Westphal break;
6956771bfd9SFlorian Westphal } else {
6966771bfd9SFlorian Westphal WARN_ON_ONCE(!fin);
6976771bfd9SFlorian Westphal sk_eat_skb(ssk, skb);
6986771bfd9SFlorian Westphal done = true;
6996771bfd9SFlorian Westphal }
7006771bfd9SFlorian Westphal
7016771bfd9SFlorian Westphal WRITE_ONCE(tp->copied_seq, seq);
7026771bfd9SFlorian Westphal more_data_avail = mptcp_subflow_data_available(ssk);
703600911ffSFlorian Westphal
70413c7ba0cSFlorian Westphal if (atomic_read(&sk->sk_rmem_alloc) > sk_rbuf) {
705600911ffSFlorian Westphal done = true;
706600911ffSFlorian Westphal break;
707600911ffSFlorian Westphal }
7086771bfd9SFlorian Westphal } while (more_data_avail);
7096771bfd9SFlorian Westphal
71018d82cdeSGeliang Tang if (moved > 0)
71118d82cdeSGeliang Tang msk->last_data_recv = tcp_jiffies32;
7126719331cSPaolo Abeni *bytes += moved;
7136771bfd9SFlorian Westphal return done;
7146771bfd9SFlorian Westphal }
7156771bfd9SFlorian Westphal
__mptcp_ofo_queue(struct mptcp_sock * msk)71687952603SPaolo Abeni static bool __mptcp_ofo_queue(struct mptcp_sock *msk)
717ab174ad8SPaolo Abeni {
718ab174ad8SPaolo Abeni struct sock *sk = (struct sock *)msk;
719ab174ad8SPaolo Abeni struct sk_buff *skb, *tail;
720ab174ad8SPaolo Abeni bool moved = false;
721ab174ad8SPaolo Abeni struct rb_node *p;
722ab174ad8SPaolo Abeni u64 end_seq;
723ab174ad8SPaolo Abeni
724ab174ad8SPaolo Abeni p = rb_first(&msk->out_of_order_queue);
72506242e44SPaolo Abeni pr_debug("msk=%p empty=%d", msk, RB_EMPTY_ROOT(&msk->out_of_order_queue));
726ab174ad8SPaolo Abeni while (p) {
727ab174ad8SPaolo Abeni skb = rb_to_skb(p);
728ab174ad8SPaolo Abeni if (after64(MPTCP_SKB_CB(skb)->map_seq, msk->ack_seq))
729ab174ad8SPaolo Abeni break;
730ab174ad8SPaolo Abeni
731ab174ad8SPaolo Abeni p = rb_next(p);
732ab174ad8SPaolo Abeni rb_erase(&skb->rbnode, &msk->out_of_order_queue);
733ab174ad8SPaolo Abeni
734ab174ad8SPaolo Abeni if (unlikely(!after64(MPTCP_SKB_CB(skb)->end_seq,
735ab174ad8SPaolo Abeni msk->ack_seq))) {
736ab174ad8SPaolo Abeni mptcp_drop(sk, skb);
73706242e44SPaolo Abeni MPTCP_INC_STATS(sock_net(sk), MPTCP_MIB_DUPDATA);
738ab174ad8SPaolo Abeni continue;
739ab174ad8SPaolo Abeni }
740ab174ad8SPaolo Abeni
741ab174ad8SPaolo Abeni end_seq = MPTCP_SKB_CB(skb)->end_seq;
742ab174ad8SPaolo Abeni tail = skb_peek_tail(&sk->sk_receive_queue);
743ab174ad8SPaolo Abeni if (!tail || !mptcp_ooo_try_coalesce(msk, tail, skb)) {
744ab174ad8SPaolo Abeni int delta = msk->ack_seq - MPTCP_SKB_CB(skb)->map_seq;
745ab174ad8SPaolo Abeni
746ab174ad8SPaolo Abeni /* skip overlapping data, if any */
74706242e44SPaolo Abeni pr_debug("uncoalesced seq=%llx ack seq=%llx delta=%d",
74806242e44SPaolo Abeni MPTCP_SKB_CB(skb)->map_seq, msk->ack_seq,
74906242e44SPaolo Abeni delta);
750ab174ad8SPaolo Abeni MPTCP_SKB_CB(skb)->offset += delta;
7512c9e7765SFlorian Westphal MPTCP_SKB_CB(skb)->map_seq += delta;
752ab174ad8SPaolo Abeni __skb_queue_tail(&sk->sk_receive_queue, skb);
753ab174ad8SPaolo Abeni }
75438967f42SPaolo Abeni msk->bytes_received += end_seq - msk->ack_seq;
7559426ce47SPaolo Abeni WRITE_ONCE(msk->ack_seq, end_seq);
756ab174ad8SPaolo Abeni moved = true;
757ab174ad8SPaolo Abeni }
758ab174ad8SPaolo Abeni return moved;
759ab174ad8SPaolo Abeni }
760ab174ad8SPaolo Abeni
__mptcp_subflow_error_report(struct sock * sk,struct sock * ssk)7619f1a9881SPaolo Abeni static bool __mptcp_subflow_error_report(struct sock *sk, struct sock *ssk)
762d5fbeff1SPaolo Abeni {
763d5fbeff1SPaolo Abeni int err = sock_error(ssk);
764d5fbeff1SPaolo Abeni int ssk_state;
765d5fbeff1SPaolo Abeni
766d5fbeff1SPaolo Abeni if (!err)
7679f1a9881SPaolo Abeni return false;
768d5fbeff1SPaolo Abeni
769d5fbeff1SPaolo Abeni /* only propagate errors on fallen-back sockets or
770d5fbeff1SPaolo Abeni * on MPC connect
771d5fbeff1SPaolo Abeni */
7729f1a9881SPaolo Abeni if (sk->sk_state != TCP_SYN_SENT && !__mptcp_check_fallback(mptcp_sk(sk)))
7739f1a9881SPaolo Abeni return false;
774d5fbeff1SPaolo Abeni
775d5fbeff1SPaolo Abeni /* We need to propagate only transition to CLOSE state.
776d5fbeff1SPaolo Abeni * Orphaned socket will see such state change via
777d5fbeff1SPaolo Abeni * subflow_sched_work_if_closed() and that path will properly
778d5fbeff1SPaolo Abeni * destroy the msk as needed.
779d5fbeff1SPaolo Abeni */
780d5fbeff1SPaolo Abeni ssk_state = inet_sk_state_load(ssk);
781d5fbeff1SPaolo Abeni if (ssk_state == TCP_CLOSE && !sock_flag(sk, SOCK_DEAD))
782c693a851SGeliang Tang mptcp_set_state(sk, ssk_state);
783d5fbeff1SPaolo Abeni WRITE_ONCE(sk->sk_err, -err);
784d5fbeff1SPaolo Abeni
785d5fbeff1SPaolo Abeni /* This barrier is coupled with smp_rmb() in mptcp_poll() */
786d5fbeff1SPaolo Abeni smp_wmb();
787d5fbeff1SPaolo Abeni sk_error_report(sk);
7889f1a9881SPaolo Abeni return true;
789d5fbeff1SPaolo Abeni }
7909f1a9881SPaolo Abeni
__mptcp_error_report(struct sock * sk)7919f1a9881SPaolo Abeni void __mptcp_error_report(struct sock *sk)
7929f1a9881SPaolo Abeni {
7939f1a9881SPaolo Abeni struct mptcp_subflow_context *subflow;
7949f1a9881SPaolo Abeni struct mptcp_sock *msk = mptcp_sk(sk);
7959f1a9881SPaolo Abeni
7969f1a9881SPaolo Abeni mptcp_for_each_subflow(msk, subflow)
7979f1a9881SPaolo Abeni if (__mptcp_subflow_error_report(sk, mptcp_subflow_tcp_sock(subflow)))
7989f1a9881SPaolo Abeni break;
799d5fbeff1SPaolo Abeni }
800d5fbeff1SPaolo Abeni
8012e52213cSFlorian Westphal /* In most cases we will be able to lock the mptcp socket. If its already
8022e52213cSFlorian Westphal * owned, we need to defer to the work queue to avoid ABBA deadlock.
8032e52213cSFlorian Westphal */
move_skbs_to_msk(struct mptcp_sock * msk,struct sock * ssk)80499d1055cSPaolo Abeni static bool move_skbs_to_msk(struct mptcp_sock *msk, struct sock *ssk)
8052e52213cSFlorian Westphal {
8062e52213cSFlorian Westphal struct sock *sk = (struct sock *)msk;
8072e52213cSFlorian Westphal unsigned int moved = 0;
8082e52213cSFlorian Westphal
8092e52213cSFlorian Westphal __mptcp_move_skbs_from_subflow(msk, ssk, &moved);
81087952603SPaolo Abeni __mptcp_ofo_queue(msk);
811499ada50SPaolo Abeni if (unlikely(ssk->sk_err)) {
812499ada50SPaolo Abeni if (!sock_owned_by_user(sk))
813499ada50SPaolo Abeni __mptcp_error_report(sk);
814499ada50SPaolo Abeni else
815e9d09bacSPaolo Abeni __set_bit(MPTCP_ERROR_REPORT, &msk->cb_flags);
816499ada50SPaolo Abeni }
817ab174ad8SPaolo Abeni
818ab174ad8SPaolo Abeni /* If the moves have caught up with the DATA_FIN sequence number
819ab174ad8SPaolo Abeni * it's time to ack the DATA_FIN and change socket state, but
820ab174ad8SPaolo Abeni * this is not a good place to change state. Let the workqueue
821ab174ad8SPaolo Abeni * do it.
822ab174ad8SPaolo Abeni */
823ba8f48f7SPaolo Abeni if (mptcp_pending_data_fin(sk, NULL))
824ba8f48f7SPaolo Abeni mptcp_schedule_work(sk);
82599d1055cSPaolo Abeni return moved > 0;
8262e52213cSFlorian Westphal }
8272e52213cSFlorian Westphal
mptcp_data_ready(struct sock * sk,struct sock * ssk)8282e52213cSFlorian Westphal void mptcp_data_ready(struct sock *sk, struct sock *ssk)
829101f6f85SFlorian Westphal {
8306719331cSPaolo Abeni struct mptcp_subflow_context *subflow = mptcp_subflow_ctx(ssk);
831101f6f85SFlorian Westphal struct mptcp_sock *msk = mptcp_sk(sk);
83213c7ba0cSFlorian Westphal int sk_rbuf, ssk_rbuf;
833101f6f85SFlorian Westphal
834d7b1bfd0SPaolo Abeni /* The peer can send data while we are shutting down this
835d7b1bfd0SPaolo Abeni * subflow at msk destruction time, but we must avoid enqueuing
836d7b1bfd0SPaolo Abeni * more data to the msk receive queue
837d7b1bfd0SPaolo Abeni */
838d7b1bfd0SPaolo Abeni if (unlikely(subflow->disposable))
839d7b1bfd0SPaolo Abeni return;
840d7b1bfd0SPaolo Abeni
84113c7ba0cSFlorian Westphal ssk_rbuf = READ_ONCE(ssk->sk_rcvbuf);
84213c7ba0cSFlorian Westphal sk_rbuf = READ_ONCE(sk->sk_rcvbuf);
84313c7ba0cSFlorian Westphal if (unlikely(ssk_rbuf > sk_rbuf))
84413c7ba0cSFlorian Westphal sk_rbuf = ssk_rbuf;
84513c7ba0cSFlorian Westphal
84699d1055cSPaolo Abeni /* over limit? can't append more skbs to msk, Also, no need to wake-up*/
847ce599c51SPaolo Abeni if (__mptcp_rmem(sk) > sk_rbuf) {
848ce599c51SPaolo Abeni MPTCP_INC_STATS(sock_net(sk), MPTCP_MIB_RCVPRUNED);
84999d1055cSPaolo Abeni return;
850ce599c51SPaolo Abeni }
8512e52213cSFlorian Westphal
85299d1055cSPaolo Abeni /* Wake-up the reader only for in-sequence data */
85399d1055cSPaolo Abeni mptcp_data_lock(sk);
8545684ab1aSPaolo Abeni if (move_skbs_to_msk(msk, ssk) && mptcp_epollin_ready(sk))
855101f6f85SFlorian Westphal sk->sk_data_ready(sk);
85699d1055cSPaolo Abeni mptcp_data_unlock(sk);
85799d1055cSPaolo Abeni }
858101f6f85SFlorian Westphal
mptcp_subflow_joined(struct mptcp_sock * msk,struct sock * ssk)85955b47ca7SPaolo Abeni static void mptcp_subflow_joined(struct mptcp_sock *msk, struct sock *ssk)
86055b47ca7SPaolo Abeni {
86155b47ca7SPaolo Abeni mptcp_subflow_ctx(ssk)->map_seq = READ_ONCE(msk->ack_seq);
86255b47ca7SPaolo Abeni WRITE_ONCE(msk->allow_infinite_fallback, false);
86355b47ca7SPaolo Abeni mptcp_event(MPTCP_EVENT_SUB_ESTABLISHED, msk, ssk, GFP_ATOMIC);
86455b47ca7SPaolo Abeni }
86555b47ca7SPaolo Abeni
__mptcp_finish_join(struct mptcp_sock * msk,struct sock * ssk)8663e501490SPaolo Abeni static bool __mptcp_finish_join(struct mptcp_sock *msk, struct sock *ssk)
867ec3edaa7SPeter Krystad {
8683e501490SPaolo Abeni struct sock *sk = (struct sock *)msk;
8695cf92bbaSPaolo Abeni
8703e501490SPaolo Abeni if (sk->sk_state != TCP_ESTABLISHED)
87178962489SFlorian Westphal return false;
872ec3edaa7SPeter Krystad
8733e501490SPaolo Abeni /* attach to msk socket only after we are sure we will deal with it
8743e501490SPaolo Abeni * at close time
8753e501490SPaolo Abeni */
8763e501490SPaolo Abeni if (sk->sk_socket && !ssk->sk_socket)
8773e501490SPaolo Abeni mptcp_sock_graft(ssk, sk->sk_socket);
87878962489SFlorian Westphal
8796f06b4d4SPaolo Abeni mptcp_subflow_ctx(ssk)->subflow_id = msk->subflow_id++;
8803e501490SPaolo Abeni mptcp_sockopt_sync_locked(msk, ssk);
88155b47ca7SPaolo Abeni mptcp_subflow_joined(msk, ssk);
88227e5ccc2SPaolo Abeni mptcp_stop_tout_timer(sk);
8838005184fSPaolo Abeni __mptcp_propagate_sndbuf(sk, ssk);
8843e501490SPaolo Abeni return true;
88578962489SFlorian Westphal }
88678962489SFlorian Westphal
__mptcp_flush_join_list(struct sock * sk,struct list_head * join_list)88756a666c4SPaolo Abeni static void __mptcp_flush_join_list(struct sock *sk, struct list_head *join_list)
88878962489SFlorian Westphal {
8893e501490SPaolo Abeni struct mptcp_subflow_context *tmp, *subflow;
8903e501490SPaolo Abeni struct mptcp_sock *msk = mptcp_sk(sk);
89178962489SFlorian Westphal
89256a666c4SPaolo Abeni list_for_each_entry_safe(subflow, tmp, join_list, node) {
8933e501490SPaolo Abeni struct sock *ssk = mptcp_subflow_tcp_sock(subflow);
8943e501490SPaolo Abeni bool slow = lock_sock_fast(ssk);
8953e501490SPaolo Abeni
8963e501490SPaolo Abeni list_move_tail(&subflow->node, &msk->conn_list);
8973e501490SPaolo Abeni if (!__mptcp_finish_join(msk, ssk))
8983e501490SPaolo Abeni mptcp_subflow_reset(ssk);
8993e501490SPaolo Abeni unlock_sock_fast(ssk, slow);
90078962489SFlorian Westphal }
901ec3edaa7SPeter Krystad }
902ec3edaa7SPeter Krystad
mptcp_rtx_timer_pending(struct sock * sk)903f6909dc1SPaolo Abeni static bool mptcp_rtx_timer_pending(struct sock *sk)
904b51f9b80SPaolo Abeni {
905b51f9b80SPaolo Abeni return timer_pending(&inet_csk(sk)->icsk_retransmit_timer);
906b51f9b80SPaolo Abeni }
907b51f9b80SPaolo Abeni
mptcp_reset_rtx_timer(struct sock * sk)908f6909dc1SPaolo Abeni static void mptcp_reset_rtx_timer(struct sock *sk)
909b51f9b80SPaolo Abeni {
910b51f9b80SPaolo Abeni struct inet_connection_sock *icsk = inet_csk(sk);
911b51f9b80SPaolo Abeni unsigned long tout;
912b51f9b80SPaolo Abeni
913e16163b6SPaolo Abeni /* prevent rescheduling on close */
914e16163b6SPaolo Abeni if (unlikely(inet_sk_state_load(sk) == TCP_CLOSE))
915e16163b6SPaolo Abeni return;
916e16163b6SPaolo Abeni
91733d41c9cSPaolo Abeni tout = mptcp_sk(sk)->timer_ival;
918b51f9b80SPaolo Abeni sk_reset_timer(sk, &icsk->icsk_retransmit_timer, jiffies + tout);
919b51f9b80SPaolo Abeni }
920b51f9b80SPaolo Abeni
mptcp_schedule_work(struct sock * sk)921ba8f48f7SPaolo Abeni bool mptcp_schedule_work(struct sock *sk)
922ba8f48f7SPaolo Abeni {
923ba8f48f7SPaolo Abeni if (inet_sk_state_load(sk) != TCP_CLOSE &&
924ba8f48f7SPaolo Abeni schedule_work(&mptcp_sk(sk)->work)) {
925ba8f48f7SPaolo Abeni /* each subflow already holds a reference to the sk, and the
926ba8f48f7SPaolo Abeni * workqueue is invoked by a subflow, so sk can't go away here.
927ba8f48f7SPaolo Abeni */
928ba8f48f7SPaolo Abeni sock_hold(sk);
929ba8f48f7SPaolo Abeni return true;
930ba8f48f7SPaolo Abeni }
931ba8f48f7SPaolo Abeni return false;
932ba8f48f7SPaolo Abeni }
933ba8f48f7SPaolo Abeni
mptcp_subflow_recv_lookup(const struct mptcp_sock * msk)9347a6a6cbcSPaolo Abeni static struct sock *mptcp_subflow_recv_lookup(const struct mptcp_sock *msk)
9357a6a6cbcSPaolo Abeni {
9367a6a6cbcSPaolo Abeni struct mptcp_subflow_context *subflow;
9377a6a6cbcSPaolo Abeni
938109cdeb8SGeliang Tang msk_owned_by_me(msk);
9397a6a6cbcSPaolo Abeni
9407a6a6cbcSPaolo Abeni mptcp_for_each_subflow(msk, subflow) {
94199d1055cSPaolo Abeni if (READ_ONCE(subflow->data_avail))
9427a6a6cbcSPaolo Abeni return mptcp_subflow_tcp_sock(subflow);
9437a6a6cbcSPaolo Abeni }
9447a6a6cbcSPaolo Abeni
9457a6a6cbcSPaolo Abeni return NULL;
9467a6a6cbcSPaolo Abeni }
9477a6a6cbcSPaolo Abeni
mptcp_skb_can_collapse_to(u64 write_seq,const struct sk_buff * skb,const struct mptcp_ext * mpext)9483f8e0aaeSPaolo Abeni static bool mptcp_skb_can_collapse_to(u64 write_seq,
94957040755SPaolo Abeni const struct sk_buff *skb,
95057040755SPaolo Abeni const struct mptcp_ext *mpext)
9516d0060f6SMat Martineau {
95257040755SPaolo Abeni if (!tcp_skb_can_collapse_to(skb))
95357040755SPaolo Abeni return false;
95457040755SPaolo Abeni
9555a369ca6SPaolo Abeni /* can collapse only if MPTCP level sequence is in order and this
9565a369ca6SPaolo Abeni * mapping has not been xmitted yet
9575a369ca6SPaolo Abeni */
9585a369ca6SPaolo Abeni return mpext && mpext->data_seq + mpext->data_len == write_seq &&
9595a369ca6SPaolo Abeni !mpext->frozen;
96057040755SPaolo Abeni }
96157040755SPaolo Abeni
96229249eacSPaolo Abeni /* we can append data to the given data frag if:
96329249eacSPaolo Abeni * - there is space available in the backing page_frag
96429249eacSPaolo Abeni * - the data frag tail matches the current page_frag free offset
96529249eacSPaolo Abeni * - the data frag end sequence number matches the current write seq
96629249eacSPaolo Abeni */
mptcp_frag_can_collapse_to(const struct mptcp_sock * msk,const struct page_frag * pfrag,const struct mptcp_data_frag * df)96718b683bfSPaolo Abeni static bool mptcp_frag_can_collapse_to(const struct mptcp_sock *msk,
96818b683bfSPaolo Abeni const struct page_frag *pfrag,
96918b683bfSPaolo Abeni const struct mptcp_data_frag *df)
97018b683bfSPaolo Abeni {
97118b683bfSPaolo Abeni return df && pfrag->page == df->page &&
972d9ca1de8SPaolo Abeni pfrag->size - pfrag->offset > 0 &&
97329249eacSPaolo Abeni pfrag->offset == (df->offset + df->data_len) &&
97418b683bfSPaolo Abeni df->data_seq + df->data_len == msk->write_seq;
97518b683bfSPaolo Abeni }
97618b683bfSPaolo Abeni
dfrag_uncharge(struct sock * sk,int len)977d027236cSPaolo Abeni static void dfrag_uncharge(struct sock *sk, int len)
97818b683bfSPaolo Abeni {
979d027236cSPaolo Abeni sk_mem_uncharge(sk, len);
9807948f6ccSFlorian Westphal sk_wmem_queued_add(sk, -len);
981d027236cSPaolo Abeni }
982d027236cSPaolo Abeni
dfrag_clear(struct sock * sk,struct mptcp_data_frag * dfrag)983d027236cSPaolo Abeni static void dfrag_clear(struct sock *sk, struct mptcp_data_frag *dfrag)
984d027236cSPaolo Abeni {
985d027236cSPaolo Abeni int len = dfrag->data_len + dfrag->overhead;
986d027236cSPaolo Abeni
98718b683bfSPaolo Abeni list_del(&dfrag->list);
988d027236cSPaolo Abeni dfrag_uncharge(sk, len);
98918b683bfSPaolo Abeni put_page(dfrag->page);
99018b683bfSPaolo Abeni }
99118b683bfSPaolo Abeni
9929426ce47SPaolo Abeni /* called under both the msk socket lock and the data lock */
__mptcp_clean_una(struct sock * sk)9936e628cd3SPaolo Abeni static void __mptcp_clean_una(struct sock *sk)
99418b683bfSPaolo Abeni {
99518b683bfSPaolo Abeni struct mptcp_sock *msk = mptcp_sk(sk);
99618b683bfSPaolo Abeni struct mptcp_data_frag *dtmp, *dfrag;
997e1ff9e82SDavide Caratti u64 snd_una;
998e1ff9e82SDavide Caratti
9997439d687SPaolo Abeni snd_una = msk->snd_una;
100018b683bfSPaolo Abeni list_for_each_entry_safe(dfrag, dtmp, &msk->rtx_queue, list) {
100118b683bfSPaolo Abeni if (after64(dfrag->data_seq + dfrag->data_len, snd_una))
100218b683bfSPaolo Abeni break;
100318b683bfSPaolo Abeni
10041e1d9d6fSPaolo Abeni if (unlikely(dfrag == msk->first_pending)) {
10051e1d9d6fSPaolo Abeni /* in recovery mode can see ack after the current snd head */
10061e1d9d6fSPaolo Abeni if (WARN_ON_ONCE(!msk->recovery))
1007d9ca1de8SPaolo Abeni break;
10081e1d9d6fSPaolo Abeni
10091e1d9d6fSPaolo Abeni WRITE_ONCE(msk->first_pending, mptcp_send_next(sk));
10101e1d9d6fSPaolo Abeni }
10111e1d9d6fSPaolo Abeni
1012d027236cSPaolo Abeni dfrag_clear(sk, dfrag);
1013d027236cSPaolo Abeni }
1014d027236cSPaolo Abeni
10157948f6ccSFlorian Westphal dfrag = mptcp_rtx_head(sk);
10167948f6ccSFlorian Westphal if (dfrag && after64(snd_una, dfrag->data_seq)) {
101753eb4c38SPaolo Abeni u64 delta = snd_una - dfrag->data_seq;
101853eb4c38SPaolo Abeni
10191e1d9d6fSPaolo Abeni /* prevent wrap around in recovery mode */
10201e1d9d6fSPaolo Abeni if (unlikely(delta > dfrag->already_sent)) {
10211e1d9d6fSPaolo Abeni if (WARN_ON_ONCE(!msk->recovery))
102253eb4c38SPaolo Abeni goto out;
10231e1d9d6fSPaolo Abeni if (WARN_ON_ONCE(delta > dfrag->data_len))
10241e1d9d6fSPaolo Abeni goto out;
10251e1d9d6fSPaolo Abeni dfrag->already_sent += delta - dfrag->already_sent;
10261e1d9d6fSPaolo Abeni }
10277948f6ccSFlorian Westphal
10287948f6ccSFlorian Westphal dfrag->data_seq += delta;
102953eb4c38SPaolo Abeni dfrag->offset += delta;
10307948f6ccSFlorian Westphal dfrag->data_len -= delta;
1031d9ca1de8SPaolo Abeni dfrag->already_sent -= delta;
10327948f6ccSFlorian Westphal
10337948f6ccSFlorian Westphal dfrag_uncharge(sk, delta);
10347948f6ccSFlorian Westphal }
10357948f6ccSFlorian Westphal
10361e1d9d6fSPaolo Abeni /* all retransmitted data acked, recovery completed */
10371e1d9d6fSPaolo Abeni if (unlikely(msk->recovery) && after64(msk->snd_una, msk->recovery_snd_nxt))
10381e1d9d6fSPaolo Abeni msk->recovery = false;
10391e1d9d6fSPaolo Abeni
104053eb4c38SPaolo Abeni out:
1041d440a4e2SPaolo Abeni if (snd_una == msk->snd_nxt && snd_una == msk->write_seq) {
1042f6909dc1SPaolo Abeni if (mptcp_rtx_timer_pending(sk) && !mptcp_data_fin_enabled(msk))
1043f6909dc1SPaolo Abeni mptcp_stop_rtx_timer(sk);
10446e628cd3SPaolo Abeni } else {
1045f6909dc1SPaolo Abeni mptcp_reset_rtx_timer(sk);
10466e628cd3SPaolo Abeni }
1047d440a4e2SPaolo Abeni
1048d440a4e2SPaolo Abeni if (mptcp_pending_data_fin_ack(sk))
1049d440a4e2SPaolo Abeni mptcp_schedule_work(sk);
10506e628cd3SPaolo Abeni }
10516e628cd3SPaolo Abeni
__mptcp_clean_una_wakeup(struct sock * sk)1052417789dfSPaolo Abeni static void __mptcp_clean_una_wakeup(struct sock *sk)
1053417789dfSPaolo Abeni {
1054765ff425SPaolo Abeni lockdep_assert_held_once(&sk->sk_lock.slock);
1055765ff425SPaolo Abeni
1056417789dfSPaolo Abeni __mptcp_clean_una(sk);
1057417789dfSPaolo Abeni mptcp_write_space(sk);
1058417789dfSPaolo Abeni }
1059417789dfSPaolo Abeni
mptcp_clean_una_wakeup(struct sock * sk)1060b5941f06SPaolo Abeni static void mptcp_clean_una_wakeup(struct sock *sk)
1061b5941f06SPaolo Abeni {
1062b5941f06SPaolo Abeni mptcp_data_lock(sk);
1063b5941f06SPaolo Abeni __mptcp_clean_una_wakeup(sk);
1064b5941f06SPaolo Abeni mptcp_data_unlock(sk);
1065b5941f06SPaolo Abeni }
1066b5941f06SPaolo Abeni
mptcp_enter_memory_pressure(struct sock * sk)1067724cfd2eSPaolo Abeni static void mptcp_enter_memory_pressure(struct sock *sk)
106818b683bfSPaolo Abeni {
1069d9ca1de8SPaolo Abeni struct mptcp_subflow_context *subflow;
1070d9ca1de8SPaolo Abeni struct mptcp_sock *msk = mptcp_sk(sk);
1071d9ca1de8SPaolo Abeni bool first = true;
1072d9ca1de8SPaolo Abeni
1073d9ca1de8SPaolo Abeni mptcp_for_each_subflow(msk, subflow) {
1074d9ca1de8SPaolo Abeni struct sock *ssk = mptcp_subflow_tcp_sock(subflow);
1075d9ca1de8SPaolo Abeni
1076d9ca1de8SPaolo Abeni if (first)
1077d9ca1de8SPaolo Abeni tcp_enter_memory_pressure(ssk);
1078d9ca1de8SPaolo Abeni sk_stream_moderate_sndbuf(ssk);
10798005184fSPaolo Abeni
1080d9ca1de8SPaolo Abeni first = false;
1081d9ca1de8SPaolo Abeni }
10828005184fSPaolo Abeni __mptcp_sync_sndbuf(sk);
1083724cfd2eSPaolo Abeni }
1084724cfd2eSPaolo Abeni
1085724cfd2eSPaolo Abeni /* ensure we get enough memory for the frag hdr, beyond some minimal amount of
1086724cfd2eSPaolo Abeni * data
1087724cfd2eSPaolo Abeni */
mptcp_page_frag_refill(struct sock * sk,struct page_frag * pfrag)1088724cfd2eSPaolo Abeni static bool mptcp_page_frag_refill(struct sock *sk, struct page_frag *pfrag)
1089724cfd2eSPaolo Abeni {
1090724cfd2eSPaolo Abeni if (likely(skb_page_frag_refill(32U + sizeof(struct mptcp_data_frag),
1091724cfd2eSPaolo Abeni pfrag, sk->sk_allocation)))
1092724cfd2eSPaolo Abeni return true;
1093724cfd2eSPaolo Abeni
1094724cfd2eSPaolo Abeni mptcp_enter_memory_pressure(sk);
109518b683bfSPaolo Abeni return false;
109618b683bfSPaolo Abeni }
109718b683bfSPaolo Abeni
109818b683bfSPaolo Abeni static struct mptcp_data_frag *
mptcp_carve_data_frag(const struct mptcp_sock * msk,struct page_frag * pfrag,int orig_offset)109918b683bfSPaolo Abeni mptcp_carve_data_frag(const struct mptcp_sock *msk, struct page_frag *pfrag,
110018b683bfSPaolo Abeni int orig_offset)
110118b683bfSPaolo Abeni {
110218b683bfSPaolo Abeni int offset = ALIGN(orig_offset, sizeof(long));
110318b683bfSPaolo Abeni struct mptcp_data_frag *dfrag;
110418b683bfSPaolo Abeni
110518b683bfSPaolo Abeni dfrag = (struct mptcp_data_frag *)(page_to_virt(pfrag->page) + offset);
110618b683bfSPaolo Abeni dfrag->data_len = 0;
110718b683bfSPaolo Abeni dfrag->data_seq = msk->write_seq;
110818b683bfSPaolo Abeni dfrag->overhead = offset - orig_offset + sizeof(struct mptcp_data_frag);
110918b683bfSPaolo Abeni dfrag->offset = offset + sizeof(struct mptcp_data_frag);
1110d9ca1de8SPaolo Abeni dfrag->already_sent = 0;
111118b683bfSPaolo Abeni dfrag->page = pfrag->page;
111218b683bfSPaolo Abeni
111318b683bfSPaolo Abeni return dfrag;
111418b683bfSPaolo Abeni }
111518b683bfSPaolo Abeni
1116caf971dfSPaolo Abeni struct mptcp_sendmsg_info {
1117caf971dfSPaolo Abeni int mss_now;
1118caf971dfSPaolo Abeni int size_goal;
1119d9ca1de8SPaolo Abeni u16 limit;
1120d9ca1de8SPaolo Abeni u16 sent;
1121d9ca1de8SPaolo Abeni unsigned int flags;
11221094c6feSPaolo Abeni bool data_lock_held;
1123caf971dfSPaolo Abeni };
1124caf971dfSPaolo Abeni
mptcp_check_allowed_size(const struct mptcp_sock * msk,struct sock * ssk,u64 data_seq,int avail_size)1125b713d006SPaolo Abeni static int mptcp_check_allowed_size(const struct mptcp_sock *msk, struct sock *ssk,
1126b713d006SPaolo Abeni u64 data_seq, int avail_size)
11276f8a612aSFlorian Westphal {
11286f8a612aSFlorian Westphal u64 window_end = mptcp_wnd_end(msk);
1129b713d006SPaolo Abeni u64 mptcp_snd_wnd;
11306f8a612aSFlorian Westphal
11316f8a612aSFlorian Westphal if (__mptcp_check_fallback(msk))
11326f8a612aSFlorian Westphal return avail_size;
11336f8a612aSFlorian Westphal
1134b713d006SPaolo Abeni mptcp_snd_wnd = window_end - data_seq;
1135b713d006SPaolo Abeni avail_size = min_t(unsigned int, mptcp_snd_wnd, avail_size);
11366f8a612aSFlorian Westphal
113792be2f52SPaolo Abeni if (unlikely(tcp_sk(ssk)->snd_wnd < mptcp_snd_wnd)) {
1138b713d006SPaolo Abeni tcp_sk(ssk)->snd_wnd = min_t(u64, U32_MAX, mptcp_snd_wnd);
113992be2f52SPaolo Abeni MPTCP_INC_STATS(sock_net(ssk), MPTCP_MIB_SNDWNDSHARED);
114092be2f52SPaolo Abeni }
11416f8a612aSFlorian Westphal
11426f8a612aSFlorian Westphal return avail_size;
11436f8a612aSFlorian Westphal }
11446f8a612aSFlorian Westphal
__mptcp_add_ext(struct sk_buff * skb,gfp_t gfp)1145724cfd2eSPaolo Abeni static bool __mptcp_add_ext(struct sk_buff *skb, gfp_t gfp)
1146724cfd2eSPaolo Abeni {
1147724cfd2eSPaolo Abeni struct skb_ext *mpext = __skb_ext_alloc(gfp);
1148724cfd2eSPaolo Abeni
1149724cfd2eSPaolo Abeni if (!mpext)
1150724cfd2eSPaolo Abeni return false;
1151724cfd2eSPaolo Abeni __skb_ext_set(skb, SKB_EXT_MPTCP, mpext);
1152724cfd2eSPaolo Abeni return true;
1153724cfd2eSPaolo Abeni }
1154724cfd2eSPaolo Abeni
__mptcp_do_alloc_tx_skb(struct sock * sk,gfp_t gfp)11556e628cd3SPaolo Abeni static struct sk_buff *__mptcp_do_alloc_tx_skb(struct sock *sk, gfp_t gfp)
1156724cfd2eSPaolo Abeni {
1157724cfd2eSPaolo Abeni struct sk_buff *skb;
1158724cfd2eSPaolo Abeni
11596e628cd3SPaolo Abeni skb = alloc_skb_fclone(MAX_TCP_HEADER, gfp);
1160724cfd2eSPaolo Abeni if (likely(skb)) {
11616e628cd3SPaolo Abeni if (likely(__mptcp_add_ext(skb, gfp))) {
1162724cfd2eSPaolo Abeni skb_reserve(skb, MAX_TCP_HEADER);
1163a52fe46eSEric Dumazet skb->ip_summed = CHECKSUM_PARTIAL;
1164f70cad10SPaolo Abeni INIT_LIST_HEAD(&skb->tcp_tsorted_anchor);
1165724cfd2eSPaolo Abeni return skb;
1166724cfd2eSPaolo Abeni }
1167724cfd2eSPaolo Abeni __kfree_skb(skb);
1168724cfd2eSPaolo Abeni } else {
1169724cfd2eSPaolo Abeni mptcp_enter_memory_pressure(sk);
1170724cfd2eSPaolo Abeni }
1171724cfd2eSPaolo Abeni return NULL;
1172724cfd2eSPaolo Abeni }
1173724cfd2eSPaolo Abeni
__mptcp_alloc_tx_skb(struct sock * sk,struct sock * ssk,gfp_t gfp)1174f70cad10SPaolo Abeni static struct sk_buff *__mptcp_alloc_tx_skb(struct sock *sk, struct sock *ssk, gfp_t gfp)
1175724cfd2eSPaolo Abeni {
1176724cfd2eSPaolo Abeni struct sk_buff *skb;
1177724cfd2eSPaolo Abeni
11786e628cd3SPaolo Abeni skb = __mptcp_do_alloc_tx_skb(sk, gfp);
1179724cfd2eSPaolo Abeni if (!skb)
1180f70cad10SPaolo Abeni return NULL;
1181724cfd2eSPaolo Abeni
1182724cfd2eSPaolo Abeni if (likely(sk_wmem_schedule(ssk, skb->truesize))) {
1183f70cad10SPaolo Abeni tcp_skb_entail(ssk, skb);
1184f70cad10SPaolo Abeni return skb;
1185724cfd2eSPaolo Abeni }
11863ef3905aSYonglong Li tcp_skb_tsorted_anchor_cleanup(skb);
1187724cfd2eSPaolo Abeni kfree_skb(skb);
1188f70cad10SPaolo Abeni return NULL;
1189724cfd2eSPaolo Abeni }
1190724cfd2eSPaolo Abeni
mptcp_alloc_tx_skb(struct sock * sk,struct sock * ssk,bool data_lock_held)1191f70cad10SPaolo Abeni static struct sk_buff *mptcp_alloc_tx_skb(struct sock *sk, struct sock *ssk, bool data_lock_held)
1192724cfd2eSPaolo Abeni {
11931094c6feSPaolo Abeni gfp_t gfp = data_lock_held ? GFP_ATOMIC : sk->sk_allocation;
1194724cfd2eSPaolo Abeni
11951094c6feSPaolo Abeni return __mptcp_alloc_tx_skb(sk, ssk, gfp);
1196724cfd2eSPaolo Abeni }
1197724cfd2eSPaolo Abeni
1198d0cc2987SGeliang Tang /* note: this always recompute the csum on the whole skb, even
1199d0cc2987SGeliang Tang * if we just appended a single frag. More status info needed
1200d0cc2987SGeliang Tang */
mptcp_update_data_checksum(struct sk_buff * skb,int added)1201d0cc2987SGeliang Tang static void mptcp_update_data_checksum(struct sk_buff *skb, int added)
1202d0cc2987SGeliang Tang {
1203d0cc2987SGeliang Tang struct mptcp_ext *mpext = mptcp_get_ext(skb);
1204d0cc2987SGeliang Tang __wsum csum = ~csum_unfold(mpext->csum);
1205d0cc2987SGeliang Tang int offset = skb->len - added;
1206d0cc2987SGeliang Tang
1207d0cc2987SGeliang Tang mpext->csum = csum_fold(csum_block_add(csum, skb_checksum(skb, offset, added, 0), offset));
1208d0cc2987SGeliang Tang }
1209d0cc2987SGeliang Tang
mptcp_update_infinite_map(struct mptcp_sock * msk,struct sock * ssk,struct mptcp_ext * mpext)12101e39e5a3SGeliang Tang static void mptcp_update_infinite_map(struct mptcp_sock *msk,
12111e39e5a3SGeliang Tang struct sock *ssk,
12121e39e5a3SGeliang Tang struct mptcp_ext *mpext)
12131e39e5a3SGeliang Tang {
12141e39e5a3SGeliang Tang if (!mpext)
12151e39e5a3SGeliang Tang return;
12161e39e5a3SGeliang Tang
12171e39e5a3SGeliang Tang mpext->infinite_map = 1;
12181e39e5a3SGeliang Tang mpext->data_len = 0;
12191e39e5a3SGeliang Tang
1220104125b8SGeliang Tang MPTCP_INC_STATS(sock_net(ssk), MPTCP_MIB_INFINITEMAPTX);
12211e39e5a3SGeliang Tang mptcp_subflow_ctx(ssk)->send_infinite_map = 0;
12221e39e5a3SGeliang Tang pr_fallback(msk);
1223d51991e2SPaolo Abeni mptcp_do_fallback(ssk);
12241e39e5a3SGeliang Tang }
12251e39e5a3SGeliang Tang
12269fce92f0SPaolo Abeni #define MPTCP_MAX_GSO_SIZE (GSO_LEGACY_MAX_SIZE - (MAX_TCP_HEADER + 1))
12279fce92f0SPaolo Abeni
mptcp_sendmsg_frag(struct sock * sk,struct sock * ssk,struct mptcp_data_frag * dfrag,struct mptcp_sendmsg_info * info)122857040755SPaolo Abeni static int mptcp_sendmsg_frag(struct sock *sk, struct sock *ssk,
1229d9ca1de8SPaolo Abeni struct mptcp_data_frag *dfrag,
1230caf971dfSPaolo Abeni struct mptcp_sendmsg_info *info)
123157040755SPaolo Abeni {
1232d9ca1de8SPaolo Abeni u64 data_seq = dfrag->data_seq + info->sent;
1233f70cad10SPaolo Abeni int offset = dfrag->offset + info->sent;
12346d0060f6SMat Martineau struct mptcp_sock *msk = mptcp_sk(sk);
12356f8a612aSFlorian Westphal bool zero_window_probe = false;
12366d0060f6SMat Martineau struct mptcp_ext *mpext = NULL;
1237f70cad10SPaolo Abeni bool can_coalesce = false;
1238f70cad10SPaolo Abeni bool reuse_skb = true;
1239f70cad10SPaolo Abeni struct sk_buff *skb;
1240f70cad10SPaolo Abeni size_t copy;
1241f70cad10SPaolo Abeni int i;
12426d0060f6SMat Martineau
1243e4b61351SGeliang Tang pr_debug("msk=%p ssk=%p sending dfrag at seq=%llu len=%u already sent=%u",
1244d9ca1de8SPaolo Abeni msk, ssk, dfrag->data_seq, dfrag->data_len, info->sent);
12456d0060f6SMat Martineau
1246f70cad10SPaolo Abeni if (WARN_ON_ONCE(info->sent > info->limit ||
1247f70cad10SPaolo Abeni info->limit > dfrag->data_len))
1248f70cad10SPaolo Abeni return 0;
1249f70cad10SPaolo Abeni
1250c886d702SPaolo Abeni if (unlikely(!__tcp_can_send(ssk)))
1251c886d702SPaolo Abeni return -EAGAIN;
1252c886d702SPaolo Abeni
1253d9ca1de8SPaolo Abeni /* compute send limit */
12549fce92f0SPaolo Abeni if (unlikely(ssk->sk_gso_max_size > MPTCP_MAX_GSO_SIZE))
12559fce92f0SPaolo Abeni ssk->sk_gso_max_size = MPTCP_MAX_GSO_SIZE;
1256d9ca1de8SPaolo Abeni info->mss_now = tcp_send_mss(ssk, &info->size_goal, info->flags);
1257f70cad10SPaolo Abeni copy = info->size_goal;
1258f70cad10SPaolo Abeni
125957040755SPaolo Abeni skb = tcp_write_queue_tail(ssk);
1260f70cad10SPaolo Abeni if (skb && copy > skb->len) {
126157040755SPaolo Abeni /* Limit the write to the size available in the
126257040755SPaolo Abeni * current skb, if any, so that we create at most a new skb.
126357040755SPaolo Abeni * Explicitly tells TCP internals to avoid collapsing on later
126457040755SPaolo Abeni * queue management operation, to avoid breaking the ext <->
126557040755SPaolo Abeni * SSN association set here
126657040755SPaolo Abeni */
1267a16c054bSGeliang Tang mpext = mptcp_get_ext(skb);
12681094c6feSPaolo Abeni if (!mptcp_skb_can_collapse_to(data_seq, skb, mpext)) {
126957040755SPaolo Abeni TCP_SKB_CB(skb)->eor = 1;
1270b9cd26f6SPaolo Abeni tcp_mark_push(tcp_sk(ssk), skb);
12711094c6feSPaolo Abeni goto alloc_skb;
12721094c6feSPaolo Abeni }
12731094c6feSPaolo Abeni
1274f70cad10SPaolo Abeni i = skb_shinfo(skb)->nr_frags;
1275f70cad10SPaolo Abeni can_coalesce = skb_can_coalesce(skb, i, dfrag->page, offset);
1276a86a0661SEric Dumazet if (!can_coalesce && i >= READ_ONCE(net_hotdata.sysctl_max_skb_frags)) {
1277f70cad10SPaolo Abeni tcp_mark_push(tcp_sk(ssk), skb);
1278f70cad10SPaolo Abeni goto alloc_skb;
127915e6ca97SPaolo Abeni }
128018b683bfSPaolo Abeni
1281f70cad10SPaolo Abeni copy -= skb->len;
1282f70cad10SPaolo Abeni } else {
12831094c6feSPaolo Abeni alloc_skb:
1284f70cad10SPaolo Abeni skb = mptcp_alloc_tx_skb(sk, ssk, info->data_lock_held);
1285f70cad10SPaolo Abeni if (!skb)
1286f70cad10SPaolo Abeni return -ENOMEM;
1287f70cad10SPaolo Abeni
1288f70cad10SPaolo Abeni i = skb_shinfo(skb)->nr_frags;
1289f70cad10SPaolo Abeni reuse_skb = false;
1290a16c054bSGeliang Tang mpext = mptcp_get_ext(skb);
1291f70cad10SPaolo Abeni }
12921094c6feSPaolo Abeni
12936f8a612aSFlorian Westphal /* Zero window and all data acked? Probe. */
1294b713d006SPaolo Abeni copy = mptcp_check_allowed_size(msk, ssk, data_seq, copy);
1295f70cad10SPaolo Abeni if (copy == 0) {
12967439d687SPaolo Abeni u64 snd_una = READ_ONCE(msk->snd_una);
12977439d687SPaolo Abeni
129872377ab2SPaolo Abeni if (snd_una != msk->snd_nxt || tcp_write_queue_tail(ssk)) {
129927728ba8SEric Dumazet tcp_remove_empty_skb(ssk);
13006f8a612aSFlorian Westphal return 0;
13016f8a612aSFlorian Westphal }
13026f8a612aSFlorian Westphal
1303f70cad10SPaolo Abeni zero_window_probe = true;
1304f70cad10SPaolo Abeni data_seq = snd_una - 1;
1305f70cad10SPaolo Abeni copy = 1;
1306f70cad10SPaolo Abeni }
1307f70cad10SPaolo Abeni
1308f70cad10SPaolo Abeni copy = min_t(size_t, copy, info->limit - info->sent);
1309f70cad10SPaolo Abeni if (!sk_wmem_schedule(ssk, copy)) {
131027728ba8SEric Dumazet tcp_remove_empty_skb(ssk);
1311e2223995SPaolo Abeni return -ENOMEM;
131235759383SFlorian Westphal }
131318b683bfSPaolo Abeni
1314f70cad10SPaolo Abeni if (can_coalesce) {
1315f70cad10SPaolo Abeni skb_frag_size_add(&skb_shinfo(skb)->frags[i - 1], copy);
1316f70cad10SPaolo Abeni } else {
1317f70cad10SPaolo Abeni get_page(dfrag->page);
1318f70cad10SPaolo Abeni skb_fill_page_desc(skb, i, dfrag->page, offset, copy);
131957040755SPaolo Abeni }
132057040755SPaolo Abeni
1321f70cad10SPaolo Abeni skb->len += copy;
1322f70cad10SPaolo Abeni skb->data_len += copy;
1323f70cad10SPaolo Abeni skb->truesize += copy;
1324f70cad10SPaolo Abeni sk_wmem_queued_add(ssk, copy);
1325f70cad10SPaolo Abeni sk_mem_charge(ssk, copy);
1326f70cad10SPaolo Abeni WRITE_ONCE(tcp_sk(ssk)->write_seq, tcp_sk(ssk)->write_seq + copy);
1327f70cad10SPaolo Abeni TCP_SKB_CB(skb)->end_seq += copy;
1328f70cad10SPaolo Abeni tcp_skb_pcount_set(skb, 0);
1329f70cad10SPaolo Abeni
1330f70cad10SPaolo Abeni /* on skb reuse we just need to update the DSS len */
1331f70cad10SPaolo Abeni if (reuse_skb) {
1332f70cad10SPaolo Abeni TCP_SKB_CB(skb)->tcp_flags &= ~TCPHDR_PSH;
1333f70cad10SPaolo Abeni mpext->data_len += copy;
1334f70cad10SPaolo Abeni goto out;
1335724cfd2eSPaolo Abeni }
13366d0060f6SMat Martineau
13376d0060f6SMat Martineau memset(mpext, 0, sizeof(*mpext));
1338d9ca1de8SPaolo Abeni mpext->data_seq = data_seq;
13396d0060f6SMat Martineau mpext->subflow_seq = mptcp_subflow_ctx(ssk)->rel_write_seq;
1340f70cad10SPaolo Abeni mpext->data_len = copy;
13416d0060f6SMat Martineau mpext->use_map = 1;
13426d0060f6SMat Martineau mpext->dsn64 = 1;
13436d0060f6SMat Martineau
13446d0060f6SMat Martineau pr_debug("data_seq=%llu subflow_seq=%u data_len=%u dsn64=%d",
13456d0060f6SMat Martineau mpext->data_seq, mpext->subflow_seq, mpext->data_len,
13466d0060f6SMat Martineau mpext->dsn64);
13476d0060f6SMat Martineau
13486f8a612aSFlorian Westphal if (zero_window_probe) {
1349f70cad10SPaolo Abeni mptcp_subflow_ctx(ssk)->rel_write_seq += copy;
13506f8a612aSFlorian Westphal mpext->frozen = 1;
1351d0cc2987SGeliang Tang if (READ_ONCE(msk->csum_enabled))
1352f70cad10SPaolo Abeni mptcp_update_data_checksum(skb, copy);
13536f8a612aSFlorian Westphal tcp_push_pending_frames(ssk);
1354d0cc2987SGeliang Tang return 0;
13556f8a612aSFlorian Westphal }
135657040755SPaolo Abeni out:
1357d0cc2987SGeliang Tang if (READ_ONCE(msk->csum_enabled))
1358f70cad10SPaolo Abeni mptcp_update_data_checksum(skb, copy);
13591e39e5a3SGeliang Tang if (mptcp_subflow_ctx(ssk)->send_infinite_map)
13601e39e5a3SGeliang Tang mptcp_update_infinite_map(msk, ssk, mpext);
13610eb4e7eeSGeliang Tang trace_mptcp_sendmsg_frag(mpext);
1362f70cad10SPaolo Abeni mptcp_subflow_ctx(ssk)->rel_write_seq += copy;
1363f70cad10SPaolo Abeni return copy;
13646d0060f6SMat Martineau }
13656d0060f6SMat Martineau
1366d5f49190SPaolo Abeni #define MPTCP_SEND_BURST_SIZE ((1 << 16) - \
1367d5f49190SPaolo Abeni sizeof(struct tcphdr) - \
1368d5f49190SPaolo Abeni MAX_TCP_OPTION_SPACE - \
1369d5f49190SPaolo Abeni sizeof(struct ipv6hdr) - \
1370d5f49190SPaolo Abeni sizeof(struct frag_hdr))
1371d5f49190SPaolo Abeni
1372d5f49190SPaolo Abeni struct subflow_send_info {
1373d5f49190SPaolo Abeni struct sock *ssk;
13743ce0852cSPaolo Abeni u64 linger_time;
1375d5f49190SPaolo Abeni };
1376d5f49190SPaolo Abeni
mptcp_subflow_set_active(struct mptcp_subflow_context * subflow)1377ff5a0b42SPaolo Abeni void mptcp_subflow_set_active(struct mptcp_subflow_context *subflow)
1378ff5a0b42SPaolo Abeni {
1379ff5a0b42SPaolo Abeni if (!subflow->stale)
1380ff5a0b42SPaolo Abeni return;
1381ff5a0b42SPaolo Abeni
1382ff5a0b42SPaolo Abeni subflow->stale = 0;
1383fc1b4e3bSPaolo Abeni MPTCP_INC_STATS(sock_net(mptcp_subflow_tcp_sock(subflow)), MPTCP_MIB_SUBFLOWRECOVER);
1384ff5a0b42SPaolo Abeni }
1385ff5a0b42SPaolo Abeni
mptcp_subflow_active(struct mptcp_subflow_context * subflow)1386ff5a0b42SPaolo Abeni bool mptcp_subflow_active(struct mptcp_subflow_context *subflow)
1387ff5a0b42SPaolo Abeni {
1388ff5a0b42SPaolo Abeni if (unlikely(subflow->stale)) {
1389ff5a0b42SPaolo Abeni u32 rcv_tstamp = READ_ONCE(tcp_sk(mptcp_subflow_tcp_sock(subflow))->rcv_tstamp);
1390ff5a0b42SPaolo Abeni
1391ff5a0b42SPaolo Abeni if (subflow->stale_rcv_tstamp == rcv_tstamp)
1392ff5a0b42SPaolo Abeni return false;
1393ff5a0b42SPaolo Abeni
1394ff5a0b42SPaolo Abeni mptcp_subflow_set_active(subflow);
1395ff5a0b42SPaolo Abeni }
1396ff5a0b42SPaolo Abeni return __mptcp_subflow_active(subflow);
1397ff5a0b42SPaolo Abeni }
1398ff5a0b42SPaolo Abeni
13993ce0852cSPaolo Abeni #define SSK_MODE_ACTIVE 0
14003ce0852cSPaolo Abeni #define SSK_MODE_BACKUP 1
14013ce0852cSPaolo Abeni #define SSK_MODE_MAX 2
14023ce0852cSPaolo Abeni
140333d41c9cSPaolo Abeni /* implement the mptcp packet scheduler;
140433d41c9cSPaolo Abeni * returns the subflow that will transmit the next DSS
140533d41c9cSPaolo Abeni * additionally updates the rtx timeout
140633d41c9cSPaolo Abeni */
mptcp_subflow_get_send(struct mptcp_sock * msk)140707336a87SGeliang Tang struct sock *mptcp_subflow_get_send(struct mptcp_sock *msk)
1408f296234cSPeter Krystad {
14093ce0852cSPaolo Abeni struct subflow_send_info send_info[SSK_MODE_MAX];
1410f296234cSPeter Krystad struct mptcp_subflow_context *subflow;
141133d41c9cSPaolo Abeni struct sock *sk = (struct sock *)msk;
14123ce0852cSPaolo Abeni u32 pace, burst, wmem;
1413d5f49190SPaolo Abeni int i, nr_active = 0;
1414d5f49190SPaolo Abeni struct sock *ssk;
14153ce0852cSPaolo Abeni u64 linger_time;
141633d41c9cSPaolo Abeni long tout = 0;
1417f296234cSPeter Krystad
1418d5f49190SPaolo Abeni /* pick the subflow with the lower wmem/wspace ratio */
14193ce0852cSPaolo Abeni for (i = 0; i < SSK_MODE_MAX; ++i) {
1420d5f49190SPaolo Abeni send_info[i].ssk = NULL;
14213ce0852cSPaolo Abeni send_info[i].linger_time = -1;
1422d5f49190SPaolo Abeni }
14233ce0852cSPaolo Abeni
1424d5f49190SPaolo Abeni mptcp_for_each_subflow(msk, subflow) {
1425e10a9892SGeliang Tang trace_mptcp_subflow_get_send(subflow);
1426d5f49190SPaolo Abeni ssk = mptcp_subflow_tcp_sock(subflow);
1427d5f49190SPaolo Abeni if (!mptcp_subflow_active(subflow))
1428f296234cSPeter Krystad continue;
1429d5f49190SPaolo Abeni
143033d41c9cSPaolo Abeni tout = max(tout, mptcp_timeout_from_subflow(subflow));
1431d5f49190SPaolo Abeni nr_active += !subflow->backup;
14323ce0852cSPaolo Abeni pace = subflow->avg_pacing_rate;
14333ce0852cSPaolo Abeni if (unlikely(!pace)) {
14343ce0852cSPaolo Abeni /* init pacing rate from socket */
14353ce0852cSPaolo Abeni subflow->avg_pacing_rate = READ_ONCE(ssk->sk_pacing_rate);
14363ce0852cSPaolo Abeni pace = subflow->avg_pacing_rate;
1437d5f49190SPaolo Abeni if (!pace)
1438d5f49190SPaolo Abeni continue;
14393ce0852cSPaolo Abeni }
1440d5f49190SPaolo Abeni
14413ce0852cSPaolo Abeni linger_time = div_u64((u64)READ_ONCE(ssk->sk_wmem_queued) << 32, pace);
14423ce0852cSPaolo Abeni if (linger_time < send_info[subflow->backup].linger_time) {
1443d5f49190SPaolo Abeni send_info[subflow->backup].ssk = ssk;
14443ce0852cSPaolo Abeni send_info[subflow->backup].linger_time = linger_time;
1445d5f49190SPaolo Abeni }
1446f296234cSPeter Krystad }
144733d41c9cSPaolo Abeni __mptcp_set_timeout(sk, tout);
1448f296234cSPeter Krystad
1449d5f49190SPaolo Abeni /* pick the best backup if no other subflow is active */
1450d5f49190SPaolo Abeni if (!nr_active)
14513ce0852cSPaolo Abeni send_info[SSK_MODE_ACTIVE].ssk = send_info[SSK_MODE_BACKUP].ssk;
1452d5f49190SPaolo Abeni
14533ce0852cSPaolo Abeni /* According to the blest algorithm, to avoid HoL blocking for the
14543ce0852cSPaolo Abeni * faster flow, we need to:
14553ce0852cSPaolo Abeni * - estimate the faster flow linger time
14563ce0852cSPaolo Abeni * - use the above to estimate the amount of byte transferred
14573ce0852cSPaolo Abeni * by the faster flow
14583ce0852cSPaolo Abeni * - check that the amount of queued data is greter than the above,
14593ce0852cSPaolo Abeni * otherwise do not use the picked, slower, subflow
14603ce0852cSPaolo Abeni * We select the subflow with the shorter estimated time to flush
14613ce0852cSPaolo Abeni * the queued mem, which basically ensure the above. We just need
14623ce0852cSPaolo Abeni * to check that subflow has a non empty cwin.
14633ce0852cSPaolo Abeni */
14643ce0852cSPaolo Abeni ssk = send_info[SSK_MODE_ACTIVE].ssk;
1465b713d006SPaolo Abeni if (!ssk || !sk_stream_memory_free(ssk))
1466d5f49190SPaolo Abeni return NULL;
14673ce0852cSPaolo Abeni
1468b713d006SPaolo Abeni burst = min_t(int, MPTCP_SEND_BURST_SIZE, mptcp_wnd_end(msk) - msk->snd_nxt);
14693ce0852cSPaolo Abeni wmem = READ_ONCE(ssk->sk_wmem_queued);
1470ebc1e08fSGeliang Tang if (!burst)
1471b713d006SPaolo Abeni return ssk;
1472b713d006SPaolo Abeni
14733ce0852cSPaolo Abeni subflow = mptcp_subflow_ctx(ssk);
14743ce0852cSPaolo Abeni subflow->avg_pacing_rate = div_u64((u64)subflow->avg_pacing_rate * wmem +
14753ce0852cSPaolo Abeni READ_ONCE(ssk->sk_pacing_rate) * burst,
14763ce0852cSPaolo Abeni burst + wmem);
14773ce0852cSPaolo Abeni msk->snd_burst = burst;
14783ce0852cSPaolo Abeni return ssk;
1479f296234cSPeter Krystad }
1480f296234cSPeter Krystad
mptcp_push_release(struct sock * ssk,struct mptcp_sendmsg_info * info)1481b8e0def3SGeliang Tang static void mptcp_push_release(struct sock *ssk, struct mptcp_sendmsg_info *info)
1482d9ca1de8SPaolo Abeni {
1483d9ca1de8SPaolo Abeni tcp_push(ssk, 0, info->mss_now, tcp_sk(ssk)->nonagle, info->size_goal);
1484d9ca1de8SPaolo Abeni release_sock(ssk);
1485d9ca1de8SPaolo Abeni }
1486d9ca1de8SPaolo Abeni
mptcp_update_post_push(struct mptcp_sock * msk,struct mptcp_data_frag * dfrag,u32 sent)14870d199e43SFlorian Westphal static void mptcp_update_post_push(struct mptcp_sock *msk,
14880d199e43SFlorian Westphal struct mptcp_data_frag *dfrag,
14890d199e43SFlorian Westphal u32 sent)
14900d199e43SFlorian Westphal {
14910d199e43SFlorian Westphal u64 snd_nxt_new = dfrag->data_seq;
14920d199e43SFlorian Westphal
14930d199e43SFlorian Westphal dfrag->already_sent += sent;
14940d199e43SFlorian Westphal
14950d199e43SFlorian Westphal msk->snd_burst -= sent;
14960d199e43SFlorian Westphal
14970d199e43SFlorian Westphal snd_nxt_new += dfrag->already_sent;
14980d199e43SFlorian Westphal
14990d199e43SFlorian Westphal /* snd_nxt_new can be smaller than snd_nxt in case mptcp
15000d199e43SFlorian Westphal * is recovering after a failover. In that event, this re-sends
15010d199e43SFlorian Westphal * old segments.
15020d199e43SFlorian Westphal *
15030d199e43SFlorian Westphal * Thus compute snd_nxt_new candidate based on
15040d199e43SFlorian Westphal * the dfrag->data_seq that was sent and the data
15050d199e43SFlorian Westphal * that has been handed to the subflow for transmission
15060d199e43SFlorian Westphal * and skip update in case it was old dfrag.
15070d199e43SFlorian Westphal */
150838967f42SPaolo Abeni if (likely(after64(snd_nxt_new, msk->snd_nxt))) {
150938967f42SPaolo Abeni msk->bytes_sent += snd_nxt_new - msk->snd_nxt;
1510d440a4e2SPaolo Abeni WRITE_ONCE(msk->snd_nxt, snd_nxt_new);
15110d199e43SFlorian Westphal }
151238967f42SPaolo Abeni }
15130d199e43SFlorian Westphal
mptcp_check_and_set_pending(struct sock * sk)15148b38217aSMaxim Galaganov void mptcp_check_and_set_pending(struct sock *sk)
15153241a9c0SFlorian Westphal {
1516bdd70eb6SPaolo Abeni if (mptcp_send_head(sk)) {
1517bdd70eb6SPaolo Abeni mptcp_data_lock(sk);
1518bdd70eb6SPaolo Abeni mptcp_sk(sk)->cb_flags |= BIT(MPTCP_PUSH_PENDING);
1519bdd70eb6SPaolo Abeni mptcp_data_unlock(sk);
1520bdd70eb6SPaolo Abeni }
15213241a9c0SFlorian Westphal }
15223241a9c0SFlorian Westphal
__subflow_push_pending(struct sock * sk,struct sock * ssk,struct mptcp_sendmsg_info * info)1523c5b4297dSGeliang Tang static int __subflow_push_pending(struct sock *sk, struct sock *ssk,
1524c5b4297dSGeliang Tang struct mptcp_sendmsg_info *info)
1525c5b4297dSGeliang Tang {
1526c5b4297dSGeliang Tang struct mptcp_sock *msk = mptcp_sk(sk);
1527c5b4297dSGeliang Tang struct mptcp_data_frag *dfrag;
1528c5b4297dSGeliang Tang int len, copied = 0, err = 0;
1529c5b4297dSGeliang Tang
1530c5b4297dSGeliang Tang while ((dfrag = mptcp_send_head(sk))) {
1531c5b4297dSGeliang Tang info->sent = dfrag->already_sent;
1532c5b4297dSGeliang Tang info->limit = dfrag->data_len;
1533c5b4297dSGeliang Tang len = dfrag->data_len - dfrag->already_sent;
1534c5b4297dSGeliang Tang while (len > 0) {
1535c5b4297dSGeliang Tang int ret = 0;
1536c5b4297dSGeliang Tang
1537c5b4297dSGeliang Tang ret = mptcp_sendmsg_frag(sk, ssk, dfrag, info);
1538c5b4297dSGeliang Tang if (ret <= 0) {
1539c5b4297dSGeliang Tang err = copied ? : ret;
1540c5b4297dSGeliang Tang goto out;
1541c5b4297dSGeliang Tang }
1542c5b4297dSGeliang Tang
1543c5b4297dSGeliang Tang info->sent += ret;
1544c5b4297dSGeliang Tang copied += ret;
1545c5b4297dSGeliang Tang len -= ret;
1546c5b4297dSGeliang Tang
1547c5b4297dSGeliang Tang mptcp_update_post_push(msk, dfrag, ret);
1548c5b4297dSGeliang Tang }
1549c5b4297dSGeliang Tang WRITE_ONCE(msk->first_pending, mptcp_send_next(sk));
1550c5b4297dSGeliang Tang
1551c5b4297dSGeliang Tang if (msk->snd_burst <= 0 ||
1552c5b4297dSGeliang Tang !sk_stream_memory_free(ssk) ||
1553c5b4297dSGeliang Tang !mptcp_subflow_active(mptcp_subflow_ctx(ssk))) {
1554c5b4297dSGeliang Tang err = copied;
1555c5b4297dSGeliang Tang goto out;
1556c5b4297dSGeliang Tang }
1557c5b4297dSGeliang Tang mptcp_set_timeout(sk);
1558c5b4297dSGeliang Tang }
1559c5b4297dSGeliang Tang err = copied;
1560c5b4297dSGeliang Tang
1561c5b4297dSGeliang Tang out:
156218d82cdeSGeliang Tang if (err > 0)
156318d82cdeSGeliang Tang msk->last_data_sent = tcp_jiffies32;
1564c5b4297dSGeliang Tang return err;
1565c5b4297dSGeliang Tang }
1566c5b4297dSGeliang Tang
__mptcp_push_pending(struct sock * sk,unsigned int flags)1567ff5a0b42SPaolo Abeni void __mptcp_push_pending(struct sock *sk, unsigned int flags)
1568d9ca1de8SPaolo Abeni {
1569d9ca1de8SPaolo Abeni struct sock *prev_ssk = NULL, *ssk = NULL;
1570d9ca1de8SPaolo Abeni struct mptcp_sock *msk = mptcp_sk(sk);
1571d9ca1de8SPaolo Abeni struct mptcp_sendmsg_info info = {
1572d9ca1de8SPaolo Abeni .flags = flags,
1573d9ca1de8SPaolo Abeni };
15740522b424SGeliang Tang bool do_check_data_fin = false;
15750fa1b378SGeliang Tang int push_count = 1;
1576d9ca1de8SPaolo Abeni
15770fa1b378SGeliang Tang while (mptcp_send_head(sk) && (push_count > 0)) {
15780fa1b378SGeliang Tang struct mptcp_subflow_context *subflow;
1579d9ca1de8SPaolo Abeni int ret = 0;
1580d9ca1de8SPaolo Abeni
15810fa1b378SGeliang Tang if (mptcp_sched_get_send(msk))
15820fa1b378SGeliang Tang break;
1583d9ca1de8SPaolo Abeni
15840fa1b378SGeliang Tang push_count = 0;
15850fa1b378SGeliang Tang
15860fa1b378SGeliang Tang mptcp_for_each_subflow(msk, subflow) {
15870fa1b378SGeliang Tang if (READ_ONCE(subflow->scheduled)) {
15880fa1b378SGeliang Tang mptcp_subflow_set_scheduled(subflow, false);
15890fa1b378SGeliang Tang
15900fa1b378SGeliang Tang prev_ssk = ssk;
15910fa1b378SGeliang Tang ssk = mptcp_subflow_tcp_sock(subflow);
15920fa1b378SGeliang Tang if (ssk != prev_ssk) {
15939758f40eSPaolo Abeni /* First check. If the ssk has changed since
15949758f40eSPaolo Abeni * the last round, release prev_ssk
1595d9ca1de8SPaolo Abeni */
15960fa1b378SGeliang Tang if (prev_ssk)
1597b8e0def3SGeliang Tang mptcp_push_release(prev_ssk, &info);
1598d9ca1de8SPaolo Abeni
15999758f40eSPaolo Abeni /* Need to lock the new subflow only if different
16009758f40eSPaolo Abeni * from the previous one, otherwise we are still
16019758f40eSPaolo Abeni * helding the relevant lock
16029758f40eSPaolo Abeni */
1603d9ca1de8SPaolo Abeni lock_sock(ssk);
16040fa1b378SGeliang Tang }
16050fa1b378SGeliang Tang
16060fa1b378SGeliang Tang push_count++;
1607d9ca1de8SPaolo Abeni
1608c5b4297dSGeliang Tang ret = __subflow_push_pending(sk, ssk, &info);
1609d9ca1de8SPaolo Abeni if (ret <= 0) {
16100fa1b378SGeliang Tang if (ret != -EAGAIN ||
16110fa1b378SGeliang Tang (1 << ssk->sk_state) &
16120fa1b378SGeliang Tang (TCPF_FIN_WAIT1 | TCPF_FIN_WAIT2 | TCPF_CLOSE))
16130fa1b378SGeliang Tang push_count--;
1614c886d702SPaolo Abeni continue;
1615d9ca1de8SPaolo Abeni }
16160522b424SGeliang Tang do_check_data_fin = true;
1617d9ca1de8SPaolo Abeni }
16180fa1b378SGeliang Tang }
16190fa1b378SGeliang Tang }
1620d9ca1de8SPaolo Abeni
1621d9ca1de8SPaolo Abeni /* at this point we held the socket lock for the last subflow we used */
1622d9ca1de8SPaolo Abeni if (ssk)
1623b8e0def3SGeliang Tang mptcp_push_release(ssk, &info);
1624d9ca1de8SPaolo Abeni
162533d41c9cSPaolo Abeni /* ensure the rtx timer is running */
1626f6909dc1SPaolo Abeni if (!mptcp_rtx_timer_pending(sk))
1627f6909dc1SPaolo Abeni mptcp_reset_rtx_timer(sk);
16280522b424SGeliang Tang if (do_check_data_fin)
162981c1d029SPaolo Abeni mptcp_check_send_data_fin(sk);
1630d9ca1de8SPaolo Abeni }
1631d9ca1de8SPaolo Abeni
__mptcp_subflow_push_pending(struct sock * sk,struct sock * ssk,bool first)163273a0052aSGeliang Tang static void __mptcp_subflow_push_pending(struct sock *sk, struct sock *ssk, bool first)
16336e628cd3SPaolo Abeni {
16346e628cd3SPaolo Abeni struct mptcp_sock *msk = mptcp_sk(sk);
16351094c6feSPaolo Abeni struct mptcp_sendmsg_info info = {
16361094c6feSPaolo Abeni .data_lock_held = true,
16371094c6feSPaolo Abeni };
16380fa1b378SGeliang Tang bool keep_pushing = true;
1639b19bc294SPaolo Abeni struct sock *xmit_ssk;
1640c5b4297dSGeliang Tang int copied = 0;
16416e628cd3SPaolo Abeni
16426e628cd3SPaolo Abeni info.flags = 0;
16430fa1b378SGeliang Tang while (mptcp_send_head(sk) && keep_pushing) {
16440fa1b378SGeliang Tang struct mptcp_subflow_context *subflow = mptcp_subflow_ctx(ssk);
16456e628cd3SPaolo Abeni int ret = 0;
16466e628cd3SPaolo Abeni
164773a0052aSGeliang Tang /* check for a different subflow usage only after
1648b19bc294SPaolo Abeni * spooling the first chunk of data
1649b19bc294SPaolo Abeni */
16500fa1b378SGeliang Tang if (first) {
16510fa1b378SGeliang Tang mptcp_subflow_set_scheduled(subflow, false);
1652c5b4297dSGeliang Tang ret = __subflow_push_pending(sk, ssk, &info);
1653b19bc294SPaolo Abeni first = false;
1654c5b4297dSGeliang Tang if (ret <= 0)
1655c5b4297dSGeliang Tang break;
1656c5b4297dSGeliang Tang copied += ret;
16570fa1b378SGeliang Tang continue;
16580fa1b378SGeliang Tang }
16590fa1b378SGeliang Tang
16600fa1b378SGeliang Tang if (mptcp_sched_get_send(msk))
16610fa1b378SGeliang Tang goto out;
16620fa1b378SGeliang Tang
16630fa1b378SGeliang Tang if (READ_ONCE(subflow->scheduled)) {
16640fa1b378SGeliang Tang mptcp_subflow_set_scheduled(subflow, false);
16650fa1b378SGeliang Tang ret = __subflow_push_pending(sk, ssk, &info);
16660fa1b378SGeliang Tang if (ret <= 0)
16670fa1b378SGeliang Tang keep_pushing = false;
16680fa1b378SGeliang Tang copied += ret;
16690fa1b378SGeliang Tang }
16700fa1b378SGeliang Tang
16710fa1b378SGeliang Tang mptcp_for_each_subflow(msk, subflow) {
16720fa1b378SGeliang Tang if (READ_ONCE(subflow->scheduled)) {
16730fa1b378SGeliang Tang xmit_ssk = mptcp_subflow_tcp_sock(subflow);
16740fa1b378SGeliang Tang if (xmit_ssk != ssk) {
16750fa1b378SGeliang Tang mptcp_subflow_delegate(subflow,
16760fa1b378SGeliang Tang MPTCP_DELEGATE_SEND);
16770fa1b378SGeliang Tang keep_pushing = false;
16780fa1b378SGeliang Tang }
16790fa1b378SGeliang Tang }
16800fa1b378SGeliang Tang }
16816e628cd3SPaolo Abeni }
16826e628cd3SPaolo Abeni
16836e628cd3SPaolo Abeni out:
16846e628cd3SPaolo Abeni /* __mptcp_alloc_tx_skb could have released some wmem and we are
16856e628cd3SPaolo Abeni * not going to flush it via release_sock()
16866e628cd3SPaolo Abeni */
16876e628cd3SPaolo Abeni if (copied) {
16886e628cd3SPaolo Abeni tcp_push(ssk, 0, info.mss_now, tcp_sk(ssk)->nonagle,
16896e628cd3SPaolo Abeni info.size_goal);
1690f6909dc1SPaolo Abeni if (!mptcp_rtx_timer_pending(sk))
1691f6909dc1SPaolo Abeni mptcp_reset_rtx_timer(sk);
1692d09d818eSPaolo Abeni
16936e628cd3SPaolo Abeni if (msk->snd_data_fin_enable &&
16946e628cd3SPaolo Abeni msk->snd_nxt + 1 == msk->write_seq)
16956e628cd3SPaolo Abeni mptcp_schedule_work(sk);
16966e628cd3SPaolo Abeni }
16976e628cd3SPaolo Abeni }
16986e628cd3SPaolo Abeni
16997d803344SPaolo Abeni static int mptcp_disconnect(struct sock *sk, int flags);
17007d803344SPaolo Abeni
mptcp_sendmsg_fastopen(struct sock * sk,struct msghdr * msg,size_t len,int * copied_syn)1701a2702a07SPaolo Abeni static int mptcp_sendmsg_fastopen(struct sock *sk, struct msghdr *msg,
1702fa9e5746SPaolo Abeni size_t len, int *copied_syn)
1703fa9e5746SPaolo Abeni {
1704fa9e5746SPaolo Abeni unsigned int saved_flags = msg->msg_flags;
1705fa9e5746SPaolo Abeni struct mptcp_sock *msk = mptcp_sk(sk);
1706a2702a07SPaolo Abeni struct sock *ssk;
1707fa9e5746SPaolo Abeni int ret;
1708fa9e5746SPaolo Abeni
1709a2702a07SPaolo Abeni /* on flags based fastopen the mptcp is supposed to create the
1710a2702a07SPaolo Abeni * first subflow right now. Otherwise we are in the defer_connect
1711a2702a07SPaolo Abeni * path, and the first subflow must be already present.
1712a2702a07SPaolo Abeni * Since the defer_connect flag is cleared after the first succsful
1713a2702a07SPaolo Abeni * fastopen attempt, no need to check for additional subflow status.
1714a2702a07SPaolo Abeni */
1715ddb1a072SPaolo Abeni if (msg->msg_flags & MSG_FASTOPEN) {
17163f326a82SPaolo Abeni ssk = __mptcp_nmpc_sk(msk);
17173f326a82SPaolo Abeni if (IS_ERR(ssk))
17183f326a82SPaolo Abeni return PTR_ERR(ssk);
1719ddb1a072SPaolo Abeni }
1720a2702a07SPaolo Abeni if (!msk->first)
1721a2702a07SPaolo Abeni return -EINVAL;
1722a2702a07SPaolo Abeni
1723a2702a07SPaolo Abeni ssk = msk->first;
1724a2702a07SPaolo Abeni
1725fa9e5746SPaolo Abeni lock_sock(ssk);
1726fa9e5746SPaolo Abeni msg->msg_flags |= MSG_DONTWAIT;
17277d803344SPaolo Abeni msk->fastopening = 1;
1728fa9e5746SPaolo Abeni ret = tcp_sendmsg_fastopen(ssk, msg, copied_syn, len, NULL);
17297d803344SPaolo Abeni msk->fastopening = 0;
1730fa9e5746SPaolo Abeni msg->msg_flags = saved_flags;
1731fa9e5746SPaolo Abeni release_sock(ssk);
1732fa9e5746SPaolo Abeni
1733fa9e5746SPaolo Abeni /* do the blocking bits of inet_stream_connect outside the ssk socket lock */
1734fa9e5746SPaolo Abeni if (ret == -EINPROGRESS && !(msg->msg_flags & MSG_DONTWAIT)) {
1735fa9e5746SPaolo Abeni ret = __inet_stream_connect(sk->sk_socket, msg->msg_name,
1736fa9e5746SPaolo Abeni msg->msg_namelen, msg->msg_flags, 1);
1737fa9e5746SPaolo Abeni
1738fa9e5746SPaolo Abeni /* Keep the same behaviour of plain TCP: zero the copied bytes in
1739fa9e5746SPaolo Abeni * case of any error, except timeout or signal
1740fa9e5746SPaolo Abeni */
1741fa9e5746SPaolo Abeni if (ret && ret != -EINPROGRESS && ret != -ERESTARTSYS && ret != -EINTR)
1742fa9e5746SPaolo Abeni *copied_syn = 0;
17437d803344SPaolo Abeni } else if (ret && ret != -EINPROGRESS) {
1744c2b2ae39SPaolo Abeni /* The disconnect() op called by tcp_sendmsg_fastopen()/
1745c2b2ae39SPaolo Abeni * __inet_stream_connect() can fail, due to looking check,
1746c2b2ae39SPaolo Abeni * see mptcp_disconnect().
1747c2b2ae39SPaolo Abeni * Attempt it again outside the problematic scope.
1748c2b2ae39SPaolo Abeni */
1749c2b2ae39SPaolo Abeni if (!mptcp_disconnect(sk, 0))
1750c2b2ae39SPaolo Abeni sk->sk_socket->state = SS_UNCONNECTED;
1751fa9e5746SPaolo Abeni }
175208e39c0dSEric Dumazet inet_clear_bit(DEFER_CONNECT, sk);
1753fa9e5746SPaolo Abeni
1754fa9e5746SPaolo Abeni return ret;
1755fa9e5746SPaolo Abeni }
1756fa9e5746SPaolo Abeni
do_copy_data_nocache(struct sock * sk,int copy,struct iov_iter * from,char * to)17570ffe8e74SPaolo Abeni static int do_copy_data_nocache(struct sock *sk, int copy,
17580ffe8e74SPaolo Abeni struct iov_iter *from, char *to)
17590ffe8e74SPaolo Abeni {
17600ffe8e74SPaolo Abeni if (sk->sk_route_caps & NETIF_F_NOCACHE_COPY) {
17610ffe8e74SPaolo Abeni if (!copy_from_iter_full_nocache(to, copy, from))
17620ffe8e74SPaolo Abeni return -EFAULT;
17630ffe8e74SPaolo Abeni } else if (!copy_from_iter_full(to, copy, from)) {
17640ffe8e74SPaolo Abeni return -EFAULT;
17650ffe8e74SPaolo Abeni }
17660ffe8e74SPaolo Abeni return 0;
17670ffe8e74SPaolo Abeni }
17680ffe8e74SPaolo Abeni
176929b5e5efSPaolo Abeni /* open-code sk_stream_memory_free() plus sent limit computation to
177029b5e5efSPaolo Abeni * avoid indirect calls in fast-path.
177129b5e5efSPaolo Abeni * Called under the msk socket lock, so we can avoid a bunch of ONCE
177229b5e5efSPaolo Abeni * annotations.
177329b5e5efSPaolo Abeni */
mptcp_send_limit(const struct sock * sk)177429b5e5efSPaolo Abeni static u32 mptcp_send_limit(const struct sock *sk)
177529b5e5efSPaolo Abeni {
177629b5e5efSPaolo Abeni const struct mptcp_sock *msk = mptcp_sk(sk);
177729b5e5efSPaolo Abeni u32 limit, not_sent;
177829b5e5efSPaolo Abeni
177929b5e5efSPaolo Abeni if (sk->sk_wmem_queued >= READ_ONCE(sk->sk_sndbuf))
178029b5e5efSPaolo Abeni return 0;
178129b5e5efSPaolo Abeni
178229b5e5efSPaolo Abeni limit = mptcp_notsent_lowat(sk);
178329b5e5efSPaolo Abeni if (limit == UINT_MAX)
178429b5e5efSPaolo Abeni return UINT_MAX;
178529b5e5efSPaolo Abeni
178629b5e5efSPaolo Abeni not_sent = msk->write_seq - msk->snd_nxt;
178729b5e5efSPaolo Abeni if (not_sent >= limit)
178829b5e5efSPaolo Abeni return 0;
178929b5e5efSPaolo Abeni
179029b5e5efSPaolo Abeni return limit - not_sent;
179129b5e5efSPaolo Abeni }
179229b5e5efSPaolo Abeni
mptcp_sendmsg(struct sock * sk,struct msghdr * msg,size_t len)1793f870fa0bSMat Martineau static int mptcp_sendmsg(struct sock *sk, struct msghdr *msg, size_t len)
1794f870fa0bSMat Martineau {
1795f870fa0bSMat Martineau struct mptcp_sock *msk = mptcp_sk(sk);
179617091708SFlorian Westphal struct page_frag *pfrag;
17976d0060f6SMat Martineau size_t copied = 0;
1798caf971dfSPaolo Abeni int ret = 0;
17996d0060f6SMat Martineau long timeo;
1800f870fa0bSMat Martineau
1801987858e5SPaolo Abeni /* silently ignore everything else */
18021e777f39SDmytro Shytyi msg->msg_flags &= MSG_MORE | MSG_DONTWAIT | MSG_NOSIGNAL | MSG_FASTOPEN;
1803987858e5SPaolo Abeni
18046511882cSPaolo Abeni lock_sock(sk);
18051954b860SMat Martineau
180608e39c0dSEric Dumazet if (unlikely(inet_test_bit(DEFER_CONNECT, sk) ||
180708e39c0dSEric Dumazet msg->msg_flags & MSG_FASTOPEN)) {
1808d98a82a6SDmytro Shytyi int copied_syn = 0;
1809d98a82a6SDmytro Shytyi
1810a2702a07SPaolo Abeni ret = mptcp_sendmsg_fastopen(sk, msg, len, &copied_syn);
1811d98a82a6SDmytro Shytyi copied += copied_syn;
1812fa9e5746SPaolo Abeni if (ret == -EINPROGRESS && copied_syn > 0)
1813d98a82a6SDmytro Shytyi goto out;
1814fa9e5746SPaolo Abeni else if (ret)
181569800e51SPaolo Abeni goto do_error;
1816d98a82a6SDmytro Shytyi }
1817d98a82a6SDmytro Shytyi
18181954b860SMat Martineau timeo = sock_sndtimeo(sk, msg->msg_flags & MSG_DONTWAIT);
18191954b860SMat Martineau
18201954b860SMat Martineau if ((1 << sk->sk_state) & ~(TCPF_ESTABLISHED | TCPF_CLOSE_WAIT)) {
18211954b860SMat Martineau ret = sk_stream_wait_connect(sk, &timeo);
18221954b860SMat Martineau if (ret)
182369800e51SPaolo Abeni goto do_error;
18241954b860SMat Martineau }
18251954b860SMat Martineau
182669800e51SPaolo Abeni ret = -EPIPE;
182769800e51SPaolo Abeni if (unlikely(sk->sk_err || (sk->sk_shutdown & SEND_SHUTDOWN)))
182869800e51SPaolo Abeni goto do_error;
182969800e51SPaolo Abeni
183017091708SFlorian Westphal pfrag = sk_page_frag(sk);
183118b683bfSPaolo Abeni
1832d9ca1de8SPaolo Abeni while (msg_data_left(msg)) {
1833724cfd2eSPaolo Abeni int total_ts, frag_truesize = 0;
1834d9ca1de8SPaolo Abeni struct mptcp_data_frag *dfrag;
1835d9ca1de8SPaolo Abeni bool dfrag_collapsed;
1836d9ca1de8SPaolo Abeni size_t psize, offset;
183729b5e5efSPaolo Abeni u32 copy_limit;
183829b5e5efSPaolo Abeni
183929b5e5efSPaolo Abeni /* ensure fitting the notsent_lowat() constraint */
184029b5e5efSPaolo Abeni copy_limit = mptcp_send_limit(sk);
184129b5e5efSPaolo Abeni if (!copy_limit)
184229b5e5efSPaolo Abeni goto wait_for_memory;
1843d9ca1de8SPaolo Abeni
1844d9ca1de8SPaolo Abeni /* reuse tail pfrag, if possible, or carve a new one from the
1845d9ca1de8SPaolo Abeni * page allocator
1846fb529e62SFlorian Westphal */
1847d9ca1de8SPaolo Abeni dfrag = mptcp_pending_tail(sk);
1848d9ca1de8SPaolo Abeni dfrag_collapsed = mptcp_frag_can_collapse_to(msk, pfrag, dfrag);
1849d9ca1de8SPaolo Abeni if (!dfrag_collapsed) {
1850d9ca1de8SPaolo Abeni if (!mptcp_page_frag_refill(sk, pfrag))
1851d9ca1de8SPaolo Abeni goto wait_for_memory;
1852d9ca1de8SPaolo Abeni
1853d9ca1de8SPaolo Abeni dfrag = mptcp_carve_data_frag(msk, pfrag, pfrag->offset);
1854d9ca1de8SPaolo Abeni frag_truesize = dfrag->overhead;
1855fb529e62SFlorian Westphal }
1856fb529e62SFlorian Westphal
1857d9ca1de8SPaolo Abeni /* we do not bound vs wspace, to allow a single packet.
1858d9ca1de8SPaolo Abeni * memory accounting will prevent execessive memory usage
1859d9ca1de8SPaolo Abeni * anyway
1860d9ca1de8SPaolo Abeni */
1861d9ca1de8SPaolo Abeni offset = dfrag->offset + dfrag->data_len;
1862d9ca1de8SPaolo Abeni psize = pfrag->size - offset;
1863d9ca1de8SPaolo Abeni psize = min_t(size_t, psize, msg_data_left(msg));
186429b5e5efSPaolo Abeni psize = min_t(size_t, psize, copy_limit);
1865724cfd2eSPaolo Abeni total_ts = psize + frag_truesize;
18668ce568edSPaolo Abeni
18676511882cSPaolo Abeni if (!sk_wmem_schedule(sk, total_ts))
1868d9ca1de8SPaolo Abeni goto wait_for_memory;
1869d9ca1de8SPaolo Abeni
18700ffe8e74SPaolo Abeni ret = do_copy_data_nocache(sk, psize, &msg->msg_iter,
18710ffe8e74SPaolo Abeni page_address(dfrag->page) + offset);
18720ffe8e74SPaolo Abeni if (ret)
187369800e51SPaolo Abeni goto do_error;
1874d9ca1de8SPaolo Abeni
1875d9ca1de8SPaolo Abeni /* data successfully copied into the write queue */
18765e6300e7SEric Dumazet sk_forward_alloc_add(sk, -total_ts);
1877d9ca1de8SPaolo Abeni copied += psize;
1878d9ca1de8SPaolo Abeni dfrag->data_len += psize;
1879d9ca1de8SPaolo Abeni frag_truesize += psize;
1880d9ca1de8SPaolo Abeni pfrag->offset += frag_truesize;
1881d9ca1de8SPaolo Abeni WRITE_ONCE(msk->write_seq, msk->write_seq + psize);
1882d9ca1de8SPaolo Abeni
1883d9ca1de8SPaolo Abeni /* charge data on mptcp pending queue to the msk socket
1884d9ca1de8SPaolo Abeni * Note: we charge such data both to sk and ssk
1885d9ca1de8SPaolo Abeni */
1886d9ca1de8SPaolo Abeni sk_wmem_queued_add(sk, frag_truesize);
1887d9ca1de8SPaolo Abeni if (!dfrag_collapsed) {
1888d9ca1de8SPaolo Abeni get_page(dfrag->page);
1889d9ca1de8SPaolo Abeni list_add_tail(&dfrag->list, &msk->rtx_queue);
1890d9ca1de8SPaolo Abeni if (!msk->first_pending)
1891d9ca1de8SPaolo Abeni WRITE_ONCE(msk->first_pending, dfrag);
1892d9ca1de8SPaolo Abeni }
1893e4b61351SGeliang Tang pr_debug("msk=%p dfrag at seq=%llu len=%u sent=%u new=%d", msk,
1894d9ca1de8SPaolo Abeni dfrag->data_seq, dfrag->data_len, dfrag->already_sent,
1895d9ca1de8SPaolo Abeni !dfrag_collapsed);
1896d9ca1de8SPaolo Abeni
1897d9ca1de8SPaolo Abeni continue;
1898d9ca1de8SPaolo Abeni
1899d9ca1de8SPaolo Abeni wait_for_memory:
1900037db6eaSPaolo Abeni set_bit(SOCK_NOSPACE, &sk->sk_socket->flags);
1901c2e6048fSPaolo Abeni __mptcp_push_pending(sk, msg->msg_flags);
1902f296234cSPeter Krystad ret = sk_stream_wait_memory(sk, &timeo);
1903f296234cSPeter Krystad if (ret)
190469800e51SPaolo Abeni goto do_error;
1905cec37a6eSPeter Krystad }
1906cec37a6eSPeter Krystad
190713e16037SPaolo Abeni if (copied)
1908c2e6048fSPaolo Abeni __mptcp_push_pending(sk, msg->msg_flags);
1909da51aef5SPaolo Abeni
19101954b860SMat Martineau out:
1911cec37a6eSPeter Krystad release_sock(sk);
191269800e51SPaolo Abeni return copied;
191369800e51SPaolo Abeni
191469800e51SPaolo Abeni do_error:
191569800e51SPaolo Abeni if (copied)
191669800e51SPaolo Abeni goto out;
191769800e51SPaolo Abeni
191869800e51SPaolo Abeni copied = sk_stream_error(sk, msg->msg_flags, ret);
191969800e51SPaolo Abeni goto out;
1920f870fa0bSMat Martineau }
1921f870fa0bSMat Martineau
__mptcp_recvmsg_mskq(struct mptcp_sock * msk,struct msghdr * msg,size_t len,int flags,struct scm_timestamping_internal * tss,int * cmsg_flags)19226771bfd9SFlorian Westphal static int __mptcp_recvmsg_mskq(struct mptcp_sock *msk,
19236771bfd9SFlorian Westphal struct msghdr *msg,
1924b7f653b2SFlorian Westphal size_t len, int flags,
1925b7f653b2SFlorian Westphal struct scm_timestamping_internal *tss,
1926b7f653b2SFlorian Westphal int *cmsg_flags)
19276771bfd9SFlorian Westphal {
1928ca4fb892SYonglong Li struct sk_buff *skb, *tmp;
19296771bfd9SFlorian Westphal int copied = 0;
19306771bfd9SFlorian Westphal
1931ca4fb892SYonglong Li skb_queue_walk_safe(&msk->receive_queue, skb, tmp) {
19326771bfd9SFlorian Westphal u32 offset = MPTCP_SKB_CB(skb)->offset;
19336771bfd9SFlorian Westphal u32 data_len = skb->len - offset;
19346771bfd9SFlorian Westphal u32 count = min_t(size_t, len - copied, data_len);
19356771bfd9SFlorian Westphal int err;
19366771bfd9SFlorian Westphal
1937d976092cSPaolo Abeni if (!(flags & MSG_TRUNC)) {
19386771bfd9SFlorian Westphal err = skb_copy_datagram_msg(skb, offset, msg, count);
19396771bfd9SFlorian Westphal if (unlikely(err < 0)) {
19406771bfd9SFlorian Westphal if (!copied)
19416771bfd9SFlorian Westphal return err;
19426771bfd9SFlorian Westphal break;
19436771bfd9SFlorian Westphal }
1944d976092cSPaolo Abeni }
19456771bfd9SFlorian Westphal
1946b7f653b2SFlorian Westphal if (MPTCP_SKB_CB(skb)->has_rxtstamp) {
1947b7f653b2SFlorian Westphal tcp_update_recv_tstamps(skb, tss);
1948b7f653b2SFlorian Westphal *cmsg_flags |= MPTCP_CMSG_TS;
1949b7f653b2SFlorian Westphal }
1950b7f653b2SFlorian Westphal
19516771bfd9SFlorian Westphal copied += count;
19526771bfd9SFlorian Westphal
19536771bfd9SFlorian Westphal if (count < data_len) {
19542c9e7765SFlorian Westphal if (!(flags & MSG_PEEK)) {
19556771bfd9SFlorian Westphal MPTCP_SKB_CB(skb)->offset += count;
19562c9e7765SFlorian Westphal MPTCP_SKB_CB(skb)->map_seq += count;
19575684ab1aSPaolo Abeni msk->bytes_consumed += count;
19582c9e7765SFlorian Westphal }
19596771bfd9SFlorian Westphal break;
19606771bfd9SFlorian Westphal }
19616771bfd9SFlorian Westphal
1962ca4fb892SYonglong Li if (!(flags & MSG_PEEK)) {
196387952603SPaolo Abeni /* we will bulk release the skb memory later */
196487952603SPaolo Abeni skb->destructor = NULL;
1965ce599c51SPaolo Abeni WRITE_ONCE(msk->rmem_released, msk->rmem_released + skb->truesize);
196687952603SPaolo Abeni __skb_unlink(skb, &msk->receive_queue);
19676771bfd9SFlorian Westphal __kfree_skb(skb);
19685684ab1aSPaolo Abeni msk->bytes_consumed += count;
1969ca4fb892SYonglong Li }
19706771bfd9SFlorian Westphal
19716771bfd9SFlorian Westphal if (copied >= len)
19726771bfd9SFlorian Westphal break;
19736771bfd9SFlorian Westphal }
19746771bfd9SFlorian Westphal
19756771bfd9SFlorian Westphal return copied;
19766771bfd9SFlorian Westphal }
19776771bfd9SFlorian Westphal
1978a6b118feSFlorian Westphal /* receive buffer autotuning. See tcp_rcv_space_adjust for more information.
1979a6b118feSFlorian Westphal *
1980a6b118feSFlorian Westphal * Only difference: Use highest rtt estimate of the subflows in use.
1981a6b118feSFlorian Westphal */
mptcp_rcv_space_adjust(struct mptcp_sock * msk,int copied)1982a6b118feSFlorian Westphal static void mptcp_rcv_space_adjust(struct mptcp_sock *msk, int copied)
1983a6b118feSFlorian Westphal {
1984a6b118feSFlorian Westphal struct mptcp_subflow_context *subflow;
1985a6b118feSFlorian Westphal struct sock *sk = (struct sock *)msk;
1986b8dc6d6cSPaolo Abeni u8 scaling_ratio = U8_MAX;
1987a6b118feSFlorian Westphal u32 time, advmss = 1;
1988a6b118feSFlorian Westphal u64 rtt_us, mstamp;
1989a6b118feSFlorian Westphal
1990109cdeb8SGeliang Tang msk_owned_by_me(msk);
1991a6b118feSFlorian Westphal
1992a6b118feSFlorian Westphal if (copied <= 0)
1993a6b118feSFlorian Westphal return;
1994a6b118feSFlorian Westphal
1995013e3179SPaolo Abeni if (!msk->rcvspace_init)
1996013e3179SPaolo Abeni mptcp_rcv_space_init(msk, msk->first);
1997013e3179SPaolo Abeni
1998a6b118feSFlorian Westphal msk->rcvq_space.copied += copied;
1999a6b118feSFlorian Westphal
2000a6b118feSFlorian Westphal mstamp = div_u64(tcp_clock_ns(), NSEC_PER_USEC);
2001a6b118feSFlorian Westphal time = tcp_stamp_us_delta(mstamp, msk->rcvq_space.time);
2002a6b118feSFlorian Westphal
2003a6b118feSFlorian Westphal rtt_us = msk->rcvq_space.rtt_us;
2004a6b118feSFlorian Westphal if (rtt_us && time < (rtt_us >> 3))
2005a6b118feSFlorian Westphal return;
2006a6b118feSFlorian Westphal
2007a6b118feSFlorian Westphal rtt_us = 0;
2008a6b118feSFlorian Westphal mptcp_for_each_subflow(msk, subflow) {
2009a6b118feSFlorian Westphal const struct tcp_sock *tp;
2010a6b118feSFlorian Westphal u64 sf_rtt_us;
2011a6b118feSFlorian Westphal u32 sf_advmss;
2012a6b118feSFlorian Westphal
2013a6b118feSFlorian Westphal tp = tcp_sk(mptcp_subflow_tcp_sock(subflow));
2014a6b118feSFlorian Westphal
2015a6b118feSFlorian Westphal sf_rtt_us = READ_ONCE(tp->rcv_rtt_est.rtt_us);
2016a6b118feSFlorian Westphal sf_advmss = READ_ONCE(tp->advmss);
2017a6b118feSFlorian Westphal
2018a6b118feSFlorian Westphal rtt_us = max(sf_rtt_us, rtt_us);
2019a6b118feSFlorian Westphal advmss = max(sf_advmss, advmss);
2020b8dc6d6cSPaolo Abeni scaling_ratio = min(tp->scaling_ratio, scaling_ratio);
2021a6b118feSFlorian Westphal }
2022a6b118feSFlorian Westphal
2023a6b118feSFlorian Westphal msk->rcvq_space.rtt_us = rtt_us;
2024b8dc6d6cSPaolo Abeni msk->scaling_ratio = scaling_ratio;
2025a6b118feSFlorian Westphal if (time < (rtt_us >> 3) || rtt_us == 0)
2026a6b118feSFlorian Westphal return;
2027a6b118feSFlorian Westphal
2028a6b118feSFlorian Westphal if (msk->rcvq_space.copied <= msk->rcvq_space.space)
2029a6b118feSFlorian Westphal goto new_measure;
2030a6b118feSFlorian Westphal
203178047648SKuniyuki Iwashima if (READ_ONCE(sock_net(sk)->ipv4.sysctl_tcp_moderate_rcvbuf) &&
2032a6b118feSFlorian Westphal !(sk->sk_userlocks & SOCK_RCVBUF_LOCK)) {
2033a6b118feSFlorian Westphal u64 rcvwin, grow;
2034b8dc6d6cSPaolo Abeni int rcvbuf;
2035a6b118feSFlorian Westphal
2036a6b118feSFlorian Westphal rcvwin = ((u64)msk->rcvq_space.copied << 1) + 16 * advmss;
2037a6b118feSFlorian Westphal
2038a6b118feSFlorian Westphal grow = rcvwin * (msk->rcvq_space.copied - msk->rcvq_space.space);
2039a6b118feSFlorian Westphal
2040a6b118feSFlorian Westphal do_div(grow, msk->rcvq_space.space);
2041a6b118feSFlorian Westphal rcvwin += (grow << 1);
2042a6b118feSFlorian Westphal
2043b8dc6d6cSPaolo Abeni rcvbuf = min_t(u64, __tcp_space_from_win(scaling_ratio, rcvwin),
204402739545SKuniyuki Iwashima READ_ONCE(sock_net(sk)->ipv4.sysctl_tcp_rmem[2]));
2045a6b118feSFlorian Westphal
2046a6b118feSFlorian Westphal if (rcvbuf > sk->sk_rcvbuf) {
2047a6b118feSFlorian Westphal u32 window_clamp;
2048a6b118feSFlorian Westphal
2049b8dc6d6cSPaolo Abeni window_clamp = __tcp_win_from_space(scaling_ratio, rcvbuf);
2050a6b118feSFlorian Westphal WRITE_ONCE(sk->sk_rcvbuf, rcvbuf);
2051a6b118feSFlorian Westphal
2052a6b118feSFlorian Westphal /* Make subflows follow along. If we do not do this, we
2053a6b118feSFlorian Westphal * get drops at subflow level if skbs can't be moved to
2054a6b118feSFlorian Westphal * the mptcp rx queue fast enough (announced rcv_win can
2055a6b118feSFlorian Westphal * exceed ssk->sk_rcvbuf).
2056a6b118feSFlorian Westphal */
2057a6b118feSFlorian Westphal mptcp_for_each_subflow(msk, subflow) {
2058a6b118feSFlorian Westphal struct sock *ssk;
2059c76c6956SPaolo Abeni bool slow;
2060a6b118feSFlorian Westphal
2061a6b118feSFlorian Westphal ssk = mptcp_subflow_tcp_sock(subflow);
2062c76c6956SPaolo Abeni slow = lock_sock_fast(ssk);
2063a6b118feSFlorian Westphal WRITE_ONCE(ssk->sk_rcvbuf, rcvbuf);
2064f410cbeaSEric Dumazet WRITE_ONCE(tcp_sk(ssk)->window_clamp, window_clamp);
2065c76c6956SPaolo Abeni tcp_cleanup_rbuf(ssk, 1);
2066c76c6956SPaolo Abeni unlock_sock_fast(ssk, slow);
2067a6b118feSFlorian Westphal }
2068a6b118feSFlorian Westphal }
2069a6b118feSFlorian Westphal }
2070a6b118feSFlorian Westphal
2071a6b118feSFlorian Westphal msk->rcvq_space.space = msk->rcvq_space.copied;
2072a6b118feSFlorian Westphal new_measure:
2073a6b118feSFlorian Westphal msk->rcvq_space.copied = 0;
2074a6b118feSFlorian Westphal msk->rcvq_space.time = mstamp;
2075a6b118feSFlorian Westphal }
2076a6b118feSFlorian Westphal
__mptcp_update_rmem(struct sock * sk)207787952603SPaolo Abeni static void __mptcp_update_rmem(struct sock *sk)
207887952603SPaolo Abeni {
207987952603SPaolo Abeni struct mptcp_sock *msk = mptcp_sk(sk);
208087952603SPaolo Abeni
208187952603SPaolo Abeni if (!msk->rmem_released)
208287952603SPaolo Abeni return;
208387952603SPaolo Abeni
208487952603SPaolo Abeni atomic_sub(msk->rmem_released, &sk->sk_rmem_alloc);
20856511882cSPaolo Abeni mptcp_rmem_uncharge(sk, msk->rmem_released);
2086ce599c51SPaolo Abeni WRITE_ONCE(msk->rmem_released, 0);
208787952603SPaolo Abeni }
208887952603SPaolo Abeni
__mptcp_splice_receive_queue(struct sock * sk)208987952603SPaolo Abeni static void __mptcp_splice_receive_queue(struct sock *sk)
209087952603SPaolo Abeni {
209187952603SPaolo Abeni struct mptcp_sock *msk = mptcp_sk(sk);
209287952603SPaolo Abeni
209387952603SPaolo Abeni skb_queue_splice_tail_init(&sk->sk_receive_queue, &msk->receive_queue);
209487952603SPaolo Abeni }
209587952603SPaolo Abeni
__mptcp_move_skbs(struct mptcp_sock * msk)2096e3859603SPaolo Abeni static bool __mptcp_move_skbs(struct mptcp_sock *msk)
20976771bfd9SFlorian Westphal {
209887952603SPaolo Abeni struct sock *sk = (struct sock *)msk;
20996771bfd9SFlorian Westphal unsigned int moved = 0;
210087952603SPaolo Abeni bool ret, done;
2101d5f49190SPaolo Abeni
21026771bfd9SFlorian Westphal do {
21036771bfd9SFlorian Westphal struct sock *ssk = mptcp_subflow_recv_lookup(msk);
210465f49fe7SFlorian Westphal bool slowpath;
21056771bfd9SFlorian Westphal
210687952603SPaolo Abeni /* we can have data pending in the subflows only if the msk
210787952603SPaolo Abeni * receive buffer was full at subflow_data_ready() time,
210887952603SPaolo Abeni * that is an unlikely slow path.
210987952603SPaolo Abeni */
211087952603SPaolo Abeni if (likely(!ssk))
21116771bfd9SFlorian Westphal break;
21126771bfd9SFlorian Westphal
211365f49fe7SFlorian Westphal slowpath = lock_sock_fast(ssk);
211487952603SPaolo Abeni mptcp_data_lock(sk);
2115e3859603SPaolo Abeni __mptcp_update_rmem(sk);
21166771bfd9SFlorian Westphal done = __mptcp_move_skbs_from_subflow(msk, ssk, &moved);
211787952603SPaolo Abeni mptcp_data_unlock(sk);
2118499ada50SPaolo Abeni
2119499ada50SPaolo Abeni if (unlikely(ssk->sk_err))
2120499ada50SPaolo Abeni __mptcp_error_report(sk);
212165f49fe7SFlorian Westphal unlock_sock_fast(ssk, slowpath);
21226771bfd9SFlorian Westphal } while (!done);
21236771bfd9SFlorian Westphal
212487952603SPaolo Abeni /* acquire the data lock only if some input data is pending */
212587952603SPaolo Abeni ret = moved > 0;
212687952603SPaolo Abeni if (!RB_EMPTY_ROOT(&msk->out_of_order_queue) ||
212787952603SPaolo Abeni !skb_queue_empty_lockless(&sk->sk_receive_queue)) {
212887952603SPaolo Abeni mptcp_data_lock(sk);
212987952603SPaolo Abeni __mptcp_update_rmem(sk);
213087952603SPaolo Abeni ret |= __mptcp_ofo_queue(msk);
213187952603SPaolo Abeni __mptcp_splice_receive_queue(sk);
213287952603SPaolo Abeni mptcp_data_unlock(sk);
2133ab174ad8SPaolo Abeni }
213487952603SPaolo Abeni if (ret)
213587952603SPaolo Abeni mptcp_check_data_fin((struct sock *)msk);
213687952603SPaolo Abeni return !skb_queue_empty(&msk->receive_queue);
21376771bfd9SFlorian Westphal }
21386771bfd9SFlorian Westphal
mptcp_inq_hint(const struct sock * sk)21392c9e7765SFlorian Westphal static unsigned int mptcp_inq_hint(const struct sock *sk)
21402c9e7765SFlorian Westphal {
21412c9e7765SFlorian Westphal const struct mptcp_sock *msk = mptcp_sk(sk);
21422c9e7765SFlorian Westphal const struct sk_buff *skb;
21432c9e7765SFlorian Westphal
21442c9e7765SFlorian Westphal skb = skb_peek(&msk->receive_queue);
21452c9e7765SFlorian Westphal if (skb) {
21469426ce47SPaolo Abeni u64 hint_val = READ_ONCE(msk->ack_seq) - MPTCP_SKB_CB(skb)->map_seq;
21472c9e7765SFlorian Westphal
21482c9e7765SFlorian Westphal if (hint_val >= INT_MAX)
21492c9e7765SFlorian Westphal return INT_MAX;
21502c9e7765SFlorian Westphal
21512c9e7765SFlorian Westphal return (unsigned int)hint_val;
21522c9e7765SFlorian Westphal }
21532c9e7765SFlorian Westphal
21542c9e7765SFlorian Westphal if (sk->sk_state == TCP_CLOSE || (sk->sk_shutdown & RCV_SHUTDOWN))
21552c9e7765SFlorian Westphal return 1;
21562c9e7765SFlorian Westphal
21572c9e7765SFlorian Westphal return 0;
21582c9e7765SFlorian Westphal }
21592c9e7765SFlorian Westphal
mptcp_recvmsg(struct sock * sk,struct msghdr * msg,size_t len,int flags,int * addr_len)2160f870fa0bSMat Martineau static int mptcp_recvmsg(struct sock *sk, struct msghdr *msg, size_t len,
2161ec095263SOliver Hartkopp int flags, int *addr_len)
2162f870fa0bSMat Martineau {
2163f870fa0bSMat Martineau struct mptcp_sock *msk = mptcp_sk(sk);
2164b7f653b2SFlorian Westphal struct scm_timestamping_internal tss;
2165b7f653b2SFlorian Westphal int copied = 0, cmsg_flags = 0;
21667a6a6cbcSPaolo Abeni int target;
21677a6a6cbcSPaolo Abeni long timeo;
2168f870fa0bSMat Martineau
2169cb9d80f4SPaolo Abeni /* MSG_ERRQUEUE is really a no-op till we support IP_RECVERR */
2170cb9d80f4SPaolo Abeni if (unlikely(flags & MSG_ERRQUEUE))
2171cb9d80f4SPaolo Abeni return inet_recv_error(sk, msg, len, addr_len);
2172cb9d80f4SPaolo Abeni
21736511882cSPaolo Abeni lock_sock(sk);
2174fd897679SPaolo Abeni if (unlikely(sk->sk_state == TCP_LISTEN)) {
2175fd897679SPaolo Abeni copied = -ENOTCONN;
2176fd897679SPaolo Abeni goto out_err;
2177fd897679SPaolo Abeni }
2178fd897679SPaolo Abeni
2179ec095263SOliver Hartkopp timeo = sock_rcvtimeo(sk, flags & MSG_DONTWAIT);
21807a6a6cbcSPaolo Abeni
21817a6a6cbcSPaolo Abeni len = min_t(size_t, len, INT_MAX);
21827a6a6cbcSPaolo Abeni target = sock_rcvlowat(sk, flags & MSG_WAITALL, len);
21837a6a6cbcSPaolo Abeni
21842c9e7765SFlorian Westphal if (unlikely(msk->recvmsg_inq))
21852c9e7765SFlorian Westphal cmsg_flags = MPTCP_CMSG_INQ;
21862c9e7765SFlorian Westphal
218705e3eceaSEric Dumazet while (copied < len) {
2188e3859603SPaolo Abeni int bytes_read;
21897a6a6cbcSPaolo Abeni
2190b7f653b2SFlorian Westphal bytes_read = __mptcp_recvmsg_mskq(msk, msg, len - copied, flags, &tss, &cmsg_flags);
21916771bfd9SFlorian Westphal if (unlikely(bytes_read < 0)) {
21927a6a6cbcSPaolo Abeni if (!copied)
21937a6a6cbcSPaolo Abeni copied = bytes_read;
21946771bfd9SFlorian Westphal goto out_err;
2195cec37a6eSPeter Krystad }
2196cec37a6eSPeter Krystad
21977a6a6cbcSPaolo Abeni copied += bytes_read;
21987a6a6cbcSPaolo Abeni
2199ea4ca586SPaolo Abeni /* be sure to advertise window change */
2200fd897679SPaolo Abeni mptcp_cleanup_rbuf(msk);
2201ea4ca586SPaolo Abeni
2202e3859603SPaolo Abeni if (skb_queue_empty(&msk->receive_queue) && __mptcp_move_skbs(msk))
2203e3859603SPaolo Abeni continue;
2204e3859603SPaolo Abeni
22057a6a6cbcSPaolo Abeni /* only the master socket status is relevant here. The exit
22067a6a6cbcSPaolo Abeni * conditions mirror closely tcp_recvmsg()
22077a6a6cbcSPaolo Abeni */
22087a6a6cbcSPaolo Abeni if (copied >= target)
22097a6a6cbcSPaolo Abeni break;
22107a6a6cbcSPaolo Abeni
22117a6a6cbcSPaolo Abeni if (copied) {
22127a6a6cbcSPaolo Abeni if (sk->sk_err ||
22137a6a6cbcSPaolo Abeni sk->sk_state == TCP_CLOSE ||
22147a6a6cbcSPaolo Abeni (sk->sk_shutdown & RCV_SHUTDOWN) ||
22157a6a6cbcSPaolo Abeni !timeo ||
22167a6a6cbcSPaolo Abeni signal_pending(current))
22177a6a6cbcSPaolo Abeni break;
22187a6a6cbcSPaolo Abeni } else {
22197a6a6cbcSPaolo Abeni if (sk->sk_err) {
22207a6a6cbcSPaolo Abeni copied = sock_error(sk);
22217a6a6cbcSPaolo Abeni break;
22227a6a6cbcSPaolo Abeni }
22237a6a6cbcSPaolo Abeni
222487952603SPaolo Abeni if (sk->sk_shutdown & RCV_SHUTDOWN) {
222587952603SPaolo Abeni /* race breaker: the shutdown could be after the
222687952603SPaolo Abeni * previous receive queue check
222787952603SPaolo Abeni */
2228e3859603SPaolo Abeni if (__mptcp_move_skbs(msk))
222987952603SPaolo Abeni continue;
22307a6a6cbcSPaolo Abeni break;
223187952603SPaolo Abeni }
22327a6a6cbcSPaolo Abeni
22337a6a6cbcSPaolo Abeni if (sk->sk_state == TCP_CLOSE) {
22347a6a6cbcSPaolo Abeni copied = -ENOTCONN;
22357a6a6cbcSPaolo Abeni break;
22367a6a6cbcSPaolo Abeni }
22377a6a6cbcSPaolo Abeni
22387a6a6cbcSPaolo Abeni if (!timeo) {
22397a6a6cbcSPaolo Abeni copied = -EAGAIN;
22407a6a6cbcSPaolo Abeni break;
22417a6a6cbcSPaolo Abeni }
22427a6a6cbcSPaolo Abeni
22437a6a6cbcSPaolo Abeni if (signal_pending(current)) {
22447a6a6cbcSPaolo Abeni copied = sock_intr_errno(timeo);
22457a6a6cbcSPaolo Abeni break;
22467a6a6cbcSPaolo Abeni }
22477a6a6cbcSPaolo Abeni }
22487a6a6cbcSPaolo Abeni
22497a6a6cbcSPaolo Abeni pr_debug("block timeout %ld", timeo);
2250612f71d7SPaolo Abeni sk_wait_data(sk, &timeo, NULL);
22517a6a6cbcSPaolo Abeni }
22523c90e377SPaolo Abeni
22536771bfd9SFlorian Westphal out_err:
2254b7f653b2SFlorian Westphal if (cmsg_flags && copied >= 0) {
2255b7f653b2SFlorian Westphal if (cmsg_flags & MPTCP_CMSG_TS)
2256b7f653b2SFlorian Westphal tcp_recv_timestamp(msg, sk, &tss);
22572c9e7765SFlorian Westphal
22582c9e7765SFlorian Westphal if (cmsg_flags & MPTCP_CMSG_INQ) {
22592c9e7765SFlorian Westphal unsigned int inq = mptcp_inq_hint(sk);
22602c9e7765SFlorian Westphal
22612c9e7765SFlorian Westphal put_cmsg(msg, SOL_TCP, TCP_CM_INQ, sizeof(inq), &inq);
22622c9e7765SFlorian Westphal }
2263b7f653b2SFlorian Westphal }
2264b7f653b2SFlorian Westphal
2265612f71d7SPaolo Abeni pr_debug("msk=%p rx queue empty=%d:%d copied=%d",
2266612f71d7SPaolo Abeni msk, skb_queue_empty_lockless(&sk->sk_receive_queue),
2267612f71d7SPaolo Abeni skb_queue_empty(&msk->receive_queue), copied);
2268ca4fb892SYonglong Li if (!(flags & MSG_PEEK))
2269a6b118feSFlorian Westphal mptcp_rcv_space_adjust(msk, copied);
2270a6b118feSFlorian Westphal
2271cec37a6eSPeter Krystad release_sock(sk);
2272cec37a6eSPeter Krystad return copied;
2273cec37a6eSPeter Krystad }
2274cec37a6eSPeter Krystad
mptcp_retransmit_timer(struct timer_list * t)2275b51f9b80SPaolo Abeni static void mptcp_retransmit_timer(struct timer_list *t)
2276b51f9b80SPaolo Abeni {
2277b51f9b80SPaolo Abeni struct inet_connection_sock *icsk = from_timer(icsk, t,
2278b51f9b80SPaolo Abeni icsk_retransmit_timer);
2279b51f9b80SPaolo Abeni struct sock *sk = &icsk->icsk_inet.sk;
22802d6f5a2bSPaolo Abeni struct mptcp_sock *msk = mptcp_sk(sk);
2281b51f9b80SPaolo Abeni
2282b51f9b80SPaolo Abeni bh_lock_sock(sk);
2283b51f9b80SPaolo Abeni if (!sock_owned_by_user(sk)) {
22842d6f5a2bSPaolo Abeni /* we need a process context to retransmit */
22852d6f5a2bSPaolo Abeni if (!test_and_set_bit(MPTCP_WORK_RTX, &msk->flags))
22862d6f5a2bSPaolo Abeni mptcp_schedule_work(sk);
2287b51f9b80SPaolo Abeni } else {
2288b51f9b80SPaolo Abeni /* delegate our work to tcp_release_cb() */
2289e9d09bacSPaolo Abeni __set_bit(MPTCP_RETRANSMIT, &msk->cb_flags);
2290b51f9b80SPaolo Abeni }
2291b51f9b80SPaolo Abeni bh_unlock_sock(sk);
2292b51f9b80SPaolo Abeni sock_put(sk);
2293b51f9b80SPaolo Abeni }
2294b51f9b80SPaolo Abeni
mptcp_tout_timer(struct timer_list * t)2295f6909dc1SPaolo Abeni static void mptcp_tout_timer(struct timer_list *t)
2296e16163b6SPaolo Abeni {
2297e16163b6SPaolo Abeni struct sock *sk = from_timer(sk, t, sk_timer);
2298e16163b6SPaolo Abeni
2299e16163b6SPaolo Abeni mptcp_schedule_work(sk);
2300b6d69fc8SFlorian Westphal sock_put(sk);
2301e16163b6SPaolo Abeni }
2302e16163b6SPaolo Abeni
23033b1d6210SPaolo Abeni /* Find an idle subflow. Return NULL if there is unacked data at tcp
23043b1d6210SPaolo Abeni * level.
23053b1d6210SPaolo Abeni *
23063b1d6210SPaolo Abeni * A backup subflow is returned only if that is the only kind available.
23073b1d6210SPaolo Abeni */
mptcp_subflow_get_retrans(struct mptcp_sock * msk)230807336a87SGeliang Tang struct sock *mptcp_subflow_get_retrans(struct mptcp_sock *msk)
23093b1d6210SPaolo Abeni {
231071b7dec2SPaolo Abeni struct sock *backup = NULL, *pick = NULL;
23113b1d6210SPaolo Abeni struct mptcp_subflow_context *subflow;
231271b7dec2SPaolo Abeni int min_stale_count = INT_MAX;
23133b1d6210SPaolo Abeni
23143b1d6210SPaolo Abeni mptcp_for_each_subflow(msk, subflow) {
23153b1d6210SPaolo Abeni struct sock *ssk = mptcp_subflow_tcp_sock(subflow);
23163b1d6210SPaolo Abeni
2317ff5a0b42SPaolo Abeni if (!__mptcp_subflow_active(subflow))
2318d5f49190SPaolo Abeni continue;
2319d5f49190SPaolo Abeni
232071b7dec2SPaolo Abeni /* still data outstanding at TCP level? skip this */
232171b7dec2SPaolo Abeni if (!tcp_rtx_and_write_queues_empty(ssk)) {
232271b7dec2SPaolo Abeni mptcp_pm_subflow_chk_stale(msk, ssk);
232371b7dec2SPaolo Abeni min_stale_count = min_t(int, min_stale_count, subflow->stale_count);
2324860975c6SFlorian Westphal continue;
2325860975c6SFlorian Westphal }
23263b1d6210SPaolo Abeni
23273b1d6210SPaolo Abeni if (subflow->backup) {
23283b1d6210SPaolo Abeni if (!backup)
23293b1d6210SPaolo Abeni backup = ssk;
23303b1d6210SPaolo Abeni continue;
23313b1d6210SPaolo Abeni }
23323b1d6210SPaolo Abeni
233371b7dec2SPaolo Abeni if (!pick)
233471b7dec2SPaolo Abeni pick = ssk;
23353b1d6210SPaolo Abeni }
23363b1d6210SPaolo Abeni
233771b7dec2SPaolo Abeni if (pick)
233871b7dec2SPaolo Abeni return pick;
233971b7dec2SPaolo Abeni
234071b7dec2SPaolo Abeni /* use backup only if there are no progresses anywhere */
234171b7dec2SPaolo Abeni return min_stale_count > 1 ? backup : NULL;
23423b1d6210SPaolo Abeni }
23433b1d6210SPaolo Abeni
__mptcp_retransmit_pending_data(struct sock * sk)23441e1d9d6fSPaolo Abeni bool __mptcp_retransmit_pending_data(struct sock *sk)
23451e1d9d6fSPaolo Abeni {
23461e1d9d6fSPaolo Abeni struct mptcp_data_frag *cur, *rtx_head;
23471e1d9d6fSPaolo Abeni struct mptcp_sock *msk = mptcp_sk(sk);
23481e1d9d6fSPaolo Abeni
234900df24f1SGeliang Tang if (__mptcp_check_fallback(msk))
23501e1d9d6fSPaolo Abeni return false;
23511e1d9d6fSPaolo Abeni
23521e1d9d6fSPaolo Abeni /* the closing socket has some data untransmitted and/or unacked:
23531e1d9d6fSPaolo Abeni * some data in the mptcp rtx queue has not really xmitted yet.
23541e1d9d6fSPaolo Abeni * keep it simple and re-inject the whole mptcp level rtx queue
23551e1d9d6fSPaolo Abeni */
23561e1d9d6fSPaolo Abeni mptcp_data_lock(sk);
23571e1d9d6fSPaolo Abeni __mptcp_clean_una_wakeup(sk);
23581e1d9d6fSPaolo Abeni rtx_head = mptcp_rtx_head(sk);
23591e1d9d6fSPaolo Abeni if (!rtx_head) {
23601e1d9d6fSPaolo Abeni mptcp_data_unlock(sk);
23611e1d9d6fSPaolo Abeni return false;
23621e1d9d6fSPaolo Abeni }
23631e1d9d6fSPaolo Abeni
23641e1d9d6fSPaolo Abeni msk->recovery_snd_nxt = msk->snd_nxt;
23651e1d9d6fSPaolo Abeni msk->recovery = true;
23661e1d9d6fSPaolo Abeni mptcp_data_unlock(sk);
23671e1d9d6fSPaolo Abeni
23681e1d9d6fSPaolo Abeni msk->first_pending = rtx_head;
23691e1d9d6fSPaolo Abeni msk->snd_burst = 0;
23701e1d9d6fSPaolo Abeni
23711e1d9d6fSPaolo Abeni /* be sure to clear the "sent status" on all re-injected fragments */
23721e1d9d6fSPaolo Abeni list_for_each_entry(cur, &msk->rtx_queue, list) {
23731e1d9d6fSPaolo Abeni if (!cur->already_sent)
23741e1d9d6fSPaolo Abeni break;
23751e1d9d6fSPaolo Abeni cur->already_sent = 0;
23761e1d9d6fSPaolo Abeni }
23771e1d9d6fSPaolo Abeni
23781e1d9d6fSPaolo Abeni return true;
23791e1d9d6fSPaolo Abeni }
23801e1d9d6fSPaolo Abeni
2381b29fcfb5SPaolo Abeni /* flags for __mptcp_close_ssk() */
2382b29fcfb5SPaolo Abeni #define MPTCP_CF_PUSH BIT(1)
2383b29fcfb5SPaolo Abeni #define MPTCP_CF_FASTCLOSE BIT(2)
2384b29fcfb5SPaolo Abeni
238514c56686SGeliang Tang /* be sure to send a reset only if the caller asked for it, also
238614c56686SGeliang Tang * clean completely the subflow status when the subflow reaches
238714c56686SGeliang Tang * TCP_CLOSE state
238814c56686SGeliang Tang */
__mptcp_subflow_disconnect(struct sock * ssk,struct mptcp_subflow_context * subflow,unsigned int flags)238914c56686SGeliang Tang static void __mptcp_subflow_disconnect(struct sock *ssk,
239014c56686SGeliang Tang struct mptcp_subflow_context *subflow,
239114c56686SGeliang Tang unsigned int flags)
239214c56686SGeliang Tang {
239314c56686SGeliang Tang if (((1 << ssk->sk_state) & (TCPF_CLOSE | TCPF_LISTEN)) ||
239414c56686SGeliang Tang (flags & MPTCP_CF_FASTCLOSE)) {
239514c56686SGeliang Tang /* The MPTCP code never wait on the subflow sockets, TCP-level
239614c56686SGeliang Tang * disconnect should never fail
239714c56686SGeliang Tang */
239814c56686SGeliang Tang WARN_ON_ONCE(tcp_disconnect(ssk, 0));
239914c56686SGeliang Tang mptcp_subflow_ctx_reset(subflow);
240014c56686SGeliang Tang } else {
240114c56686SGeliang Tang tcp_shutdown(ssk, SEND_SHUTDOWN);
240214c56686SGeliang Tang }
240314c56686SGeliang Tang }
240414c56686SGeliang Tang
2405cec37a6eSPeter Krystad /* subflow sockets can be either outgoing (connect) or incoming
2406cec37a6eSPeter Krystad * (accept).
2407cec37a6eSPeter Krystad *
2408cec37a6eSPeter Krystad * Outgoing subflows use in-kernel sockets.
2409cec37a6eSPeter Krystad * Incoming subflows do not have their own 'struct socket' allocated,
2410cec37a6eSPeter Krystad * so we need to use tcp_close() after detaching them from the mptcp
2411cec37a6eSPeter Krystad * parent socket.
2412cec37a6eSPeter Krystad */
__mptcp_close_ssk(struct sock * sk,struct sock * ssk,struct mptcp_subflow_context * subflow,unsigned int flags)2413a141e02eSFlorian Westphal static void __mptcp_close_ssk(struct sock *sk, struct sock *ssk,
2414b29fcfb5SPaolo Abeni struct mptcp_subflow_context *subflow,
2415b29fcfb5SPaolo Abeni unsigned int flags)
2416cec37a6eSPeter Krystad {
2417e0be4931SFlorian Westphal struct mptcp_sock *msk = mptcp_sk(sk);
241863740448SPaolo Abeni bool dispose_it, need_push = false;
241963740448SPaolo Abeni
242063740448SPaolo Abeni /* If the first subflow moved to a close state before accept, e.g. due
242127e5ccc2SPaolo Abeni * to an incoming reset or listener shutdown, the subflow socket is
242227e5ccc2SPaolo Abeni * already deleted by inet_child_forget() and the mptcp socket can't
242327e5ccc2SPaolo Abeni * survive too.
242463740448SPaolo Abeni */
242527e5ccc2SPaolo Abeni if (msk->in_accept_queue && msk->first == ssk &&
242627e5ccc2SPaolo Abeni (sock_flag(sk, SOCK_DEAD) || sock_flag(ssk, SOCK_DEAD))) {
242763740448SPaolo Abeni /* ensure later check in mptcp_worker() will dispose the msk */
242863740448SPaolo Abeni sock_set_flag(sk, SOCK_DEAD);
2429d866ae9aSPaolo Abeni mptcp_set_close_tout(sk, tcp_jiffies32 - (mptcp_close_timeout(sk) + 1));
243063740448SPaolo Abeni lock_sock_nested(ssk, SINGLE_DEPTH_NESTING);
243163740448SPaolo Abeni mptcp_subflow_drop_ctx(ssk);
243263740448SPaolo Abeni goto out_release;
243363740448SPaolo Abeni }
2434e0be4931SFlorian Westphal
243539880bd8SPaolo Abeni dispose_it = msk->free_first || ssk != msk->first;
2436b29fcfb5SPaolo Abeni if (dispose_it)
2437cec37a6eSPeter Krystad list_del(&subflow->node);
2438cec37a6eSPeter Krystad
24393f8b2667SPaolo Abeni lock_sock_nested(ssk, SINGLE_DEPTH_NESTING);
2440e16163b6SPaolo Abeni
2441ff18f9efSPaolo Abeni if ((flags & MPTCP_CF_FASTCLOSE) && !__mptcp_check_fallback(msk)) {
244214c56686SGeliang Tang /* be sure to force the tcp_close path
2443d21f8348SPaolo Abeni * to generate the egress reset
2444d21f8348SPaolo Abeni */
2445d21f8348SPaolo Abeni ssk->sk_lingertime = 0;
2446d21f8348SPaolo Abeni sock_set_flag(ssk, SOCK_LINGER);
2447b29fcfb5SPaolo Abeni subflow->send_fastclose = 1;
2448d21f8348SPaolo Abeni }
2449b29fcfb5SPaolo Abeni
2450b29fcfb5SPaolo Abeni need_push = (flags & MPTCP_CF_PUSH) && __mptcp_retransmit_pending_data(sk);
2451b29fcfb5SPaolo Abeni if (!dispose_it) {
245214c56686SGeliang Tang __mptcp_subflow_disconnect(ssk, subflow, flags);
2453b29fcfb5SPaolo Abeni release_sock(ssk);
2454b29fcfb5SPaolo Abeni
2455b29fcfb5SPaolo Abeni goto out;
2456b29fcfb5SPaolo Abeni }
2457b29fcfb5SPaolo Abeni
2458d7b1bfd0SPaolo Abeni subflow->disposable = 1;
2459d7b1bfd0SPaolo Abeni
2460e16163b6SPaolo Abeni /* if ssk hit tcp_done(), tcp_cleanup_ulp() cleared the related ops
2461e16163b6SPaolo Abeni * the ssk has been already destroyed, we just need to release the
2462e16163b6SPaolo Abeni * reference owned by msk;
2463e16163b6SPaolo Abeni */
2464e16163b6SPaolo Abeni if (!inet_csk(ssk)->icsk_ulp_ops) {
2465b6985b9bSPaolo Abeni WARN_ON_ONCE(!sock_flag(ssk, SOCK_DEAD));
2466e16163b6SPaolo Abeni kfree_rcu(subflow, rcu);
2467e16163b6SPaolo Abeni } else {
2468d7b1bfd0SPaolo Abeni /* otherwise tcp will dispose of the ssk and subflow ctx */
2469e16163b6SPaolo Abeni __tcp_close(ssk, 0);
2470e16163b6SPaolo Abeni
2471e16163b6SPaolo Abeni /* close acquired an extra ref */
2472e16163b6SPaolo Abeni __sock_put(ssk);
2473e16163b6SPaolo Abeni }
247463740448SPaolo Abeni
247563740448SPaolo Abeni out_release:
24769f1a9881SPaolo Abeni __mptcp_subflow_error_report(sk, ssk);
2477e16163b6SPaolo Abeni release_sock(ssk);
2478e16163b6SPaolo Abeni
2479e16163b6SPaolo Abeni sock_put(ssk);
2480e0be4931SFlorian Westphal
2481c8fe62f0SFlorian Westphal if (ssk == msk->first)
24821b1b43eeSPaolo Abeni WRITE_ONCE(msk->first, NULL);
2483c8fe62f0SFlorian Westphal
2484b29fcfb5SPaolo Abeni out:
24858005184fSPaolo Abeni __mptcp_sync_sndbuf(sk);
24861e1d9d6fSPaolo Abeni if (need_push)
24871e1d9d6fSPaolo Abeni __mptcp_push_pending(sk, 0);
248827e5ccc2SPaolo Abeni
248927e5ccc2SPaolo Abeni /* Catch every 'all subflows closed' scenario, including peers silently
249027e5ccc2SPaolo Abeni * closing them, e.g. due to timeout.
249127e5ccc2SPaolo Abeni * For established sockets, allow an additional timeout before closing,
249227e5ccc2SPaolo Abeni * as the protocol can still create more subflows.
249327e5ccc2SPaolo Abeni */
249427e5ccc2SPaolo Abeni if (list_is_singular(&msk->conn_list) && msk->first &&
249527e5ccc2SPaolo Abeni inet_sk_state_load(msk->first) == TCP_CLOSE) {
249627e5ccc2SPaolo Abeni if (sk->sk_state != TCP_ESTABLISHED ||
249727e5ccc2SPaolo Abeni msk->in_accept_queue || sock_flag(sk, SOCK_DEAD)) {
2498c693a851SGeliang Tang mptcp_set_state(sk, TCP_CLOSE);
249927e5ccc2SPaolo Abeni mptcp_close_wake_up(sk);
250027e5ccc2SPaolo Abeni } else {
250127e5ccc2SPaolo Abeni mptcp_start_tout_timer(sk);
250227e5ccc2SPaolo Abeni }
250327e5ccc2SPaolo Abeni }
2504f870fa0bSMat Martineau }
2505f870fa0bSMat Martineau
mptcp_close_ssk(struct sock * sk,struct sock * ssk,struct mptcp_subflow_context * subflow)2506a141e02eSFlorian Westphal void mptcp_close_ssk(struct sock *sk, struct sock *ssk,
2507a141e02eSFlorian Westphal struct mptcp_subflow_context *subflow)
2508a141e02eSFlorian Westphal {
2509b911c97cSFlorian Westphal if (sk->sk_state == TCP_ESTABLISHED)
2510b911c97cSFlorian Westphal mptcp_event(MPTCP_EVENT_SUB_CLOSED, mptcp_sk(sk), ssk, GFP_KERNEL);
2511a88c9e49SPaolo Abeni
2512a88c9e49SPaolo Abeni /* subflow aborted before reaching the fully_established status
2513a88c9e49SPaolo Abeni * attempt the creation of the next subflow
2514a88c9e49SPaolo Abeni */
251574cbb0c6SGeliang Tang mptcp_pm_subflow_check_next(mptcp_sk(sk), subflow);
2516a88c9e49SPaolo Abeni
2517b29fcfb5SPaolo Abeni __mptcp_close_ssk(sk, ssk, subflow, MPTCP_CF_PUSH);
2518a141e02eSFlorian Westphal }
2519a141e02eSFlorian Westphal
mptcp_sync_mss(struct sock * sk,u32 pmtu)2520dc24f8b4SPaolo Abeni static unsigned int mptcp_sync_mss(struct sock *sk, u32 pmtu)
2521dc24f8b4SPaolo Abeni {
2522dc24f8b4SPaolo Abeni return 0;
2523dc24f8b4SPaolo Abeni }
2524dc24f8b4SPaolo Abeni
__mptcp_close_subflow(struct sock * sk)2525b6985b9bSPaolo Abeni static void __mptcp_close_subflow(struct sock *sk)
25260e4f35d7SPaolo Abeni {
25270e4f35d7SPaolo Abeni struct mptcp_subflow_context *subflow, *tmp;
2528b6985b9bSPaolo Abeni struct mptcp_sock *msk = mptcp_sk(sk);
25290e4f35d7SPaolo Abeni
25303abc05d9SFlorian Westphal might_sleep();
25313abc05d9SFlorian Westphal
25325efbf6f7SMatthieu Baerts mptcp_for_each_subflow_safe(msk, subflow, tmp) {
25330e4f35d7SPaolo Abeni struct sock *ssk = mptcp_subflow_tcp_sock(subflow);
25340e4f35d7SPaolo Abeni
25350e4f35d7SPaolo Abeni if (inet_sk_state_load(ssk) != TCP_CLOSE)
25360e4f35d7SPaolo Abeni continue;
25370e4f35d7SPaolo Abeni
253840947e13SFlorian Westphal /* 'subflow_data_ready' will re-sched once rx queue is empty */
253940947e13SFlorian Westphal if (!skb_queue_empty_lockless(&ssk->sk_receive_queue))
254040947e13SFlorian Westphal continue;
254140947e13SFlorian Westphal
2542b6985b9bSPaolo Abeni mptcp_close_ssk(sk, ssk, subflow);
2543b6985b9bSPaolo Abeni }
2544b6985b9bSPaolo Abeni
25450e4f35d7SPaolo Abeni }
25460e4f35d7SPaolo Abeni
mptcp_close_tout_expired(const struct sock * sk)254727e5ccc2SPaolo Abeni static bool mptcp_close_tout_expired(const struct sock *sk)
2548e16163b6SPaolo Abeni {
254927e5ccc2SPaolo Abeni if (!inet_csk(sk)->icsk_mtup.probe_timestamp ||
255027e5ccc2SPaolo Abeni sk->sk_state == TCP_CLOSE)
2551e16163b6SPaolo Abeni return false;
255227e5ccc2SPaolo Abeni
255327e5ccc2SPaolo Abeni return time_after32(tcp_jiffies32,
2554d866ae9aSPaolo Abeni inet_csk(sk)->icsk_mtup.probe_timestamp + mptcp_close_timeout(sk));
2555e16163b6SPaolo Abeni }
2556e16163b6SPaolo Abeni
mptcp_check_fastclose(struct mptcp_sock * msk)255750c504a2SFlorian Westphal static void mptcp_check_fastclose(struct mptcp_sock *msk)
255850c504a2SFlorian Westphal {
255950c504a2SFlorian Westphal struct mptcp_subflow_context *subflow, *tmp;
256080638684SGeliang Tang struct sock *sk = (struct sock *)msk;
256150c504a2SFlorian Westphal
256250c504a2SFlorian Westphal if (likely(!READ_ONCE(msk->rcv_fastclose)))
256350c504a2SFlorian Westphal return;
256450c504a2SFlorian Westphal
256550c504a2SFlorian Westphal mptcp_token_destroy(msk);
256650c504a2SFlorian Westphal
25675efbf6f7SMatthieu Baerts mptcp_for_each_subflow_safe(msk, subflow, tmp) {
256850c504a2SFlorian Westphal struct sock *tcp_sk = mptcp_subflow_tcp_sock(subflow);
256975e908c3SPaolo Abeni bool slow;
257050c504a2SFlorian Westphal
257175e908c3SPaolo Abeni slow = lock_sock_fast(tcp_sk);
257250c504a2SFlorian Westphal if (tcp_sk->sk_state != TCP_CLOSE) {
2573215d4024SJason Xing mptcp_send_active_reset_reason(tcp_sk);
257450c504a2SFlorian Westphal tcp_set_state(tcp_sk, TCP_CLOSE);
257550c504a2SFlorian Westphal }
257675e908c3SPaolo Abeni unlock_sock_fast(tcp_sk, slow);
257750c504a2SFlorian Westphal }
257850c504a2SFlorian Westphal
257969800e51SPaolo Abeni /* Mirror the tcp_reset() error propagation */
258069800e51SPaolo Abeni switch (sk->sk_state) {
258169800e51SPaolo Abeni case TCP_SYN_SENT:
25829ae8e5adSEric Dumazet WRITE_ONCE(sk->sk_err, ECONNREFUSED);
258369800e51SPaolo Abeni break;
258469800e51SPaolo Abeni case TCP_CLOSE_WAIT:
25859ae8e5adSEric Dumazet WRITE_ONCE(sk->sk_err, EPIPE);
258669800e51SPaolo Abeni break;
258769800e51SPaolo Abeni case TCP_CLOSE:
258869800e51SPaolo Abeni return;
258969800e51SPaolo Abeni default:
25909ae8e5adSEric Dumazet WRITE_ONCE(sk->sk_err, ECONNRESET);
259169800e51SPaolo Abeni }
259269800e51SPaolo Abeni
2593c693a851SGeliang Tang mptcp_set_state(sk, TCP_CLOSE);
25946b9831bfSPaolo Abeni WRITE_ONCE(sk->sk_shutdown, SHUTDOWN_MASK);
259550c504a2SFlorian Westphal smp_mb__before_atomic(); /* SHUTDOWN must be visible first */
259650c504a2SFlorian Westphal set_bit(MPTCP_WORK_CLOSE_SUBFLOW, &msk->flags);
259750c504a2SFlorian Westphal
259869800e51SPaolo Abeni /* the calling mptcp_worker will properly destroy the socket */
259969800e51SPaolo Abeni if (sock_flag(sk, SOCK_DEAD))
260069800e51SPaolo Abeni return;
260169800e51SPaolo Abeni
260269800e51SPaolo Abeni sk->sk_state_change(sk);
260369800e51SPaolo Abeni sk_error_report(sk);
260450c504a2SFlorian Westphal }
260550c504a2SFlorian Westphal
__mptcp_retrans(struct sock * sk)26062948d0a1SPaolo Abeni static void __mptcp_retrans(struct sock *sk)
260780992017SPaolo Abeni {
26082948d0a1SPaolo Abeni struct mptcp_sock *msk = mptcp_sk(sk);
2609ee2708aeSGeliang Tang struct mptcp_subflow_context *subflow;
2610caf971dfSPaolo Abeni struct mptcp_sendmsg_info info = {};
26113b1d6210SPaolo Abeni struct mptcp_data_frag *dfrag;
26122948d0a1SPaolo Abeni struct sock *ssk;
2613ee2708aeSGeliang Tang int ret, err;
2614ee2708aeSGeliang Tang u16 len = 0;
26152948d0a1SPaolo Abeni
2616b5941f06SPaolo Abeni mptcp_clean_una_wakeup(sk);
26173241a9c0SFlorian Westphal
26183241a9c0SFlorian Westphal /* first check ssk: need to kick "stale" logic */
2619ee2708aeSGeliang Tang err = mptcp_sched_get_retrans(msk);
26202948d0a1SPaolo Abeni dfrag = mptcp_rtx_head(sk);
26216477dd39SMat Martineau if (!dfrag) {
26226477dd39SMat Martineau if (mptcp_data_fin_enabled(msk)) {
26236477dd39SMat Martineau struct inet_connection_sock *icsk = inet_csk(sk);
26246477dd39SMat Martineau
26256477dd39SMat Martineau icsk->icsk_retransmits++;
26266477dd39SMat Martineau mptcp_set_datafin_timeout(sk);
26276477dd39SMat Martineau mptcp_send_ack(msk);
26286477dd39SMat Martineau
26296477dd39SMat Martineau goto reset_timer;
26306477dd39SMat Martineau }
26316477dd39SMat Martineau
26323241a9c0SFlorian Westphal if (!mptcp_send_head(sk))
26332948d0a1SPaolo Abeni return;
26343241a9c0SFlorian Westphal
26353241a9c0SFlorian Westphal goto reset_timer;
26366477dd39SMat Martineau }
26372948d0a1SPaolo Abeni
2638ee2708aeSGeliang Tang if (err)
26392948d0a1SPaolo Abeni goto reset_timer;
26402948d0a1SPaolo Abeni
2641ee2708aeSGeliang Tang mptcp_for_each_subflow(msk, subflow) {
2642ee2708aeSGeliang Tang if (READ_ONCE(subflow->scheduled)) {
2643ee2708aeSGeliang Tang u16 copied = 0;
2644ee2708aeSGeliang Tang
2645ee2708aeSGeliang Tang mptcp_subflow_set_scheduled(subflow, false);
2646ee2708aeSGeliang Tang
2647ee2708aeSGeliang Tang ssk = mptcp_subflow_tcp_sock(subflow);
2648ee2708aeSGeliang Tang
26492948d0a1SPaolo Abeni lock_sock(ssk);
26502948d0a1SPaolo Abeni
26512948d0a1SPaolo Abeni /* limit retransmission to the bytes already sent on some subflows */
26522948d0a1SPaolo Abeni info.sent = 0;
2653ee2708aeSGeliang Tang info.limit = READ_ONCE(msk->csum_enabled) ? dfrag->data_len :
2654ee2708aeSGeliang Tang dfrag->already_sent;
26554e14867dSPaolo Abeni while (info.sent < info.limit) {
26562948d0a1SPaolo Abeni ret = mptcp_sendmsg_frag(sk, ssk, dfrag, &info);
26572948d0a1SPaolo Abeni if (ret <= 0)
26582948d0a1SPaolo Abeni break;
26592948d0a1SPaolo Abeni
26602948d0a1SPaolo Abeni MPTCP_INC_STATS(sock_net(sk), MPTCP_MIB_RETRANSSEGS);
26612948d0a1SPaolo Abeni copied += ret;
26622948d0a1SPaolo Abeni info.sent += ret;
26632948d0a1SPaolo Abeni }
26644e14867dSPaolo Abeni if (copied) {
2665ee2708aeSGeliang Tang len = max(copied, len);
26662948d0a1SPaolo Abeni tcp_push(ssk, 0, info.mss_now, tcp_sk(ssk)->nonagle,
26672948d0a1SPaolo Abeni info.size_goal);
26680530020aSGeliang Tang WRITE_ONCE(msk->allow_infinite_fallback, false);
26694e14867dSPaolo Abeni }
26702948d0a1SPaolo Abeni
26712948d0a1SPaolo Abeni release_sock(ssk);
2672ee2708aeSGeliang Tang }
2673ee2708aeSGeliang Tang }
2674ee2708aeSGeliang Tang
2675ee2708aeSGeliang Tang msk->bytes_retrans += len;
2676ee2708aeSGeliang Tang dfrag->already_sent = max(dfrag->already_sent, len);
26772948d0a1SPaolo Abeni
26782948d0a1SPaolo Abeni reset_timer:
26793241a9c0SFlorian Westphal mptcp_check_and_set_pending(sk);
26803241a9c0SFlorian Westphal
2681f6909dc1SPaolo Abeni if (!mptcp_rtx_timer_pending(sk))
2682f6909dc1SPaolo Abeni mptcp_reset_rtx_timer(sk);
26832948d0a1SPaolo Abeni }
26842948d0a1SPaolo Abeni
268576a13b31SGeliang Tang /* schedule the timeout timer for the relevant event: either close timeout
268676a13b31SGeliang Tang * or mp_fail timeout. The close timeout takes precedence on the mp_fail one
268776a13b31SGeliang Tang */
mptcp_reset_tout_timer(struct mptcp_sock * msk,unsigned long fail_tout)2688f6909dc1SPaolo Abeni void mptcp_reset_tout_timer(struct mptcp_sock *msk, unsigned long fail_tout)
268976a13b31SGeliang Tang {
269076a13b31SGeliang Tang struct sock *sk = (struct sock *)msk;
269176a13b31SGeliang Tang unsigned long timeout, close_timeout;
269276a13b31SGeliang Tang
269327e5ccc2SPaolo Abeni if (!fail_tout && !inet_csk(sk)->icsk_mtup.probe_timestamp)
269476a13b31SGeliang Tang return;
269576a13b31SGeliang Tang
269627e5ccc2SPaolo Abeni close_timeout = inet_csk(sk)->icsk_mtup.probe_timestamp - tcp_jiffies32 + jiffies +
2697d866ae9aSPaolo Abeni mptcp_close_timeout(sk);
269876a13b31SGeliang Tang
269976a13b31SGeliang Tang /* the close timeout takes precedence on the fail one, and here at least one of
270076a13b31SGeliang Tang * them is active
270176a13b31SGeliang Tang */
270227e5ccc2SPaolo Abeni timeout = inet_csk(sk)->icsk_mtup.probe_timestamp ? close_timeout : fail_tout;
270376a13b31SGeliang Tang
270476a13b31SGeliang Tang sk_reset_timer(sk, &sk->sk_timer, timeout);
270576a13b31SGeliang Tang }
270676a13b31SGeliang Tang
mptcp_mp_fail_no_response(struct mptcp_sock * msk)270749fa1919SGeliang Tang static void mptcp_mp_fail_no_response(struct mptcp_sock *msk)
270849fa1919SGeliang Tang {
270976a13b31SGeliang Tang struct sock *ssk = msk->first;
271049fa1919SGeliang Tang bool slow;
271149fa1919SGeliang Tang
271276a13b31SGeliang Tang if (!ssk)
271376a13b31SGeliang Tang return;
271476a13b31SGeliang Tang
271549fa1919SGeliang Tang pr_debug("MP_FAIL doesn't respond, reset the subflow");
271649fa1919SGeliang Tang
271749fa1919SGeliang Tang slow = lock_sock_fast(ssk);
271849fa1919SGeliang Tang mptcp_subflow_reset(ssk);
271976a13b31SGeliang Tang WRITE_ONCE(mptcp_subflow_ctx(ssk)->fail_tout, 0);
272049fa1919SGeliang Tang unlock_sock_fast(ssk, slow);
272149fa1919SGeliang Tang }
272249fa1919SGeliang Tang
mptcp_do_fastclose(struct sock * sk)2723d21f8348SPaolo Abeni static void mptcp_do_fastclose(struct sock *sk)
2724d21f8348SPaolo Abeni {
2725d21f8348SPaolo Abeni struct mptcp_subflow_context *subflow, *tmp;
2726d21f8348SPaolo Abeni struct mptcp_sock *msk = mptcp_sk(sk);
2727d21f8348SPaolo Abeni
2728c693a851SGeliang Tang mptcp_set_state(sk, TCP_CLOSE);
2729d21f8348SPaolo Abeni mptcp_for_each_subflow_safe(msk, subflow, tmp)
2730d21f8348SPaolo Abeni __mptcp_close_ssk(sk, mptcp_subflow_tcp_sock(subflow),
2731d21f8348SPaolo Abeni subflow, MPTCP_CF_FASTCLOSE);
2732d21f8348SPaolo Abeni }
2733d21f8348SPaolo Abeni
mptcp_worker(struct work_struct * work)27342948d0a1SPaolo Abeni static void mptcp_worker(struct work_struct *work)
27352948d0a1SPaolo Abeni {
27362948d0a1SPaolo Abeni struct mptcp_sock *msk = container_of(work, struct mptcp_sock, work);
273780638684SGeliang Tang struct sock *sk = (struct sock *)msk;
273876a13b31SGeliang Tang unsigned long fail_tout;
27392948d0a1SPaolo Abeni int state;
274080992017SPaolo Abeni
274180992017SPaolo Abeni lock_sock(sk);
2742e16163b6SPaolo Abeni state = sk->sk_state;
2743d6a04437SPaolo Abeni if (unlikely((1 << state) & (TCPF_CLOSE | TCPF_LISTEN)))
2744e16163b6SPaolo Abeni goto unlock;
2745e16163b6SPaolo Abeni
274650c504a2SFlorian Westphal mptcp_check_fastclose(msk);
274750c504a2SFlorian Westphal
2748e9801430SFlorian Westphal mptcp_pm_nl_work(msk);
2749b416268bSFlorian Westphal
275081c1d029SPaolo Abeni mptcp_check_send_data_fin(sk);
275181c1d029SPaolo Abeni mptcp_check_data_fin_ack(sk);
275243b54c6eSMat Martineau mptcp_check_data_fin(sk);
275343b54c6eSMat Martineau
2754b6985b9bSPaolo Abeni if (test_and_clear_bit(MPTCP_WORK_CLOSE_SUBFLOW, &msk->flags))
2755b6985b9bSPaolo Abeni __mptcp_close_subflow(sk);
2756b6985b9bSPaolo Abeni
275727e5ccc2SPaolo Abeni if (mptcp_close_tout_expired(sk)) {
2758d21f8348SPaolo Abeni mptcp_do_fastclose(sk);
275927e5ccc2SPaolo Abeni mptcp_close_wake_up(sk);
276027e5ccc2SPaolo Abeni }
2761bbd49d11SPaolo Abeni
276227e5ccc2SPaolo Abeni if (sock_flag(sk, SOCK_DEAD) && sk->sk_state == TCP_CLOSE) {
2763e16163b6SPaolo Abeni __mptcp_destroy_sock(sk);
2764e16163b6SPaolo Abeni goto unlock;
2765e16163b6SPaolo Abeni }
2766e16163b6SPaolo Abeni
27672948d0a1SPaolo Abeni if (test_and_clear_bit(MPTCP_WORK_RTX, &msk->flags))
27682948d0a1SPaolo Abeni __mptcp_retrans(sk);
27693b1d6210SPaolo Abeni
277076a13b31SGeliang Tang fail_tout = msk->first ? READ_ONCE(mptcp_subflow_ctx(msk->first)->fail_tout) : 0;
277176a13b31SGeliang Tang if (fail_tout && time_after(jiffies, fail_tout))
277249fa1919SGeliang Tang mptcp_mp_fail_no_response(msk);
277349fa1919SGeliang Tang
27743b1d6210SPaolo Abeni unlock:
277580992017SPaolo Abeni release_sock(sk);
277680992017SPaolo Abeni sock_put(sk);
277780992017SPaolo Abeni }
277880992017SPaolo Abeni
__mptcp_init_sock(struct sock * sk)2779e2636917SKuniyuki Iwashima static void __mptcp_init_sock(struct sock *sk)
2780f870fa0bSMat Martineau {
2781cec37a6eSPeter Krystad struct mptcp_sock *msk = mptcp_sk(sk);
2782cec37a6eSPeter Krystad
2783cec37a6eSPeter Krystad INIT_LIST_HEAD(&msk->conn_list);
2784ec3edaa7SPeter Krystad INIT_LIST_HEAD(&msk->join_list);
278518b683bfSPaolo Abeni INIT_LIST_HEAD(&msk->rtx_queue);
278680992017SPaolo Abeni INIT_WORK(&msk->work, mptcp_worker);
278787952603SPaolo Abeni __skb_queue_head_init(&msk->receive_queue);
2788ab174ad8SPaolo Abeni msk->out_of_order_queue = RB_ROOT;
2789f0e6a4cfSPaolo Abeni msk->first_pending = NULL;
27909426ce47SPaolo Abeni WRITE_ONCE(msk->rmem_fwd_alloc, 0);
2791ce599c51SPaolo Abeni WRITE_ONCE(msk->rmem_released, 0);
279233d41c9cSPaolo Abeni msk->timer_ival = TCP_RTO_MIN;
27935684ab1aSPaolo Abeni msk->scaling_ratio = TCP_DEFAULT_SCALING_RATIO;
2794cec37a6eSPeter Krystad
27951b1b43eeSPaolo Abeni WRITE_ONCE(msk->first, NULL);
2796dc24f8b4SPaolo Abeni inet_csk(sk)->icsk_sync_mss = mptcp_sync_mss;
2797752e9067SGeliang Tang WRITE_ONCE(msk->csum_enabled, mptcp_is_checksum_enabled(sock_net(sk)));
27980530020aSGeliang Tang WRITE_ONCE(msk->allow_infinite_fallback, true);
27991e1d9d6fSPaolo Abeni msk->recovery = false;
28006f06b4d4SPaolo Abeni msk->subflow_id = 1;
280118d82cdeSGeliang Tang msk->last_data_sent = tcp_jiffies32;
280218d82cdeSGeliang Tang msk->last_data_recv = tcp_jiffies32;
280318d82cdeSGeliang Tang msk->last_ack_recv = tcp_jiffies32;
28048ab183deSPaolo Abeni
28051b1c7a0eSPeter Krystad mptcp_pm_data_init(msk);
28061b1c7a0eSPeter Krystad
2807b51f9b80SPaolo Abeni /* re-use the csk retrans timer for MPTCP-level retrans */
2808b51f9b80SPaolo Abeni timer_setup(&msk->sk.icsk_retransmit_timer, mptcp_retransmit_timer, 0);
2809f6909dc1SPaolo Abeni timer_setup(&sk->sk_timer, mptcp_tout_timer, 0);
2810f870fa0bSMat Martineau }
2811f870fa0bSMat Martineau
mptcp_ca_reset(struct sock * sk)2812b29fcfb5SPaolo Abeni static void mptcp_ca_reset(struct sock *sk)
2813784325e9SMatthieu Baerts {
281420b5759fSPaolo Abeni struct inet_connection_sock *icsk = inet_csk(sk);
2815b29fcfb5SPaolo Abeni
2816b29fcfb5SPaolo Abeni tcp_assign_congestion_control(sk);
28175eae7a82SMatthieu Baerts (NGI0) strscpy(mptcp_sk(sk)->ca_name, icsk->icsk_ca_ops->name,
28185eae7a82SMatthieu Baerts (NGI0) sizeof(mptcp_sk(sk)->ca_name));
2819b29fcfb5SPaolo Abeni
2820b29fcfb5SPaolo Abeni /* no need to keep a reference to the ops, the name will suffice */
2821b29fcfb5SPaolo Abeni tcp_cleanup_congestion_control(sk);
2822b29fcfb5SPaolo Abeni icsk->icsk_ca_ops = NULL;
2823b29fcfb5SPaolo Abeni }
2824b29fcfb5SPaolo Abeni
mptcp_init_sock(struct sock * sk)2825b29fcfb5SPaolo Abeni static int mptcp_init_sock(struct sock *sk)
2826b29fcfb5SPaolo Abeni {
2827fc518953SFlorian Westphal struct net *net = sock_net(sk);
28281730b2b2SGeliang Tang int ret;
282918b683bfSPaolo Abeni
2830e2636917SKuniyuki Iwashima __mptcp_init_sock(sk);
2831b6c08380SGeliang Tang
2832fc518953SFlorian Westphal if (!mptcp_is_enabled(net))
2833fc518953SFlorian Westphal return -ENOPROTOOPT;
2834fc518953SFlorian Westphal
2835fc518953SFlorian Westphal if (unlikely(!net->mib.mptcp_statistics) && !mptcp_mib_alloc(net))
2836fc518953SFlorian Westphal return -ENOMEM;
2837fc518953SFlorian Westphal
28381730b2b2SGeliang Tang ret = mptcp_init_sched(mptcp_sk(sk),
28391730b2b2SGeliang Tang mptcp_sched_find(mptcp_get_scheduler(net)));
28401730b2b2SGeliang Tang if (ret)
28411730b2b2SGeliang Tang return ret;
28421730b2b2SGeliang Tang
2843a5ef058dSPaolo Abeni set_bit(SOCK_CUSTOM_SOCKOPT, &sk->sk_socket->flags);
2844a5ef058dSPaolo Abeni
284520b5759fSPaolo Abeni /* fetch the ca name; do it outside __mptcp_init_sock(), so that clone will
284620b5759fSPaolo Abeni * propagate the correct value
284720b5759fSPaolo Abeni */
2848b29fcfb5SPaolo Abeni mptcp_ca_reset(sk);
284920b5759fSPaolo Abeni
2850d027236cSPaolo Abeni sk_sockets_allocated_inc(sk);
2851a963853fSGeliang Tang sk->sk_rcvbuf = READ_ONCE(net->ipv4.sysctl_tcp_rmem[1]);
2852a963853fSGeliang Tang sk->sk_sndbuf = READ_ONCE(net->ipv4.sysctl_tcp_wmem[1]);
2853d027236cSPaolo Abeni
285418b683bfSPaolo Abeni return 0;
285518b683bfSPaolo Abeni }
285618b683bfSPaolo Abeni
__mptcp_clear_xmit(struct sock * sk)285718b683bfSPaolo Abeni static void __mptcp_clear_xmit(struct sock *sk)
285818b683bfSPaolo Abeni {
285918b683bfSPaolo Abeni struct mptcp_sock *msk = mptcp_sk(sk);
286018b683bfSPaolo Abeni struct mptcp_data_frag *dtmp, *dfrag;
286118b683bfSPaolo Abeni
2862d9ca1de8SPaolo Abeni WRITE_ONCE(msk->first_pending, NULL);
286318b683bfSPaolo Abeni list_for_each_entry_safe(dfrag, dtmp, &msk->rtx_queue, list)
2864d027236cSPaolo Abeni dfrag_clear(sk, dfrag);
2865784325e9SMatthieu Baerts }
2866784325e9SMatthieu Baerts
mptcp_cancel_work(struct sock * sk)286730e51b92SMenglong Dong void mptcp_cancel_work(struct sock *sk)
286880992017SPaolo Abeni {
286980992017SPaolo Abeni struct mptcp_sock *msk = mptcp_sk(sk);
287080992017SPaolo Abeni
2871b2771d24SPaolo Abeni if (cancel_work_sync(&msk->work))
2872e16163b6SPaolo Abeni __sock_put(sk);
287380992017SPaolo Abeni }
287480992017SPaolo Abeni
mptcp_subflow_shutdown(struct sock * sk,struct sock * ssk,int how)2875d0876b22SGeliang Tang void mptcp_subflow_shutdown(struct sock *sk, struct sock *ssk, int how)
287621498490SPeter Krystad {
287721498490SPeter Krystad lock_sock(ssk);
287821498490SPeter Krystad
287921498490SPeter Krystad switch (ssk->sk_state) {
288021498490SPeter Krystad case TCP_LISTEN:
288121498490SPeter Krystad if (!(how & RCV_SHUTDOWN))
288221498490SPeter Krystad break;
2883df561f66SGustavo A. R. Silva fallthrough;
288421498490SPeter Krystad case TCP_SYN_SENT:
2885c2b2ae39SPaolo Abeni WARN_ON_ONCE(tcp_disconnect(ssk, O_NONBLOCK));
288621498490SPeter Krystad break;
288721498490SPeter Krystad default:
288843b54c6eSMat Martineau if (__mptcp_check_fallback(mptcp_sk(sk))) {
288943b54c6eSMat Martineau pr_debug("Fallback");
289021498490SPeter Krystad ssk->sk_shutdown |= how;
289121498490SPeter Krystad tcp_shutdown(ssk, how);
289281c1d029SPaolo Abeni
289381c1d029SPaolo Abeni /* simulate the data_fin ack reception to let the state
289481c1d029SPaolo Abeni * machine move forward
289581c1d029SPaolo Abeni */
289681c1d029SPaolo Abeni WRITE_ONCE(mptcp_sk(sk)->snd_una, mptcp_sk(sk)->snd_nxt);
289781c1d029SPaolo Abeni mptcp_schedule_work(sk);
289843b54c6eSMat Martineau } else {
289943b54c6eSMat Martineau pr_debug("Sending DATA_FIN on subflow %p", ssk);
290043b54c6eSMat Martineau tcp_send_ack(ssk);
2901f6909dc1SPaolo Abeni if (!mptcp_rtx_timer_pending(sk))
2902f6909dc1SPaolo Abeni mptcp_reset_rtx_timer(sk);
290343b54c6eSMat Martineau }
290421498490SPeter Krystad break;
290521498490SPeter Krystad }
290621498490SPeter Krystad
290721498490SPeter Krystad release_sock(ssk);
290821498490SPeter Krystad }
290921498490SPeter Krystad
mptcp_set_state(struct sock * sk,int state)2910d9cd27b8SGeliang Tang void mptcp_set_state(struct sock *sk, int state)
2911d9cd27b8SGeliang Tang {
2912d9cd27b8SGeliang Tang int oldstate = sk->sk_state;
2913d9cd27b8SGeliang Tang
2914d9cd27b8SGeliang Tang switch (state) {
2915d9cd27b8SGeliang Tang case TCP_ESTABLISHED:
2916d9cd27b8SGeliang Tang if (oldstate != TCP_ESTABLISHED)
2917d9cd27b8SGeliang Tang MPTCP_INC_STATS(sock_net(sk), MPTCP_MIB_CURRESTAB);
2918d9cd27b8SGeliang Tang break;
2919*9633e937SJason Xing case TCP_CLOSE_WAIT:
2920*9633e937SJason Xing /* Unlike TCP, MPTCP sk would not have the TCP_SYN_RECV state:
2921*9633e937SJason Xing * MPTCP "accepted" sockets will be created later on. So no
2922*9633e937SJason Xing * transition from TCP_SYN_RECV to TCP_CLOSE_WAIT.
2923*9633e937SJason Xing */
2924*9633e937SJason Xing break;
2925d9cd27b8SGeliang Tang default:
2926*9633e937SJason Xing if (oldstate == TCP_ESTABLISHED || oldstate == TCP_CLOSE_WAIT)
2927d9cd27b8SGeliang Tang MPTCP_DEC_STATS(sock_net(sk), MPTCP_MIB_CURRESTAB);
2928d9cd27b8SGeliang Tang }
2929d9cd27b8SGeliang Tang
2930d9cd27b8SGeliang Tang inet_sk_state_store(sk, state);
2931d9cd27b8SGeliang Tang }
2932d9cd27b8SGeliang Tang
29336920b851SMat Martineau static const unsigned char new_state[16] = {
29346920b851SMat Martineau /* current state: new state: action: */
29356920b851SMat Martineau [0 /* (Invalid) */] = TCP_CLOSE,
29366920b851SMat Martineau [TCP_ESTABLISHED] = TCP_FIN_WAIT1 | TCP_ACTION_FIN,
29376920b851SMat Martineau [TCP_SYN_SENT] = TCP_CLOSE,
29386920b851SMat Martineau [TCP_SYN_RECV] = TCP_FIN_WAIT1 | TCP_ACTION_FIN,
29396920b851SMat Martineau [TCP_FIN_WAIT1] = TCP_FIN_WAIT1,
29406920b851SMat Martineau [TCP_FIN_WAIT2] = TCP_FIN_WAIT2,
29416920b851SMat Martineau [TCP_TIME_WAIT] = TCP_CLOSE, /* should not happen ! */
29426920b851SMat Martineau [TCP_CLOSE] = TCP_CLOSE,
29436920b851SMat Martineau [TCP_CLOSE_WAIT] = TCP_LAST_ACK | TCP_ACTION_FIN,
29446920b851SMat Martineau [TCP_LAST_ACK] = TCP_LAST_ACK,
29456920b851SMat Martineau [TCP_LISTEN] = TCP_CLOSE,
29466920b851SMat Martineau [TCP_CLOSING] = TCP_CLOSING,
29476920b851SMat Martineau [TCP_NEW_SYN_RECV] = TCP_CLOSE, /* should not happen ! */
29486920b851SMat Martineau };
29496920b851SMat Martineau
mptcp_close_state(struct sock * sk)29506920b851SMat Martineau static int mptcp_close_state(struct sock *sk)
29516920b851SMat Martineau {
29526920b851SMat Martineau int next = (int)new_state[sk->sk_state];
29536920b851SMat Martineau int ns = next & TCP_STATE_MASK;
29546920b851SMat Martineau
2955c693a851SGeliang Tang mptcp_set_state(sk, ns);
29566920b851SMat Martineau
29576920b851SMat Martineau return next & TCP_ACTION_FIN;
29586920b851SMat Martineau }
29596920b851SMat Martineau
mptcp_check_send_data_fin(struct sock * sk)296081c1d029SPaolo Abeni static void mptcp_check_send_data_fin(struct sock *sk)
2961e16163b6SPaolo Abeni {
2962e16163b6SPaolo Abeni struct mptcp_subflow_context *subflow;
2963e16163b6SPaolo Abeni struct mptcp_sock *msk = mptcp_sk(sk);
2964e16163b6SPaolo Abeni
2965e16163b6SPaolo Abeni pr_debug("msk=%p snd_data_fin_enable=%d pending=%d snd_nxt=%llu write_seq=%llu",
2966e16163b6SPaolo Abeni msk, msk->snd_data_fin_enable, !!mptcp_send_head(sk),
2967e16163b6SPaolo Abeni msk->snd_nxt, msk->write_seq);
2968e16163b6SPaolo Abeni
2969e16163b6SPaolo Abeni /* we still need to enqueue subflows or not really shutting down,
2970e16163b6SPaolo Abeni * skip this
2971e16163b6SPaolo Abeni */
2972e16163b6SPaolo Abeni if (!msk->snd_data_fin_enable || msk->snd_nxt + 1 != msk->write_seq ||
2973e16163b6SPaolo Abeni mptcp_send_head(sk))
2974e16163b6SPaolo Abeni return;
2975e16163b6SPaolo Abeni
2976e16163b6SPaolo Abeni WRITE_ONCE(msk->snd_nxt, msk->write_seq);
2977e16163b6SPaolo Abeni
2978e16163b6SPaolo Abeni mptcp_for_each_subflow(msk, subflow) {
2979e16163b6SPaolo Abeni struct sock *tcp_sk = mptcp_subflow_tcp_sock(subflow);
2980e16163b6SPaolo Abeni
2981e16163b6SPaolo Abeni mptcp_subflow_shutdown(sk, tcp_sk, SEND_SHUTDOWN);
2982e16163b6SPaolo Abeni }
2983e16163b6SPaolo Abeni }
2984e16163b6SPaolo Abeni
__mptcp_wr_shutdown(struct sock * sk)2985e16163b6SPaolo Abeni static void __mptcp_wr_shutdown(struct sock *sk)
2986e16163b6SPaolo Abeni {
2987e16163b6SPaolo Abeni struct mptcp_sock *msk = mptcp_sk(sk);
2988e16163b6SPaolo Abeni
2989e16163b6SPaolo Abeni pr_debug("msk=%p snd_data_fin_enable=%d shutdown=%x state=%d pending=%d",
2990e16163b6SPaolo Abeni msk, msk->snd_data_fin_enable, sk->sk_shutdown, sk->sk_state,
2991e16163b6SPaolo Abeni !!mptcp_send_head(sk));
2992e16163b6SPaolo Abeni
2993e16163b6SPaolo Abeni /* will be ignored by fallback sockets */
2994e16163b6SPaolo Abeni WRITE_ONCE(msk->write_seq, msk->write_seq + 1);
2995e16163b6SPaolo Abeni WRITE_ONCE(msk->snd_data_fin_enable, 1);
2996e16163b6SPaolo Abeni
299781c1d029SPaolo Abeni mptcp_check_send_data_fin(sk);
2998e16163b6SPaolo Abeni }
2999e16163b6SPaolo Abeni
__mptcp_destroy_sock(struct sock * sk)3000e16163b6SPaolo Abeni static void __mptcp_destroy_sock(struct sock *sk)
3001f870fa0bSMat Martineau {
3002f870fa0bSMat Martineau struct mptcp_sock *msk = mptcp_sk(sk);
3003f870fa0bSMat Martineau
3004e16163b6SPaolo Abeni pr_debug("msk=%p", msk);
30052c22c06cSFlorian Westphal
30063abc05d9SFlorian Westphal might_sleep();
30073abc05d9SFlorian Westphal
3008f6909dc1SPaolo Abeni mptcp_stop_rtx_timer(sk);
3009e16163b6SPaolo Abeni sk_stop_timer(sk, &sk->sk_timer);
3010e16163b6SPaolo Abeni msk->pm.status = 0;
30111730b2b2SGeliang Tang mptcp_release_sched(msk);
3012b2c5b614SFlorian Westphal
3013e16163b6SPaolo Abeni sk->sk_prot->destroy(sk);
3014e16163b6SPaolo Abeni
30159426ce47SPaolo Abeni WARN_ON_ONCE(READ_ONCE(msk->rmem_fwd_alloc));
301687952603SPaolo Abeni WARN_ON_ONCE(msk->rmem_released);
3017e16163b6SPaolo Abeni sk_stream_kill_queues(sk);
3018e16163b6SPaolo Abeni xfrm_sk_free_policy(sk);
3019aa1fbd94SFlorian Westphal
3020e16163b6SPaolo Abeni sock_put(sk);
3021e16163b6SPaolo Abeni }
3022e16163b6SPaolo Abeni
__mptcp_unaccepted_force_close(struct sock * sk)302363740448SPaolo Abeni void __mptcp_unaccepted_force_close(struct sock *sk)
302463740448SPaolo Abeni {
302563740448SPaolo Abeni sock_set_flag(sk, SOCK_DEAD);
302663740448SPaolo Abeni mptcp_do_fastclose(sk);
302763740448SPaolo Abeni __mptcp_destroy_sock(sk);
302863740448SPaolo Abeni }
302963740448SPaolo Abeni
mptcp_check_readable(struct sock * sk)30305684ab1aSPaolo Abeni static __poll_t mptcp_check_readable(struct sock *sk)
3031d21f8348SPaolo Abeni {
30325684ab1aSPaolo Abeni return mptcp_epollin_ready(sk) ? EPOLLIN | EPOLLRDNORM : 0;
3033d21f8348SPaolo Abeni }
3034d21f8348SPaolo Abeni
mptcp_check_listen_stop(struct sock * sk)303557fc0f1cSPaolo Abeni static void mptcp_check_listen_stop(struct sock *sk)
3036c558246eSMenglong Dong {
303757fc0f1cSPaolo Abeni struct sock *ssk;
303857fc0f1cSPaolo Abeni
303957fc0f1cSPaolo Abeni if (inet_sk_state_load(sk) != TCP_LISTEN)
304057fc0f1cSPaolo Abeni return;
304157fc0f1cSPaolo Abeni
3042c558246eSMenglong Dong sock_prot_inuse_add(sock_net(sk), sk->sk_prot, -1);
304357fc0f1cSPaolo Abeni ssk = mptcp_sk(sk)->first;
304457fc0f1cSPaolo Abeni if (WARN_ON_ONCE(!ssk || inet_sk_state_load(ssk) != TCP_LISTEN))
304557fc0f1cSPaolo Abeni return;
304657fc0f1cSPaolo Abeni
304757fc0f1cSPaolo Abeni lock_sock_nested(ssk, SINGLE_DEPTH_NESTING);
30483fffa15bSPaolo Abeni tcp_set_state(ssk, TCP_CLOSE);
304957fc0f1cSPaolo Abeni mptcp_subflow_queue_clean(sk, ssk);
305057fc0f1cSPaolo Abeni inet_csk_listen_stop(ssk);
305157fc0f1cSPaolo Abeni mptcp_event_pm_listener(ssk, MPTCP_EVENT_LISTENER_CLOSED);
305257fc0f1cSPaolo Abeni release_sock(ssk);
3053c558246eSMenglong Dong }
3054c558246eSMenglong Dong
__mptcp_close(struct sock * sk,long timeout)305526d3e21cSMenglong Dong bool __mptcp_close(struct sock *sk, long timeout)
3056e16163b6SPaolo Abeni {
3057e16163b6SPaolo Abeni struct mptcp_subflow_context *subflow;
305876a13b31SGeliang Tang struct mptcp_sock *msk = mptcp_sk(sk);
3059e16163b6SPaolo Abeni bool do_cancel_work = false;
3060d4e85922SPaolo Abeni int subflows_alive = 0;
3061e16163b6SPaolo Abeni
30626b9831bfSPaolo Abeni WRITE_ONCE(sk->sk_shutdown, SHUTDOWN_MASK);
3063e16163b6SPaolo Abeni
3064e16163b6SPaolo Abeni if ((1 << sk->sk_state) & (TCPF_LISTEN | TCPF_CLOSE)) {
306557fc0f1cSPaolo Abeni mptcp_check_listen_stop(sk);
3066c693a851SGeliang Tang mptcp_set_state(sk, TCP_CLOSE);
3067e16163b6SPaolo Abeni goto cleanup;
3068e16163b6SPaolo Abeni }
3069e16163b6SPaolo Abeni
30705684ab1aSPaolo Abeni if (mptcp_data_avail(msk) || timeout < 0) {
30718d547809SPaolo Abeni /* If the msk has read data, or the caller explicitly ask it,
30728d547809SPaolo Abeni * do the MPTCP equivalent of TCP reset, aka MPTCP fastclose
30738d547809SPaolo Abeni */
3074d21f8348SPaolo Abeni mptcp_do_fastclose(sk);
30758d547809SPaolo Abeni timeout = 0;
3076d21f8348SPaolo Abeni } else if (mptcp_close_state(sk)) {
3077e16163b6SPaolo Abeni __mptcp_wr_shutdown(sk);
3078d21f8348SPaolo Abeni }
3079e16163b6SPaolo Abeni
3080e16163b6SPaolo Abeni sk_stream_wait_close(sk, timeout);
3081e16163b6SPaolo Abeni
3082e16163b6SPaolo Abeni cleanup:
3083e16163b6SPaolo Abeni /* orphan all the subflows */
308476a13b31SGeliang Tang mptcp_for_each_subflow(msk, subflow) {
3085e16163b6SPaolo Abeni struct sock *ssk = mptcp_subflow_tcp_sock(subflow);
308649054556SPaolo Abeni bool slow = lock_sock_fast_nested(ssk);
3087e16163b6SPaolo Abeni
3088d4e85922SPaolo Abeni subflows_alive += ssk->sk_state != TCP_CLOSE;
3089d4e85922SPaolo Abeni
309076a13b31SGeliang Tang /* since the close timeout takes precedence on the fail one,
309176a13b31SGeliang Tang * cancel the latter
309276a13b31SGeliang Tang */
309376a13b31SGeliang Tang if (ssk == msk->first)
309476a13b31SGeliang Tang subflow->fail_tout = 0;
309576a13b31SGeliang Tang
3096fe948001SMenglong Dong /* detach from the parent socket, but allow data_ready to
3097fe948001SMenglong Dong * push incoming data into the mptcp stack, to properly ack it
3098fe948001SMenglong Dong */
3099fe948001SMenglong Dong ssk->sk_socket = NULL;
3100fe948001SMenglong Dong ssk->sk_wq = NULL;
3101e16163b6SPaolo Abeni unlock_sock_fast(ssk, slow);
3102e16163b6SPaolo Abeni }
3103e16163b6SPaolo Abeni sock_orphan(sk);
3104e16163b6SPaolo Abeni
3105d4e85922SPaolo Abeni /* all the subflows are closed, only timeout can change the msk
3106d4e85922SPaolo Abeni * state, let's not keep resources busy for no reasons
3107d4e85922SPaolo Abeni */
3108d4e85922SPaolo Abeni if (subflows_alive == 0)
3109c693a851SGeliang Tang mptcp_set_state(sk, TCP_CLOSE);
3110d4e85922SPaolo Abeni
3111e16163b6SPaolo Abeni sock_hold(sk);
3112e16163b6SPaolo Abeni pr_debug("msk=%p state=%d", sk, sk->sk_state);
311300df24f1SGeliang Tang if (msk->token)
311476a13b31SGeliang Tang mptcp_event(MPTCP_EVENT_CLOSED, msk, NULL, GFP_KERNEL);
3115b29fcfb5SPaolo Abeni
3116e16163b6SPaolo Abeni if (sk->sk_state == TCP_CLOSE) {
3117e16163b6SPaolo Abeni __mptcp_destroy_sock(sk);
3118e16163b6SPaolo Abeni do_cancel_work = true;
3119e16163b6SPaolo Abeni } else {
312027e5ccc2SPaolo Abeni mptcp_start_tout_timer(sk);
3121e16163b6SPaolo Abeni }
312226d3e21cSMenglong Dong
312326d3e21cSMenglong Dong return do_cancel_work;
312426d3e21cSMenglong Dong }
312526d3e21cSMenglong Dong
mptcp_close(struct sock * sk,long timeout)312626d3e21cSMenglong Dong static void mptcp_close(struct sock *sk, long timeout)
312726d3e21cSMenglong Dong {
312826d3e21cSMenglong Dong bool do_cancel_work;
312926d3e21cSMenglong Dong
313026d3e21cSMenglong Dong lock_sock(sk);
313126d3e21cSMenglong Dong
313226d3e21cSMenglong Dong do_cancel_work = __mptcp_close(sk, timeout);
3133e16163b6SPaolo Abeni release_sock(sk);
3134e16163b6SPaolo Abeni if (do_cancel_work)
313580992017SPaolo Abeni mptcp_cancel_work(sk);
3136b911c97cSFlorian Westphal
3137e16163b6SPaolo Abeni sock_put(sk);
3138f870fa0bSMat Martineau }
3139f870fa0bSMat Martineau
mptcp_copy_inaddrs(struct sock * msk,const struct sock * ssk)31407e8b88ecSPaolo Abeni static void mptcp_copy_inaddrs(struct sock *msk, const struct sock *ssk)
3141cf7da0d6SPeter Krystad {
3142cf7da0d6SPeter Krystad #if IS_ENABLED(CONFIG_MPTCP_IPV6)
3143cf7da0d6SPeter Krystad const struct ipv6_pinfo *ssk6 = inet6_sk(ssk);
3144cf7da0d6SPeter Krystad struct ipv6_pinfo *msk6 = inet6_sk(msk);
3145cf7da0d6SPeter Krystad
3146cf7da0d6SPeter Krystad msk->sk_v6_daddr = ssk->sk_v6_daddr;
3147cf7da0d6SPeter Krystad msk->sk_v6_rcv_saddr = ssk->sk_v6_rcv_saddr;
3148cf7da0d6SPeter Krystad
3149cf7da0d6SPeter Krystad if (msk6 && ssk6) {
3150cf7da0d6SPeter Krystad msk6->saddr = ssk6->saddr;
3151cf7da0d6SPeter Krystad msk6->flow_label = ssk6->flow_label;
3152cf7da0d6SPeter Krystad }
3153cf7da0d6SPeter Krystad #endif
3154cf7da0d6SPeter Krystad
3155cf7da0d6SPeter Krystad inet_sk(msk)->inet_num = inet_sk(ssk)->inet_num;
3156cf7da0d6SPeter Krystad inet_sk(msk)->inet_dport = inet_sk(ssk)->inet_dport;
3157cf7da0d6SPeter Krystad inet_sk(msk)->inet_sport = inet_sk(ssk)->inet_sport;
3158cf7da0d6SPeter Krystad inet_sk(msk)->inet_daddr = inet_sk(ssk)->inet_daddr;
3159cf7da0d6SPeter Krystad inet_sk(msk)->inet_saddr = inet_sk(ssk)->inet_saddr;
3160cf7da0d6SPeter Krystad inet_sk(msk)->inet_rcv_saddr = inet_sk(ssk)->inet_rcv_saddr;
3161cf7da0d6SPeter Krystad }
3162cf7da0d6SPeter Krystad
mptcp_disconnect(struct sock * sk,int flags)316318b683bfSPaolo Abeni static int mptcp_disconnect(struct sock *sk, int flags)
316418b683bfSPaolo Abeni {
316576e2a55dSPaolo Abeni struct mptcp_sock *msk = mptcp_sk(sk);
316676e2a55dSPaolo Abeni
31677d803344SPaolo Abeni /* We are on the fastopen error path. We can't call straight into the
31687d803344SPaolo Abeni * subflows cleanup code due to lock nesting (we are already under
3169c2b2ae39SPaolo Abeni * msk->firstsocket lock).
31707d803344SPaolo Abeni */
31717d803344SPaolo Abeni if (msk->fastopening)
3172c2b2ae39SPaolo Abeni return -EBUSY;
31737d803344SPaolo Abeni
317457fc0f1cSPaolo Abeni mptcp_check_listen_stop(sk);
3175c693a851SGeliang Tang mptcp_set_state(sk, TCP_CLOSE);
3176b29fcfb5SPaolo Abeni
3177f6909dc1SPaolo Abeni mptcp_stop_rtx_timer(sk);
317827e5ccc2SPaolo Abeni mptcp_stop_tout_timer(sk);
3179b29fcfb5SPaolo Abeni
318000df24f1SGeliang Tang if (msk->token)
318100df24f1SGeliang Tang mptcp_event(MPTCP_EVENT_CLOSED, msk, NULL, GFP_KERNEL);
3182b29fcfb5SPaolo Abeni
3183c0bf3c6aSPaolo Abeni /* msk->subflow is still intact, the following will not free the first
3184c0bf3c6aSPaolo Abeni * subflow
3185c0bf3c6aSPaolo Abeni */
3186c0bf3c6aSPaolo Abeni mptcp_destroy_common(msk, MPTCP_CF_FASTCLOSE);
3187e9d09bacSPaolo Abeni WRITE_ONCE(msk->flags, 0);
3188e9d09bacSPaolo Abeni msk->cb_flags = 0;
3189b29fcfb5SPaolo Abeni msk->recovery = false;
319028e5c138SPaolo Abeni WRITE_ONCE(msk->can_ack, false);
319128e5c138SPaolo Abeni WRITE_ONCE(msk->fully_established, false);
319228e5c138SPaolo Abeni WRITE_ONCE(msk->rcv_data_fin, false);
319328e5c138SPaolo Abeni WRITE_ONCE(msk->snd_data_fin_enable, false);
319428e5c138SPaolo Abeni WRITE_ONCE(msk->rcv_fastclose, false);
319528e5c138SPaolo Abeni WRITE_ONCE(msk->use_64bit_ack, false);
3196b29fcfb5SPaolo Abeni WRITE_ONCE(msk->csum_enabled, mptcp_is_checksum_enabled(sock_net(sk)));
3197b29fcfb5SPaolo Abeni mptcp_pm_data_reset(msk);
3198b29fcfb5SPaolo Abeni mptcp_ca_reset(sk);
319928e5c138SPaolo Abeni msk->bytes_consumed = 0;
320038967f42SPaolo Abeni msk->bytes_acked = 0;
320138967f42SPaolo Abeni msk->bytes_received = 0;
320238967f42SPaolo Abeni msk->bytes_sent = 0;
320338967f42SPaolo Abeni msk->bytes_retrans = 0;
3204013e3179SPaolo Abeni msk->rcvspace_init = 0;
3205b29fcfb5SPaolo Abeni
32066b9831bfSPaolo Abeni WRITE_ONCE(sk->sk_shutdown, 0);
3207b29fcfb5SPaolo Abeni sk_error_report(sk);
320842c556feSFlorian Westphal return 0;
320918b683bfSPaolo Abeni }
321018b683bfSPaolo Abeni
3211b0519de8SFlorian Westphal #if IS_ENABLED(CONFIG_MPTCP_IPV6)
mptcp_inet6_sk(const struct sock * sk)3212b0519de8SFlorian Westphal static struct ipv6_pinfo *mptcp_inet6_sk(const struct sock *sk)
3213b0519de8SFlorian Westphal {
3214b0519de8SFlorian Westphal unsigned int offset = sizeof(struct mptcp6_sock) - sizeof(struct ipv6_pinfo);
3215b0519de8SFlorian Westphal
3216b0519de8SFlorian Westphal return (struct ipv6_pinfo *)(((u8 *)sk) + offset);
3217b0519de8SFlorian Westphal }
321810048689SDavide Caratti
mptcp_copy_ip6_options(struct sock * newsk,const struct sock * sk)321910048689SDavide Caratti static void mptcp_copy_ip6_options(struct sock *newsk, const struct sock *sk)
322010048689SDavide Caratti {
322110048689SDavide Caratti const struct ipv6_pinfo *np = inet6_sk(sk);
322210048689SDavide Caratti struct ipv6_txoptions *opt;
322310048689SDavide Caratti struct ipv6_pinfo *newnp;
322410048689SDavide Caratti
322510048689SDavide Caratti newnp = inet6_sk(newsk);
322610048689SDavide Caratti
322710048689SDavide Caratti rcu_read_lock();
322810048689SDavide Caratti opt = rcu_dereference(np->opt);
322910048689SDavide Caratti if (opt) {
323010048689SDavide Caratti opt = ipv6_dup_options(newsk, opt);
323110048689SDavide Caratti if (!opt)
323210048689SDavide Caratti net_warn_ratelimited("%s: Failed to copy ip6 options\n", __func__);
323310048689SDavide Caratti }
323410048689SDavide Caratti RCU_INIT_POINTER(newnp->opt, opt);
323510048689SDavide Caratti rcu_read_unlock();
323610048689SDavide Caratti }
3237b0519de8SFlorian Westphal #endif
3238b0519de8SFlorian Westphal
mptcp_copy_ip_options(struct sock * newsk,const struct sock * sk)323910048689SDavide Caratti static void mptcp_copy_ip_options(struct sock *newsk, const struct sock *sk)
324010048689SDavide Caratti {
324110048689SDavide Caratti struct ip_options_rcu *inet_opt, *newopt = NULL;
324210048689SDavide Caratti const struct inet_sock *inet = inet_sk(sk);
324310048689SDavide Caratti struct inet_sock *newinet;
324410048689SDavide Caratti
324510048689SDavide Caratti newinet = inet_sk(newsk);
324610048689SDavide Caratti
324710048689SDavide Caratti rcu_read_lock();
324810048689SDavide Caratti inet_opt = rcu_dereference(inet->inet_opt);
324910048689SDavide Caratti if (inet_opt) {
325010048689SDavide Caratti newopt = sock_kmalloc(newsk, sizeof(*inet_opt) +
325110048689SDavide Caratti inet_opt->opt.optlen, GFP_ATOMIC);
325210048689SDavide Caratti if (newopt)
325310048689SDavide Caratti memcpy(newopt, inet_opt, sizeof(*inet_opt) +
325410048689SDavide Caratti inet_opt->opt.optlen);
325510048689SDavide Caratti else
325610048689SDavide Caratti net_warn_ratelimited("%s: Failed to copy ip options\n", __func__);
325710048689SDavide Caratti }
325810048689SDavide Caratti RCU_INIT_POINTER(newinet->inet_opt, newopt);
325910048689SDavide Caratti rcu_read_unlock();
326010048689SDavide Caratti }
326110048689SDavide Caratti
mptcp_sk_clone_init(const struct sock * sk,const struct mptcp_options_received * mp_opt,struct sock * ssk,struct request_sock * req)32627e8b88ecSPaolo Abeni struct sock *mptcp_sk_clone_init(const struct sock *sk,
3263cfde141eSPaolo Abeni const struct mptcp_options_received *mp_opt,
32647e8b88ecSPaolo Abeni struct sock *ssk,
3265fca5c82cSPaolo Abeni struct request_sock *req)
3266b0519de8SFlorian Westphal {
326758b09919SPaolo Abeni struct mptcp_subflow_request_sock *subflow_req = mptcp_subflow_rsk(req);
3268b0519de8SFlorian Westphal struct sock *nsk = sk_clone_lock(sk, GFP_ATOMIC);
3269e4a0fa47SPaolo Abeni struct mptcp_subflow_context *subflow;
327058b09919SPaolo Abeni struct mptcp_sock *msk;
3271b0519de8SFlorian Westphal
3272b0519de8SFlorian Westphal if (!nsk)
3273b0519de8SFlorian Westphal return NULL;
3274b0519de8SFlorian Westphal
3275b0519de8SFlorian Westphal #if IS_ENABLED(CONFIG_MPTCP_IPV6)
3276b0519de8SFlorian Westphal if (nsk->sk_family == AF_INET6)
3277b0519de8SFlorian Westphal inet_sk(nsk)->pinet6 = mptcp_inet6_sk(nsk);
3278b0519de8SFlorian Westphal #endif
3279b0519de8SFlorian Westphal
328058b09919SPaolo Abeni __mptcp_init_sock(nsk);
328158b09919SPaolo Abeni
328210048689SDavide Caratti #if IS_ENABLED(CONFIG_MPTCP_IPV6)
328310048689SDavide Caratti if (nsk->sk_family == AF_INET6)
328410048689SDavide Caratti mptcp_copy_ip6_options(nsk, sk);
328510048689SDavide Caratti else
328610048689SDavide Caratti #endif
328710048689SDavide Caratti mptcp_copy_ip_options(nsk, sk);
328810048689SDavide Caratti
328958b09919SPaolo Abeni msk = mptcp_sk(nsk);
32901c09d7cbSPaolo Abeni WRITE_ONCE(msk->local_key, subflow_req->local_key);
3291b9f45543SPaolo Abeni WRITE_ONCE(msk->token, subflow_req->token);
3292b6985b9bSPaolo Abeni msk->in_accept_queue = 1;
3293b93df08cSPaolo Abeni WRITE_ONCE(msk->fully_established, false);
329474c7dfbeSPaolo Abeni if (mp_opt->suboptions & OPTION_MPTCP_CSUMREQD)
329506251181SGeliang Tang WRITE_ONCE(msk->csum_enabled, true);
329658b09919SPaolo Abeni
3297d440a4e2SPaolo Abeni WRITE_ONCE(msk->write_seq, subflow_req->idsn + 1);
3298d440a4e2SPaolo Abeni WRITE_ONCE(msk->snd_nxt, msk->write_seq);
32999426ce47SPaolo Abeni WRITE_ONCE(msk->snd_una, msk->write_seq);
330065f5dd4fSJakub Kicinski WRITE_ONCE(msk->wnd_end, msk->snd_nxt + tcp_sk(ssk)->snd_wnd);
3301df00b087SFlorian Westphal msk->setsockopt_seq = mptcp_sk(sk)->setsockopt_seq;
33021730b2b2SGeliang Tang mptcp_init_sched(msk, mptcp_sk(sk)->sched);
33036f8a612aSFlorian Westphal
33046f06b4d4SPaolo Abeni /* passive msk is created after the first/MPC subflow */
33056f06b4d4SPaolo Abeni msk->subflow_id = 2;
33066f06b4d4SPaolo Abeni
33075e20087dSFlorian Westphal sock_reset_flag(nsk, SOCK_RCU_FREE);
33080c148460SPaolo Abeni security_inet_csk_clone(nsk, req);
33097e8b88ecSPaolo Abeni
33107e8b88ecSPaolo Abeni /* this can't race with mptcp_close(), as the msk is
33117e8b88ecSPaolo Abeni * not yet exposted to user-space
33127e8b88ecSPaolo Abeni */
3313c693a851SGeliang Tang mptcp_set_state(nsk, TCP_ESTABLISHED);
33147e8b88ecSPaolo Abeni
33157e8b88ecSPaolo Abeni /* The msk maintain a ref to each subflow in the connections list */
33167e8b88ecSPaolo Abeni WRITE_ONCE(msk->first, ssk);
3317e4a0fa47SPaolo Abeni subflow = mptcp_subflow_ctx(ssk);
3318e4a0fa47SPaolo Abeni list_add(&subflow->node, &msk->conn_list);
33197e8b88ecSPaolo Abeni sock_hold(ssk);
33207e8b88ecSPaolo Abeni
33217e8b88ecSPaolo Abeni /* new mpc subflow takes ownership of the newly
33227e8b88ecSPaolo Abeni * created mptcp socket
33237e8b88ecSPaolo Abeni */
33247e8b88ecSPaolo Abeni mptcp_token_accept(subflow_req, msk);
33257e8b88ecSPaolo Abeni
33267e8b88ecSPaolo Abeni /* set msk addresses early to ensure mptcp_pm_get_local_id()
33277e8b88ecSPaolo Abeni * uses the correct data
33287e8b88ecSPaolo Abeni */
33297e8b88ecSPaolo Abeni mptcp_copy_inaddrs(nsk, ssk);
33308005184fSPaolo Abeni __mptcp_propagate_sndbuf(nsk, ssk);
33317e8b88ecSPaolo Abeni
33327e8b88ecSPaolo Abeni mptcp_rcv_space_init(msk, ssk);
3333e4a0fa47SPaolo Abeni
3334e4a0fa47SPaolo Abeni if (mp_opt->suboptions & OPTION_MPTCP_MPC_ACK)
3335e4a0fa47SPaolo Abeni __mptcp_subflow_fully_established(msk, subflow, mp_opt);
333658b09919SPaolo Abeni bh_unlock_sock(nsk);
333758b09919SPaolo Abeni
3338b6985b9bSPaolo Abeni /* note: the newly allocated socket refcount is 2 now */
3339b0519de8SFlorian Westphal return nsk;
3340b0519de8SFlorian Westphal }
3341b0519de8SFlorian Westphal
mptcp_rcv_space_init(struct mptcp_sock * msk,const struct sock * ssk)3342a6b118feSFlorian Westphal void mptcp_rcv_space_init(struct mptcp_sock *msk, const struct sock *ssk)
3343a6b118feSFlorian Westphal {
3344a6b118feSFlorian Westphal const struct tcp_sock *tp = tcp_sk(ssk);
3345a6b118feSFlorian Westphal
3346013e3179SPaolo Abeni msk->rcvspace_init = 1;
3347a6b118feSFlorian Westphal msk->rcvq_space.copied = 0;
3348a6b118feSFlorian Westphal msk->rcvq_space.rtt_us = 0;
3349a6b118feSFlorian Westphal
3350a6b118feSFlorian Westphal msk->rcvq_space.time = tp->tcp_mstamp;
3351a6b118feSFlorian Westphal
3352a6b118feSFlorian Westphal /* initial rcv_space offering made to peer */
3353a6b118feSFlorian Westphal msk->rcvq_space.space = min_t(u32, tp->rcv_wnd,
3354a6b118feSFlorian Westphal TCP_INIT_CWND * tp->advmss);
3355a6b118feSFlorian Westphal if (msk->rcvq_space.space == 0)
3356a6b118feSFlorian Westphal msk->rcvq_space.space = TCP_INIT_CWND * TCP_MSS_DEFAULT;
3357a6b118feSFlorian Westphal }
3358a6b118feSFlorian Westphal
mptcp_destroy_common(struct mptcp_sock * msk,unsigned int flags)3359c0bf3c6aSPaolo Abeni void mptcp_destroy_common(struct mptcp_sock *msk, unsigned int flags)
33605c8c1640SGeliang Tang {
3361c0bf3c6aSPaolo Abeni struct mptcp_subflow_context *subflow, *tmp;
336287952603SPaolo Abeni struct sock *sk = (struct sock *)msk;
336387952603SPaolo Abeni
33646e628cd3SPaolo Abeni __mptcp_clear_xmit(sk);
33656e628cd3SPaolo Abeni
3366c0bf3c6aSPaolo Abeni /* join list will be eventually flushed (with rst) at sock lock release time */
33675efbf6f7SMatthieu Baerts mptcp_for_each_subflow_safe(msk, subflow, tmp)
3368c0bf3c6aSPaolo Abeni __mptcp_close_ssk(sk, mptcp_subflow_tcp_sock(subflow), subflow, flags);
3369c0bf3c6aSPaolo Abeni
337087952603SPaolo Abeni /* move to sk_receive_queue, sk_stream_kill_queues will purge it */
3371b29fcfb5SPaolo Abeni mptcp_data_lock(sk);
337287952603SPaolo Abeni skb_queue_splice_tail_init(&msk->receive_queue, &sk->sk_receive_queue);
33736511882cSPaolo Abeni __skb_queue_purge(&sk->sk_receive_queue);
33745c8c1640SGeliang Tang skb_rbtree_purge(&msk->out_of_order_queue);
3375b29fcfb5SPaolo Abeni mptcp_data_unlock(sk);
33766511882cSPaolo Abeni
33776511882cSPaolo Abeni /* move all the rx fwd alloc into the sk_mem_reclaim_final in
33786511882cSPaolo Abeni * inet_sock_destruct() will dispose it
33796511882cSPaolo Abeni */
33805e6300e7SEric Dumazet sk_forward_alloc_add(sk, msk->rmem_fwd_alloc);
33819531e4a8SEric Dumazet WRITE_ONCE(msk->rmem_fwd_alloc, 0);
33825c8c1640SGeliang Tang mptcp_token_destroy(msk);
33835c8c1640SGeliang Tang mptcp_pm_free_anno_list(msk);
33844638de5aSKishen Maloor mptcp_free_local_addr_list(msk);
33855c8c1640SGeliang Tang }
33865c8c1640SGeliang Tang
mptcp_destroy(struct sock * sk)338779c0949eSPeter Krystad static void mptcp_destroy(struct sock *sk)
338879c0949eSPeter Krystad {
3389c9fd9c5fSFlorian Westphal struct mptcp_sock *msk = mptcp_sk(sk);
3390c9fd9c5fSFlorian Westphal
339139880bd8SPaolo Abeni /* allow the following to close even the initial subflow */
339239880bd8SPaolo Abeni msk->free_first = 1;
3393c0bf3c6aSPaolo Abeni mptcp_destroy_common(msk, 0);
3394d027236cSPaolo Abeni sk_sockets_allocated_dec(sk);
339579c0949eSPeter Krystad }
339679c0949eSPeter Krystad
__mptcp_data_acked(struct sock * sk)33976e628cd3SPaolo Abeni void __mptcp_data_acked(struct sock *sk)
33986e628cd3SPaolo Abeni {
33996e628cd3SPaolo Abeni if (!sock_owned_by_user(sk))
34006e628cd3SPaolo Abeni __mptcp_clean_una(sk);
34016e628cd3SPaolo Abeni else
3402e9d09bacSPaolo Abeni __set_bit(MPTCP_CLEAN_UNA, &mptcp_sk(sk)->cb_flags);
34036e628cd3SPaolo Abeni }
34046e628cd3SPaolo Abeni
__mptcp_check_push(struct sock * sk,struct sock * ssk)3405219d0499SPaolo Abeni void __mptcp_check_push(struct sock *sk, struct sock *ssk)
34066e628cd3SPaolo Abeni {
34076e628cd3SPaolo Abeni if (!mptcp_send_head(sk))
34086e628cd3SPaolo Abeni return;
34096e628cd3SPaolo Abeni
341073a0052aSGeliang Tang if (!sock_owned_by_user(sk))
341173a0052aSGeliang Tang __mptcp_subflow_push_pending(sk, ssk, false);
341273a0052aSGeliang Tang else
3413e9d09bacSPaolo Abeni __set_bit(MPTCP_PUSH_PENDING, &mptcp_sk(sk)->cb_flags);
34146e628cd3SPaolo Abeni }
34156e628cd3SPaolo Abeni
3416e9d09bacSPaolo Abeni #define MPTCP_FLAGS_PROCESS_CTX_NEED (BIT(MPTCP_PUSH_PENDING) | \
3417e9d09bacSPaolo Abeni BIT(MPTCP_RETRANSMIT) | \
3418e9d09bacSPaolo Abeni BIT(MPTCP_FLUSH_JOIN_LIST))
3419e9d09bacSPaolo Abeni
3420e93da928SPaolo Abeni /* processes deferred events and flush wmem */
mptcp_release_cb(struct sock * sk)342114c441b5SPaolo Abeni static void mptcp_release_cb(struct sock *sk)
3422e9d09bacSPaolo Abeni __must_hold(&sk->sk_lock.slock)
342314c441b5SPaolo Abeni {
3424e9d09bacSPaolo Abeni struct mptcp_sock *msk = mptcp_sk(sk);
34252d6f5a2bSPaolo Abeni
3426e9d09bacSPaolo Abeni for (;;) {
3427bdd70eb6SPaolo Abeni unsigned long flags = (msk->cb_flags & MPTCP_FLAGS_PROCESS_CTX_NEED);
342856a666c4SPaolo Abeni struct list_head join_list;
342956a666c4SPaolo Abeni
3430c2e6048fSPaolo Abeni if (!flags)
3431c2e6048fSPaolo Abeni break;
3432c2e6048fSPaolo Abeni
343356a666c4SPaolo Abeni INIT_LIST_HEAD(&join_list);
343456a666c4SPaolo Abeni list_splice_init(&msk->join_list, &join_list);
343556a666c4SPaolo Abeni
3436c2e6048fSPaolo Abeni /* the following actions acquire the subflow socket lock
34376e628cd3SPaolo Abeni *
34386e628cd3SPaolo Abeni * 1) can't be invoked in atomic scope
34396e628cd3SPaolo Abeni * 2) must avoid ABBA deadlock with msk socket spinlock: the RX
34406e628cd3SPaolo Abeni * datapath acquires the msk socket spinlock while helding
34416e628cd3SPaolo Abeni * the subflow socket lock
34426e628cd3SPaolo Abeni */
3443e9d09bacSPaolo Abeni msk->cb_flags &= ~flags;
34446e628cd3SPaolo Abeni spin_unlock_bh(&sk->sk_lock.slock);
344556a666c4SPaolo Abeni
34463e501490SPaolo Abeni if (flags & BIT(MPTCP_FLUSH_JOIN_LIST))
344756a666c4SPaolo Abeni __mptcp_flush_join_list(sk, &join_list);
34482e5de7e0SDan Carpenter if (flags & BIT(MPTCP_PUSH_PENDING))
3449c2e6048fSPaolo Abeni __mptcp_push_pending(sk, 0);
34502d6f5a2bSPaolo Abeni if (flags & BIT(MPTCP_RETRANSMIT))
34512d6f5a2bSPaolo Abeni __mptcp_retrans(sk);
3452c2e6048fSPaolo Abeni
3453c2e6048fSPaolo Abeni cond_resched();
34546e628cd3SPaolo Abeni spin_lock_bh(&sk->sk_lock.slock);
34556e628cd3SPaolo Abeni }
3456c2e6048fSPaolo Abeni
345765a569b0SPaolo Abeni if (__test_and_clear_bit(MPTCP_CLEAN_UNA, &msk->cb_flags))
345865a569b0SPaolo Abeni __mptcp_clean_una_wakeup(sk);
345917ebf8a4SXiang Yang if (unlikely(msk->cb_flags)) {
34604fd19a30SPaolo Abeni /* be sure to sync the msk state before taking actions
3461d109a776SPaolo Abeni * depending on sk_state (MPTCP_ERROR_REPORT)
3462d109a776SPaolo Abeni * On sk release avoid actions depending on the first subflow
3463490274b4SPaolo Abeni */
34644fd19a30SPaolo Abeni if (__test_and_clear_bit(MPTCP_SYNC_STATE, &msk->cb_flags) && msk->first)
34654fd19a30SPaolo Abeni __mptcp_sync_state(sk, msk->pending_state);
3466e9d09bacSPaolo Abeni if (__test_and_clear_bit(MPTCP_ERROR_REPORT, &msk->cb_flags))
346715cc1045SPaolo Abeni __mptcp_error_report(sk);
34688005184fSPaolo Abeni if (__test_and_clear_bit(MPTCP_SYNC_SNDBUF, &msk->cb_flags))
34698005184fSPaolo Abeni __mptcp_sync_sndbuf(sk);
347065a569b0SPaolo Abeni }
34716e628cd3SPaolo Abeni
347287952603SPaolo Abeni __mptcp_update_rmem(sk);
347314c441b5SPaolo Abeni }
347414c441b5SPaolo Abeni
3475bcd97734SPaolo Abeni /* MP_JOIN client subflow must wait for 4th ack before sending any data:
3476bcd97734SPaolo Abeni * TCP can't schedule delack timer before the subflow is fully established.
3477bcd97734SPaolo Abeni * MPTCP uses the delack timer to do 3rd ack retransmissions
3478bcd97734SPaolo Abeni */
schedule_3rdack_retransmission(struct sock * ssk)3479bcd97734SPaolo Abeni static void schedule_3rdack_retransmission(struct sock *ssk)
3480bcd97734SPaolo Abeni {
3481bcd97734SPaolo Abeni struct inet_connection_sock *icsk = inet_csk(ssk);
3482bcd97734SPaolo Abeni struct tcp_sock *tp = tcp_sk(ssk);
3483bcd97734SPaolo Abeni unsigned long timeout;
3484bcd97734SPaolo Abeni
3485bcd97734SPaolo Abeni if (mptcp_subflow_ctx(ssk)->fully_established)
3486bcd97734SPaolo Abeni return;
3487bcd97734SPaolo Abeni
3488bcd97734SPaolo Abeni /* reschedule with a timeout above RTT, as we must look only for drop */
3489bcd97734SPaolo Abeni if (tp->srtt_us)
3490bcd97734SPaolo Abeni timeout = usecs_to_jiffies(tp->srtt_us >> (3 - 1));
3491bcd97734SPaolo Abeni else
3492bcd97734SPaolo Abeni timeout = TCP_TIMEOUT_INIT;
3493bcd97734SPaolo Abeni timeout += jiffies;
3494bcd97734SPaolo Abeni
3495bcd97734SPaolo Abeni WARN_ON_ONCE(icsk->icsk_ack.pending & ICSK_ACK_TIMER);
3496bcd97734SPaolo Abeni icsk->icsk_ack.pending |= ICSK_ACK_SCHED | ICSK_ACK_TIMER;
3497bcd97734SPaolo Abeni icsk->icsk_ack.timeout = timeout;
3498bcd97734SPaolo Abeni sk_reset_timer(ssk, &icsk->icsk_delack_timer, timeout);
3499bcd97734SPaolo Abeni }
3500bcd97734SPaolo Abeni
mptcp_subflow_process_delegated(struct sock * ssk,long status)3501a5efdbceSPaolo Abeni void mptcp_subflow_process_delegated(struct sock *ssk, long status)
3502b19bc294SPaolo Abeni {
3503b19bc294SPaolo Abeni struct mptcp_subflow_context *subflow = mptcp_subflow_ctx(ssk);
3504b19bc294SPaolo Abeni struct sock *sk = subflow->conn;
3505b19bc294SPaolo Abeni
3506a5efdbceSPaolo Abeni if (status & BIT(MPTCP_DELEGATE_SEND)) {
3507b19bc294SPaolo Abeni mptcp_data_lock(sk);
3508b19bc294SPaolo Abeni if (!sock_owned_by_user(sk))
350973a0052aSGeliang Tang __mptcp_subflow_push_pending(sk, ssk, true);
3510b19bc294SPaolo Abeni else
3511e9d09bacSPaolo Abeni __set_bit(MPTCP_PUSH_PENDING, &mptcp_sk(sk)->cb_flags);
3512b19bc294SPaolo Abeni mptcp_data_unlock(sk);
3513bcd97734SPaolo Abeni }
35148005184fSPaolo Abeni if (status & BIT(MPTCP_DELEGATE_SNDBUF)) {
35158005184fSPaolo Abeni mptcp_data_lock(sk);
35168005184fSPaolo Abeni if (!sock_owned_by_user(sk))
35178005184fSPaolo Abeni __mptcp_sync_sndbuf(sk);
35188005184fSPaolo Abeni else
35198005184fSPaolo Abeni __set_bit(MPTCP_SYNC_SNDBUF, &mptcp_sk(sk)->cb_flags);
35208005184fSPaolo Abeni mptcp_data_unlock(sk);
35218005184fSPaolo Abeni }
3522a5efdbceSPaolo Abeni if (status & BIT(MPTCP_DELEGATE_ACK))
3523bcd97734SPaolo Abeni schedule_3rdack_retransmission(ssk);
3524b19bc294SPaolo Abeni }
3525b19bc294SPaolo Abeni
mptcp_hash(struct sock * sk)35262c5ebd00SPaolo Abeni static int mptcp_hash(struct sock *sk)
35272c5ebd00SPaolo Abeni {
35282c5ebd00SPaolo Abeni /* should never be called,
35292c5ebd00SPaolo Abeni * we hash the TCP subflows not the master socket
35302c5ebd00SPaolo Abeni */
35312c5ebd00SPaolo Abeni WARN_ON_ONCE(1);
35322c5ebd00SPaolo Abeni return 0;
35332c5ebd00SPaolo Abeni }
35342c5ebd00SPaolo Abeni
mptcp_unhash(struct sock * sk)35352c5ebd00SPaolo Abeni static void mptcp_unhash(struct sock *sk)
35362c5ebd00SPaolo Abeni {
35372c5ebd00SPaolo Abeni /* called from sk_common_release(), but nothing to do here */
35382c5ebd00SPaolo Abeni }
35392c5ebd00SPaolo Abeni
mptcp_get_port(struct sock * sk,unsigned short snum)3540cec37a6eSPeter Krystad static int mptcp_get_port(struct sock *sk, unsigned short snum)
3541f870fa0bSMat Martineau {
3542f870fa0bSMat Martineau struct mptcp_sock *msk = mptcp_sk(sk);
3543f870fa0bSMat Martineau
3544cfb63e50SPaolo Abeni pr_debug("msk=%p, ssk=%p", msk, msk->first);
3545cfb63e50SPaolo Abeni if (WARN_ON_ONCE(!msk->first))
3546cec37a6eSPeter Krystad return -EINVAL;
3547f870fa0bSMat Martineau
3548cfb63e50SPaolo Abeni return inet_csk_get_port(msk->first, snum);
3549cec37a6eSPeter Krystad }
3550f870fa0bSMat Martineau
mptcp_finish_connect(struct sock * ssk)3551cec37a6eSPeter Krystad void mptcp_finish_connect(struct sock *ssk)
3552cec37a6eSPeter Krystad {
3553cec37a6eSPeter Krystad struct mptcp_subflow_context *subflow;
3554cec37a6eSPeter Krystad struct mptcp_sock *msk;
3555cec37a6eSPeter Krystad struct sock *sk;
3556f870fa0bSMat Martineau
3557cec37a6eSPeter Krystad subflow = mptcp_subflow_ctx(ssk);
3558cec37a6eSPeter Krystad sk = subflow->conn;
3559cec37a6eSPeter Krystad msk = mptcp_sk(sk);
3560cec37a6eSPeter Krystad
3561648ef4b8SMat Martineau pr_debug("msk=%p, token=%u", sk, subflow->token);
3562648ef4b8SMat Martineau
3563b3ea6b27SPaolo Abeni subflow->map_seq = subflow->iasn;
3564648ef4b8SMat Martineau subflow->map_subflow_seq = 1;
35656d0060f6SMat Martineau
3566cec37a6eSPeter Krystad /* the socket is not connected yet, no msk/subflow ops can access/race
3567cec37a6eSPeter Krystad * accessing the field below
3568cec37a6eSPeter Krystad */
3569cec37a6eSPeter Krystad WRITE_ONCE(msk->local_key, subflow->local_key);
35701b1c7a0eSPeter Krystad
35716c714f1bSFlorian Westphal mptcp_pm_new_connection(msk, ssk, 0);
3572f870fa0bSMat Martineau }
3573f870fa0bSMat Martineau
mptcp_sock_graft(struct sock * sk,struct socket * parent)3574866f26f2SPaolo Abeni void mptcp_sock_graft(struct sock *sk, struct socket *parent)
3575cf7da0d6SPeter Krystad {
3576cf7da0d6SPeter Krystad write_lock_bh(&sk->sk_callback_lock);
3577cf7da0d6SPeter Krystad rcu_assign_pointer(sk->sk_wq, &parent->wq);
3578cf7da0d6SPeter Krystad sk_set_socket(sk, parent);
3579cf7da0d6SPeter Krystad sk->sk_uid = SOCK_INODE(parent)->i_uid;
3580cf7da0d6SPeter Krystad write_unlock_bh(&sk->sk_callback_lock);
3581cf7da0d6SPeter Krystad }
3582cf7da0d6SPeter Krystad
mptcp_finish_join(struct sock * ssk)3583e16163b6SPaolo Abeni bool mptcp_finish_join(struct sock *ssk)
3584f296234cSPeter Krystad {
3585e16163b6SPaolo Abeni struct mptcp_subflow_context *subflow = mptcp_subflow_ctx(ssk);
3586f296234cSPeter Krystad struct mptcp_sock *msk = mptcp_sk(subflow->conn);
3587f296234cSPeter Krystad struct sock *parent = (void *)msk;
35883e501490SPaolo Abeni bool ret = true;
3589f296234cSPeter Krystad
3590f296234cSPeter Krystad pr_debug("msk=%p, subflow=%p", msk, subflow);
3591f296234cSPeter Krystad
3592f296234cSPeter Krystad /* mptcp socket already closing? */
3593dc87efdbSFlorian Westphal if (!mptcp_is_fully_established(parent)) {
3594dc87efdbSFlorian Westphal subflow->reset_reason = MPTCP_RST_EMPTCP;
3595f296234cSPeter Krystad return false;
3596dc87efdbSFlorian Westphal }
3597f296234cSPeter Krystad
359855b47ca7SPaolo Abeni /* active subflow, already present inside the conn_list */
359955b47ca7SPaolo Abeni if (!list_empty(&subflow->node)) {
360055b47ca7SPaolo Abeni mptcp_subflow_joined(msk, ssk);
36018005184fSPaolo Abeni mptcp_propagate_sndbuf(parent, ssk);
360255b47ca7SPaolo Abeni return true;
360355b47ca7SPaolo Abeni }
3604f296234cSPeter Krystad
36053e501490SPaolo Abeni if (!mptcp_pm_allow_new_subflow(msk))
36063e501490SPaolo Abeni goto err_prohibited;
360710f6d46cSPaolo Abeni
360855b47ca7SPaolo Abeni /* If we can't acquire msk socket lock here, let the release callback
36093e501490SPaolo Abeni * handle it
361010f6d46cSPaolo Abeni */
36113e501490SPaolo Abeni mptcp_data_lock(parent);
36123e501490SPaolo Abeni if (!sock_owned_by_user(parent)) {
36133e501490SPaolo Abeni ret = __mptcp_finish_join(msk, ssk);
36143e501490SPaolo Abeni if (ret) {
3615e16163b6SPaolo Abeni sock_hold(ssk);
36163e501490SPaolo Abeni list_add_tail(&subflow->node, &msk->conn_list);
3617e16163b6SPaolo Abeni }
36183e501490SPaolo Abeni } else {
36193e501490SPaolo Abeni sock_hold(ssk);
36203e501490SPaolo Abeni list_add_tail(&subflow->node, &msk->join_list);
3621e9d09bacSPaolo Abeni __set_bit(MPTCP_FLUSH_JOIN_LIST, &msk->cb_flags);
36223e501490SPaolo Abeni }
36233e501490SPaolo Abeni mptcp_data_unlock(parent);
36243e501490SPaolo Abeni
3625dc87efdbSFlorian Westphal if (!ret) {
36263e501490SPaolo Abeni err_prohibited:
3627dc87efdbSFlorian Westphal subflow->reset_reason = MPTCP_RST_EPROHIBIT;
362810f6d46cSPaolo Abeni return false;
3629dc87efdbSFlorian Westphal }
363010f6d46cSPaolo Abeni
363110f6d46cSPaolo Abeni return true;
3632f296234cSPeter Krystad }
3633f296234cSPeter Krystad
mptcp_shutdown(struct sock * sk,int how)363476e2a55dSPaolo Abeni static void mptcp_shutdown(struct sock *sk, int how)
363576e2a55dSPaolo Abeni {
363676e2a55dSPaolo Abeni pr_debug("sk=%p, how=%d", sk, how);
363776e2a55dSPaolo Abeni
363876e2a55dSPaolo Abeni if ((how & SEND_SHUTDOWN) && mptcp_close_state(sk))
363976e2a55dSPaolo Abeni __mptcp_wr_shutdown(sk);
364076e2a55dSPaolo Abeni }
364176e2a55dSPaolo Abeni
mptcp_forward_alloc_get(const struct sock * sk)36426511882cSPaolo Abeni static int mptcp_forward_alloc_get(const struct sock *sk)
36436511882cSPaolo Abeni {
36449531e4a8SEric Dumazet return READ_ONCE(sk->sk_forward_alloc) +
36459531e4a8SEric Dumazet READ_ONCE(mptcp_sk(sk)->rmem_fwd_alloc);
36466511882cSPaolo Abeni }
36476511882cSPaolo Abeni
mptcp_ioctl_outq(const struct mptcp_sock * msk,u64 v)3648644807e3SFlorian Westphal static int mptcp_ioctl_outq(const struct mptcp_sock *msk, u64 v)
3649644807e3SFlorian Westphal {
3650644807e3SFlorian Westphal const struct sock *sk = (void *)msk;
3651644807e3SFlorian Westphal u64 delta;
3652644807e3SFlorian Westphal
3653644807e3SFlorian Westphal if (sk->sk_state == TCP_LISTEN)
3654644807e3SFlorian Westphal return -EINVAL;
3655644807e3SFlorian Westphal
3656644807e3SFlorian Westphal if ((1 << sk->sk_state) & (TCPF_SYN_SENT | TCPF_SYN_RECV))
3657644807e3SFlorian Westphal return 0;
3658644807e3SFlorian Westphal
3659644807e3SFlorian Westphal delta = msk->write_seq - v;
366007c2c7a3SPaolo Abeni if (__mptcp_check_fallback(msk) && msk->first) {
366107c2c7a3SPaolo Abeni struct tcp_sock *tp = tcp_sk(msk->first);
366207c2c7a3SPaolo Abeni
366307c2c7a3SPaolo Abeni /* the first subflow is disconnected after close - see
366407c2c7a3SPaolo Abeni * __mptcp_close_ssk(). tcp_disconnect() moves the write_seq
366507c2c7a3SPaolo Abeni * so ignore that status, too.
366607c2c7a3SPaolo Abeni */
366707c2c7a3SPaolo Abeni if (!((1 << msk->first->sk_state) &
366807c2c7a3SPaolo Abeni (TCPF_SYN_SENT | TCPF_SYN_RECV | TCPF_CLOSE)))
366907c2c7a3SPaolo Abeni delta += READ_ONCE(tp->write_seq) - tp->snd_una;
367007c2c7a3SPaolo Abeni }
3671644807e3SFlorian Westphal if (delta > INT_MAX)
3672644807e3SFlorian Westphal delta = INT_MAX;
3673644807e3SFlorian Westphal
3674644807e3SFlorian Westphal return (int)delta;
3675644807e3SFlorian Westphal }
3676644807e3SFlorian Westphal
mptcp_ioctl(struct sock * sk,int cmd,int * karg)3677e1d001faSBreno Leitao static int mptcp_ioctl(struct sock *sk, int cmd, int *karg)
3678644807e3SFlorian Westphal {
3679644807e3SFlorian Westphal struct mptcp_sock *msk = mptcp_sk(sk);
3680644807e3SFlorian Westphal bool slow;
3681644807e3SFlorian Westphal
3682644807e3SFlorian Westphal switch (cmd) {
3683644807e3SFlorian Westphal case SIOCINQ:
3684644807e3SFlorian Westphal if (sk->sk_state == TCP_LISTEN)
3685644807e3SFlorian Westphal return -EINVAL;
3686644807e3SFlorian Westphal
3687644807e3SFlorian Westphal lock_sock(sk);
3688644807e3SFlorian Westphal __mptcp_move_skbs(msk);
3689e1d001faSBreno Leitao *karg = mptcp_inq_hint(sk);
3690644807e3SFlorian Westphal release_sock(sk);
3691644807e3SFlorian Westphal break;
3692644807e3SFlorian Westphal case SIOCOUTQ:
3693644807e3SFlorian Westphal slow = lock_sock_fast(sk);
3694e1d001faSBreno Leitao *karg = mptcp_ioctl_outq(msk, READ_ONCE(msk->snd_una));
3695644807e3SFlorian Westphal unlock_sock_fast(sk, slow);
3696644807e3SFlorian Westphal break;
3697644807e3SFlorian Westphal case SIOCOUTQNSD:
3698644807e3SFlorian Westphal slow = lock_sock_fast(sk);
3699e1d001faSBreno Leitao *karg = mptcp_ioctl_outq(msk, msk->snd_nxt);
3700644807e3SFlorian Westphal unlock_sock_fast(sk, slow);
3701644807e3SFlorian Westphal break;
3702644807e3SFlorian Westphal default:
3703644807e3SFlorian Westphal return -ENOIOCTLCMD;
3704644807e3SFlorian Westphal }
3705644807e3SFlorian Westphal
3706e1d001faSBreno Leitao return 0;
3707644807e3SFlorian Westphal }
3708644807e3SFlorian Westphal
mptcp_subflow_early_fallback(struct mptcp_sock * msk,struct mptcp_subflow_context * subflow)370954f1944eSPaolo Abeni static void mptcp_subflow_early_fallback(struct mptcp_sock *msk,
371054f1944eSPaolo Abeni struct mptcp_subflow_context *subflow)
371154f1944eSPaolo Abeni {
371254f1944eSPaolo Abeni subflow->request_mptcp = 0;
371354f1944eSPaolo Abeni __mptcp_do_fallback(msk);
371454f1944eSPaolo Abeni }
371554f1944eSPaolo Abeni
mptcp_connect(struct sock * sk,struct sockaddr * uaddr,int addr_len)371654f1944eSPaolo Abeni static int mptcp_connect(struct sock *sk, struct sockaddr *uaddr, int addr_len)
371754f1944eSPaolo Abeni {
371854f1944eSPaolo Abeni struct mptcp_subflow_context *subflow;
371954f1944eSPaolo Abeni struct mptcp_sock *msk = mptcp_sk(sk);
372054f1944eSPaolo Abeni int err = -EINVAL;
3721ccae357cSPaolo Abeni struct sock *ssk;
372254f1944eSPaolo Abeni
37233f326a82SPaolo Abeni ssk = __mptcp_nmpc_sk(msk);
37243f326a82SPaolo Abeni if (IS_ERR(ssk))
37253f326a82SPaolo Abeni return PTR_ERR(ssk);
372654f1944eSPaolo Abeni
3727c693a851SGeliang Tang mptcp_set_state(sk, TCP_SYN_SENT);
3728ccae357cSPaolo Abeni subflow = mptcp_subflow_ctx(ssk);
372954f1944eSPaolo Abeni #ifdef CONFIG_TCP_MD5SIG
373054f1944eSPaolo Abeni /* no MPTCP if MD5SIG is enabled on this socket or we may run out of
373154f1944eSPaolo Abeni * TCP option space.
373254f1944eSPaolo Abeni */
3733ccae357cSPaolo Abeni if (rcu_access_pointer(tcp_sk(ssk)->md5sig_info))
373454f1944eSPaolo Abeni mptcp_subflow_early_fallback(msk, subflow);
373554f1944eSPaolo Abeni #endif
3736ccae357cSPaolo Abeni if (subflow->request_mptcp && mptcp_token_new_connect(ssk)) {
3737ccae357cSPaolo Abeni MPTCP_INC_STATS(sock_net(ssk), MPTCP_MIB_TOKENFALLBACKINIT);
373854f1944eSPaolo Abeni mptcp_subflow_early_fallback(msk, subflow);
373954f1944eSPaolo Abeni }
3740fb7a0d33SPaolo Abeni
3741fb7a0d33SPaolo Abeni WRITE_ONCE(msk->write_seq, subflow->idsn);
3742fb7a0d33SPaolo Abeni WRITE_ONCE(msk->snd_nxt, subflow->idsn);
374354f1944eSPaolo Abeni if (likely(!__mptcp_check_fallback(msk)))
374454f1944eSPaolo Abeni MPTCP_INC_STATS(sock_net(sk), MPTCP_MIB_MPCAPABLEACTIVE);
374554f1944eSPaolo Abeni
374654f1944eSPaolo Abeni /* if reaching here via the fastopen/sendmsg path, the caller already
374754f1944eSPaolo Abeni * acquired the subflow socket lock, too.
374854f1944eSPaolo Abeni */
3749ccae357cSPaolo Abeni if (!msk->fastopening)
3750ccae357cSPaolo Abeni lock_sock(ssk);
3751ccae357cSPaolo Abeni
3752ccae357cSPaolo Abeni /* the following mirrors closely a very small chunk of code from
3753ccae357cSPaolo Abeni * __inet_stream_connect()
3754ccae357cSPaolo Abeni */
3755ccae357cSPaolo Abeni if (ssk->sk_state != TCP_CLOSE)
3756ccae357cSPaolo Abeni goto out;
3757ccae357cSPaolo Abeni
3758ccae357cSPaolo Abeni if (BPF_CGROUP_PRE_CONNECT_ENABLED(ssk)) {
3759ccae357cSPaolo Abeni err = ssk->sk_prot->pre_connect(ssk, uaddr, addr_len);
3760ccae357cSPaolo Abeni if (err)
3761ccae357cSPaolo Abeni goto out;
3762ccae357cSPaolo Abeni }
3763ccae357cSPaolo Abeni
3764ccae357cSPaolo Abeni err = ssk->sk_prot->connect(ssk, uaddr, addr_len);
3765ccae357cSPaolo Abeni if (err < 0)
3766ccae357cSPaolo Abeni goto out;
3767ccae357cSPaolo Abeni
376808e39c0dSEric Dumazet inet_assign_bit(DEFER_CONNECT, sk, inet_test_bit(DEFER_CONNECT, ssk));
3769ccae357cSPaolo Abeni
3770ccae357cSPaolo Abeni out:
3771ccae357cSPaolo Abeni if (!msk->fastopening)
3772ccae357cSPaolo Abeni release_sock(ssk);
377354f1944eSPaolo Abeni
377454f1944eSPaolo Abeni /* on successful connect, the msk state will be moved to established by
377554f1944eSPaolo Abeni * subflow_finish_connect()
377654f1944eSPaolo Abeni */
3777ccae357cSPaolo Abeni if (unlikely(err)) {
3778131a6277SPaolo Abeni /* avoid leaving a dangling token in an unconnected socket */
3779131a6277SPaolo Abeni mptcp_token_destroy(msk);
3780c693a851SGeliang Tang mptcp_set_state(sk, TCP_CLOSE);
378154f1944eSPaolo Abeni return err;
378254f1944eSPaolo Abeni }
378354f1944eSPaolo Abeni
3784ccae357cSPaolo Abeni mptcp_copy_inaddrs(sk, ssk);
3785786fc124SPaolo Abeni return 0;
378654f1944eSPaolo Abeni }
378754f1944eSPaolo Abeni
3788f870fa0bSMat Martineau static struct proto mptcp_prot = {
3789f870fa0bSMat Martineau .name = "MPTCP",
3790f870fa0bSMat Martineau .owner = THIS_MODULE,
3791f870fa0bSMat Martineau .init = mptcp_init_sock,
379254f1944eSPaolo Abeni .connect = mptcp_connect,
379318b683bfSPaolo Abeni .disconnect = mptcp_disconnect,
3794f870fa0bSMat Martineau .close = mptcp_close,
3795717e79c8SPeter Krystad .setsockopt = mptcp_setsockopt,
3796717e79c8SPeter Krystad .getsockopt = mptcp_getsockopt,
379776e2a55dSPaolo Abeni .shutdown = mptcp_shutdown,
379879c0949eSPeter Krystad .destroy = mptcp_destroy,
3799f870fa0bSMat Martineau .sendmsg = mptcp_sendmsg,
3800644807e3SFlorian Westphal .ioctl = mptcp_ioctl,
3801f870fa0bSMat Martineau .recvmsg = mptcp_recvmsg,
380214c441b5SPaolo Abeni .release_cb = mptcp_release_cb,
38032c5ebd00SPaolo Abeni .hash = mptcp_hash,
38042c5ebd00SPaolo Abeni .unhash = mptcp_unhash,
3805cec37a6eSPeter Krystad .get_port = mptcp_get_port,
38066511882cSPaolo Abeni .forward_alloc_get = mptcp_forward_alloc_get,
380729b5e5efSPaolo Abeni .stream_memory_free = mptcp_stream_memory_free,
3808d027236cSPaolo Abeni .sockets_allocated = &mptcp_sockets_allocated,
38090defbb0aSEric Dumazet
3810d027236cSPaolo Abeni .memory_allocated = &tcp_memory_allocated,
38110defbb0aSEric Dumazet .per_cpu_fw_alloc = &tcp_memory_per_cpu_fw_alloc,
38120defbb0aSEric Dumazet
3813d027236cSPaolo Abeni .memory_pressure = &tcp_memory_pressure,
3814d027236cSPaolo Abeni .sysctl_wmem_offset = offsetof(struct net, ipv4.sysctl_tcp_wmem),
3815989ef49bSPaolo Abeni .sysctl_rmem_offset = offsetof(struct net, ipv4.sysctl_tcp_rmem),
3816d027236cSPaolo Abeni .sysctl_mem = sysctl_tcp_mem,
3817f870fa0bSMat Martineau .obj_size = sizeof(struct mptcp_sock),
38182c5ebd00SPaolo Abeni .slab_flags = SLAB_TYPESAFE_BY_RCU,
3819f870fa0bSMat Martineau .no_autobind = true,
3820f870fa0bSMat Martineau };
3821f870fa0bSMat Martineau
mptcp_bind(struct socket * sock,struct sockaddr * uaddr,int addr_len)38222303f994SPeter Krystad static int mptcp_bind(struct socket *sock, struct sockaddr *uaddr, int addr_len)
38232303f994SPeter Krystad {
38242303f994SPeter Krystad struct mptcp_sock *msk = mptcp_sk(sock->sk);
38258cf2ebdcSPaolo Abeni struct sock *ssk, *sk = sock->sk;
38268cf2ebdcSPaolo Abeni int err = -EINVAL;
38272303f994SPeter Krystad
38288cf2ebdcSPaolo Abeni lock_sock(sk);
38293f326a82SPaolo Abeni ssk = __mptcp_nmpc_sk(msk);
38303f326a82SPaolo Abeni if (IS_ERR(ssk)) {
38313f326a82SPaolo Abeni err = PTR_ERR(ssk);
38322303f994SPeter Krystad goto unlock;
38332303f994SPeter Krystad }
38342303f994SPeter Krystad
38358cf2ebdcSPaolo Abeni if (sk->sk_family == AF_INET)
38368cf2ebdcSPaolo Abeni err = inet_bind_sk(ssk, uaddr, addr_len);
38378cf2ebdcSPaolo Abeni #if IS_ENABLED(CONFIG_MPTCP_IPV6)
38388cf2ebdcSPaolo Abeni else if (sk->sk_family == AF_INET6)
38398cf2ebdcSPaolo Abeni err = inet6_bind_sk(ssk, uaddr, addr_len);
38408cf2ebdcSPaolo Abeni #endif
3841cf7da0d6SPeter Krystad if (!err)
38428cf2ebdcSPaolo Abeni mptcp_copy_inaddrs(sk, ssk);
38432303f994SPeter Krystad
38442303f994SPeter Krystad unlock:
38458cf2ebdcSPaolo Abeni release_sock(sk);
38462303f994SPeter Krystad return err;
38472303f994SPeter Krystad }
38482303f994SPeter Krystad
mptcp_listen(struct socket * sock,int backlog)3849cf7da0d6SPeter Krystad static int mptcp_listen(struct socket *sock, int backlog)
3850cf7da0d6SPeter Krystad {
3851cf7da0d6SPeter Krystad struct mptcp_sock *msk = mptcp_sk(sock->sk);
3852cfdcfeedSMenglong Dong struct sock *sk = sock->sk;
385340f56d0cSPaolo Abeni struct sock *ssk;
3854cf7da0d6SPeter Krystad int err;
3855cf7da0d6SPeter Krystad
3856cf7da0d6SPeter Krystad pr_debug("msk=%p", msk);
3857cf7da0d6SPeter Krystad
3858cfdcfeedSMenglong Dong lock_sock(sk);
38590226436aSPaolo Abeni
38600226436aSPaolo Abeni err = -EINVAL;
38610226436aSPaolo Abeni if (sock->state != SS_UNCONNECTED || sock->type != SOCK_STREAM)
38620226436aSPaolo Abeni goto unlock;
38630226436aSPaolo Abeni
38643f326a82SPaolo Abeni ssk = __mptcp_nmpc_sk(msk);
38653f326a82SPaolo Abeni if (IS_ERR(ssk)) {
38663f326a82SPaolo Abeni err = PTR_ERR(ssk);
3867cf7da0d6SPeter Krystad goto unlock;
3868cf7da0d6SPeter Krystad }
3869cf7da0d6SPeter Krystad
3870c693a851SGeliang Tang mptcp_set_state(sk, TCP_LISTEN);
3871cfdcfeedSMenglong Dong sock_set_flag(sk, SOCK_RCU_FREE);
38725e20087dSFlorian Westphal
387340f56d0cSPaolo Abeni lock_sock(ssk);
387440f56d0cSPaolo Abeni err = __inet_listen_sk(ssk, backlog);
387540f56d0cSPaolo Abeni release_sock(ssk);
3876c693a851SGeliang Tang mptcp_set_state(sk, inet_sk_state_load(ssk));
387740f56d0cSPaolo Abeni
3878c558246eSMenglong Dong if (!err) {
3879c558246eSMenglong Dong sock_prot_inuse_add(sock_net(sk), sk->sk_prot, 1);
388040f56d0cSPaolo Abeni mptcp_copy_inaddrs(sk, ssk);
388140f56d0cSPaolo Abeni mptcp_event_pm_listener(ssk, MPTCP_EVENT_LISTENER_CREATED);
388221d9b73aSPaolo Abeni }
3883f8c9dfbdSGeliang Tang
3884cf7da0d6SPeter Krystad unlock:
3885cfdcfeedSMenglong Dong release_sock(sk);
3886cf7da0d6SPeter Krystad return err;
3887cf7da0d6SPeter Krystad }
3888cf7da0d6SPeter Krystad
mptcp_stream_accept(struct socket * sock,struct socket * newsock,struct proto_accept_arg * arg)3889cf7da0d6SPeter Krystad static int mptcp_stream_accept(struct socket *sock, struct socket *newsock,
389092ef0fd5SJens Axboe struct proto_accept_arg *arg)
3891cf7da0d6SPeter Krystad {
3892cf7da0d6SPeter Krystad struct mptcp_sock *msk = mptcp_sk(sock->sk);
38931f6610b9SPaolo Abeni struct sock *ssk, *newsk;
3894cf7da0d6SPeter Krystad
3895cf7da0d6SPeter Krystad pr_debug("msk=%p", msk);
3896cf7da0d6SPeter Krystad
38975b825727SPaolo Abeni /* Buggy applications can call accept on socket states other then LISTEN
389861761231SPaolo Abeni * but no need to allocate the first subflow just to error out.
389961761231SPaolo Abeni */
39001f6610b9SPaolo Abeni ssk = READ_ONCE(msk->first);
39011f6610b9SPaolo Abeni if (!ssk)
390271ba088cSPaolo Abeni return -EINVAL;
3903cf7da0d6SPeter Krystad
39048e2b8a9fSDavide Caratti pr_debug("ssk=%p, listener=%p", ssk, mptcp_subflow_ctx(ssk));
390592ef0fd5SJens Axboe newsk = inet_csk_accept(ssk, arg);
3906e76c8ef5SPaolo Abeni if (!newsk)
390792ef0fd5SJens Axboe return arg->err;
3908e76c8ef5SPaolo Abeni
39098e2b8a9fSDavide Caratti pr_debug("newsk=%p, subflow is mptcp=%d", newsk, sk_is_mptcp(newsk));
39108e2b8a9fSDavide Caratti if (sk_is_mptcp(newsk)) {
3911cf7da0d6SPeter Krystad struct mptcp_subflow_context *subflow;
39128e2b8a9fSDavide Caratti struct sock *new_mptcp_sock;
39138e2b8a9fSDavide Caratti
39148e2b8a9fSDavide Caratti subflow = mptcp_subflow_ctx(newsk);
39158e2b8a9fSDavide Caratti new_mptcp_sock = subflow->conn;
39168e2b8a9fSDavide Caratti
39178e2b8a9fSDavide Caratti /* is_mptcp should be false if subflow->conn is missing, see
39188e2b8a9fSDavide Caratti * subflow_syn_recv_sock()
39198e2b8a9fSDavide Caratti */
39208e2b8a9fSDavide Caratti if (WARN_ON_ONCE(!new_mptcp_sock)) {
39218e2b8a9fSDavide Caratti tcp_sk(newsk)->is_mptcp = 0;
39228e2b8a9fSDavide Caratti goto tcpfallback;
39238e2b8a9fSDavide Caratti }
39248e2b8a9fSDavide Caratti
39258e2b8a9fSDavide Caratti newsk = new_mptcp_sock;
39268e2b8a9fSDavide Caratti MPTCP_INC_STATS(sock_net(ssk), MPTCP_MIB_MPCAPABLEPASSIVEACK);
39278e2b8a9fSDavide Caratti
392892ef0fd5SJens Axboe newsk->sk_kern_sock = arg->kern;
39298e2b8a9fSDavide Caratti lock_sock(newsk);
39308e2b8a9fSDavide Caratti __inet_accept(sock, newsock, newsk);
39310397c6d8SPaolo Abeni
3932a5ef058dSPaolo Abeni set_bit(SOCK_CUSTOM_SOCKOPT, &newsock->flags);
39338e2b8a9fSDavide Caratti msk = mptcp_sk(newsk);
3934b6985b9bSPaolo Abeni msk->in_accept_queue = 0;
3935a5ef058dSPaolo Abeni
3936cf7da0d6SPeter Krystad /* set ssk->sk_socket of accept()ed flows to mptcp socket.
3937cf7da0d6SPeter Krystad * This is needed so NOSPACE flag can be set from tcp stack.
3938cf7da0d6SPeter Krystad */
3939190f8b06SGeliang Tang mptcp_for_each_subflow(msk, subflow) {
3940cf7da0d6SPeter Krystad struct sock *ssk = mptcp_subflow_tcp_sock(subflow);
3941cf7da0d6SPeter Krystad
3942cf7da0d6SPeter Krystad if (!ssk->sk_socket)
3943cf7da0d6SPeter Krystad mptcp_sock_graft(ssk, newsock);
3944cf7da0d6SPeter Krystad }
394563740448SPaolo Abeni
394663740448SPaolo Abeni /* Do late cleanup for the first subflow as necessary. Also
394763740448SPaolo Abeni * deal with bad peers not doing a complete shutdown.
394863740448SPaolo Abeni */
394939880bd8SPaolo Abeni if (unlikely(inet_sk_state_load(msk->first) == TCP_CLOSE)) {
395063740448SPaolo Abeni __mptcp_close_ssk(newsk, msk->first,
395163740448SPaolo Abeni mptcp_subflow_ctx(msk->first), 0);
395239880bd8SPaolo Abeni if (unlikely(list_is_singular(&msk->conn_list)))
3953c693a851SGeliang Tang mptcp_set_state(newsk, TCP_CLOSE);
395463740448SPaolo Abeni }
39558e2b8a9fSDavide Caratti } else {
39568e2b8a9fSDavide Caratti tcpfallback:
395792ef0fd5SJens Axboe newsk->sk_kern_sock = arg->kern;
39588e2b8a9fSDavide Caratti lock_sock(newsk);
39598e2b8a9fSDavide Caratti __inet_accept(sock, newsock, newsk);
39608e2b8a9fSDavide Caratti /* we are being invoked after accepting a non-mp-capable
39618e2b8a9fSDavide Caratti * flow: sk is a tcp_sk, not an mptcp one.
39628e2b8a9fSDavide Caratti *
39638e2b8a9fSDavide Caratti * Hand the socket over to tcp so all further socket ops
39648e2b8a9fSDavide Caratti * bypass mptcp.
39658e2b8a9fSDavide Caratti */
39668e2b8a9fSDavide Caratti WRITE_ONCE(newsock->sk->sk_socket->ops,
39678e2b8a9fSDavide Caratti mptcp_fallback_tcp_ops(newsock->sk));
3968cf7da0d6SPeter Krystad }
3969e76c8ef5SPaolo Abeni release_sock(newsk);
3970cf7da0d6SPeter Krystad
3971e76c8ef5SPaolo Abeni return 0;
3972cf7da0d6SPeter Krystad }
3973cf7da0d6SPeter Krystad
mptcp_check_writeable(struct mptcp_sock * msk)39748edf0864SFlorian Westphal static __poll_t mptcp_check_writeable(struct mptcp_sock *msk)
39758edf0864SFlorian Westphal {
39768edf0864SFlorian Westphal struct sock *sk = (struct sock *)msk;
39778edf0864SFlorian Westphal
397829b5e5efSPaolo Abeni if (__mptcp_stream_is_writeable(sk, 1))
39798edf0864SFlorian Westphal return EPOLLOUT | EPOLLWRNORM;
39808edf0864SFlorian Westphal
3981037db6eaSPaolo Abeni set_bit(SOCK_NOSPACE, &sk->sk_socket->flags);
3982037db6eaSPaolo Abeni smp_mb__after_atomic(); /* NOSPACE is changed by mptcp_write_space() */
398329b5e5efSPaolo Abeni if (__mptcp_stream_is_writeable(sk, 1))
39846e628cd3SPaolo Abeni return EPOLLOUT | EPOLLWRNORM;
39858edf0864SFlorian Westphal
39866e628cd3SPaolo Abeni return 0;
39878edf0864SFlorian Westphal }
39888edf0864SFlorian Westphal
mptcp_poll(struct file * file,struct socket * sock,struct poll_table_struct * wait)39892303f994SPeter Krystad static __poll_t mptcp_poll(struct file *file, struct socket *sock,
39902303f994SPeter Krystad struct poll_table_struct *wait)
39912303f994SPeter Krystad {
39921891c4a0SFlorian Westphal struct sock *sk = sock->sk;
39938ab183deSPaolo Abeni struct mptcp_sock *msk;
39942303f994SPeter Krystad __poll_t mask = 0;
39956b9831bfSPaolo Abeni u8 shutdown;
39968a05661bSPaolo Abeni int state;
39972303f994SPeter Krystad
39981891c4a0SFlorian Westphal msk = mptcp_sk(sk);
39991891c4a0SFlorian Westphal sock_poll_wait(file, sock, wait);
40001891c4a0SFlorian Westphal
40018a05661bSPaolo Abeni state = inet_sk_state_load(sk);
40026719331cSPaolo Abeni pr_debug("msk=%p state=%d flags=%lx", msk, state, msk->flags);
400371ba088cSPaolo Abeni if (state == TCP_LISTEN) {
40045426a4efSPaolo Abeni struct sock *ssk = READ_ONCE(msk->first);
40055b825727SPaolo Abeni
40065426a4efSPaolo Abeni if (WARN_ON_ONCE(!ssk))
400771ba088cSPaolo Abeni return 0;
400871ba088cSPaolo Abeni
40095426a4efSPaolo Abeni return inet_csk_listen_poll(ssk);
401071ba088cSPaolo Abeni }
40118a05661bSPaolo Abeni
40126b9831bfSPaolo Abeni shutdown = READ_ONCE(sk->sk_shutdown);
40136b9831bfSPaolo Abeni if (shutdown == SHUTDOWN_MASK || state == TCP_CLOSE)
40146b9831bfSPaolo Abeni mask |= EPOLLHUP;
40156b9831bfSPaolo Abeni if (shutdown & RCV_SHUTDOWN)
40166b9831bfSPaolo Abeni mask |= EPOLLIN | EPOLLRDNORM | EPOLLRDHUP;
40176b9831bfSPaolo Abeni
40188a05661bSPaolo Abeni if (state != TCP_SYN_SENT && state != TCP_SYN_RECV) {
40195684ab1aSPaolo Abeni mask |= mptcp_check_readable(sk);
40206b9831bfSPaolo Abeni if (shutdown & SEND_SHUTDOWN)
40216b9831bfSPaolo Abeni mask |= EPOLLOUT | EPOLLWRNORM;
40226b9831bfSPaolo Abeni else
40238edf0864SFlorian Westphal mask |= mptcp_check_writeable(msk);
402408e39c0dSEric Dumazet } else if (state == TCP_SYN_SENT &&
402508e39c0dSEric Dumazet inet_test_bit(DEFER_CONNECT, sk)) {
4026a42cf9d1SBenjamin Hesmans /* cf tcp_poll() note about TFO */
4027a42cf9d1SBenjamin Hesmans mask |= EPOLLOUT | EPOLLWRNORM;
40288a05661bSPaolo Abeni }
40291891c4a0SFlorian Westphal
4030d89e3ed7SPaolo Abeni /* This barrier is coupled with smp_wmb() in __mptcp_error_report() */
403115cc1045SPaolo Abeni smp_rmb();
40329ae8e5adSEric Dumazet if (READ_ONCE(sk->sk_err))
403315cc1045SPaolo Abeni mask |= EPOLLERR;
403415cc1045SPaolo Abeni
40352303f994SPeter Krystad return mask;
40362303f994SPeter Krystad }
40372303f994SPeter Krystad
4038e42f1ac6SFlorian Westphal static const struct proto_ops mptcp_stream_ops = {
4039e42f1ac6SFlorian Westphal .family = PF_INET,
4040e42f1ac6SFlorian Westphal .owner = THIS_MODULE,
40410a3cc579SPaolo Abeni .release = inet_release,
4042e42f1ac6SFlorian Westphal .bind = mptcp_bind,
4043786fc124SPaolo Abeni .connect = inet_stream_connect,
4044e42f1ac6SFlorian Westphal .socketpair = sock_no_socketpair,
4045e42f1ac6SFlorian Westphal .accept = mptcp_stream_accept,
4046d2f77c53SPaolo Abeni .getname = inet_getname,
4047e42f1ac6SFlorian Westphal .poll = mptcp_poll,
4048e42f1ac6SFlorian Westphal .ioctl = inet_ioctl,
4049e42f1ac6SFlorian Westphal .gettstamp = sock_gettstamp,
4050e42f1ac6SFlorian Westphal .listen = mptcp_listen,
405176e2a55dSPaolo Abeni .shutdown = inet_shutdown,
4052e42f1ac6SFlorian Westphal .setsockopt = sock_common_setsockopt,
4053e42f1ac6SFlorian Westphal .getsockopt = sock_common_getsockopt,
4054e42f1ac6SFlorian Westphal .sendmsg = inet_sendmsg,
4055e42f1ac6SFlorian Westphal .recvmsg = inet_recvmsg,
4056e42f1ac6SFlorian Westphal .mmap = sock_no_mmap,
40575684ab1aSPaolo Abeni .set_rcvlowat = mptcp_set_rcvlowat,
4058e42f1ac6SFlorian Westphal };
40592303f994SPeter Krystad
4060f870fa0bSMat Martineau static struct inet_protosw mptcp_protosw = {
4061f870fa0bSMat Martineau .type = SOCK_STREAM,
4062f870fa0bSMat Martineau .protocol = IPPROTO_MPTCP,
4063f870fa0bSMat Martineau .prot = &mptcp_prot,
40642303f994SPeter Krystad .ops = &mptcp_stream_ops,
40652303f994SPeter Krystad .flags = INET_PROTOSW_ICSK,
4066f870fa0bSMat Martineau };
4067f870fa0bSMat Martineau
mptcp_napi_poll(struct napi_struct * napi,int budget)4068b19bc294SPaolo Abeni static int mptcp_napi_poll(struct napi_struct *napi, int budget)
4069b19bc294SPaolo Abeni {
4070b19bc294SPaolo Abeni struct mptcp_delegated_action *delegated;
4071b19bc294SPaolo Abeni struct mptcp_subflow_context *subflow;
4072b19bc294SPaolo Abeni int work_done = 0;
4073b19bc294SPaolo Abeni
4074b19bc294SPaolo Abeni delegated = container_of(napi, struct mptcp_delegated_action, napi);
4075b19bc294SPaolo Abeni while ((subflow = mptcp_subflow_delegated_next(delegated)) != NULL) {
4076b19bc294SPaolo Abeni struct sock *ssk = mptcp_subflow_tcp_sock(subflow);
4077b19bc294SPaolo Abeni
4078b19bc294SPaolo Abeni bh_lock_sock_nested(ssk);
4079a5efdbceSPaolo Abeni if (!sock_owned_by_user(ssk)) {
4080a5efdbceSPaolo Abeni mptcp_subflow_process_delegated(ssk, xchg(&subflow->delegated_status, 0));
4081a5efdbceSPaolo Abeni } else {
4082a5efdbceSPaolo Abeni /* tcp_release_cb_override already processed
4083b19bc294SPaolo Abeni * the action or will do at next release_sock().
4084b19bc294SPaolo Abeni * In both case must dequeue the subflow here - on the same
4085b19bc294SPaolo Abeni * CPU that scheduled it.
4086b19bc294SPaolo Abeni */
4087a5efdbceSPaolo Abeni smp_wmb();
4088a5efdbceSPaolo Abeni clear_bit(MPTCP_DELEGATE_SCHEDULED, &subflow->delegated_status);
4089a5efdbceSPaolo Abeni }
4090b19bc294SPaolo Abeni bh_unlock_sock(ssk);
4091b19bc294SPaolo Abeni sock_put(ssk);
4092b19bc294SPaolo Abeni
4093b19bc294SPaolo Abeni if (++work_done == budget)
4094b19bc294SPaolo Abeni return budget;
4095b19bc294SPaolo Abeni }
4096b19bc294SPaolo Abeni
4097b19bc294SPaolo Abeni /* always provide a 0 'work_done' argument, so that napi_complete_done
4098b19bc294SPaolo Abeni * will not try accessing the NULL napi->dev ptr
4099b19bc294SPaolo Abeni */
4100b19bc294SPaolo Abeni napi_complete_done(napi, 0);
4101b19bc294SPaolo Abeni return work_done;
4102b19bc294SPaolo Abeni }
4103b19bc294SPaolo Abeni
mptcp_proto_init(void)4104d39dcecaSPaolo Abeni void __init mptcp_proto_init(void)
4105f870fa0bSMat Martineau {
4106b19bc294SPaolo Abeni struct mptcp_delegated_action *delegated;
4107b19bc294SPaolo Abeni int cpu;
4108b19bc294SPaolo Abeni
41092303f994SPeter Krystad mptcp_prot.h.hashinfo = tcp_prot.h.hashinfo;
41102303f994SPeter Krystad
4111d027236cSPaolo Abeni if (percpu_counter_init(&mptcp_sockets_allocated, 0, GFP_KERNEL))
4112d027236cSPaolo Abeni panic("Failed to allocate MPTCP pcpu counter\n");
4113d027236cSPaolo Abeni
4114b19bc294SPaolo Abeni init_dummy_netdev(&mptcp_napi_dev);
4115b19bc294SPaolo Abeni for_each_possible_cpu(cpu) {
4116b19bc294SPaolo Abeni delegated = per_cpu_ptr(&mptcp_delegated_actions, cpu);
4117b19bc294SPaolo Abeni INIT_LIST_HEAD(&delegated->head);
411816d083e2SJakub Kicinski netif_napi_add_tx(&mptcp_napi_dev, &delegated->napi,
411916d083e2SJakub Kicinski mptcp_napi_poll);
4120b19bc294SPaolo Abeni napi_enable(&delegated->napi);
4121b19bc294SPaolo Abeni }
4122b19bc294SPaolo Abeni
41232303f994SPeter Krystad mptcp_subflow_init();
41241b1c7a0eSPeter Krystad mptcp_pm_init();
4125ed1ad86bSGeliang Tang mptcp_sched_init();
41262c5ebd00SPaolo Abeni mptcp_token_init();
41272303f994SPeter Krystad
4128f870fa0bSMat Martineau if (proto_register(&mptcp_prot, 1) != 0)
4129f870fa0bSMat Martineau panic("Failed to register MPTCP proto.\n");
4130f870fa0bSMat Martineau
4131f870fa0bSMat Martineau inet_register_protosw(&mptcp_protosw);
41326771bfd9SFlorian Westphal
41336771bfd9SFlorian Westphal BUILD_BUG_ON(sizeof(struct mptcp_skb_cb) > sizeof_field(struct sk_buff, cb));
4134f870fa0bSMat Martineau }
4135f870fa0bSMat Martineau
4136f870fa0bSMat Martineau #if IS_ENABLED(CONFIG_MPTCP_IPV6)
4137e42f1ac6SFlorian Westphal static const struct proto_ops mptcp_v6_stream_ops = {
4138e42f1ac6SFlorian Westphal .family = PF_INET6,
4139e42f1ac6SFlorian Westphal .owner = THIS_MODULE,
41400a3cc579SPaolo Abeni .release = inet6_release,
4141e42f1ac6SFlorian Westphal .bind = mptcp_bind,
4142786fc124SPaolo Abeni .connect = inet_stream_connect,
4143e42f1ac6SFlorian Westphal .socketpair = sock_no_socketpair,
4144e42f1ac6SFlorian Westphal .accept = mptcp_stream_accept,
4145d2f77c53SPaolo Abeni .getname = inet6_getname,
4146e42f1ac6SFlorian Westphal .poll = mptcp_poll,
4147e42f1ac6SFlorian Westphal .ioctl = inet6_ioctl,
4148e42f1ac6SFlorian Westphal .gettstamp = sock_gettstamp,
4149e42f1ac6SFlorian Westphal .listen = mptcp_listen,
415076e2a55dSPaolo Abeni .shutdown = inet_shutdown,
4151e42f1ac6SFlorian Westphal .setsockopt = sock_common_setsockopt,
4152e42f1ac6SFlorian Westphal .getsockopt = sock_common_getsockopt,
4153e42f1ac6SFlorian Westphal .sendmsg = inet6_sendmsg,
4154e42f1ac6SFlorian Westphal .recvmsg = inet6_recvmsg,
4155e42f1ac6SFlorian Westphal .mmap = sock_no_mmap,
4156e42f1ac6SFlorian Westphal #ifdef CONFIG_COMPAT
41573986912fSChristoph Hellwig .compat_ioctl = inet6_compat_ioctl,
4158e42f1ac6SFlorian Westphal #endif
41595684ab1aSPaolo Abeni .set_rcvlowat = mptcp_set_rcvlowat,
4160e42f1ac6SFlorian Westphal };
4161e42f1ac6SFlorian Westphal
4162f870fa0bSMat Martineau static struct proto mptcp_v6_prot;
4163f870fa0bSMat Martineau
4164f870fa0bSMat Martineau static struct inet_protosw mptcp_v6_protosw = {
4165f870fa0bSMat Martineau .type = SOCK_STREAM,
4166f870fa0bSMat Martineau .protocol = IPPROTO_MPTCP,
4167f870fa0bSMat Martineau .prot = &mptcp_v6_prot,
41682303f994SPeter Krystad .ops = &mptcp_v6_stream_ops,
4169f870fa0bSMat Martineau .flags = INET_PROTOSW_ICSK,
4170f870fa0bSMat Martineau };
4171f870fa0bSMat Martineau
mptcp_proto_v6_init(void)4172d39dcecaSPaolo Abeni int __init mptcp_proto_v6_init(void)
4173f870fa0bSMat Martineau {
4174f870fa0bSMat Martineau int err;
4175f870fa0bSMat Martineau
4176f870fa0bSMat Martineau mptcp_v6_prot = mptcp_prot;
41775eae7a82SMatthieu Baerts (NGI0) strscpy(mptcp_v6_prot.name, "MPTCPv6", sizeof(mptcp_v6_prot.name));
4178f870fa0bSMat Martineau mptcp_v6_prot.slab = NULL;
4179b0519de8SFlorian Westphal mptcp_v6_prot.obj_size = sizeof(struct mptcp6_sock);
4180f5f80e32SEric Dumazet mptcp_v6_prot.ipv6_pinfo_offset = offsetof(struct mptcp6_sock, np);
4181f870fa0bSMat Martineau
4182f870fa0bSMat Martineau err = proto_register(&mptcp_v6_prot, 1);
4183f870fa0bSMat Martineau if (err)
4184f870fa0bSMat Martineau return err;
4185f870fa0bSMat Martineau
4186f870fa0bSMat Martineau err = inet6_register_protosw(&mptcp_v6_protosw);
4187f870fa0bSMat Martineau if (err)
4188f870fa0bSMat Martineau proto_unregister(&mptcp_v6_prot);
4189f870fa0bSMat Martineau
4190f870fa0bSMat Martineau return err;
4191f870fa0bSMat Martineau }
4192f870fa0bSMat Martineau #endif
4193