1f870fa0bSMat Martineau // SPDX-License-Identifier: GPL-2.0 2f870fa0bSMat Martineau /* Multipath TCP 3f870fa0bSMat Martineau * 4f870fa0bSMat Martineau * Copyright (c) 2017 - 2019, Intel Corporation. 5f870fa0bSMat Martineau */ 6f870fa0bSMat Martineau 7f870fa0bSMat Martineau #define pr_fmt(fmt) "MPTCP: " fmt 8f870fa0bSMat Martineau 9f870fa0bSMat Martineau #include <linux/kernel.h> 10f870fa0bSMat Martineau #include <linux/module.h> 11f870fa0bSMat Martineau #include <linux/netdevice.h> 127a6a6cbcSPaolo Abeni #include <linux/sched/signal.h> 137a6a6cbcSPaolo Abeni #include <linux/atomic.h> 14f870fa0bSMat Martineau #include <net/sock.h> 15f870fa0bSMat Martineau #include <net/inet_common.h> 16f870fa0bSMat Martineau #include <net/inet_hashtables.h> 17f870fa0bSMat Martineau #include <net/protocol.h> 18f870fa0bSMat Martineau #include <net/tcp.h> 193721b9b6SMat Martineau #include <net/tcp_states.h> 20cf7da0d6SPeter Krystad #if IS_ENABLED(CONFIG_MPTCP_IPV6) 21cf7da0d6SPeter Krystad #include <net/transp_v6.h> 22cf7da0d6SPeter Krystad #endif 23f870fa0bSMat Martineau #include <net/mptcp.h> 24f870fa0bSMat Martineau #include "protocol.h" 25fc518953SFlorian Westphal #include "mib.h" 26f870fa0bSMat Martineau 27b0519de8SFlorian Westphal #if IS_ENABLED(CONFIG_MPTCP_IPV6) 28b0519de8SFlorian Westphal struct mptcp6_sock { 29b0519de8SFlorian Westphal struct mptcp_sock msk; 30b0519de8SFlorian Westphal struct ipv6_pinfo np; 31b0519de8SFlorian Westphal }; 32b0519de8SFlorian Westphal #endif 33b0519de8SFlorian Westphal 346771bfd9SFlorian Westphal struct mptcp_skb_cb { 356771bfd9SFlorian Westphal u32 offset; 366771bfd9SFlorian Westphal }; 376771bfd9SFlorian Westphal 386771bfd9SFlorian Westphal #define MPTCP_SKB_CB(__skb) ((struct mptcp_skb_cb *)&((__skb)->cb[0])) 396771bfd9SFlorian Westphal 40d027236cSPaolo Abeni static struct percpu_counter mptcp_sockets_allocated; 41d027236cSPaolo Abeni 422303f994SPeter Krystad /* If msk has an initial subflow socket, and the MP_CAPABLE handshake has not 432303f994SPeter Krystad * completed yet or has failed, return the subflow socket. 442303f994SPeter Krystad * Otherwise return NULL. 452303f994SPeter Krystad */ 462303f994SPeter Krystad static struct socket *__mptcp_nmpc_socket(const struct mptcp_sock *msk) 472303f994SPeter Krystad { 48d22f4988SChristoph Paasch if (!msk->subflow || READ_ONCE(msk->can_ack)) 492303f994SPeter Krystad return NULL; 502303f994SPeter Krystad 512303f994SPeter Krystad return msk->subflow; 522303f994SPeter Krystad } 532303f994SPeter Krystad 54d2f77c53SPaolo Abeni static bool mptcp_is_tcpsk(struct sock *sk) 550b4f33deSFlorian Westphal { 560b4f33deSFlorian Westphal struct socket *sock = sk->sk_socket; 570b4f33deSFlorian Westphal 580b4f33deSFlorian Westphal if (unlikely(sk->sk_prot == &tcp_prot)) { 590b4f33deSFlorian Westphal /* we are being invoked after mptcp_accept() has 600b4f33deSFlorian Westphal * accepted a non-mp-capable flow: sk is a tcp_sk, 610b4f33deSFlorian Westphal * not an mptcp one. 620b4f33deSFlorian Westphal * 630b4f33deSFlorian Westphal * Hand the socket over to tcp so all further socket ops 640b4f33deSFlorian Westphal * bypass mptcp. 650b4f33deSFlorian Westphal */ 660b4f33deSFlorian Westphal sock->ops = &inet_stream_ops; 67d2f77c53SPaolo Abeni return true; 680b4f33deSFlorian Westphal #if IS_ENABLED(CONFIG_MPTCP_IPV6) 690b4f33deSFlorian Westphal } else if (unlikely(sk->sk_prot == &tcpv6_prot)) { 700b4f33deSFlorian Westphal sock->ops = &inet6_stream_ops; 71d2f77c53SPaolo Abeni return true; 720b4f33deSFlorian Westphal #endif 730b4f33deSFlorian Westphal } 740b4f33deSFlorian Westphal 75d2f77c53SPaolo Abeni return false; 760b4f33deSFlorian Westphal } 770b4f33deSFlorian Westphal 7876660afbSPaolo Abeni static struct sock *__mptcp_tcp_fallback(struct mptcp_sock *msk) 79cec37a6eSPeter Krystad { 80cec37a6eSPeter Krystad sock_owned_by_me((const struct sock *)msk); 81cec37a6eSPeter Krystad 82e1ff9e82SDavide Caratti if (likely(!__mptcp_check_fallback(msk))) 83cec37a6eSPeter Krystad return NULL; 84cec37a6eSPeter Krystad 8576660afbSPaolo Abeni return msk->first; 868ab183deSPaolo Abeni } 878ab183deSPaolo Abeni 88fa68018dSPaolo Abeni static int __mptcp_socket_create(struct mptcp_sock *msk) 892303f994SPeter Krystad { 902303f994SPeter Krystad struct mptcp_subflow_context *subflow; 912303f994SPeter Krystad struct sock *sk = (struct sock *)msk; 922303f994SPeter Krystad struct socket *ssock; 932303f994SPeter Krystad int err; 942303f994SPeter Krystad 952303f994SPeter Krystad err = mptcp_subflow_create_socket(sk, &ssock); 962303f994SPeter Krystad if (err) 97fa68018dSPaolo Abeni return err; 982303f994SPeter Krystad 998ab183deSPaolo Abeni msk->first = ssock->sk; 1002303f994SPeter Krystad msk->subflow = ssock; 1012303f994SPeter Krystad subflow = mptcp_subflow_ctx(ssock->sk); 102cec37a6eSPeter Krystad list_add(&subflow->node, &msk->conn_list); 1032303f994SPeter Krystad subflow->request_mptcp = 1; 1042303f994SPeter Krystad 105e1ff9e82SDavide Caratti /* accept() will wait on first subflow sk_wq, and we always wakes up 106e1ff9e82SDavide Caratti * via msk->sk_socket 107e1ff9e82SDavide Caratti */ 108e1ff9e82SDavide Caratti RCU_INIT_POINTER(msk->first->sk_wq, &sk->sk_socket->wq); 109e1ff9e82SDavide Caratti 110fa68018dSPaolo Abeni return 0; 1112303f994SPeter Krystad } 1122303f994SPeter Krystad 1136771bfd9SFlorian Westphal static void __mptcp_move_skb(struct mptcp_sock *msk, struct sock *ssk, 1146771bfd9SFlorian Westphal struct sk_buff *skb, 1156771bfd9SFlorian Westphal unsigned int offset, size_t copy_len) 1166771bfd9SFlorian Westphal { 1176771bfd9SFlorian Westphal struct sock *sk = (struct sock *)msk; 1184e637c70SFlorian Westphal struct sk_buff *tail; 1196771bfd9SFlorian Westphal 1206771bfd9SFlorian Westphal __skb_unlink(skb, &ssk->sk_receive_queue); 1214e637c70SFlorian Westphal 1224e637c70SFlorian Westphal skb_ext_reset(skb); 1234e637c70SFlorian Westphal skb_orphan(skb); 1244e637c70SFlorian Westphal msk->ack_seq += copy_len; 1254e637c70SFlorian Westphal 1264e637c70SFlorian Westphal tail = skb_peek_tail(&sk->sk_receive_queue); 1274e637c70SFlorian Westphal if (offset == 0 && tail) { 1284e637c70SFlorian Westphal bool fragstolen; 1294e637c70SFlorian Westphal int delta; 1304e637c70SFlorian Westphal 1314e637c70SFlorian Westphal if (skb_try_coalesce(tail, skb, &fragstolen, &delta)) { 1324e637c70SFlorian Westphal kfree_skb_partial(skb, fragstolen); 1334e637c70SFlorian Westphal atomic_add(delta, &sk->sk_rmem_alloc); 1344e637c70SFlorian Westphal sk_mem_charge(sk, delta); 1354e637c70SFlorian Westphal return; 1364e637c70SFlorian Westphal } 1374e637c70SFlorian Westphal } 1384e637c70SFlorian Westphal 139600911ffSFlorian Westphal skb_set_owner_r(skb, sk); 1406771bfd9SFlorian Westphal __skb_queue_tail(&sk->sk_receive_queue, skb); 1416771bfd9SFlorian Westphal MPTCP_SKB_CB(skb)->offset = offset; 1426771bfd9SFlorian Westphal } 1436771bfd9SFlorian Westphal 14416a9a9daSMat Martineau static void mptcp_stop_timer(struct sock *sk) 14516a9a9daSMat Martineau { 14616a9a9daSMat Martineau struct inet_connection_sock *icsk = inet_csk(sk); 14716a9a9daSMat Martineau 14816a9a9daSMat Martineau sk_stop_timer(sk, &icsk->icsk_retransmit_timer); 14916a9a9daSMat Martineau mptcp_sk(sk)->timer_ival = 0; 15016a9a9daSMat Martineau } 15116a9a9daSMat Martineau 152de06f573SFlorian Westphal /* both sockets must be locked */ 153de06f573SFlorian Westphal static bool mptcp_subflow_dsn_valid(const struct mptcp_sock *msk, 154de06f573SFlorian Westphal struct sock *ssk) 155de06f573SFlorian Westphal { 156de06f573SFlorian Westphal struct mptcp_subflow_context *subflow = mptcp_subflow_ctx(ssk); 157de06f573SFlorian Westphal u64 dsn = mptcp_subflow_get_mapped_dsn(subflow); 158de06f573SFlorian Westphal 159de06f573SFlorian Westphal /* revalidate data sequence number. 160de06f573SFlorian Westphal * 161de06f573SFlorian Westphal * mptcp_subflow_data_available() is usually called 162de06f573SFlorian Westphal * without msk lock. Its unlikely (but possible) 163de06f573SFlorian Westphal * that msk->ack_seq has been advanced since the last 164de06f573SFlorian Westphal * call found in-sequence data. 165de06f573SFlorian Westphal */ 166de06f573SFlorian Westphal if (likely(dsn == msk->ack_seq)) 167de06f573SFlorian Westphal return true; 168de06f573SFlorian Westphal 169de06f573SFlorian Westphal subflow->data_avail = 0; 1706719331cSPaolo Abeni mptcp_subflow_data_available(ssk); 1716719331cSPaolo Abeni return subflow->data_avail == MPTCP_SUBFLOW_DATA_AVAIL; 172de06f573SFlorian Westphal } 173de06f573SFlorian Westphal 17416a9a9daSMat Martineau static void mptcp_check_data_fin_ack(struct sock *sk) 17516a9a9daSMat Martineau { 17616a9a9daSMat Martineau struct mptcp_sock *msk = mptcp_sk(sk); 17716a9a9daSMat Martineau 17816a9a9daSMat Martineau if (__mptcp_check_fallback(msk)) 17916a9a9daSMat Martineau return; 18016a9a9daSMat Martineau 18116a9a9daSMat Martineau /* Look for an acknowledged DATA_FIN */ 18216a9a9daSMat Martineau if (((1 << sk->sk_state) & 18316a9a9daSMat Martineau (TCPF_FIN_WAIT1 | TCPF_CLOSING | TCPF_LAST_ACK)) && 18416a9a9daSMat Martineau msk->write_seq == atomic64_read(&msk->snd_una)) { 18516a9a9daSMat Martineau mptcp_stop_timer(sk); 18616a9a9daSMat Martineau 18716a9a9daSMat Martineau WRITE_ONCE(msk->snd_data_fin_enable, 0); 18816a9a9daSMat Martineau 18916a9a9daSMat Martineau switch (sk->sk_state) { 19016a9a9daSMat Martineau case TCP_FIN_WAIT1: 19116a9a9daSMat Martineau inet_sk_state_store(sk, TCP_FIN_WAIT2); 19216a9a9daSMat Martineau sk->sk_state_change(sk); 19316a9a9daSMat Martineau break; 19416a9a9daSMat Martineau case TCP_CLOSING: 19516a9a9daSMat Martineau case TCP_LAST_ACK: 19616a9a9daSMat Martineau inet_sk_state_store(sk, TCP_CLOSE); 19716a9a9daSMat Martineau sk->sk_state_change(sk); 19816a9a9daSMat Martineau break; 19916a9a9daSMat Martineau } 20016a9a9daSMat Martineau 20116a9a9daSMat Martineau if (sk->sk_shutdown == SHUTDOWN_MASK || 20216a9a9daSMat Martineau sk->sk_state == TCP_CLOSE) 20316a9a9daSMat Martineau sk_wake_async(sk, SOCK_WAKE_WAITD, POLL_HUP); 20416a9a9daSMat Martineau else 20516a9a9daSMat Martineau sk_wake_async(sk, SOCK_WAKE_WAITD, POLL_IN); 20616a9a9daSMat Martineau } 20716a9a9daSMat Martineau } 20816a9a9daSMat Martineau 2093721b9b6SMat Martineau static bool mptcp_pending_data_fin(struct sock *sk, u64 *seq) 2103721b9b6SMat Martineau { 2113721b9b6SMat Martineau struct mptcp_sock *msk = mptcp_sk(sk); 2123721b9b6SMat Martineau 2133721b9b6SMat Martineau if (READ_ONCE(msk->rcv_data_fin) && 2143721b9b6SMat Martineau ((1 << sk->sk_state) & 2153721b9b6SMat Martineau (TCPF_ESTABLISHED | TCPF_FIN_WAIT1 | TCPF_FIN_WAIT2))) { 2163721b9b6SMat Martineau u64 rcv_data_fin_seq = READ_ONCE(msk->rcv_data_fin_seq); 2173721b9b6SMat Martineau 2183721b9b6SMat Martineau if (msk->ack_seq == rcv_data_fin_seq) { 2193721b9b6SMat Martineau if (seq) 2203721b9b6SMat Martineau *seq = rcv_data_fin_seq; 2213721b9b6SMat Martineau 2223721b9b6SMat Martineau return true; 2233721b9b6SMat Martineau } 2243721b9b6SMat Martineau } 2253721b9b6SMat Martineau 2263721b9b6SMat Martineau return false; 2273721b9b6SMat Martineau } 2283721b9b6SMat Martineau 2293721b9b6SMat Martineau static void mptcp_set_timeout(const struct sock *sk, const struct sock *ssk) 2303721b9b6SMat Martineau { 2313721b9b6SMat Martineau long tout = ssk && inet_csk(ssk)->icsk_pending ? 2323721b9b6SMat Martineau inet_csk(ssk)->icsk_timeout - jiffies : 0; 2333721b9b6SMat Martineau 2343721b9b6SMat Martineau if (tout <= 0) 2353721b9b6SMat Martineau tout = mptcp_sk(sk)->timer_ival; 2363721b9b6SMat Martineau mptcp_sk(sk)->timer_ival = tout > 0 ? tout : TCP_RTO_MIN; 2373721b9b6SMat Martineau } 2383721b9b6SMat Martineau 2393721b9b6SMat Martineau static void mptcp_check_data_fin(struct sock *sk) 2403721b9b6SMat Martineau { 2413721b9b6SMat Martineau struct mptcp_sock *msk = mptcp_sk(sk); 2423721b9b6SMat Martineau u64 rcv_data_fin_seq; 2433721b9b6SMat Martineau 2443721b9b6SMat Martineau if (__mptcp_check_fallback(msk) || !msk->first) 2453721b9b6SMat Martineau return; 2463721b9b6SMat Martineau 2473721b9b6SMat Martineau /* Need to ack a DATA_FIN received from a peer while this side 2483721b9b6SMat Martineau * of the connection is in ESTABLISHED, FIN_WAIT1, or FIN_WAIT2. 2493721b9b6SMat Martineau * msk->rcv_data_fin was set when parsing the incoming options 2503721b9b6SMat Martineau * at the subflow level and the msk lock was not held, so this 2513721b9b6SMat Martineau * is the first opportunity to act on the DATA_FIN and change 2523721b9b6SMat Martineau * the msk state. 2533721b9b6SMat Martineau * 2543721b9b6SMat Martineau * If we are caught up to the sequence number of the incoming 2553721b9b6SMat Martineau * DATA_FIN, send the DATA_ACK now and do state transition. If 2563721b9b6SMat Martineau * not caught up, do nothing and let the recv code send DATA_ACK 2573721b9b6SMat Martineau * when catching up. 2583721b9b6SMat Martineau */ 2593721b9b6SMat Martineau 2603721b9b6SMat Martineau if (mptcp_pending_data_fin(sk, &rcv_data_fin_seq)) { 2613721b9b6SMat Martineau struct mptcp_subflow_context *subflow; 2623721b9b6SMat Martineau 2633721b9b6SMat Martineau msk->ack_seq++; 2643721b9b6SMat Martineau WRITE_ONCE(msk->rcv_data_fin, 0); 2653721b9b6SMat Martineau 2663721b9b6SMat Martineau sk->sk_shutdown |= RCV_SHUTDOWN; 26716a9a9daSMat Martineau smp_mb__before_atomic(); /* SHUTDOWN must be visible first */ 26816a9a9daSMat Martineau set_bit(MPTCP_DATA_READY, &msk->flags); 2693721b9b6SMat Martineau 2703721b9b6SMat Martineau switch (sk->sk_state) { 2713721b9b6SMat Martineau case TCP_ESTABLISHED: 2723721b9b6SMat Martineau inet_sk_state_store(sk, TCP_CLOSE_WAIT); 2733721b9b6SMat Martineau break; 2743721b9b6SMat Martineau case TCP_FIN_WAIT1: 2753721b9b6SMat Martineau inet_sk_state_store(sk, TCP_CLOSING); 2763721b9b6SMat Martineau break; 2773721b9b6SMat Martineau case TCP_FIN_WAIT2: 2783721b9b6SMat Martineau inet_sk_state_store(sk, TCP_CLOSE); 2793721b9b6SMat Martineau // @@ Close subflows now? 2803721b9b6SMat Martineau break; 2813721b9b6SMat Martineau default: 2823721b9b6SMat Martineau /* Other states not expected */ 2833721b9b6SMat Martineau WARN_ON_ONCE(1); 2843721b9b6SMat Martineau break; 2853721b9b6SMat Martineau } 2863721b9b6SMat Martineau 2873721b9b6SMat Martineau mptcp_set_timeout(sk, NULL); 2883721b9b6SMat Martineau mptcp_for_each_subflow(msk, subflow) { 2893721b9b6SMat Martineau struct sock *ssk = mptcp_subflow_tcp_sock(subflow); 2903721b9b6SMat Martineau 2913721b9b6SMat Martineau lock_sock(ssk); 2923721b9b6SMat Martineau tcp_send_ack(ssk); 2933721b9b6SMat Martineau release_sock(ssk); 2943721b9b6SMat Martineau } 2953721b9b6SMat Martineau 2963721b9b6SMat Martineau sk->sk_state_change(sk); 2973721b9b6SMat Martineau 2983721b9b6SMat Martineau if (sk->sk_shutdown == SHUTDOWN_MASK || 2993721b9b6SMat Martineau sk->sk_state == TCP_CLOSE) 3003721b9b6SMat Martineau sk_wake_async(sk, SOCK_WAKE_WAITD, POLL_HUP); 3013721b9b6SMat Martineau else 3023721b9b6SMat Martineau sk_wake_async(sk, SOCK_WAKE_WAITD, POLL_IN); 3033721b9b6SMat Martineau } 3043721b9b6SMat Martineau } 3053721b9b6SMat Martineau 3066771bfd9SFlorian Westphal static bool __mptcp_move_skbs_from_subflow(struct mptcp_sock *msk, 3076771bfd9SFlorian Westphal struct sock *ssk, 3086771bfd9SFlorian Westphal unsigned int *bytes) 3096771bfd9SFlorian Westphal { 3106771bfd9SFlorian Westphal struct mptcp_subflow_context *subflow = mptcp_subflow_ctx(ssk); 311600911ffSFlorian Westphal struct sock *sk = (struct sock *)msk; 3126771bfd9SFlorian Westphal unsigned int moved = 0; 3136771bfd9SFlorian Westphal bool more_data_avail; 3146771bfd9SFlorian Westphal struct tcp_sock *tp; 3156771bfd9SFlorian Westphal bool done = false; 316600911ffSFlorian Westphal 3176719331cSPaolo Abeni pr_debug("msk=%p ssk=%p data avail=%d valid=%d empty=%d", 3186719331cSPaolo Abeni msk, ssk, subflow->data_avail, 3196719331cSPaolo Abeni mptcp_subflow_dsn_valid(msk, ssk), 3206719331cSPaolo Abeni !skb_peek(&ssk->sk_receive_queue)); 3216719331cSPaolo Abeni if (subflow->data_avail == MPTCP_SUBFLOW_OOO_DATA) { 3226719331cSPaolo Abeni mptcp_subflow_discard_data(ssk, subflow->map_data_len); 323de06f573SFlorian Westphal return false; 324de06f573SFlorian Westphal } 325de06f573SFlorian Westphal 3266719331cSPaolo Abeni if (!mptcp_subflow_dsn_valid(msk, ssk)) 3276719331cSPaolo Abeni return false; 3286719331cSPaolo Abeni 3296771bfd9SFlorian Westphal tp = tcp_sk(ssk); 3306771bfd9SFlorian Westphal do { 3316771bfd9SFlorian Westphal u32 map_remaining, offset; 3326771bfd9SFlorian Westphal u32 seq = tp->copied_seq; 3336771bfd9SFlorian Westphal struct sk_buff *skb; 3346771bfd9SFlorian Westphal bool fin; 3356771bfd9SFlorian Westphal 3366771bfd9SFlorian Westphal /* try to move as much data as available */ 3376771bfd9SFlorian Westphal map_remaining = subflow->map_data_len - 3386771bfd9SFlorian Westphal mptcp_subflow_get_map_offset(subflow); 3396771bfd9SFlorian Westphal 3406771bfd9SFlorian Westphal skb = skb_peek(&ssk->sk_receive_queue); 3416771bfd9SFlorian Westphal if (!skb) 3426771bfd9SFlorian Westphal break; 3436771bfd9SFlorian Westphal 344e1ff9e82SDavide Caratti if (__mptcp_check_fallback(msk)) { 345e1ff9e82SDavide Caratti /* if we are running under the workqueue, TCP could have 346e1ff9e82SDavide Caratti * collapsed skbs between dummy map creation and now 347e1ff9e82SDavide Caratti * be sure to adjust the size 348e1ff9e82SDavide Caratti */ 349e1ff9e82SDavide Caratti map_remaining = skb->len; 350e1ff9e82SDavide Caratti subflow->map_data_len = skb->len; 351e1ff9e82SDavide Caratti } 352e1ff9e82SDavide Caratti 3536771bfd9SFlorian Westphal offset = seq - TCP_SKB_CB(skb)->seq; 3546771bfd9SFlorian Westphal fin = TCP_SKB_CB(skb)->tcp_flags & TCPHDR_FIN; 3556771bfd9SFlorian Westphal if (fin) { 3566771bfd9SFlorian Westphal done = true; 3576771bfd9SFlorian Westphal seq++; 3586771bfd9SFlorian Westphal } 3596771bfd9SFlorian Westphal 3606771bfd9SFlorian Westphal if (offset < skb->len) { 3616771bfd9SFlorian Westphal size_t len = skb->len - offset; 3626771bfd9SFlorian Westphal 3636771bfd9SFlorian Westphal if (tp->urg_data) 3646771bfd9SFlorian Westphal done = true; 3656771bfd9SFlorian Westphal 3666771bfd9SFlorian Westphal __mptcp_move_skb(msk, ssk, skb, offset, len); 3676771bfd9SFlorian Westphal seq += len; 3686771bfd9SFlorian Westphal moved += len; 3696771bfd9SFlorian Westphal 3706771bfd9SFlorian Westphal if (WARN_ON_ONCE(map_remaining < len)) 3716771bfd9SFlorian Westphal break; 3726771bfd9SFlorian Westphal } else { 3736771bfd9SFlorian Westphal WARN_ON_ONCE(!fin); 3746771bfd9SFlorian Westphal sk_eat_skb(ssk, skb); 3756771bfd9SFlorian Westphal done = true; 3766771bfd9SFlorian Westphal } 3776771bfd9SFlorian Westphal 3786771bfd9SFlorian Westphal WRITE_ONCE(tp->copied_seq, seq); 3796771bfd9SFlorian Westphal more_data_avail = mptcp_subflow_data_available(ssk); 380600911ffSFlorian Westphal 381600911ffSFlorian Westphal if (atomic_read(&sk->sk_rmem_alloc) > READ_ONCE(sk->sk_rcvbuf)) { 382600911ffSFlorian Westphal done = true; 383600911ffSFlorian Westphal break; 384600911ffSFlorian Westphal } 3856771bfd9SFlorian Westphal } while (more_data_avail); 3866771bfd9SFlorian Westphal 3876719331cSPaolo Abeni *bytes += moved; 3886771bfd9SFlorian Westphal 38943b54c6eSMat Martineau /* If the moves have caught up with the DATA_FIN sequence number 39043b54c6eSMat Martineau * it's time to ack the DATA_FIN and change socket state, but 39143b54c6eSMat Martineau * this is not a good place to change state. Let the workqueue 39243b54c6eSMat Martineau * do it. 39343b54c6eSMat Martineau */ 39443b54c6eSMat Martineau if (mptcp_pending_data_fin(sk, NULL) && 39543b54c6eSMat Martineau schedule_work(&msk->work)) 39643b54c6eSMat Martineau sock_hold(sk); 39743b54c6eSMat Martineau 3986771bfd9SFlorian Westphal return done; 3996771bfd9SFlorian Westphal } 4006771bfd9SFlorian Westphal 4012e52213cSFlorian Westphal /* In most cases we will be able to lock the mptcp socket. If its already 4022e52213cSFlorian Westphal * owned, we need to defer to the work queue to avoid ABBA deadlock. 4032e52213cSFlorian Westphal */ 4042e52213cSFlorian Westphal static bool move_skbs_to_msk(struct mptcp_sock *msk, struct sock *ssk) 4052e52213cSFlorian Westphal { 4062e52213cSFlorian Westphal struct sock *sk = (struct sock *)msk; 4072e52213cSFlorian Westphal unsigned int moved = 0; 4082e52213cSFlorian Westphal 4092e52213cSFlorian Westphal if (READ_ONCE(sk->sk_lock.owned)) 4102e52213cSFlorian Westphal return false; 4112e52213cSFlorian Westphal 4122e52213cSFlorian Westphal if (unlikely(!spin_trylock_bh(&sk->sk_lock.slock))) 4132e52213cSFlorian Westphal return false; 4142e52213cSFlorian Westphal 4152e52213cSFlorian Westphal /* must re-check after taking the lock */ 4162e52213cSFlorian Westphal if (!READ_ONCE(sk->sk_lock.owned)) 4172e52213cSFlorian Westphal __mptcp_move_skbs_from_subflow(msk, ssk, &moved); 4182e52213cSFlorian Westphal 4192e52213cSFlorian Westphal spin_unlock_bh(&sk->sk_lock.slock); 4202e52213cSFlorian Westphal 4212e52213cSFlorian Westphal return moved > 0; 4222e52213cSFlorian Westphal } 4232e52213cSFlorian Westphal 4242e52213cSFlorian Westphal void mptcp_data_ready(struct sock *sk, struct sock *ssk) 425101f6f85SFlorian Westphal { 4266719331cSPaolo Abeni struct mptcp_subflow_context *subflow = mptcp_subflow_ctx(ssk); 427101f6f85SFlorian Westphal struct mptcp_sock *msk = mptcp_sk(sk); 4286719331cSPaolo Abeni bool wake; 429101f6f85SFlorian Westphal 4306719331cSPaolo Abeni /* move_skbs_to_msk below can legitly clear the data_avail flag, 4316719331cSPaolo Abeni * but we will need later to properly woke the reader, cache its 4326719331cSPaolo Abeni * value 4336719331cSPaolo Abeni */ 4346719331cSPaolo Abeni wake = subflow->data_avail == MPTCP_SUBFLOW_DATA_AVAIL; 4356719331cSPaolo Abeni if (wake) 436101f6f85SFlorian Westphal set_bit(MPTCP_DATA_READY, &msk->flags); 4376771bfd9SFlorian Westphal 4382e52213cSFlorian Westphal if (atomic_read(&sk->sk_rmem_alloc) < READ_ONCE(sk->sk_rcvbuf) && 4392e52213cSFlorian Westphal move_skbs_to_msk(msk, ssk)) 4402e52213cSFlorian Westphal goto wake; 4412e52213cSFlorian Westphal 442600911ffSFlorian Westphal /* don't schedule if mptcp sk is (still) over limit */ 443600911ffSFlorian Westphal if (atomic_read(&sk->sk_rmem_alloc) > READ_ONCE(sk->sk_rcvbuf)) 444600911ffSFlorian Westphal goto wake; 445600911ffSFlorian Westphal 44614c441b5SPaolo Abeni /* mptcp socket is owned, release_cb should retry */ 44714c441b5SPaolo Abeni if (!test_and_set_bit(TCP_DELACK_TIMER_DEFERRED, 44814c441b5SPaolo Abeni &sk->sk_tsq_flags)) { 44914c441b5SPaolo Abeni sock_hold(sk); 4506771bfd9SFlorian Westphal 45114c441b5SPaolo Abeni /* need to try again, its possible release_cb() has already 45214c441b5SPaolo Abeni * been called after the test_and_set_bit() above. 45314c441b5SPaolo Abeni */ 45414c441b5SPaolo Abeni move_skbs_to_msk(msk, ssk); 45514c441b5SPaolo Abeni } 456600911ffSFlorian Westphal wake: 4576719331cSPaolo Abeni if (wake) 458101f6f85SFlorian Westphal sk->sk_data_ready(sk); 459101f6f85SFlorian Westphal } 460101f6f85SFlorian Westphal 461ec3edaa7SPeter Krystad static void __mptcp_flush_join_list(struct mptcp_sock *msk) 462ec3edaa7SPeter Krystad { 463ec3edaa7SPeter Krystad if (likely(list_empty(&msk->join_list))) 464ec3edaa7SPeter Krystad return; 465ec3edaa7SPeter Krystad 466ec3edaa7SPeter Krystad spin_lock_bh(&msk->join_list_lock); 467ec3edaa7SPeter Krystad list_splice_tail_init(&msk->join_list, &msk->conn_list); 468ec3edaa7SPeter Krystad spin_unlock_bh(&msk->join_list_lock); 469ec3edaa7SPeter Krystad } 470ec3edaa7SPeter Krystad 471b51f9b80SPaolo Abeni static bool mptcp_timer_pending(struct sock *sk) 472b51f9b80SPaolo Abeni { 473b51f9b80SPaolo Abeni return timer_pending(&inet_csk(sk)->icsk_retransmit_timer); 474b51f9b80SPaolo Abeni } 475b51f9b80SPaolo Abeni 476b51f9b80SPaolo Abeni static void mptcp_reset_timer(struct sock *sk) 477b51f9b80SPaolo Abeni { 478b51f9b80SPaolo Abeni struct inet_connection_sock *icsk = inet_csk(sk); 479b51f9b80SPaolo Abeni unsigned long tout; 480b51f9b80SPaolo Abeni 481b51f9b80SPaolo Abeni /* should never be called with mptcp level timer cleared */ 482b51f9b80SPaolo Abeni tout = READ_ONCE(mptcp_sk(sk)->timer_ival); 483b51f9b80SPaolo Abeni if (WARN_ON_ONCE(!tout)) 484b51f9b80SPaolo Abeni tout = TCP_RTO_MIN; 485b51f9b80SPaolo Abeni sk_reset_timer(sk, &icsk->icsk_retransmit_timer, jiffies + tout); 486b51f9b80SPaolo Abeni } 487b51f9b80SPaolo Abeni 488b51f9b80SPaolo Abeni void mptcp_data_acked(struct sock *sk) 489b51f9b80SPaolo Abeni { 490b51f9b80SPaolo Abeni mptcp_reset_timer(sk); 4913b1d6210SPaolo Abeni 49263561a40SPaolo Abeni if ((!test_bit(MPTCP_SEND_SPACE, &mptcp_sk(sk)->flags) || 49343b54c6eSMat Martineau (inet_sk_state_load(sk) != TCP_ESTABLISHED)) && 4943b1d6210SPaolo Abeni schedule_work(&mptcp_sk(sk)->work)) 4953b1d6210SPaolo Abeni sock_hold(sk); 496b51f9b80SPaolo Abeni } 497b51f9b80SPaolo Abeni 49859832e24SFlorian Westphal void mptcp_subflow_eof(struct sock *sk) 49959832e24SFlorian Westphal { 50059832e24SFlorian Westphal struct mptcp_sock *msk = mptcp_sk(sk); 50159832e24SFlorian Westphal 50259832e24SFlorian Westphal if (!test_and_set_bit(MPTCP_WORK_EOF, &msk->flags) && 50359832e24SFlorian Westphal schedule_work(&msk->work)) 50459832e24SFlorian Westphal sock_hold(sk); 50559832e24SFlorian Westphal } 50659832e24SFlorian Westphal 5075969856aSPaolo Abeni static void mptcp_check_for_eof(struct mptcp_sock *msk) 5085969856aSPaolo Abeni { 5095969856aSPaolo Abeni struct mptcp_subflow_context *subflow; 5105969856aSPaolo Abeni struct sock *sk = (struct sock *)msk; 5115969856aSPaolo Abeni int receivers = 0; 5125969856aSPaolo Abeni 5135969856aSPaolo Abeni mptcp_for_each_subflow(msk, subflow) 5145969856aSPaolo Abeni receivers += !subflow->rx_eof; 5155969856aSPaolo Abeni 5165969856aSPaolo Abeni if (!receivers && !(sk->sk_shutdown & RCV_SHUTDOWN)) { 5175969856aSPaolo Abeni /* hopefully temporary hack: propagate shutdown status 5185969856aSPaolo Abeni * to msk, when all subflows agree on it 5195969856aSPaolo Abeni */ 5205969856aSPaolo Abeni sk->sk_shutdown |= RCV_SHUTDOWN; 5215969856aSPaolo Abeni 5225969856aSPaolo Abeni smp_mb__before_atomic(); /* SHUTDOWN must be visible first */ 5235969856aSPaolo Abeni set_bit(MPTCP_DATA_READY, &msk->flags); 5245969856aSPaolo Abeni sk->sk_data_ready(sk); 5255969856aSPaolo Abeni } 5265969856aSPaolo Abeni } 5275969856aSPaolo Abeni 5286d0060f6SMat Martineau static bool mptcp_ext_cache_refill(struct mptcp_sock *msk) 5296d0060f6SMat Martineau { 5304930f483SFlorian Westphal const struct sock *sk = (const struct sock *)msk; 5314930f483SFlorian Westphal 5326d0060f6SMat Martineau if (!msk->cached_ext) 5334930f483SFlorian Westphal msk->cached_ext = __skb_ext_alloc(sk->sk_allocation); 5346d0060f6SMat Martineau 5356d0060f6SMat Martineau return !!msk->cached_ext; 5366d0060f6SMat Martineau } 5376d0060f6SMat Martineau 5387a6a6cbcSPaolo Abeni static struct sock *mptcp_subflow_recv_lookup(const struct mptcp_sock *msk) 5397a6a6cbcSPaolo Abeni { 5407a6a6cbcSPaolo Abeni struct mptcp_subflow_context *subflow; 5417a6a6cbcSPaolo Abeni struct sock *sk = (struct sock *)msk; 5427a6a6cbcSPaolo Abeni 5437a6a6cbcSPaolo Abeni sock_owned_by_me(sk); 5447a6a6cbcSPaolo Abeni 5457a6a6cbcSPaolo Abeni mptcp_for_each_subflow(msk, subflow) { 5467a6a6cbcSPaolo Abeni if (subflow->data_avail) 5477a6a6cbcSPaolo Abeni return mptcp_subflow_tcp_sock(subflow); 5487a6a6cbcSPaolo Abeni } 5497a6a6cbcSPaolo Abeni 5507a6a6cbcSPaolo Abeni return NULL; 5517a6a6cbcSPaolo Abeni } 5527a6a6cbcSPaolo Abeni 5533f8e0aaeSPaolo Abeni static bool mptcp_skb_can_collapse_to(u64 write_seq, 55457040755SPaolo Abeni const struct sk_buff *skb, 55557040755SPaolo Abeni const struct mptcp_ext *mpext) 5566d0060f6SMat Martineau { 55757040755SPaolo Abeni if (!tcp_skb_can_collapse_to(skb)) 55857040755SPaolo Abeni return false; 55957040755SPaolo Abeni 56057040755SPaolo Abeni /* can collapse only if MPTCP level sequence is in order */ 5613f8e0aaeSPaolo Abeni return mpext && mpext->data_seq + mpext->data_len == write_seq; 56257040755SPaolo Abeni } 56357040755SPaolo Abeni 56418b683bfSPaolo Abeni static bool mptcp_frag_can_collapse_to(const struct mptcp_sock *msk, 56518b683bfSPaolo Abeni const struct page_frag *pfrag, 56618b683bfSPaolo Abeni const struct mptcp_data_frag *df) 56718b683bfSPaolo Abeni { 56818b683bfSPaolo Abeni return df && pfrag->page == df->page && 56918b683bfSPaolo Abeni df->data_seq + df->data_len == msk->write_seq; 57018b683bfSPaolo Abeni } 57118b683bfSPaolo Abeni 572d027236cSPaolo Abeni static void dfrag_uncharge(struct sock *sk, int len) 57318b683bfSPaolo Abeni { 574d027236cSPaolo Abeni sk_mem_uncharge(sk, len); 5757948f6ccSFlorian Westphal sk_wmem_queued_add(sk, -len); 576d027236cSPaolo Abeni } 577d027236cSPaolo Abeni 578d027236cSPaolo Abeni static void dfrag_clear(struct sock *sk, struct mptcp_data_frag *dfrag) 579d027236cSPaolo Abeni { 580d027236cSPaolo Abeni int len = dfrag->data_len + dfrag->overhead; 581d027236cSPaolo Abeni 58218b683bfSPaolo Abeni list_del(&dfrag->list); 583d027236cSPaolo Abeni dfrag_uncharge(sk, len); 58418b683bfSPaolo Abeni put_page(dfrag->page); 58518b683bfSPaolo Abeni } 58618b683bfSPaolo Abeni 58763561a40SPaolo Abeni static bool mptcp_is_writeable(struct mptcp_sock *msk) 58863561a40SPaolo Abeni { 58963561a40SPaolo Abeni struct mptcp_subflow_context *subflow; 59063561a40SPaolo Abeni 59163561a40SPaolo Abeni if (!sk_stream_is_writeable((struct sock *)msk)) 59263561a40SPaolo Abeni return false; 59363561a40SPaolo Abeni 59463561a40SPaolo Abeni mptcp_for_each_subflow(msk, subflow) { 59563561a40SPaolo Abeni if (sk_stream_is_writeable(subflow->tcp_sock)) 59663561a40SPaolo Abeni return true; 59763561a40SPaolo Abeni } 59863561a40SPaolo Abeni return false; 59963561a40SPaolo Abeni } 60063561a40SPaolo Abeni 60118b683bfSPaolo Abeni static void mptcp_clean_una(struct sock *sk) 60218b683bfSPaolo Abeni { 60318b683bfSPaolo Abeni struct mptcp_sock *msk = mptcp_sk(sk); 60418b683bfSPaolo Abeni struct mptcp_data_frag *dtmp, *dfrag; 605d027236cSPaolo Abeni bool cleaned = false; 606e1ff9e82SDavide Caratti u64 snd_una; 607e1ff9e82SDavide Caratti 608e1ff9e82SDavide Caratti /* on fallback we just need to ignore snd_una, as this is really 609e1ff9e82SDavide Caratti * plain TCP 610e1ff9e82SDavide Caratti */ 611e1ff9e82SDavide Caratti if (__mptcp_check_fallback(msk)) 612e1ff9e82SDavide Caratti atomic64_set(&msk->snd_una, msk->write_seq); 613e1ff9e82SDavide Caratti snd_una = atomic64_read(&msk->snd_una); 61418b683bfSPaolo Abeni 61518b683bfSPaolo Abeni list_for_each_entry_safe(dfrag, dtmp, &msk->rtx_queue, list) { 61618b683bfSPaolo Abeni if (after64(dfrag->data_seq + dfrag->data_len, snd_una)) 61718b683bfSPaolo Abeni break; 61818b683bfSPaolo Abeni 619d027236cSPaolo Abeni dfrag_clear(sk, dfrag); 620d027236cSPaolo Abeni cleaned = true; 621d027236cSPaolo Abeni } 622d027236cSPaolo Abeni 6237948f6ccSFlorian Westphal dfrag = mptcp_rtx_head(sk); 6247948f6ccSFlorian Westphal if (dfrag && after64(snd_una, dfrag->data_seq)) { 62553eb4c38SPaolo Abeni u64 delta = snd_una - dfrag->data_seq; 62653eb4c38SPaolo Abeni 62753eb4c38SPaolo Abeni if (WARN_ON_ONCE(delta > dfrag->data_len)) 62853eb4c38SPaolo Abeni goto out; 6297948f6ccSFlorian Westphal 6307948f6ccSFlorian Westphal dfrag->data_seq += delta; 63153eb4c38SPaolo Abeni dfrag->offset += delta; 6327948f6ccSFlorian Westphal dfrag->data_len -= delta; 6337948f6ccSFlorian Westphal 6347948f6ccSFlorian Westphal dfrag_uncharge(sk, delta); 6357948f6ccSFlorian Westphal cleaned = true; 6367948f6ccSFlorian Westphal } 6377948f6ccSFlorian Westphal 63853eb4c38SPaolo Abeni out: 639d027236cSPaolo Abeni if (cleaned) { 640d027236cSPaolo Abeni sk_mem_reclaim_partial(sk); 6417948f6ccSFlorian Westphal 6427948f6ccSFlorian Westphal /* Only wake up writers if a subflow is ready */ 64363561a40SPaolo Abeni if (mptcp_is_writeable(msk)) { 64463561a40SPaolo Abeni set_bit(MPTCP_SEND_SPACE, &mptcp_sk(sk)->flags); 64563561a40SPaolo Abeni smp_mb__after_atomic(); 64663561a40SPaolo Abeni 64763561a40SPaolo Abeni /* set SEND_SPACE before sk_stream_write_space clears 64863561a40SPaolo Abeni * NOSPACE 64963561a40SPaolo Abeni */ 6507948f6ccSFlorian Westphal sk_stream_write_space(sk); 65118b683bfSPaolo Abeni } 65218b683bfSPaolo Abeni } 65363561a40SPaolo Abeni } 65418b683bfSPaolo Abeni 65518b683bfSPaolo Abeni /* ensure we get enough memory for the frag hdr, beyond some minimal amount of 65618b683bfSPaolo Abeni * data 65718b683bfSPaolo Abeni */ 65818b683bfSPaolo Abeni static bool mptcp_page_frag_refill(struct sock *sk, struct page_frag *pfrag) 65918b683bfSPaolo Abeni { 66018b683bfSPaolo Abeni if (likely(skb_page_frag_refill(32U + sizeof(struct mptcp_data_frag), 66118b683bfSPaolo Abeni pfrag, sk->sk_allocation))) 66218b683bfSPaolo Abeni return true; 66318b683bfSPaolo Abeni 66418b683bfSPaolo Abeni sk->sk_prot->enter_memory_pressure(sk); 66518b683bfSPaolo Abeni sk_stream_moderate_sndbuf(sk); 66618b683bfSPaolo Abeni return false; 66718b683bfSPaolo Abeni } 66818b683bfSPaolo Abeni 66918b683bfSPaolo Abeni static struct mptcp_data_frag * 67018b683bfSPaolo Abeni mptcp_carve_data_frag(const struct mptcp_sock *msk, struct page_frag *pfrag, 67118b683bfSPaolo Abeni int orig_offset) 67218b683bfSPaolo Abeni { 67318b683bfSPaolo Abeni int offset = ALIGN(orig_offset, sizeof(long)); 67418b683bfSPaolo Abeni struct mptcp_data_frag *dfrag; 67518b683bfSPaolo Abeni 67618b683bfSPaolo Abeni dfrag = (struct mptcp_data_frag *)(page_to_virt(pfrag->page) + offset); 67718b683bfSPaolo Abeni dfrag->data_len = 0; 67818b683bfSPaolo Abeni dfrag->data_seq = msk->write_seq; 67918b683bfSPaolo Abeni dfrag->overhead = offset - orig_offset + sizeof(struct mptcp_data_frag); 68018b683bfSPaolo Abeni dfrag->offset = offset + sizeof(struct mptcp_data_frag); 68118b683bfSPaolo Abeni dfrag->page = pfrag->page; 68218b683bfSPaolo Abeni 68318b683bfSPaolo Abeni return dfrag; 68418b683bfSPaolo Abeni } 68518b683bfSPaolo Abeni 68657040755SPaolo Abeni static int mptcp_sendmsg_frag(struct sock *sk, struct sock *ssk, 6873f8e0aaeSPaolo Abeni struct msghdr *msg, struct mptcp_data_frag *dfrag, 6883f8e0aaeSPaolo Abeni long *timeo, int *pmss_now, 68957040755SPaolo Abeni int *ps_goal) 69057040755SPaolo Abeni { 69118b683bfSPaolo Abeni int mss_now, avail_size, size_goal, offset, ret, frag_truesize = 0; 69218b683bfSPaolo Abeni bool dfrag_collapsed, can_collapse = false; 6936d0060f6SMat Martineau struct mptcp_sock *msk = mptcp_sk(sk); 6946d0060f6SMat Martineau struct mptcp_ext *mpext = NULL; 6953f8e0aaeSPaolo Abeni bool retransmission = !!dfrag; 69657040755SPaolo Abeni struct sk_buff *skb, *tail; 6976d0060f6SMat Martineau struct page_frag *pfrag; 6983f8e0aaeSPaolo Abeni struct page *page; 6993f8e0aaeSPaolo Abeni u64 *write_seq; 7006d0060f6SMat Martineau size_t psize; 7016d0060f6SMat Martineau 7026d0060f6SMat Martineau /* use the mptcp page cache so that we can easily move the data 7036d0060f6SMat Martineau * from one substream to another, but do per subflow memory accounting 7043f8e0aaeSPaolo Abeni * Note: pfrag is used only !retransmission, but the compiler if 7053f8e0aaeSPaolo Abeni * fooled into a warning if we don't init here 7066d0060f6SMat Martineau */ 7076d0060f6SMat Martineau pfrag = sk_page_frag(sk); 7083f8e0aaeSPaolo Abeni if (!retransmission) { 7093f8e0aaeSPaolo Abeni write_seq = &msk->write_seq; 7103f8e0aaeSPaolo Abeni page = pfrag->page; 7113f8e0aaeSPaolo Abeni } else { 7123f8e0aaeSPaolo Abeni write_seq = &dfrag->data_seq; 7133f8e0aaeSPaolo Abeni page = dfrag->page; 7143f8e0aaeSPaolo Abeni } 7156d0060f6SMat Martineau 7166d0060f6SMat Martineau /* compute copy limit */ 7176d0060f6SMat Martineau mss_now = tcp_send_mss(ssk, &size_goal, msg->msg_flags); 71857040755SPaolo Abeni *pmss_now = mss_now; 71957040755SPaolo Abeni *ps_goal = size_goal; 72057040755SPaolo Abeni avail_size = size_goal; 72157040755SPaolo Abeni skb = tcp_write_queue_tail(ssk); 72257040755SPaolo Abeni if (skb) { 72357040755SPaolo Abeni mpext = skb_ext_find(skb, SKB_EXT_MPTCP); 7246d0060f6SMat Martineau 72557040755SPaolo Abeni /* Limit the write to the size available in the 72657040755SPaolo Abeni * current skb, if any, so that we create at most a new skb. 72757040755SPaolo Abeni * Explicitly tells TCP internals to avoid collapsing on later 72857040755SPaolo Abeni * queue management operation, to avoid breaking the ext <-> 72957040755SPaolo Abeni * SSN association set here 73057040755SPaolo Abeni */ 73157040755SPaolo Abeni can_collapse = (size_goal - skb->len > 0) && 7323f8e0aaeSPaolo Abeni mptcp_skb_can_collapse_to(*write_seq, skb, mpext); 73357040755SPaolo Abeni if (!can_collapse) 73457040755SPaolo Abeni TCP_SKB_CB(skb)->eor = 1; 73557040755SPaolo Abeni else 73657040755SPaolo Abeni avail_size = size_goal - skb->len; 73757040755SPaolo Abeni } 73818b683bfSPaolo Abeni 7393f8e0aaeSPaolo Abeni if (!retransmission) { 7403f8e0aaeSPaolo Abeni /* reuse tail pfrag, if possible, or carve a new one from the 7413f8e0aaeSPaolo Abeni * page allocator 74218b683bfSPaolo Abeni */ 74318b683bfSPaolo Abeni dfrag = mptcp_rtx_tail(sk); 74418b683bfSPaolo Abeni offset = pfrag->offset; 74518b683bfSPaolo Abeni dfrag_collapsed = mptcp_frag_can_collapse_to(msk, pfrag, dfrag); 74618b683bfSPaolo Abeni if (!dfrag_collapsed) { 74718b683bfSPaolo Abeni dfrag = mptcp_carve_data_frag(msk, pfrag, offset); 74818b683bfSPaolo Abeni offset = dfrag->offset; 74918b683bfSPaolo Abeni frag_truesize = dfrag->overhead; 75018b683bfSPaolo Abeni } 75118b683bfSPaolo Abeni psize = min_t(size_t, pfrag->size - offset, avail_size); 75257040755SPaolo Abeni 75357040755SPaolo Abeni /* Copy to page */ 7546d0060f6SMat Martineau pr_debug("left=%zu", msg_data_left(msg)); 75518b683bfSPaolo Abeni psize = copy_page_from_iter(pfrag->page, offset, 7563f8e0aaeSPaolo Abeni min_t(size_t, msg_data_left(msg), 7573f8e0aaeSPaolo Abeni psize), 7586d0060f6SMat Martineau &msg->msg_iter); 7596d0060f6SMat Martineau pr_debug("left=%zu", msg_data_left(msg)); 7606d0060f6SMat Martineau if (!psize) 7616d0060f6SMat Martineau return -EINVAL; 7626d0060f6SMat Martineau 76335759383SFlorian Westphal if (!sk_wmem_schedule(sk, psize + dfrag->overhead)) { 76435759383SFlorian Westphal iov_iter_revert(&msg->msg_iter, psize); 765d027236cSPaolo Abeni return -ENOMEM; 76635759383SFlorian Westphal } 7673f8e0aaeSPaolo Abeni } else { 7683f8e0aaeSPaolo Abeni offset = dfrag->offset; 7693f8e0aaeSPaolo Abeni psize = min_t(size_t, dfrag->data_len, avail_size); 7703f8e0aaeSPaolo Abeni } 771d027236cSPaolo Abeni 77257040755SPaolo Abeni /* tell the TCP stack to delay the push so that we can safely 77357040755SPaolo Abeni * access the skb after the sendpages call 7746d0060f6SMat Martineau */ 7753f8e0aaeSPaolo Abeni ret = do_tcp_sendpages(ssk, page, offset, psize, 77672511aabSFlorian Westphal msg->msg_flags | MSG_SENDPAGE_NOTLAST | MSG_DONTWAIT); 77735759383SFlorian Westphal if (ret <= 0) { 778b3b2854dSFlorian Westphal if (!retransmission) 77935759383SFlorian Westphal iov_iter_revert(&msg->msg_iter, psize); 7806d0060f6SMat Martineau return ret; 78135759383SFlorian Westphal } 78218b683bfSPaolo Abeni 78318b683bfSPaolo Abeni frag_truesize += ret; 7843f8e0aaeSPaolo Abeni if (!retransmission) { 7856d0060f6SMat Martineau if (unlikely(ret < psize)) 7866d0060f6SMat Martineau iov_iter_revert(&msg->msg_iter, psize - ret); 7876d0060f6SMat Martineau 78818b683bfSPaolo Abeni /* send successful, keep track of sent data for mptcp-level 78918b683bfSPaolo Abeni * retransmission 79018b683bfSPaolo Abeni */ 79118b683bfSPaolo Abeni dfrag->data_len += ret; 79218b683bfSPaolo Abeni if (!dfrag_collapsed) { 79318b683bfSPaolo Abeni get_page(dfrag->page); 79418b683bfSPaolo Abeni list_add_tail(&dfrag->list, &msk->rtx_queue); 7953f8e0aaeSPaolo Abeni sk_wmem_queued_add(sk, frag_truesize); 7963f8e0aaeSPaolo Abeni } else { 7973f8e0aaeSPaolo Abeni sk_wmem_queued_add(sk, ret); 79818b683bfSPaolo Abeni } 79918b683bfSPaolo Abeni 800d027236cSPaolo Abeni /* charge data on mptcp rtx queue to the master socket 801d027236cSPaolo Abeni * Note: we charge such data both to sk and ssk 802d027236cSPaolo Abeni */ 803d027236cSPaolo Abeni sk->sk_forward_alloc -= frag_truesize; 8043f8e0aaeSPaolo Abeni } 805d027236cSPaolo Abeni 80657040755SPaolo Abeni /* if the tail skb extension is still the cached one, collapsing 80757040755SPaolo Abeni * really happened. Note: we can't check for 'same skb' as the sk_buff 80857040755SPaolo Abeni * hdr on tail can be transmitted, freed and re-allocated by the 80957040755SPaolo Abeni * do_tcp_sendpages() call 81057040755SPaolo Abeni */ 81157040755SPaolo Abeni tail = tcp_write_queue_tail(ssk); 81257040755SPaolo Abeni if (mpext && tail && mpext == skb_ext_find(tail, SKB_EXT_MPTCP)) { 81357040755SPaolo Abeni WARN_ON_ONCE(!can_collapse); 81457040755SPaolo Abeni mpext->data_len += ret; 81557040755SPaolo Abeni goto out; 81657040755SPaolo Abeni } 81757040755SPaolo Abeni 8186d0060f6SMat Martineau skb = tcp_write_queue_tail(ssk); 8196d0060f6SMat Martineau mpext = __skb_ext_set(skb, SKB_EXT_MPTCP, msk->cached_ext); 8206d0060f6SMat Martineau msk->cached_ext = NULL; 8216d0060f6SMat Martineau 8226d0060f6SMat Martineau memset(mpext, 0, sizeof(*mpext)); 8233f8e0aaeSPaolo Abeni mpext->data_seq = *write_seq; 8246d0060f6SMat Martineau mpext->subflow_seq = mptcp_subflow_ctx(ssk)->rel_write_seq; 8256d0060f6SMat Martineau mpext->data_len = ret; 8266d0060f6SMat Martineau mpext->use_map = 1; 8276d0060f6SMat Martineau mpext->dsn64 = 1; 8286d0060f6SMat Martineau 8296d0060f6SMat Martineau pr_debug("data_seq=%llu subflow_seq=%u data_len=%u dsn64=%d", 8306d0060f6SMat Martineau mpext->data_seq, mpext->subflow_seq, mpext->data_len, 8316d0060f6SMat Martineau mpext->dsn64); 8326d0060f6SMat Martineau 83357040755SPaolo Abeni out: 8343f8e0aaeSPaolo Abeni if (!retransmission) 83518b683bfSPaolo Abeni pfrag->offset += frag_truesize; 836721e9089SMat Martineau WRITE_ONCE(*write_seq, *write_seq + ret); 8376d0060f6SMat Martineau mptcp_subflow_ctx(ssk)->rel_write_seq += ret; 8386d0060f6SMat Martineau 8396d0060f6SMat Martineau return ret; 8406d0060f6SMat Martineau } 8416d0060f6SMat Martineau 84263561a40SPaolo Abeni static void mptcp_nospace(struct mptcp_sock *msk) 843a0e17064SFlorian Westphal { 84463561a40SPaolo Abeni struct mptcp_subflow_context *subflow; 84563561a40SPaolo Abeni 846a0e17064SFlorian Westphal clear_bit(MPTCP_SEND_SPACE, &msk->flags); 847a0e17064SFlorian Westphal smp_mb__after_atomic(); /* msk->flags is changed by write_space cb */ 848a0e17064SFlorian Westphal 84963561a40SPaolo Abeni mptcp_for_each_subflow(msk, subflow) { 85063561a40SPaolo Abeni struct sock *ssk = mptcp_subflow_tcp_sock(subflow); 85163561a40SPaolo Abeni struct socket *sock = READ_ONCE(ssk->sk_socket); 85263561a40SPaolo Abeni 85363561a40SPaolo Abeni /* enables ssk->write_space() callbacks */ 85463561a40SPaolo Abeni if (sock) 855a0e17064SFlorian Westphal set_bit(SOCK_NOSPACE, &sock->flags); 856a0e17064SFlorian Westphal } 85763561a40SPaolo Abeni } 858a0e17064SFlorian Westphal 859*da51aef5SPaolo Abeni static struct sock *mptcp_subflow_get_send(struct mptcp_sock *msk, 860*da51aef5SPaolo Abeni u32 *sndbuf) 861f296234cSPeter Krystad { 862f296234cSPeter Krystad struct mptcp_subflow_context *subflow; 86363561a40SPaolo Abeni struct sock *sk = (struct sock *)msk; 864f296234cSPeter Krystad struct sock *backup = NULL; 86563561a40SPaolo Abeni bool free; 866f296234cSPeter Krystad 86763561a40SPaolo Abeni sock_owned_by_me(sk); 868f296234cSPeter Krystad 869*da51aef5SPaolo Abeni *sndbuf = 0; 870149f7c71SFlorian Westphal if (!mptcp_ext_cache_refill(msk)) 871149f7c71SFlorian Westphal return NULL; 872149f7c71SFlorian Westphal 873f296234cSPeter Krystad mptcp_for_each_subflow(msk, subflow) { 874f296234cSPeter Krystad struct sock *ssk = mptcp_subflow_tcp_sock(subflow); 875f296234cSPeter Krystad 87663561a40SPaolo Abeni free = sk_stream_is_writeable(subflow->tcp_sock); 87763561a40SPaolo Abeni if (!free) { 87863561a40SPaolo Abeni mptcp_nospace(msk); 879f296234cSPeter Krystad return NULL; 880f296234cSPeter Krystad } 881f296234cSPeter Krystad 882*da51aef5SPaolo Abeni *sndbuf = max(tcp_sk(ssk)->snd_wnd, *sndbuf); 883f296234cSPeter Krystad if (subflow->backup) { 884f296234cSPeter Krystad if (!backup) 885f296234cSPeter Krystad backup = ssk; 886f296234cSPeter Krystad 887f296234cSPeter Krystad continue; 888f296234cSPeter Krystad } 889f296234cSPeter Krystad 890f296234cSPeter Krystad return ssk; 891f296234cSPeter Krystad } 892f296234cSPeter Krystad 893f296234cSPeter Krystad return backup; 894f296234cSPeter Krystad } 895f296234cSPeter Krystad 89663561a40SPaolo Abeni static void ssk_check_wmem(struct mptcp_sock *msk) 8971891c4a0SFlorian Westphal { 89863561a40SPaolo Abeni if (unlikely(!mptcp_is_writeable(msk))) 89963561a40SPaolo Abeni mptcp_nospace(msk); 9001891c4a0SFlorian Westphal } 9011891c4a0SFlorian Westphal 902f870fa0bSMat Martineau static int mptcp_sendmsg(struct sock *sk, struct msghdr *msg, size_t len) 903f870fa0bSMat Martineau { 90457040755SPaolo Abeni int mss_now = 0, size_goal = 0, ret = 0; 905f870fa0bSMat Martineau struct mptcp_sock *msk = mptcp_sk(sk); 90617091708SFlorian Westphal struct page_frag *pfrag; 9076d0060f6SMat Martineau size_t copied = 0; 908cec37a6eSPeter Krystad struct sock *ssk; 909*da51aef5SPaolo Abeni u32 sndbuf; 91072511aabSFlorian Westphal bool tx_ok; 9116d0060f6SMat Martineau long timeo; 912f870fa0bSMat Martineau 913f870fa0bSMat Martineau if (msg->msg_flags & ~(MSG_MORE | MSG_DONTWAIT | MSG_NOSIGNAL)) 914f870fa0bSMat Martineau return -EOPNOTSUPP; 915f870fa0bSMat Martineau 916cec37a6eSPeter Krystad lock_sock(sk); 9171954b860SMat Martineau 9181954b860SMat Martineau timeo = sock_sndtimeo(sk, msg->msg_flags & MSG_DONTWAIT); 9191954b860SMat Martineau 9201954b860SMat Martineau if ((1 << sk->sk_state) & ~(TCPF_ESTABLISHED | TCPF_CLOSE_WAIT)) { 9211954b860SMat Martineau ret = sk_stream_wait_connect(sk, &timeo); 9221954b860SMat Martineau if (ret) 9231954b860SMat Martineau goto out; 9241954b860SMat Martineau } 9251954b860SMat Martineau 92617091708SFlorian Westphal pfrag = sk_page_frag(sk); 92772511aabSFlorian Westphal restart: 92818b683bfSPaolo Abeni mptcp_clean_una(sk); 92918b683bfSPaolo Abeni 93057baaf28SMat Martineau if (sk->sk_err || (sk->sk_shutdown & SEND_SHUTDOWN)) { 93157baaf28SMat Martineau ret = -EPIPE; 93257baaf28SMat Martineau goto out; 93357baaf28SMat Martineau } 93457baaf28SMat Martineau 935ec3edaa7SPeter Krystad __mptcp_flush_join_list(msk); 936*da51aef5SPaolo Abeni ssk = mptcp_subflow_get_send(msk, &sndbuf); 93717091708SFlorian Westphal while (!sk_stream_memory_free(sk) || 93817091708SFlorian Westphal !ssk || 93917091708SFlorian Westphal !mptcp_page_frag_refill(ssk, pfrag)) { 940fb529e62SFlorian Westphal if (ssk) { 941fb529e62SFlorian Westphal /* make sure retransmit timer is 942fb529e62SFlorian Westphal * running before we wait for memory. 943fb529e62SFlorian Westphal * 944fb529e62SFlorian Westphal * The retransmit timer might be needed 945fb529e62SFlorian Westphal * to make the peer send an up-to-date 946fb529e62SFlorian Westphal * MPTCP Ack. 947fb529e62SFlorian Westphal */ 948fb529e62SFlorian Westphal mptcp_set_timeout(sk, ssk); 949fb529e62SFlorian Westphal if (!mptcp_timer_pending(sk)) 950fb529e62SFlorian Westphal mptcp_reset_timer(sk); 951fb529e62SFlorian Westphal } 952fb529e62SFlorian Westphal 95363561a40SPaolo Abeni mptcp_nospace(msk); 954f296234cSPeter Krystad ret = sk_stream_wait_memory(sk, &timeo); 955f296234cSPeter Krystad if (ret) 956f296234cSPeter Krystad goto out; 957f296234cSPeter Krystad 95818b683bfSPaolo Abeni mptcp_clean_una(sk); 95918b683bfSPaolo Abeni 960*da51aef5SPaolo Abeni ssk = mptcp_subflow_get_send(msk, &sndbuf); 961f296234cSPeter Krystad if (list_empty(&msk->conn_list)) { 962f296234cSPeter Krystad ret = -ENOTCONN; 963f296234cSPeter Krystad goto out; 964f296234cSPeter Krystad } 965cec37a6eSPeter Krystad } 966cec37a6eSPeter Krystad 967*da51aef5SPaolo Abeni /* do auto tuning */ 968*da51aef5SPaolo Abeni if (!(sk->sk_userlocks & SOCK_SNDBUF_LOCK) && 969*da51aef5SPaolo Abeni sndbuf > READ_ONCE(sk->sk_sndbuf)) 970*da51aef5SPaolo Abeni WRITE_ONCE(sk->sk_sndbuf, sndbuf); 971*da51aef5SPaolo Abeni 9726d0060f6SMat Martineau pr_debug("conn_list->subflow=%p", ssk); 973cec37a6eSPeter Krystad 9746d0060f6SMat Martineau lock_sock(ssk); 97572511aabSFlorian Westphal tx_ok = msg_data_left(msg); 97672511aabSFlorian Westphal while (tx_ok) { 9773f8e0aaeSPaolo Abeni ret = mptcp_sendmsg_frag(sk, ssk, msg, NULL, &timeo, &mss_now, 97857040755SPaolo Abeni &size_goal); 97972511aabSFlorian Westphal if (ret < 0) { 98072511aabSFlorian Westphal if (ret == -EAGAIN && timeo > 0) { 98172511aabSFlorian Westphal mptcp_set_timeout(sk, ssk); 98272511aabSFlorian Westphal release_sock(ssk); 98372511aabSFlorian Westphal goto restart; 98472511aabSFlorian Westphal } 9856d0060f6SMat Martineau break; 98672511aabSFlorian Westphal } 9876d0060f6SMat Martineau 9886d0060f6SMat Martineau copied += ret; 989fb529e62SFlorian Westphal 99072511aabSFlorian Westphal tx_ok = msg_data_left(msg); 99172511aabSFlorian Westphal if (!tx_ok) 99272511aabSFlorian Westphal break; 99372511aabSFlorian Westphal 994149f7c71SFlorian Westphal if (!sk_stream_memory_free(ssk) || 99517091708SFlorian Westphal !mptcp_page_frag_refill(ssk, pfrag) || 996149f7c71SFlorian Westphal !mptcp_ext_cache_refill(msk)) { 99772511aabSFlorian Westphal tcp_push(ssk, msg->msg_flags, mss_now, 99872511aabSFlorian Westphal tcp_sk(ssk)->nonagle, size_goal); 99972511aabSFlorian Westphal mptcp_set_timeout(sk, ssk); 100072511aabSFlorian Westphal release_sock(ssk); 100172511aabSFlorian Westphal goto restart; 100272511aabSFlorian Westphal } 100372511aabSFlorian Westphal 1004fb529e62SFlorian Westphal /* memory is charged to mptcp level socket as well, i.e. 1005fb529e62SFlorian Westphal * if msg is very large, mptcp socket may run out of buffer 1006fb529e62SFlorian Westphal * space. mptcp_clean_una() will release data that has 1007fb529e62SFlorian Westphal * been acked at mptcp level in the mean time, so there is 1008fb529e62SFlorian Westphal * a good chance we can continue sending data right away. 100972511aabSFlorian Westphal * 101072511aabSFlorian Westphal * Normally, when the tcp subflow can accept more data, then 101172511aabSFlorian Westphal * so can the MPTCP socket. However, we need to cope with 101272511aabSFlorian Westphal * peers that might lag behind in their MPTCP-level 101372511aabSFlorian Westphal * acknowledgements, i.e. data might have been acked at 101472511aabSFlorian Westphal * tcp level only. So, we must also check the MPTCP socket 101572511aabSFlorian Westphal * limits before we send more data. 1016fb529e62SFlorian Westphal */ 1017fb529e62SFlorian Westphal if (unlikely(!sk_stream_memory_free(sk))) { 1018fb529e62SFlorian Westphal tcp_push(ssk, msg->msg_flags, mss_now, 1019fb529e62SFlorian Westphal tcp_sk(ssk)->nonagle, size_goal); 1020fb529e62SFlorian Westphal mptcp_clean_una(sk); 1021fb529e62SFlorian Westphal if (!sk_stream_memory_free(sk)) { 1022fb529e62SFlorian Westphal /* can't send more for now, need to wait for 1023fb529e62SFlorian Westphal * MPTCP-level ACKs from peer. 1024fb529e62SFlorian Westphal * 1025fb529e62SFlorian Westphal * Wakeup will happen via mptcp_clean_una(). 1026fb529e62SFlorian Westphal */ 1027fb529e62SFlorian Westphal mptcp_set_timeout(sk, ssk); 1028fb529e62SFlorian Westphal release_sock(ssk); 10291cec170dSFlorian Westphal goto restart; 1030fb529e62SFlorian Westphal } 1031fb529e62SFlorian Westphal } 10326d0060f6SMat Martineau } 10336d0060f6SMat Martineau 1034b51f9b80SPaolo Abeni mptcp_set_timeout(sk, ssk); 103557040755SPaolo Abeni if (copied) { 103657040755SPaolo Abeni tcp_push(ssk, msg->msg_flags, mss_now, tcp_sk(ssk)->nonagle, 103757040755SPaolo Abeni size_goal); 1038b51f9b80SPaolo Abeni 1039b51f9b80SPaolo Abeni /* start the timer, if it's not pending */ 1040b51f9b80SPaolo Abeni if (!mptcp_timer_pending(sk)) 1041b51f9b80SPaolo Abeni mptcp_reset_timer(sk); 104257040755SPaolo Abeni } 10436d0060f6SMat Martineau 10446d0060f6SMat Martineau release_sock(ssk); 10451954b860SMat Martineau out: 104663561a40SPaolo Abeni ssk_check_wmem(msk); 1047cec37a6eSPeter Krystad release_sock(sk); 10488555c6bfSPaolo Abeni return copied ? : ret; 1049f870fa0bSMat Martineau } 1050f870fa0bSMat Martineau 10517a6a6cbcSPaolo Abeni static void mptcp_wait_data(struct sock *sk, long *timeo) 10527a6a6cbcSPaolo Abeni { 10537a6a6cbcSPaolo Abeni DEFINE_WAIT_FUNC(wait, woken_wake_function); 10547a6a6cbcSPaolo Abeni struct mptcp_sock *msk = mptcp_sk(sk); 10557a6a6cbcSPaolo Abeni 10567a6a6cbcSPaolo Abeni add_wait_queue(sk_sleep(sk), &wait); 10577a6a6cbcSPaolo Abeni sk_set_bit(SOCKWQ_ASYNC_WAITDATA, sk); 10587a6a6cbcSPaolo Abeni 10597a6a6cbcSPaolo Abeni sk_wait_event(sk, timeo, 10607a6a6cbcSPaolo Abeni test_and_clear_bit(MPTCP_DATA_READY, &msk->flags), &wait); 10617a6a6cbcSPaolo Abeni 10627a6a6cbcSPaolo Abeni sk_clear_bit(SOCKWQ_ASYNC_WAITDATA, sk); 10637a6a6cbcSPaolo Abeni remove_wait_queue(sk_sleep(sk), &wait); 10647a6a6cbcSPaolo Abeni } 10657a6a6cbcSPaolo Abeni 10666771bfd9SFlorian Westphal static int __mptcp_recvmsg_mskq(struct mptcp_sock *msk, 10676771bfd9SFlorian Westphal struct msghdr *msg, 10686771bfd9SFlorian Westphal size_t len) 10696771bfd9SFlorian Westphal { 10706771bfd9SFlorian Westphal struct sock *sk = (struct sock *)msk; 10716771bfd9SFlorian Westphal struct sk_buff *skb; 10726771bfd9SFlorian Westphal int copied = 0; 10736771bfd9SFlorian Westphal 10746771bfd9SFlorian Westphal while ((skb = skb_peek(&sk->sk_receive_queue)) != NULL) { 10756771bfd9SFlorian Westphal u32 offset = MPTCP_SKB_CB(skb)->offset; 10766771bfd9SFlorian Westphal u32 data_len = skb->len - offset; 10776771bfd9SFlorian Westphal u32 count = min_t(size_t, len - copied, data_len); 10786771bfd9SFlorian Westphal int err; 10796771bfd9SFlorian Westphal 10806771bfd9SFlorian Westphal err = skb_copy_datagram_msg(skb, offset, msg, count); 10816771bfd9SFlorian Westphal if (unlikely(err < 0)) { 10826771bfd9SFlorian Westphal if (!copied) 10836771bfd9SFlorian Westphal return err; 10846771bfd9SFlorian Westphal break; 10856771bfd9SFlorian Westphal } 10866771bfd9SFlorian Westphal 10876771bfd9SFlorian Westphal copied += count; 10886771bfd9SFlorian Westphal 10896771bfd9SFlorian Westphal if (count < data_len) { 10906771bfd9SFlorian Westphal MPTCP_SKB_CB(skb)->offset += count; 10916771bfd9SFlorian Westphal break; 10926771bfd9SFlorian Westphal } 10936771bfd9SFlorian Westphal 10946771bfd9SFlorian Westphal __skb_unlink(skb, &sk->sk_receive_queue); 10956771bfd9SFlorian Westphal __kfree_skb(skb); 10966771bfd9SFlorian Westphal 10976771bfd9SFlorian Westphal if (copied >= len) 10986771bfd9SFlorian Westphal break; 10996771bfd9SFlorian Westphal } 11006771bfd9SFlorian Westphal 11016771bfd9SFlorian Westphal return copied; 11026771bfd9SFlorian Westphal } 11036771bfd9SFlorian Westphal 1104a6b118feSFlorian Westphal /* receive buffer autotuning. See tcp_rcv_space_adjust for more information. 1105a6b118feSFlorian Westphal * 1106a6b118feSFlorian Westphal * Only difference: Use highest rtt estimate of the subflows in use. 1107a6b118feSFlorian Westphal */ 1108a6b118feSFlorian Westphal static void mptcp_rcv_space_adjust(struct mptcp_sock *msk, int copied) 1109a6b118feSFlorian Westphal { 1110a6b118feSFlorian Westphal struct mptcp_subflow_context *subflow; 1111a6b118feSFlorian Westphal struct sock *sk = (struct sock *)msk; 1112a6b118feSFlorian Westphal u32 time, advmss = 1; 1113a6b118feSFlorian Westphal u64 rtt_us, mstamp; 1114a6b118feSFlorian Westphal 1115a6b118feSFlorian Westphal sock_owned_by_me(sk); 1116a6b118feSFlorian Westphal 1117a6b118feSFlorian Westphal if (copied <= 0) 1118a6b118feSFlorian Westphal return; 1119a6b118feSFlorian Westphal 1120a6b118feSFlorian Westphal msk->rcvq_space.copied += copied; 1121a6b118feSFlorian Westphal 1122a6b118feSFlorian Westphal mstamp = div_u64(tcp_clock_ns(), NSEC_PER_USEC); 1123a6b118feSFlorian Westphal time = tcp_stamp_us_delta(mstamp, msk->rcvq_space.time); 1124a6b118feSFlorian Westphal 1125a6b118feSFlorian Westphal rtt_us = msk->rcvq_space.rtt_us; 1126a6b118feSFlorian Westphal if (rtt_us && time < (rtt_us >> 3)) 1127a6b118feSFlorian Westphal return; 1128a6b118feSFlorian Westphal 1129a6b118feSFlorian Westphal rtt_us = 0; 1130a6b118feSFlorian Westphal mptcp_for_each_subflow(msk, subflow) { 1131a6b118feSFlorian Westphal const struct tcp_sock *tp; 1132a6b118feSFlorian Westphal u64 sf_rtt_us; 1133a6b118feSFlorian Westphal u32 sf_advmss; 1134a6b118feSFlorian Westphal 1135a6b118feSFlorian Westphal tp = tcp_sk(mptcp_subflow_tcp_sock(subflow)); 1136a6b118feSFlorian Westphal 1137a6b118feSFlorian Westphal sf_rtt_us = READ_ONCE(tp->rcv_rtt_est.rtt_us); 1138a6b118feSFlorian Westphal sf_advmss = READ_ONCE(tp->advmss); 1139a6b118feSFlorian Westphal 1140a6b118feSFlorian Westphal rtt_us = max(sf_rtt_us, rtt_us); 1141a6b118feSFlorian Westphal advmss = max(sf_advmss, advmss); 1142a6b118feSFlorian Westphal } 1143a6b118feSFlorian Westphal 1144a6b118feSFlorian Westphal msk->rcvq_space.rtt_us = rtt_us; 1145a6b118feSFlorian Westphal if (time < (rtt_us >> 3) || rtt_us == 0) 1146a6b118feSFlorian Westphal return; 1147a6b118feSFlorian Westphal 1148a6b118feSFlorian Westphal if (msk->rcvq_space.copied <= msk->rcvq_space.space) 1149a6b118feSFlorian Westphal goto new_measure; 1150a6b118feSFlorian Westphal 1151a6b118feSFlorian Westphal if (sock_net(sk)->ipv4.sysctl_tcp_moderate_rcvbuf && 1152a6b118feSFlorian Westphal !(sk->sk_userlocks & SOCK_RCVBUF_LOCK)) { 1153a6b118feSFlorian Westphal int rcvmem, rcvbuf; 1154a6b118feSFlorian Westphal u64 rcvwin, grow; 1155a6b118feSFlorian Westphal 1156a6b118feSFlorian Westphal rcvwin = ((u64)msk->rcvq_space.copied << 1) + 16 * advmss; 1157a6b118feSFlorian Westphal 1158a6b118feSFlorian Westphal grow = rcvwin * (msk->rcvq_space.copied - msk->rcvq_space.space); 1159a6b118feSFlorian Westphal 1160a6b118feSFlorian Westphal do_div(grow, msk->rcvq_space.space); 1161a6b118feSFlorian Westphal rcvwin += (grow << 1); 1162a6b118feSFlorian Westphal 1163a6b118feSFlorian Westphal rcvmem = SKB_TRUESIZE(advmss + MAX_TCP_HEADER); 1164a6b118feSFlorian Westphal while (tcp_win_from_space(sk, rcvmem) < advmss) 1165a6b118feSFlorian Westphal rcvmem += 128; 1166a6b118feSFlorian Westphal 1167a6b118feSFlorian Westphal do_div(rcvwin, advmss); 1168a6b118feSFlorian Westphal rcvbuf = min_t(u64, rcvwin * rcvmem, 1169a6b118feSFlorian Westphal sock_net(sk)->ipv4.sysctl_tcp_rmem[2]); 1170a6b118feSFlorian Westphal 1171a6b118feSFlorian Westphal if (rcvbuf > sk->sk_rcvbuf) { 1172a6b118feSFlorian Westphal u32 window_clamp; 1173a6b118feSFlorian Westphal 1174a6b118feSFlorian Westphal window_clamp = tcp_win_from_space(sk, rcvbuf); 1175a6b118feSFlorian Westphal WRITE_ONCE(sk->sk_rcvbuf, rcvbuf); 1176a6b118feSFlorian Westphal 1177a6b118feSFlorian Westphal /* Make subflows follow along. If we do not do this, we 1178a6b118feSFlorian Westphal * get drops at subflow level if skbs can't be moved to 1179a6b118feSFlorian Westphal * the mptcp rx queue fast enough (announced rcv_win can 1180a6b118feSFlorian Westphal * exceed ssk->sk_rcvbuf). 1181a6b118feSFlorian Westphal */ 1182a6b118feSFlorian Westphal mptcp_for_each_subflow(msk, subflow) { 1183a6b118feSFlorian Westphal struct sock *ssk; 1184a6b118feSFlorian Westphal 1185a6b118feSFlorian Westphal ssk = mptcp_subflow_tcp_sock(subflow); 1186a6b118feSFlorian Westphal WRITE_ONCE(ssk->sk_rcvbuf, rcvbuf); 1187a6b118feSFlorian Westphal tcp_sk(ssk)->window_clamp = window_clamp; 1188a6b118feSFlorian Westphal } 1189a6b118feSFlorian Westphal } 1190a6b118feSFlorian Westphal } 1191a6b118feSFlorian Westphal 1192a6b118feSFlorian Westphal msk->rcvq_space.space = msk->rcvq_space.copied; 1193a6b118feSFlorian Westphal new_measure: 1194a6b118feSFlorian Westphal msk->rcvq_space.copied = 0; 1195a6b118feSFlorian Westphal msk->rcvq_space.time = mstamp; 1196a6b118feSFlorian Westphal } 1197a6b118feSFlorian Westphal 11986771bfd9SFlorian Westphal static bool __mptcp_move_skbs(struct mptcp_sock *msk) 11996771bfd9SFlorian Westphal { 12006771bfd9SFlorian Westphal unsigned int moved = 0; 12016771bfd9SFlorian Westphal bool done; 12026771bfd9SFlorian Westphal 12036771bfd9SFlorian Westphal do { 12046771bfd9SFlorian Westphal struct sock *ssk = mptcp_subflow_recv_lookup(msk); 12056771bfd9SFlorian Westphal 12066771bfd9SFlorian Westphal if (!ssk) 12076771bfd9SFlorian Westphal break; 12086771bfd9SFlorian Westphal 12096771bfd9SFlorian Westphal lock_sock(ssk); 12106771bfd9SFlorian Westphal done = __mptcp_move_skbs_from_subflow(msk, ssk, &moved); 12116771bfd9SFlorian Westphal release_sock(ssk); 12126771bfd9SFlorian Westphal } while (!done); 12136771bfd9SFlorian Westphal 12146771bfd9SFlorian Westphal return moved > 0; 12156771bfd9SFlorian Westphal } 12166771bfd9SFlorian Westphal 1217f870fa0bSMat Martineau static int mptcp_recvmsg(struct sock *sk, struct msghdr *msg, size_t len, 1218f870fa0bSMat Martineau int nonblock, int flags, int *addr_len) 1219f870fa0bSMat Martineau { 1220f870fa0bSMat Martineau struct mptcp_sock *msk = mptcp_sk(sk); 1221cec37a6eSPeter Krystad int copied = 0; 12227a6a6cbcSPaolo Abeni int target; 12237a6a6cbcSPaolo Abeni long timeo; 1224f870fa0bSMat Martineau 1225f870fa0bSMat Martineau if (msg->msg_flags & ~(MSG_WAITALL | MSG_DONTWAIT)) 1226f870fa0bSMat Martineau return -EOPNOTSUPP; 1227f870fa0bSMat Martineau 1228cec37a6eSPeter Krystad lock_sock(sk); 12297a6a6cbcSPaolo Abeni timeo = sock_rcvtimeo(sk, nonblock); 12307a6a6cbcSPaolo Abeni 12317a6a6cbcSPaolo Abeni len = min_t(size_t, len, INT_MAX); 12327a6a6cbcSPaolo Abeni target = sock_rcvlowat(sk, flags & MSG_WAITALL, len); 1233ec3edaa7SPeter Krystad __mptcp_flush_join_list(msk); 12347a6a6cbcSPaolo Abeni 12356771bfd9SFlorian Westphal while (len > (size_t)copied) { 12367a6a6cbcSPaolo Abeni int bytes_read; 12377a6a6cbcSPaolo Abeni 12386771bfd9SFlorian Westphal bytes_read = __mptcp_recvmsg_mskq(msk, msg, len - copied); 12396771bfd9SFlorian Westphal if (unlikely(bytes_read < 0)) { 12407a6a6cbcSPaolo Abeni if (!copied) 12417a6a6cbcSPaolo Abeni copied = bytes_read; 12426771bfd9SFlorian Westphal goto out_err; 1243cec37a6eSPeter Krystad } 1244cec37a6eSPeter Krystad 12457a6a6cbcSPaolo Abeni copied += bytes_read; 12467a6a6cbcSPaolo Abeni 12476771bfd9SFlorian Westphal if (skb_queue_empty(&sk->sk_receive_queue) && 12486771bfd9SFlorian Westphal __mptcp_move_skbs(msk)) 12496771bfd9SFlorian Westphal continue; 12507a6a6cbcSPaolo Abeni 12517a6a6cbcSPaolo Abeni /* only the master socket status is relevant here. The exit 12527a6a6cbcSPaolo Abeni * conditions mirror closely tcp_recvmsg() 12537a6a6cbcSPaolo Abeni */ 12547a6a6cbcSPaolo Abeni if (copied >= target) 12557a6a6cbcSPaolo Abeni break; 12567a6a6cbcSPaolo Abeni 12577a6a6cbcSPaolo Abeni if (copied) { 12587a6a6cbcSPaolo Abeni if (sk->sk_err || 12597a6a6cbcSPaolo Abeni sk->sk_state == TCP_CLOSE || 12607a6a6cbcSPaolo Abeni (sk->sk_shutdown & RCV_SHUTDOWN) || 12617a6a6cbcSPaolo Abeni !timeo || 12627a6a6cbcSPaolo Abeni signal_pending(current)) 12637a6a6cbcSPaolo Abeni break; 12647a6a6cbcSPaolo Abeni } else { 12657a6a6cbcSPaolo Abeni if (sk->sk_err) { 12667a6a6cbcSPaolo Abeni copied = sock_error(sk); 12677a6a6cbcSPaolo Abeni break; 12687a6a6cbcSPaolo Abeni } 12697a6a6cbcSPaolo Abeni 12705969856aSPaolo Abeni if (test_and_clear_bit(MPTCP_WORK_EOF, &msk->flags)) 12715969856aSPaolo Abeni mptcp_check_for_eof(msk); 12725969856aSPaolo Abeni 12737a6a6cbcSPaolo Abeni if (sk->sk_shutdown & RCV_SHUTDOWN) 12747a6a6cbcSPaolo Abeni break; 12757a6a6cbcSPaolo Abeni 12767a6a6cbcSPaolo Abeni if (sk->sk_state == TCP_CLOSE) { 12777a6a6cbcSPaolo Abeni copied = -ENOTCONN; 12787a6a6cbcSPaolo Abeni break; 12797a6a6cbcSPaolo Abeni } 12807a6a6cbcSPaolo Abeni 12817a6a6cbcSPaolo Abeni if (!timeo) { 12827a6a6cbcSPaolo Abeni copied = -EAGAIN; 12837a6a6cbcSPaolo Abeni break; 12847a6a6cbcSPaolo Abeni } 12857a6a6cbcSPaolo Abeni 12867a6a6cbcSPaolo Abeni if (signal_pending(current)) { 12877a6a6cbcSPaolo Abeni copied = sock_intr_errno(timeo); 12887a6a6cbcSPaolo Abeni break; 12897a6a6cbcSPaolo Abeni } 12907a6a6cbcSPaolo Abeni } 12917a6a6cbcSPaolo Abeni 12927a6a6cbcSPaolo Abeni pr_debug("block timeout %ld", timeo); 12937a6a6cbcSPaolo Abeni mptcp_wait_data(sk, &timeo); 12947a6a6cbcSPaolo Abeni } 12957a6a6cbcSPaolo Abeni 12966771bfd9SFlorian Westphal if (skb_queue_empty(&sk->sk_receive_queue)) { 12976771bfd9SFlorian Westphal /* entire backlog drained, clear DATA_READY. */ 12987a6a6cbcSPaolo Abeni clear_bit(MPTCP_DATA_READY, &msk->flags); 12997a6a6cbcSPaolo Abeni 13006771bfd9SFlorian Westphal /* .. race-breaker: ssk might have gotten new data 13016771bfd9SFlorian Westphal * after last __mptcp_move_skbs() returned false. 13027a6a6cbcSPaolo Abeni */ 13036771bfd9SFlorian Westphal if (unlikely(__mptcp_move_skbs(msk))) 13046771bfd9SFlorian Westphal set_bit(MPTCP_DATA_READY, &msk->flags); 13056771bfd9SFlorian Westphal } else if (unlikely(!test_bit(MPTCP_DATA_READY, &msk->flags))) { 13066771bfd9SFlorian Westphal /* data to read but mptcp_wait_data() cleared DATA_READY */ 13077a6a6cbcSPaolo Abeni set_bit(MPTCP_DATA_READY, &msk->flags); 13087a6a6cbcSPaolo Abeni } 13096771bfd9SFlorian Westphal out_err: 13106719331cSPaolo Abeni pr_debug("msk=%p data_ready=%d rx queue empty=%d copied=%d", 13116719331cSPaolo Abeni msk, test_bit(MPTCP_DATA_READY, &msk->flags), 13126719331cSPaolo Abeni skb_queue_empty(&sk->sk_receive_queue), copied); 1313a6b118feSFlorian Westphal mptcp_rcv_space_adjust(msk, copied); 1314a6b118feSFlorian Westphal 1315cec37a6eSPeter Krystad release_sock(sk); 1316cec37a6eSPeter Krystad return copied; 1317cec37a6eSPeter Krystad } 1318cec37a6eSPeter Krystad 1319b51f9b80SPaolo Abeni static void mptcp_retransmit_handler(struct sock *sk) 1320b51f9b80SPaolo Abeni { 1321b51f9b80SPaolo Abeni struct mptcp_sock *msk = mptcp_sk(sk); 1322b51f9b80SPaolo Abeni 1323c7529392SMat Martineau if (atomic64_read(&msk->snd_una) == READ_ONCE(msk->write_seq)) { 1324b51f9b80SPaolo Abeni mptcp_stop_timer(sk); 13253b1d6210SPaolo Abeni } else { 13263b1d6210SPaolo Abeni set_bit(MPTCP_WORK_RTX, &msk->flags); 13273b1d6210SPaolo Abeni if (schedule_work(&msk->work)) 13283b1d6210SPaolo Abeni sock_hold(sk); 13293b1d6210SPaolo Abeni } 1330b51f9b80SPaolo Abeni } 1331b51f9b80SPaolo Abeni 1332b51f9b80SPaolo Abeni static void mptcp_retransmit_timer(struct timer_list *t) 1333b51f9b80SPaolo Abeni { 1334b51f9b80SPaolo Abeni struct inet_connection_sock *icsk = from_timer(icsk, t, 1335b51f9b80SPaolo Abeni icsk_retransmit_timer); 1336b51f9b80SPaolo Abeni struct sock *sk = &icsk->icsk_inet.sk; 1337b51f9b80SPaolo Abeni 1338b51f9b80SPaolo Abeni bh_lock_sock(sk); 1339b51f9b80SPaolo Abeni if (!sock_owned_by_user(sk)) { 1340b51f9b80SPaolo Abeni mptcp_retransmit_handler(sk); 1341b51f9b80SPaolo Abeni } else { 1342b51f9b80SPaolo Abeni /* delegate our work to tcp_release_cb() */ 1343b51f9b80SPaolo Abeni if (!test_and_set_bit(TCP_WRITE_TIMER_DEFERRED, 1344b51f9b80SPaolo Abeni &sk->sk_tsq_flags)) 1345b51f9b80SPaolo Abeni sock_hold(sk); 1346b51f9b80SPaolo Abeni } 1347b51f9b80SPaolo Abeni bh_unlock_sock(sk); 1348b51f9b80SPaolo Abeni sock_put(sk); 1349b51f9b80SPaolo Abeni } 1350b51f9b80SPaolo Abeni 13513b1d6210SPaolo Abeni /* Find an idle subflow. Return NULL if there is unacked data at tcp 13523b1d6210SPaolo Abeni * level. 13533b1d6210SPaolo Abeni * 13543b1d6210SPaolo Abeni * A backup subflow is returned only if that is the only kind available. 13553b1d6210SPaolo Abeni */ 13563b1d6210SPaolo Abeni static struct sock *mptcp_subflow_get_retrans(const struct mptcp_sock *msk) 13573b1d6210SPaolo Abeni { 13583b1d6210SPaolo Abeni struct mptcp_subflow_context *subflow; 13593b1d6210SPaolo Abeni struct sock *backup = NULL; 13603b1d6210SPaolo Abeni 13613b1d6210SPaolo Abeni sock_owned_by_me((const struct sock *)msk); 13623b1d6210SPaolo Abeni 13633b1d6210SPaolo Abeni mptcp_for_each_subflow(msk, subflow) { 13643b1d6210SPaolo Abeni struct sock *ssk = mptcp_subflow_tcp_sock(subflow); 13653b1d6210SPaolo Abeni 13663b1d6210SPaolo Abeni /* still data outstanding at TCP level? Don't retransmit. */ 13673b1d6210SPaolo Abeni if (!tcp_write_queue_empty(ssk)) 13683b1d6210SPaolo Abeni return NULL; 13693b1d6210SPaolo Abeni 13703b1d6210SPaolo Abeni if (subflow->backup) { 13713b1d6210SPaolo Abeni if (!backup) 13723b1d6210SPaolo Abeni backup = ssk; 13733b1d6210SPaolo Abeni continue; 13743b1d6210SPaolo Abeni } 13753b1d6210SPaolo Abeni 13763b1d6210SPaolo Abeni return ssk; 13773b1d6210SPaolo Abeni } 13783b1d6210SPaolo Abeni 13793b1d6210SPaolo Abeni return backup; 13803b1d6210SPaolo Abeni } 13813b1d6210SPaolo Abeni 1382cec37a6eSPeter Krystad /* subflow sockets can be either outgoing (connect) or incoming 1383cec37a6eSPeter Krystad * (accept). 1384cec37a6eSPeter Krystad * 1385cec37a6eSPeter Krystad * Outgoing subflows use in-kernel sockets. 1386cec37a6eSPeter Krystad * Incoming subflows do not have their own 'struct socket' allocated, 1387cec37a6eSPeter Krystad * so we need to use tcp_close() after detaching them from the mptcp 1388cec37a6eSPeter Krystad * parent socket. 1389cec37a6eSPeter Krystad */ 1390cec37a6eSPeter Krystad static void __mptcp_close_ssk(struct sock *sk, struct sock *ssk, 1391cec37a6eSPeter Krystad struct mptcp_subflow_context *subflow, 1392cec37a6eSPeter Krystad long timeout) 1393cec37a6eSPeter Krystad { 1394cec37a6eSPeter Krystad struct socket *sock = READ_ONCE(ssk->sk_socket); 1395cec37a6eSPeter Krystad 1396cec37a6eSPeter Krystad list_del(&subflow->node); 1397cec37a6eSPeter Krystad 1398cec37a6eSPeter Krystad if (sock && sock != sk->sk_socket) { 1399cec37a6eSPeter Krystad /* outgoing subflow */ 1400cec37a6eSPeter Krystad sock_release(sock); 1401cec37a6eSPeter Krystad } else { 1402cec37a6eSPeter Krystad /* incoming subflow */ 1403cec37a6eSPeter Krystad tcp_close(ssk, timeout); 1404cec37a6eSPeter Krystad } 1405f870fa0bSMat Martineau } 1406f870fa0bSMat Martineau 1407dc24f8b4SPaolo Abeni static unsigned int mptcp_sync_mss(struct sock *sk, u32 pmtu) 1408dc24f8b4SPaolo Abeni { 1409dc24f8b4SPaolo Abeni return 0; 1410dc24f8b4SPaolo Abeni } 1411dc24f8b4SPaolo Abeni 1412b416268bSFlorian Westphal static void pm_work(struct mptcp_sock *msk) 1413b416268bSFlorian Westphal { 1414b416268bSFlorian Westphal struct mptcp_pm_data *pm = &msk->pm; 1415b416268bSFlorian Westphal 1416b416268bSFlorian Westphal spin_lock_bh(&msk->pm.lock); 1417b416268bSFlorian Westphal 1418b416268bSFlorian Westphal pr_debug("msk=%p status=%x", msk, pm->status); 1419b416268bSFlorian Westphal if (pm->status & BIT(MPTCP_PM_ADD_ADDR_RECEIVED)) { 1420b416268bSFlorian Westphal pm->status &= ~BIT(MPTCP_PM_ADD_ADDR_RECEIVED); 1421b416268bSFlorian Westphal mptcp_pm_nl_add_addr_received(msk); 1422b416268bSFlorian Westphal } 1423b416268bSFlorian Westphal if (pm->status & BIT(MPTCP_PM_ESTABLISHED)) { 1424b416268bSFlorian Westphal pm->status &= ~BIT(MPTCP_PM_ESTABLISHED); 1425b416268bSFlorian Westphal mptcp_pm_nl_fully_established(msk); 1426b416268bSFlorian Westphal } 1427b416268bSFlorian Westphal if (pm->status & BIT(MPTCP_PM_SUBFLOW_ESTABLISHED)) { 1428b416268bSFlorian Westphal pm->status &= ~BIT(MPTCP_PM_SUBFLOW_ESTABLISHED); 1429b416268bSFlorian Westphal mptcp_pm_nl_subflow_established(msk); 1430b416268bSFlorian Westphal } 1431b416268bSFlorian Westphal 1432b416268bSFlorian Westphal spin_unlock_bh(&msk->pm.lock); 1433b416268bSFlorian Westphal } 1434b416268bSFlorian Westphal 143580992017SPaolo Abeni static void mptcp_worker(struct work_struct *work) 143680992017SPaolo Abeni { 143780992017SPaolo Abeni struct mptcp_sock *msk = container_of(work, struct mptcp_sock, work); 14383b1d6210SPaolo Abeni struct sock *ssk, *sk = &msk->sk.icsk_inet.sk; 1439149f7c71SFlorian Westphal int orig_len, orig_offset, mss_now = 0, size_goal = 0; 14403b1d6210SPaolo Abeni struct mptcp_data_frag *dfrag; 14413b1d6210SPaolo Abeni u64 orig_write_seq; 14423b1d6210SPaolo Abeni size_t copied = 0; 1443b3b2854dSFlorian Westphal struct msghdr msg = { 1444b3b2854dSFlorian Westphal .msg_flags = MSG_DONTWAIT, 1445b3b2854dSFlorian Westphal }; 14463b1d6210SPaolo Abeni long timeo = 0; 144780992017SPaolo Abeni 144880992017SPaolo Abeni lock_sock(sk); 14493b1d6210SPaolo Abeni mptcp_clean_una(sk); 145043b54c6eSMat Martineau mptcp_check_data_fin_ack(sk); 1451ec3edaa7SPeter Krystad __mptcp_flush_join_list(msk); 14526771bfd9SFlorian Westphal __mptcp_move_skbs(msk); 14533b1d6210SPaolo Abeni 1454b416268bSFlorian Westphal if (msk->pm.status) 1455b416268bSFlorian Westphal pm_work(msk); 1456b416268bSFlorian Westphal 145759832e24SFlorian Westphal if (test_and_clear_bit(MPTCP_WORK_EOF, &msk->flags)) 145859832e24SFlorian Westphal mptcp_check_for_eof(msk); 145959832e24SFlorian Westphal 146043b54c6eSMat Martineau mptcp_check_data_fin(sk); 146143b54c6eSMat Martineau 14623b1d6210SPaolo Abeni if (!test_and_clear_bit(MPTCP_WORK_RTX, &msk->flags)) 14633b1d6210SPaolo Abeni goto unlock; 14643b1d6210SPaolo Abeni 14653b1d6210SPaolo Abeni dfrag = mptcp_rtx_head(sk); 14663b1d6210SPaolo Abeni if (!dfrag) 14673b1d6210SPaolo Abeni goto unlock; 14683b1d6210SPaolo Abeni 1469149f7c71SFlorian Westphal if (!mptcp_ext_cache_refill(msk)) 1470149f7c71SFlorian Westphal goto reset_unlock; 1471149f7c71SFlorian Westphal 14723b1d6210SPaolo Abeni ssk = mptcp_subflow_get_retrans(msk); 14733b1d6210SPaolo Abeni if (!ssk) 14743b1d6210SPaolo Abeni goto reset_unlock; 14753b1d6210SPaolo Abeni 14763b1d6210SPaolo Abeni lock_sock(ssk); 14773b1d6210SPaolo Abeni 14783b1d6210SPaolo Abeni orig_len = dfrag->data_len; 14793b1d6210SPaolo Abeni orig_offset = dfrag->offset; 14803b1d6210SPaolo Abeni orig_write_seq = dfrag->data_seq; 14813b1d6210SPaolo Abeni while (dfrag->data_len > 0) { 1482149f7c71SFlorian Westphal int ret = mptcp_sendmsg_frag(sk, ssk, &msg, dfrag, &timeo, 1483149f7c71SFlorian Westphal &mss_now, &size_goal); 14843b1d6210SPaolo Abeni if (ret < 0) 14853b1d6210SPaolo Abeni break; 14863b1d6210SPaolo Abeni 1487fc518953SFlorian Westphal MPTCP_INC_STATS(sock_net(sk), MPTCP_MIB_RETRANSSEGS); 14883b1d6210SPaolo Abeni copied += ret; 14893b1d6210SPaolo Abeni dfrag->data_len -= ret; 14903b1d6210SPaolo Abeni dfrag->offset += ret; 1491149f7c71SFlorian Westphal 1492149f7c71SFlorian Westphal if (!mptcp_ext_cache_refill(msk)) 1493149f7c71SFlorian Westphal break; 14943b1d6210SPaolo Abeni } 14953b1d6210SPaolo Abeni if (copied) 14963b1d6210SPaolo Abeni tcp_push(ssk, msg.msg_flags, mss_now, tcp_sk(ssk)->nonagle, 14973b1d6210SPaolo Abeni size_goal); 14983b1d6210SPaolo Abeni 14993b1d6210SPaolo Abeni dfrag->data_seq = orig_write_seq; 15003b1d6210SPaolo Abeni dfrag->offset = orig_offset; 15013b1d6210SPaolo Abeni dfrag->data_len = orig_len; 15023b1d6210SPaolo Abeni 15033b1d6210SPaolo Abeni mptcp_set_timeout(sk, ssk); 15043b1d6210SPaolo Abeni release_sock(ssk); 15053b1d6210SPaolo Abeni 15063b1d6210SPaolo Abeni reset_unlock: 15073b1d6210SPaolo Abeni if (!mptcp_timer_pending(sk)) 15083b1d6210SPaolo Abeni mptcp_reset_timer(sk); 15093b1d6210SPaolo Abeni 15103b1d6210SPaolo Abeni unlock: 151180992017SPaolo Abeni release_sock(sk); 151280992017SPaolo Abeni sock_put(sk); 151380992017SPaolo Abeni } 151480992017SPaolo Abeni 1515784325e9SMatthieu Baerts static int __mptcp_init_sock(struct sock *sk) 1516f870fa0bSMat Martineau { 1517cec37a6eSPeter Krystad struct mptcp_sock *msk = mptcp_sk(sk); 1518cec37a6eSPeter Krystad 1519ec3edaa7SPeter Krystad spin_lock_init(&msk->join_list_lock); 1520ec3edaa7SPeter Krystad 1521cec37a6eSPeter Krystad INIT_LIST_HEAD(&msk->conn_list); 1522ec3edaa7SPeter Krystad INIT_LIST_HEAD(&msk->join_list); 152318b683bfSPaolo Abeni INIT_LIST_HEAD(&msk->rtx_queue); 15241891c4a0SFlorian Westphal __set_bit(MPTCP_SEND_SPACE, &msk->flags); 152580992017SPaolo Abeni INIT_WORK(&msk->work, mptcp_worker); 1526cec37a6eSPeter Krystad 15278ab183deSPaolo Abeni msk->first = NULL; 1528dc24f8b4SPaolo Abeni inet_csk(sk)->icsk_sync_mss = mptcp_sync_mss; 15298ab183deSPaolo Abeni 15301b1c7a0eSPeter Krystad mptcp_pm_data_init(msk); 15311b1c7a0eSPeter Krystad 1532b51f9b80SPaolo Abeni /* re-use the csk retrans timer for MPTCP-level retrans */ 1533b51f9b80SPaolo Abeni timer_setup(&msk->sk.icsk_retransmit_timer, mptcp_retransmit_timer, 0); 1534b51f9b80SPaolo Abeni 1535f870fa0bSMat Martineau return 0; 1536f870fa0bSMat Martineau } 1537f870fa0bSMat Martineau 1538784325e9SMatthieu Baerts static int mptcp_init_sock(struct sock *sk) 1539784325e9SMatthieu Baerts { 1540fc518953SFlorian Westphal struct net *net = sock_net(sk); 1541fc518953SFlorian Westphal int ret; 154218b683bfSPaolo Abeni 1543fc518953SFlorian Westphal if (!mptcp_is_enabled(net)) 1544fc518953SFlorian Westphal return -ENOPROTOOPT; 1545fc518953SFlorian Westphal 1546fc518953SFlorian Westphal if (unlikely(!net->mib.mptcp_statistics) && !mptcp_mib_alloc(net)) 1547fc518953SFlorian Westphal return -ENOMEM; 1548fc518953SFlorian Westphal 1549fc518953SFlorian Westphal ret = __mptcp_init_sock(sk); 155018b683bfSPaolo Abeni if (ret) 155118b683bfSPaolo Abeni return ret; 155218b683bfSPaolo Abeni 1553fa68018dSPaolo Abeni ret = __mptcp_socket_create(mptcp_sk(sk)); 1554fa68018dSPaolo Abeni if (ret) 1555fa68018dSPaolo Abeni return ret; 1556fa68018dSPaolo Abeni 1557d027236cSPaolo Abeni sk_sockets_allocated_inc(sk); 1558a6b118feSFlorian Westphal sk->sk_rcvbuf = sock_net(sk)->ipv4.sysctl_tcp_rmem[1]; 1559*da51aef5SPaolo Abeni sk->sk_sndbuf = sock_net(sk)->ipv4.sysctl_tcp_wmem[1]; 1560d027236cSPaolo Abeni 156118b683bfSPaolo Abeni return 0; 156218b683bfSPaolo Abeni } 156318b683bfSPaolo Abeni 156418b683bfSPaolo Abeni static void __mptcp_clear_xmit(struct sock *sk) 156518b683bfSPaolo Abeni { 156618b683bfSPaolo Abeni struct mptcp_sock *msk = mptcp_sk(sk); 156718b683bfSPaolo Abeni struct mptcp_data_frag *dtmp, *dfrag; 156818b683bfSPaolo Abeni 1569b51f9b80SPaolo Abeni sk_stop_timer(sk, &msk->sk.icsk_retransmit_timer); 1570b51f9b80SPaolo Abeni 157118b683bfSPaolo Abeni list_for_each_entry_safe(dfrag, dtmp, &msk->rtx_queue, list) 1572d027236cSPaolo Abeni dfrag_clear(sk, dfrag); 1573784325e9SMatthieu Baerts } 1574784325e9SMatthieu Baerts 157580992017SPaolo Abeni static void mptcp_cancel_work(struct sock *sk) 157680992017SPaolo Abeni { 157780992017SPaolo Abeni struct mptcp_sock *msk = mptcp_sk(sk); 157880992017SPaolo Abeni 157980992017SPaolo Abeni if (cancel_work_sync(&msk->work)) 158080992017SPaolo Abeni sock_put(sk); 158180992017SPaolo Abeni } 158280992017SPaolo Abeni 158343b54c6eSMat Martineau static void mptcp_subflow_shutdown(struct sock *sk, struct sock *ssk, int how) 158421498490SPeter Krystad { 158521498490SPeter Krystad lock_sock(ssk); 158621498490SPeter Krystad 158721498490SPeter Krystad switch (ssk->sk_state) { 158821498490SPeter Krystad case TCP_LISTEN: 158921498490SPeter Krystad if (!(how & RCV_SHUTDOWN)) 159021498490SPeter Krystad break; 1591df561f66SGustavo A. R. Silva fallthrough; 159221498490SPeter Krystad case TCP_SYN_SENT: 159321498490SPeter Krystad tcp_disconnect(ssk, O_NONBLOCK); 159421498490SPeter Krystad break; 159521498490SPeter Krystad default: 159643b54c6eSMat Martineau if (__mptcp_check_fallback(mptcp_sk(sk))) { 159743b54c6eSMat Martineau pr_debug("Fallback"); 159821498490SPeter Krystad ssk->sk_shutdown |= how; 159921498490SPeter Krystad tcp_shutdown(ssk, how); 160043b54c6eSMat Martineau } else { 160143b54c6eSMat Martineau pr_debug("Sending DATA_FIN on subflow %p", ssk); 160243b54c6eSMat Martineau mptcp_set_timeout(sk, ssk); 160343b54c6eSMat Martineau tcp_send_ack(ssk); 160443b54c6eSMat Martineau } 160521498490SPeter Krystad break; 160621498490SPeter Krystad } 160721498490SPeter Krystad 160821498490SPeter Krystad release_sock(ssk); 160921498490SPeter Krystad } 161021498490SPeter Krystad 16116920b851SMat Martineau static const unsigned char new_state[16] = { 16126920b851SMat Martineau /* current state: new state: action: */ 16136920b851SMat Martineau [0 /* (Invalid) */] = TCP_CLOSE, 16146920b851SMat Martineau [TCP_ESTABLISHED] = TCP_FIN_WAIT1 | TCP_ACTION_FIN, 16156920b851SMat Martineau [TCP_SYN_SENT] = TCP_CLOSE, 16166920b851SMat Martineau [TCP_SYN_RECV] = TCP_FIN_WAIT1 | TCP_ACTION_FIN, 16176920b851SMat Martineau [TCP_FIN_WAIT1] = TCP_FIN_WAIT1, 16186920b851SMat Martineau [TCP_FIN_WAIT2] = TCP_FIN_WAIT2, 16196920b851SMat Martineau [TCP_TIME_WAIT] = TCP_CLOSE, /* should not happen ! */ 16206920b851SMat Martineau [TCP_CLOSE] = TCP_CLOSE, 16216920b851SMat Martineau [TCP_CLOSE_WAIT] = TCP_LAST_ACK | TCP_ACTION_FIN, 16226920b851SMat Martineau [TCP_LAST_ACK] = TCP_LAST_ACK, 16236920b851SMat Martineau [TCP_LISTEN] = TCP_CLOSE, 16246920b851SMat Martineau [TCP_CLOSING] = TCP_CLOSING, 16256920b851SMat Martineau [TCP_NEW_SYN_RECV] = TCP_CLOSE, /* should not happen ! */ 16266920b851SMat Martineau }; 16276920b851SMat Martineau 16286920b851SMat Martineau static int mptcp_close_state(struct sock *sk) 16296920b851SMat Martineau { 16306920b851SMat Martineau int next = (int)new_state[sk->sk_state]; 16316920b851SMat Martineau int ns = next & TCP_STATE_MASK; 16326920b851SMat Martineau 16336920b851SMat Martineau inet_sk_state_store(sk, ns); 16346920b851SMat Martineau 16356920b851SMat Martineau return next & TCP_ACTION_FIN; 16366920b851SMat Martineau } 16376920b851SMat Martineau 16382c22c06cSFlorian Westphal static void mptcp_close(struct sock *sk, long timeout) 1639f870fa0bSMat Martineau { 1640cec37a6eSPeter Krystad struct mptcp_subflow_context *subflow, *tmp; 1641f870fa0bSMat Martineau struct mptcp_sock *msk = mptcp_sk(sk); 1642b2c5b614SFlorian Westphal LIST_HEAD(conn_list); 1643f870fa0bSMat Martineau 16442c22c06cSFlorian Westphal lock_sock(sk); 164543b54c6eSMat Martineau sk->sk_shutdown = SHUTDOWN_MASK; 16462c22c06cSFlorian Westphal 164743b54c6eSMat Martineau if (sk->sk_state == TCP_LISTEN) { 164843b54c6eSMat Martineau inet_sk_state_store(sk, TCP_CLOSE); 164943b54c6eSMat Martineau goto cleanup; 165043b54c6eSMat Martineau } else if (sk->sk_state == TCP_CLOSE) { 165143b54c6eSMat Martineau goto cleanup; 165243b54c6eSMat Martineau } 165343b54c6eSMat Martineau 165443b54c6eSMat Martineau if (__mptcp_check_fallback(msk)) { 165543b54c6eSMat Martineau goto update_state; 165643b54c6eSMat Martineau } else if (mptcp_close_state(sk)) { 165743b54c6eSMat Martineau pr_debug("Sending DATA_FIN sk=%p", sk); 165843b54c6eSMat Martineau WRITE_ONCE(msk->write_seq, msk->write_seq + 1); 165943b54c6eSMat Martineau WRITE_ONCE(msk->snd_data_fin_enable, 1); 166043b54c6eSMat Martineau 166143b54c6eSMat Martineau mptcp_for_each_subflow(msk, subflow) { 166243b54c6eSMat Martineau struct sock *tcp_sk = mptcp_subflow_tcp_sock(subflow); 166343b54c6eSMat Martineau 166443b54c6eSMat Martineau mptcp_subflow_shutdown(sk, tcp_sk, SHUTDOWN_MASK); 166543b54c6eSMat Martineau } 166643b54c6eSMat Martineau } 166743b54c6eSMat Martineau 166843b54c6eSMat Martineau sk_stream_wait_close(sk, timeout); 166943b54c6eSMat Martineau 167043b54c6eSMat Martineau update_state: 1671f870fa0bSMat Martineau inet_sk_state_store(sk, TCP_CLOSE); 1672f870fa0bSMat Martineau 167343b54c6eSMat Martineau cleanup: 167410f6d46cSPaolo Abeni /* be sure to always acquire the join list lock, to sync vs 167510f6d46cSPaolo Abeni * mptcp_finish_join(). 167610f6d46cSPaolo Abeni */ 167710f6d46cSPaolo Abeni spin_lock_bh(&msk->join_list_lock); 167810f6d46cSPaolo Abeni list_splice_tail_init(&msk->join_list, &msk->conn_list); 167910f6d46cSPaolo Abeni spin_unlock_bh(&msk->join_list_lock); 1680b2c5b614SFlorian Westphal list_splice_init(&msk->conn_list, &conn_list); 1681b2c5b614SFlorian Westphal 168218b683bfSPaolo Abeni __mptcp_clear_xmit(sk); 168318b683bfSPaolo Abeni 1684b2c5b614SFlorian Westphal release_sock(sk); 1685b2c5b614SFlorian Westphal 1686b2c5b614SFlorian Westphal list_for_each_entry_safe(subflow, tmp, &conn_list, node) { 1687cec37a6eSPeter Krystad struct sock *ssk = mptcp_subflow_tcp_sock(subflow); 1688cec37a6eSPeter Krystad __mptcp_close_ssk(sk, ssk, subflow, timeout); 1689f870fa0bSMat Martineau } 1690f870fa0bSMat Martineau 169180992017SPaolo Abeni mptcp_cancel_work(sk); 169280992017SPaolo Abeni 16936771bfd9SFlorian Westphal __skb_queue_purge(&sk->sk_receive_queue); 16946771bfd9SFlorian Westphal 1695cec37a6eSPeter Krystad sk_common_release(sk); 1696f870fa0bSMat Martineau } 1697f870fa0bSMat Martineau 1698cf7da0d6SPeter Krystad static void mptcp_copy_inaddrs(struct sock *msk, const struct sock *ssk) 1699cf7da0d6SPeter Krystad { 1700cf7da0d6SPeter Krystad #if IS_ENABLED(CONFIG_MPTCP_IPV6) 1701cf7da0d6SPeter Krystad const struct ipv6_pinfo *ssk6 = inet6_sk(ssk); 1702cf7da0d6SPeter Krystad struct ipv6_pinfo *msk6 = inet6_sk(msk); 1703cf7da0d6SPeter Krystad 1704cf7da0d6SPeter Krystad msk->sk_v6_daddr = ssk->sk_v6_daddr; 1705cf7da0d6SPeter Krystad msk->sk_v6_rcv_saddr = ssk->sk_v6_rcv_saddr; 1706cf7da0d6SPeter Krystad 1707cf7da0d6SPeter Krystad if (msk6 && ssk6) { 1708cf7da0d6SPeter Krystad msk6->saddr = ssk6->saddr; 1709cf7da0d6SPeter Krystad msk6->flow_label = ssk6->flow_label; 1710cf7da0d6SPeter Krystad } 1711cf7da0d6SPeter Krystad #endif 1712cf7da0d6SPeter Krystad 1713cf7da0d6SPeter Krystad inet_sk(msk)->inet_num = inet_sk(ssk)->inet_num; 1714cf7da0d6SPeter Krystad inet_sk(msk)->inet_dport = inet_sk(ssk)->inet_dport; 1715cf7da0d6SPeter Krystad inet_sk(msk)->inet_sport = inet_sk(ssk)->inet_sport; 1716cf7da0d6SPeter Krystad inet_sk(msk)->inet_daddr = inet_sk(ssk)->inet_daddr; 1717cf7da0d6SPeter Krystad inet_sk(msk)->inet_saddr = inet_sk(ssk)->inet_saddr; 1718cf7da0d6SPeter Krystad inet_sk(msk)->inet_rcv_saddr = inet_sk(ssk)->inet_rcv_saddr; 1719cf7da0d6SPeter Krystad } 1720cf7da0d6SPeter Krystad 172118b683bfSPaolo Abeni static int mptcp_disconnect(struct sock *sk, int flags) 172218b683bfSPaolo Abeni { 172342c556feSFlorian Westphal /* Should never be called. 172442c556feSFlorian Westphal * inet_stream_connect() calls ->disconnect, but that 172542c556feSFlorian Westphal * refers to the subflow socket, not the mptcp one. 172642c556feSFlorian Westphal */ 172742c556feSFlorian Westphal WARN_ON_ONCE(1); 172842c556feSFlorian Westphal return 0; 172918b683bfSPaolo Abeni } 173018b683bfSPaolo Abeni 1731b0519de8SFlorian Westphal #if IS_ENABLED(CONFIG_MPTCP_IPV6) 1732b0519de8SFlorian Westphal static struct ipv6_pinfo *mptcp_inet6_sk(const struct sock *sk) 1733b0519de8SFlorian Westphal { 1734b0519de8SFlorian Westphal unsigned int offset = sizeof(struct mptcp6_sock) - sizeof(struct ipv6_pinfo); 1735b0519de8SFlorian Westphal 1736b0519de8SFlorian Westphal return (struct ipv6_pinfo *)(((u8 *)sk) + offset); 1737b0519de8SFlorian Westphal } 1738b0519de8SFlorian Westphal #endif 1739b0519de8SFlorian Westphal 1740fca5c82cSPaolo Abeni struct sock *mptcp_sk_clone(const struct sock *sk, 1741cfde141eSPaolo Abeni const struct mptcp_options_received *mp_opt, 1742fca5c82cSPaolo Abeni struct request_sock *req) 1743b0519de8SFlorian Westphal { 174458b09919SPaolo Abeni struct mptcp_subflow_request_sock *subflow_req = mptcp_subflow_rsk(req); 1745b0519de8SFlorian Westphal struct sock *nsk = sk_clone_lock(sk, GFP_ATOMIC); 174658b09919SPaolo Abeni struct mptcp_sock *msk; 174758b09919SPaolo Abeni u64 ack_seq; 1748b0519de8SFlorian Westphal 1749b0519de8SFlorian Westphal if (!nsk) 1750b0519de8SFlorian Westphal return NULL; 1751b0519de8SFlorian Westphal 1752b0519de8SFlorian Westphal #if IS_ENABLED(CONFIG_MPTCP_IPV6) 1753b0519de8SFlorian Westphal if (nsk->sk_family == AF_INET6) 1754b0519de8SFlorian Westphal inet_sk(nsk)->pinet6 = mptcp_inet6_sk(nsk); 1755b0519de8SFlorian Westphal #endif 1756b0519de8SFlorian Westphal 175758b09919SPaolo Abeni __mptcp_init_sock(nsk); 175858b09919SPaolo Abeni 175958b09919SPaolo Abeni msk = mptcp_sk(nsk); 176058b09919SPaolo Abeni msk->local_key = subflow_req->local_key; 176158b09919SPaolo Abeni msk->token = subflow_req->token; 176258b09919SPaolo Abeni msk->subflow = NULL; 1763b93df08cSPaolo Abeni WRITE_ONCE(msk->fully_established, false); 176458b09919SPaolo Abeni 176558b09919SPaolo Abeni msk->write_seq = subflow_req->idsn + 1; 1766cc9d2566SPaolo Abeni atomic64_set(&msk->snd_una, msk->write_seq); 1767cfde141eSPaolo Abeni if (mp_opt->mp_capable) { 176858b09919SPaolo Abeni msk->can_ack = true; 1769cfde141eSPaolo Abeni msk->remote_key = mp_opt->sndr_key; 177058b09919SPaolo Abeni mptcp_crypto_key_sha(msk->remote_key, NULL, &ack_seq); 177158b09919SPaolo Abeni ack_seq++; 177258b09919SPaolo Abeni msk->ack_seq = ack_seq; 177358b09919SPaolo Abeni } 17747f20d5fcSPaolo Abeni 17755e20087dSFlorian Westphal sock_reset_flag(nsk, SOCK_RCU_FREE); 17767f20d5fcSPaolo Abeni /* will be fully established after successful MPC subflow creation */ 17777f20d5fcSPaolo Abeni inet_sk_state_store(nsk, TCP_SYN_RECV); 177858b09919SPaolo Abeni bh_unlock_sock(nsk); 177958b09919SPaolo Abeni 178058b09919SPaolo Abeni /* keep a single reference */ 178158b09919SPaolo Abeni __sock_put(nsk); 1782b0519de8SFlorian Westphal return nsk; 1783b0519de8SFlorian Westphal } 1784b0519de8SFlorian Westphal 1785a6b118feSFlorian Westphal void mptcp_rcv_space_init(struct mptcp_sock *msk, const struct sock *ssk) 1786a6b118feSFlorian Westphal { 1787a6b118feSFlorian Westphal const struct tcp_sock *tp = tcp_sk(ssk); 1788a6b118feSFlorian Westphal 1789a6b118feSFlorian Westphal msk->rcvq_space.copied = 0; 1790a6b118feSFlorian Westphal msk->rcvq_space.rtt_us = 0; 1791a6b118feSFlorian Westphal 1792a6b118feSFlorian Westphal msk->rcvq_space.time = tp->tcp_mstamp; 1793a6b118feSFlorian Westphal 1794a6b118feSFlorian Westphal /* initial rcv_space offering made to peer */ 1795a6b118feSFlorian Westphal msk->rcvq_space.space = min_t(u32, tp->rcv_wnd, 1796a6b118feSFlorian Westphal TCP_INIT_CWND * tp->advmss); 1797a6b118feSFlorian Westphal if (msk->rcvq_space.space == 0) 1798a6b118feSFlorian Westphal msk->rcvq_space.space = TCP_INIT_CWND * TCP_MSS_DEFAULT; 1799a6b118feSFlorian Westphal } 1800a6b118feSFlorian Westphal 1801cf7da0d6SPeter Krystad static struct sock *mptcp_accept(struct sock *sk, int flags, int *err, 1802cf7da0d6SPeter Krystad bool kern) 1803cf7da0d6SPeter Krystad { 1804cf7da0d6SPeter Krystad struct mptcp_sock *msk = mptcp_sk(sk); 1805cf7da0d6SPeter Krystad struct socket *listener; 1806cf7da0d6SPeter Krystad struct sock *newsk; 1807cf7da0d6SPeter Krystad 1808cf7da0d6SPeter Krystad listener = __mptcp_nmpc_socket(msk); 1809cf7da0d6SPeter Krystad if (WARN_ON_ONCE(!listener)) { 1810cf7da0d6SPeter Krystad *err = -EINVAL; 1811cf7da0d6SPeter Krystad return NULL; 1812cf7da0d6SPeter Krystad } 1813cf7da0d6SPeter Krystad 1814cf7da0d6SPeter Krystad pr_debug("msk=%p, listener=%p", msk, mptcp_subflow_ctx(listener->sk)); 1815cf7da0d6SPeter Krystad newsk = inet_csk_accept(listener->sk, flags, err, kern); 1816cf7da0d6SPeter Krystad if (!newsk) 1817cf7da0d6SPeter Krystad return NULL; 1818cf7da0d6SPeter Krystad 1819cf7da0d6SPeter Krystad pr_debug("msk=%p, subflow is mptcp=%d", msk, sk_is_mptcp(newsk)); 1820cf7da0d6SPeter Krystad if (sk_is_mptcp(newsk)) { 1821cf7da0d6SPeter Krystad struct mptcp_subflow_context *subflow; 1822cf7da0d6SPeter Krystad struct sock *new_mptcp_sock; 1823cf7da0d6SPeter Krystad struct sock *ssk = newsk; 1824cf7da0d6SPeter Krystad 1825cf7da0d6SPeter Krystad subflow = mptcp_subflow_ctx(newsk); 182658b09919SPaolo Abeni new_mptcp_sock = subflow->conn; 182758b09919SPaolo Abeni 182858b09919SPaolo Abeni /* is_mptcp should be false if subflow->conn is missing, see 182958b09919SPaolo Abeni * subflow_syn_recv_sock() 183058b09919SPaolo Abeni */ 183158b09919SPaolo Abeni if (WARN_ON_ONCE(!new_mptcp_sock)) { 183258b09919SPaolo Abeni tcp_sk(newsk)->is_mptcp = 0; 183358b09919SPaolo Abeni return newsk; 183458b09919SPaolo Abeni } 183558b09919SPaolo Abeni 183658b09919SPaolo Abeni /* acquire the 2nd reference for the owning socket */ 183758b09919SPaolo Abeni sock_hold(new_mptcp_sock); 1838cf7da0d6SPeter Krystad 1839cf7da0d6SPeter Krystad local_bh_disable(); 184058b09919SPaolo Abeni bh_lock_sock(new_mptcp_sock); 1841cf7da0d6SPeter Krystad msk = mptcp_sk(new_mptcp_sock); 18428ab183deSPaolo Abeni msk->first = newsk; 1843cf7da0d6SPeter Krystad 1844cf7da0d6SPeter Krystad newsk = new_mptcp_sock; 1845cf7da0d6SPeter Krystad mptcp_copy_inaddrs(newsk, ssk); 1846cf7da0d6SPeter Krystad list_add(&subflow->node, &msk->conn_list); 1847cf7da0d6SPeter Krystad 1848a6b118feSFlorian Westphal mptcp_rcv_space_init(msk, ssk); 1849cf7da0d6SPeter Krystad bh_unlock_sock(new_mptcp_sock); 1850fc518953SFlorian Westphal 1851fc518953SFlorian Westphal __MPTCP_INC_STATS(sock_net(sk), MPTCP_MIB_MPCAPABLEPASSIVEACK); 1852cf7da0d6SPeter Krystad local_bh_enable(); 1853fc518953SFlorian Westphal } else { 1854fc518953SFlorian Westphal MPTCP_INC_STATS(sock_net(sk), 1855fc518953SFlorian Westphal MPTCP_MIB_MPCAPABLEPASSIVEFALLBACK); 1856cf7da0d6SPeter Krystad } 1857cf7da0d6SPeter Krystad 1858cf7da0d6SPeter Krystad return newsk; 1859cf7da0d6SPeter Krystad } 1860cf7da0d6SPeter Krystad 186179c0949eSPeter Krystad static void mptcp_destroy(struct sock *sk) 186279c0949eSPeter Krystad { 1863c9fd9c5fSFlorian Westphal struct mptcp_sock *msk = mptcp_sk(sk); 1864c9fd9c5fSFlorian Westphal 18652c5ebd00SPaolo Abeni mptcp_token_destroy(msk); 1866c9fd9c5fSFlorian Westphal if (msk->cached_ext) 1867c9fd9c5fSFlorian Westphal __skb_ext_put(msk->cached_ext); 1868d027236cSPaolo Abeni 1869d027236cSPaolo Abeni sk_sockets_allocated_dec(sk); 187079c0949eSPeter Krystad } 187179c0949eSPeter Krystad 1872fd1452d8SFlorian Westphal static int mptcp_setsockopt_sol_socket(struct mptcp_sock *msk, int optname, 1873a7b75c5aSChristoph Hellwig sockptr_t optval, unsigned int optlen) 1874fd1452d8SFlorian Westphal { 1875fd1452d8SFlorian Westphal struct sock *sk = (struct sock *)msk; 1876fd1452d8SFlorian Westphal struct socket *ssock; 1877fd1452d8SFlorian Westphal int ret; 1878fd1452d8SFlorian Westphal 1879fd1452d8SFlorian Westphal switch (optname) { 1880fd1452d8SFlorian Westphal case SO_REUSEPORT: 1881fd1452d8SFlorian Westphal case SO_REUSEADDR: 1882fd1452d8SFlorian Westphal lock_sock(sk); 1883fd1452d8SFlorian Westphal ssock = __mptcp_nmpc_socket(msk); 1884fd1452d8SFlorian Westphal if (!ssock) { 1885fd1452d8SFlorian Westphal release_sock(sk); 1886fd1452d8SFlorian Westphal return -EINVAL; 1887fd1452d8SFlorian Westphal } 1888fd1452d8SFlorian Westphal 1889a7b75c5aSChristoph Hellwig ret = sock_setsockopt(ssock, SOL_SOCKET, optname, optval, optlen); 1890fd1452d8SFlorian Westphal if (ret == 0) { 1891fd1452d8SFlorian Westphal if (optname == SO_REUSEPORT) 1892fd1452d8SFlorian Westphal sk->sk_reuseport = ssock->sk->sk_reuseport; 1893fd1452d8SFlorian Westphal else if (optname == SO_REUSEADDR) 1894fd1452d8SFlorian Westphal sk->sk_reuse = ssock->sk->sk_reuse; 1895fd1452d8SFlorian Westphal } 1896fd1452d8SFlorian Westphal release_sock(sk); 1897fd1452d8SFlorian Westphal return ret; 1898fd1452d8SFlorian Westphal } 1899fd1452d8SFlorian Westphal 1900a7b75c5aSChristoph Hellwig return sock_setsockopt(sk->sk_socket, SOL_SOCKET, optname, optval, optlen); 1901fd1452d8SFlorian Westphal } 1902fd1452d8SFlorian Westphal 1903c9b95a13SFlorian Westphal static int mptcp_setsockopt_v6(struct mptcp_sock *msk, int optname, 1904a7b75c5aSChristoph Hellwig sockptr_t optval, unsigned int optlen) 1905c9b95a13SFlorian Westphal { 1906c9b95a13SFlorian Westphal struct sock *sk = (struct sock *)msk; 1907c9b95a13SFlorian Westphal int ret = -EOPNOTSUPP; 1908c9b95a13SFlorian Westphal struct socket *ssock; 1909c9b95a13SFlorian Westphal 1910c9b95a13SFlorian Westphal switch (optname) { 1911c9b95a13SFlorian Westphal case IPV6_V6ONLY: 1912c9b95a13SFlorian Westphal lock_sock(sk); 1913c9b95a13SFlorian Westphal ssock = __mptcp_nmpc_socket(msk); 1914c9b95a13SFlorian Westphal if (!ssock) { 1915c9b95a13SFlorian Westphal release_sock(sk); 1916c9b95a13SFlorian Westphal return -EINVAL; 1917c9b95a13SFlorian Westphal } 1918c9b95a13SFlorian Westphal 1919c9b95a13SFlorian Westphal ret = tcp_setsockopt(ssock->sk, SOL_IPV6, optname, optval, optlen); 1920c9b95a13SFlorian Westphal if (ret == 0) 1921c9b95a13SFlorian Westphal sk->sk_ipv6only = ssock->sk->sk_ipv6only; 1922c9b95a13SFlorian Westphal 1923c9b95a13SFlorian Westphal release_sock(sk); 1924c9b95a13SFlorian Westphal break; 1925c9b95a13SFlorian Westphal } 1926c9b95a13SFlorian Westphal 1927c9b95a13SFlorian Westphal return ret; 1928c9b95a13SFlorian Westphal } 1929c9b95a13SFlorian Westphal 1930717e79c8SPeter Krystad static int mptcp_setsockopt(struct sock *sk, int level, int optname, 1931a7b75c5aSChristoph Hellwig sockptr_t optval, unsigned int optlen) 1932717e79c8SPeter Krystad { 1933717e79c8SPeter Krystad struct mptcp_sock *msk = mptcp_sk(sk); 193476660afbSPaolo Abeni struct sock *ssk; 1935717e79c8SPeter Krystad 1936717e79c8SPeter Krystad pr_debug("msk=%p", msk); 1937717e79c8SPeter Krystad 193883f0c10bSFlorian Westphal if (level == SOL_SOCKET) 1939fd1452d8SFlorian Westphal return mptcp_setsockopt_sol_socket(msk, optname, optval, optlen); 194083f0c10bSFlorian Westphal 1941717e79c8SPeter Krystad /* @@ the meaning of setsockopt() when the socket is connected and 1942b6e4a1aeSMat Martineau * there are multiple subflows is not yet defined. It is up to the 1943b6e4a1aeSMat Martineau * MPTCP-level socket to configure the subflows until the subflow 1944b6e4a1aeSMat Martineau * is in TCP fallback, when TCP socket options are passed through 1945b6e4a1aeSMat Martineau * to the one remaining subflow. 1946717e79c8SPeter Krystad */ 1947717e79c8SPeter Krystad lock_sock(sk); 194876660afbSPaolo Abeni ssk = __mptcp_tcp_fallback(msk); 1949e154659bSFlorian Westphal release_sock(sk); 195076660afbSPaolo Abeni if (ssk) 195176660afbSPaolo Abeni return tcp_setsockopt(ssk, level, optname, optval, optlen); 195250e741bbSFlorian Westphal 1953c9b95a13SFlorian Westphal if (level == SOL_IPV6) 1954c9b95a13SFlorian Westphal return mptcp_setsockopt_v6(msk, optname, optval, optlen); 1955c9b95a13SFlorian Westphal 1956b6e4a1aeSMat Martineau return -EOPNOTSUPP; 1957717e79c8SPeter Krystad } 1958717e79c8SPeter Krystad 1959717e79c8SPeter Krystad static int mptcp_getsockopt(struct sock *sk, int level, int optname, 196050e741bbSFlorian Westphal char __user *optval, int __user *option) 1961717e79c8SPeter Krystad { 1962717e79c8SPeter Krystad struct mptcp_sock *msk = mptcp_sk(sk); 196376660afbSPaolo Abeni struct sock *ssk; 1964717e79c8SPeter Krystad 1965717e79c8SPeter Krystad pr_debug("msk=%p", msk); 1966717e79c8SPeter Krystad 1967b6e4a1aeSMat Martineau /* @@ the meaning of setsockopt() when the socket is connected and 1968b6e4a1aeSMat Martineau * there are multiple subflows is not yet defined. It is up to the 1969b6e4a1aeSMat Martineau * MPTCP-level socket to configure the subflows until the subflow 1970b6e4a1aeSMat Martineau * is in TCP fallback, when socket options are passed through 1971b6e4a1aeSMat Martineau * to the one remaining subflow. 1972717e79c8SPeter Krystad */ 1973717e79c8SPeter Krystad lock_sock(sk); 197476660afbSPaolo Abeni ssk = __mptcp_tcp_fallback(msk); 1975e154659bSFlorian Westphal release_sock(sk); 197676660afbSPaolo Abeni if (ssk) 197776660afbSPaolo Abeni return tcp_getsockopt(ssk, level, optname, optval, option); 197850e741bbSFlorian Westphal 1979b6e4a1aeSMat Martineau return -EOPNOTSUPP; 1980717e79c8SPeter Krystad } 1981717e79c8SPeter Krystad 1982b51f9b80SPaolo Abeni #define MPTCP_DEFERRED_ALL (TCPF_DELACK_TIMER_DEFERRED | \ 1983b51f9b80SPaolo Abeni TCPF_WRITE_TIMER_DEFERRED) 198414c441b5SPaolo Abeni 198514c441b5SPaolo Abeni /* this is very alike tcp_release_cb() but we must handle differently a 198614c441b5SPaolo Abeni * different set of events 198714c441b5SPaolo Abeni */ 198814c441b5SPaolo Abeni static void mptcp_release_cb(struct sock *sk) 198914c441b5SPaolo Abeni { 199014c441b5SPaolo Abeni unsigned long flags, nflags; 199114c441b5SPaolo Abeni 199214c441b5SPaolo Abeni do { 199314c441b5SPaolo Abeni flags = sk->sk_tsq_flags; 199414c441b5SPaolo Abeni if (!(flags & MPTCP_DEFERRED_ALL)) 199514c441b5SPaolo Abeni return; 199614c441b5SPaolo Abeni nflags = flags & ~MPTCP_DEFERRED_ALL; 199714c441b5SPaolo Abeni } while (cmpxchg(&sk->sk_tsq_flags, flags, nflags) != flags); 199814c441b5SPaolo Abeni 1999b51f9b80SPaolo Abeni sock_release_ownership(sk); 2000b51f9b80SPaolo Abeni 200114c441b5SPaolo Abeni if (flags & TCPF_DELACK_TIMER_DEFERRED) { 200214c441b5SPaolo Abeni struct mptcp_sock *msk = mptcp_sk(sk); 200314c441b5SPaolo Abeni struct sock *ssk; 200414c441b5SPaolo Abeni 200514c441b5SPaolo Abeni ssk = mptcp_subflow_recv_lookup(msk); 200614c441b5SPaolo Abeni if (!ssk || !schedule_work(&msk->work)) 200714c441b5SPaolo Abeni __sock_put(sk); 200814c441b5SPaolo Abeni } 2009b51f9b80SPaolo Abeni 2010b51f9b80SPaolo Abeni if (flags & TCPF_WRITE_TIMER_DEFERRED) { 2011b51f9b80SPaolo Abeni mptcp_retransmit_handler(sk); 2012b51f9b80SPaolo Abeni __sock_put(sk); 2013b51f9b80SPaolo Abeni } 201414c441b5SPaolo Abeni } 201514c441b5SPaolo Abeni 20162c5ebd00SPaolo Abeni static int mptcp_hash(struct sock *sk) 20172c5ebd00SPaolo Abeni { 20182c5ebd00SPaolo Abeni /* should never be called, 20192c5ebd00SPaolo Abeni * we hash the TCP subflows not the master socket 20202c5ebd00SPaolo Abeni */ 20212c5ebd00SPaolo Abeni WARN_ON_ONCE(1); 20222c5ebd00SPaolo Abeni return 0; 20232c5ebd00SPaolo Abeni } 20242c5ebd00SPaolo Abeni 20252c5ebd00SPaolo Abeni static void mptcp_unhash(struct sock *sk) 20262c5ebd00SPaolo Abeni { 20272c5ebd00SPaolo Abeni /* called from sk_common_release(), but nothing to do here */ 20282c5ebd00SPaolo Abeni } 20292c5ebd00SPaolo Abeni 2030cec37a6eSPeter Krystad static int mptcp_get_port(struct sock *sk, unsigned short snum) 2031f870fa0bSMat Martineau { 2032f870fa0bSMat Martineau struct mptcp_sock *msk = mptcp_sk(sk); 2033cec37a6eSPeter Krystad struct socket *ssock; 2034f870fa0bSMat Martineau 2035cec37a6eSPeter Krystad ssock = __mptcp_nmpc_socket(msk); 2036cec37a6eSPeter Krystad pr_debug("msk=%p, subflow=%p", msk, ssock); 2037cec37a6eSPeter Krystad if (WARN_ON_ONCE(!ssock)) 2038cec37a6eSPeter Krystad return -EINVAL; 2039f870fa0bSMat Martineau 2040cec37a6eSPeter Krystad return inet_csk_get_port(ssock->sk, snum); 2041cec37a6eSPeter Krystad } 2042f870fa0bSMat Martineau 2043cec37a6eSPeter Krystad void mptcp_finish_connect(struct sock *ssk) 2044cec37a6eSPeter Krystad { 2045cec37a6eSPeter Krystad struct mptcp_subflow_context *subflow; 2046cec37a6eSPeter Krystad struct mptcp_sock *msk; 2047cec37a6eSPeter Krystad struct sock *sk; 20486d0060f6SMat Martineau u64 ack_seq; 2049f870fa0bSMat Martineau 2050cec37a6eSPeter Krystad subflow = mptcp_subflow_ctx(ssk); 2051cec37a6eSPeter Krystad sk = subflow->conn; 2052cec37a6eSPeter Krystad msk = mptcp_sk(sk); 2053cec37a6eSPeter Krystad 2054648ef4b8SMat Martineau pr_debug("msk=%p, token=%u", sk, subflow->token); 2055648ef4b8SMat Martineau 20566d0060f6SMat Martineau mptcp_crypto_key_sha(subflow->remote_key, NULL, &ack_seq); 20576d0060f6SMat Martineau ack_seq++; 2058648ef4b8SMat Martineau subflow->map_seq = ack_seq; 2059648ef4b8SMat Martineau subflow->map_subflow_seq = 1; 20606d0060f6SMat Martineau 2061cec37a6eSPeter Krystad /* the socket is not connected yet, no msk/subflow ops can access/race 2062cec37a6eSPeter Krystad * accessing the field below 2063cec37a6eSPeter Krystad */ 2064cec37a6eSPeter Krystad WRITE_ONCE(msk->remote_key, subflow->remote_key); 2065cec37a6eSPeter Krystad WRITE_ONCE(msk->local_key, subflow->local_key); 20666d0060f6SMat Martineau WRITE_ONCE(msk->write_seq, subflow->idsn + 1); 20676d0060f6SMat Martineau WRITE_ONCE(msk->ack_seq, ack_seq); 2068d22f4988SChristoph Paasch WRITE_ONCE(msk->can_ack, 1); 2069cc9d2566SPaolo Abeni atomic64_set(&msk->snd_una, msk->write_seq); 20701b1c7a0eSPeter Krystad 20711b1c7a0eSPeter Krystad mptcp_pm_new_connection(msk, 0); 2072a6b118feSFlorian Westphal 2073a6b118feSFlorian Westphal mptcp_rcv_space_init(msk, ssk); 2074f870fa0bSMat Martineau } 2075f870fa0bSMat Martineau 2076cf7da0d6SPeter Krystad static void mptcp_sock_graft(struct sock *sk, struct socket *parent) 2077cf7da0d6SPeter Krystad { 2078cf7da0d6SPeter Krystad write_lock_bh(&sk->sk_callback_lock); 2079cf7da0d6SPeter Krystad rcu_assign_pointer(sk->sk_wq, &parent->wq); 2080cf7da0d6SPeter Krystad sk_set_socket(sk, parent); 2081cf7da0d6SPeter Krystad sk->sk_uid = SOCK_INODE(parent)->i_uid; 2082cf7da0d6SPeter Krystad write_unlock_bh(&sk->sk_callback_lock); 2083cf7da0d6SPeter Krystad } 2084cf7da0d6SPeter Krystad 2085f296234cSPeter Krystad bool mptcp_finish_join(struct sock *sk) 2086f296234cSPeter Krystad { 2087f296234cSPeter Krystad struct mptcp_subflow_context *subflow = mptcp_subflow_ctx(sk); 2088f296234cSPeter Krystad struct mptcp_sock *msk = mptcp_sk(subflow->conn); 2089f296234cSPeter Krystad struct sock *parent = (void *)msk; 2090f296234cSPeter Krystad struct socket *parent_sock; 2091ec3edaa7SPeter Krystad bool ret; 2092f296234cSPeter Krystad 2093f296234cSPeter Krystad pr_debug("msk=%p, subflow=%p", msk, subflow); 2094f296234cSPeter Krystad 2095f296234cSPeter Krystad /* mptcp socket already closing? */ 2096b93df08cSPaolo Abeni if (!mptcp_is_fully_established(parent)) 2097f296234cSPeter Krystad return false; 2098f296234cSPeter Krystad 2099f296234cSPeter Krystad if (!msk->pm.server_side) 2100f296234cSPeter Krystad return true; 2101f296234cSPeter Krystad 210210f6d46cSPaolo Abeni if (!mptcp_pm_allow_new_subflow(msk)) 210310f6d46cSPaolo Abeni return false; 210410f6d46cSPaolo Abeni 210510f6d46cSPaolo Abeni /* active connections are already on conn_list, and we can't acquire 210610f6d46cSPaolo Abeni * msk lock here. 210710f6d46cSPaolo Abeni * use the join list lock as synchronization point and double-check 210810f6d46cSPaolo Abeni * msk status to avoid racing with mptcp_close() 210910f6d46cSPaolo Abeni */ 211010f6d46cSPaolo Abeni spin_lock_bh(&msk->join_list_lock); 211110f6d46cSPaolo Abeni ret = inet_sk_state_load(parent) == TCP_ESTABLISHED; 211210f6d46cSPaolo Abeni if (ret && !WARN_ON_ONCE(!list_empty(&subflow->node))) 211310f6d46cSPaolo Abeni list_add_tail(&subflow->node, &msk->join_list); 211410f6d46cSPaolo Abeni spin_unlock_bh(&msk->join_list_lock); 211510f6d46cSPaolo Abeni if (!ret) 211610f6d46cSPaolo Abeni return false; 211710f6d46cSPaolo Abeni 211810f6d46cSPaolo Abeni /* attach to msk socket only after we are sure he will deal with us 211910f6d46cSPaolo Abeni * at close time 212010f6d46cSPaolo Abeni */ 2121f296234cSPeter Krystad parent_sock = READ_ONCE(parent->sk_socket); 2122f296234cSPeter Krystad if (parent_sock && !sk->sk_socket) 2123f296234cSPeter Krystad mptcp_sock_graft(sk, parent_sock); 212464d950aeSChristoph Paasch subflow->map_seq = msk->ack_seq; 212510f6d46cSPaolo Abeni return true; 2126f296234cSPeter Krystad } 2127f296234cSPeter Krystad 21281891c4a0SFlorian Westphal static bool mptcp_memory_free(const struct sock *sk, int wake) 21291891c4a0SFlorian Westphal { 21301891c4a0SFlorian Westphal struct mptcp_sock *msk = mptcp_sk(sk); 21311891c4a0SFlorian Westphal 21321891c4a0SFlorian Westphal return wake ? test_bit(MPTCP_SEND_SPACE, &msk->flags) : true; 21331891c4a0SFlorian Westphal } 21341891c4a0SFlorian Westphal 2135f870fa0bSMat Martineau static struct proto mptcp_prot = { 2136f870fa0bSMat Martineau .name = "MPTCP", 2137f870fa0bSMat Martineau .owner = THIS_MODULE, 2138f870fa0bSMat Martineau .init = mptcp_init_sock, 213918b683bfSPaolo Abeni .disconnect = mptcp_disconnect, 2140f870fa0bSMat Martineau .close = mptcp_close, 2141cf7da0d6SPeter Krystad .accept = mptcp_accept, 2142717e79c8SPeter Krystad .setsockopt = mptcp_setsockopt, 2143717e79c8SPeter Krystad .getsockopt = mptcp_getsockopt, 2144f870fa0bSMat Martineau .shutdown = tcp_shutdown, 214579c0949eSPeter Krystad .destroy = mptcp_destroy, 2146f870fa0bSMat Martineau .sendmsg = mptcp_sendmsg, 2147f870fa0bSMat Martineau .recvmsg = mptcp_recvmsg, 214814c441b5SPaolo Abeni .release_cb = mptcp_release_cb, 21492c5ebd00SPaolo Abeni .hash = mptcp_hash, 21502c5ebd00SPaolo Abeni .unhash = mptcp_unhash, 2151cec37a6eSPeter Krystad .get_port = mptcp_get_port, 2152d027236cSPaolo Abeni .sockets_allocated = &mptcp_sockets_allocated, 2153d027236cSPaolo Abeni .memory_allocated = &tcp_memory_allocated, 2154d027236cSPaolo Abeni .memory_pressure = &tcp_memory_pressure, 21551891c4a0SFlorian Westphal .stream_memory_free = mptcp_memory_free, 2156d027236cSPaolo Abeni .sysctl_wmem_offset = offsetof(struct net, ipv4.sysctl_tcp_wmem), 2157d027236cSPaolo Abeni .sysctl_mem = sysctl_tcp_mem, 2158f870fa0bSMat Martineau .obj_size = sizeof(struct mptcp_sock), 21592c5ebd00SPaolo Abeni .slab_flags = SLAB_TYPESAFE_BY_RCU, 2160f870fa0bSMat Martineau .no_autobind = true, 2161f870fa0bSMat Martineau }; 2162f870fa0bSMat Martineau 21632303f994SPeter Krystad static int mptcp_bind(struct socket *sock, struct sockaddr *uaddr, int addr_len) 21642303f994SPeter Krystad { 21652303f994SPeter Krystad struct mptcp_sock *msk = mptcp_sk(sock->sk); 21662303f994SPeter Krystad struct socket *ssock; 2167cf7da0d6SPeter Krystad int err; 21682303f994SPeter Krystad 21692303f994SPeter Krystad lock_sock(sock->sk); 2170fa68018dSPaolo Abeni ssock = __mptcp_nmpc_socket(msk); 2171fa68018dSPaolo Abeni if (!ssock) { 2172fa68018dSPaolo Abeni err = -EINVAL; 21732303f994SPeter Krystad goto unlock; 21742303f994SPeter Krystad } 21752303f994SPeter Krystad 21762303f994SPeter Krystad err = ssock->ops->bind(ssock, uaddr, addr_len); 2177cf7da0d6SPeter Krystad if (!err) 2178cf7da0d6SPeter Krystad mptcp_copy_inaddrs(sock->sk, ssock->sk); 21792303f994SPeter Krystad 21802303f994SPeter Krystad unlock: 21812303f994SPeter Krystad release_sock(sock->sk); 21822303f994SPeter Krystad return err; 21832303f994SPeter Krystad } 21842303f994SPeter Krystad 21850235d075SPaolo Abeni static void mptcp_subflow_early_fallback(struct mptcp_sock *msk, 21860235d075SPaolo Abeni struct mptcp_subflow_context *subflow) 21870235d075SPaolo Abeni { 21880235d075SPaolo Abeni subflow->request_mptcp = 0; 21890235d075SPaolo Abeni __mptcp_do_fallback(msk); 21900235d075SPaolo Abeni } 21910235d075SPaolo Abeni 21922303f994SPeter Krystad static int mptcp_stream_connect(struct socket *sock, struct sockaddr *uaddr, 21932303f994SPeter Krystad int addr_len, int flags) 21942303f994SPeter Krystad { 21952303f994SPeter Krystad struct mptcp_sock *msk = mptcp_sk(sock->sk); 21962c5ebd00SPaolo Abeni struct mptcp_subflow_context *subflow; 21972303f994SPeter Krystad struct socket *ssock; 21982303f994SPeter Krystad int err; 21992303f994SPeter Krystad 22002303f994SPeter Krystad lock_sock(sock->sk); 220141be81a8SPaolo Abeni if (sock->state != SS_UNCONNECTED && msk->subflow) { 220241be81a8SPaolo Abeni /* pending connection or invalid state, let existing subflow 220341be81a8SPaolo Abeni * cope with that 220441be81a8SPaolo Abeni */ 220541be81a8SPaolo Abeni ssock = msk->subflow; 220641be81a8SPaolo Abeni goto do_connect; 220741be81a8SPaolo Abeni } 220841be81a8SPaolo Abeni 2209fa68018dSPaolo Abeni ssock = __mptcp_nmpc_socket(msk); 2210fa68018dSPaolo Abeni if (!ssock) { 2211fa68018dSPaolo Abeni err = -EINVAL; 22122303f994SPeter Krystad goto unlock; 22132303f994SPeter Krystad } 22142303f994SPeter Krystad 2215fa68018dSPaolo Abeni mptcp_token_destroy(msk); 2216fa68018dSPaolo Abeni inet_sk_state_store(sock->sk, TCP_SYN_SENT); 22172c5ebd00SPaolo Abeni subflow = mptcp_subflow_ctx(ssock->sk); 2218cf7da0d6SPeter Krystad #ifdef CONFIG_TCP_MD5SIG 2219cf7da0d6SPeter Krystad /* no MPTCP if MD5SIG is enabled on this socket or we may run out of 2220cf7da0d6SPeter Krystad * TCP option space. 2221cf7da0d6SPeter Krystad */ 2222cf7da0d6SPeter Krystad if (rcu_access_pointer(tcp_sk(ssock->sk)->md5sig_info)) 22230235d075SPaolo Abeni mptcp_subflow_early_fallback(msk, subflow); 2224cf7da0d6SPeter Krystad #endif 22252c5ebd00SPaolo Abeni if (subflow->request_mptcp && mptcp_token_new_connect(ssock->sk)) 22260235d075SPaolo Abeni mptcp_subflow_early_fallback(msk, subflow); 2227cf7da0d6SPeter Krystad 222841be81a8SPaolo Abeni do_connect: 22292303f994SPeter Krystad err = ssock->ops->connect(ssock, uaddr, addr_len, flags); 223041be81a8SPaolo Abeni sock->state = ssock->state; 223141be81a8SPaolo Abeni 223241be81a8SPaolo Abeni /* on successful connect, the msk state will be moved to established by 223341be81a8SPaolo Abeni * subflow_finish_connect() 223441be81a8SPaolo Abeni */ 2235367fe04eSMatthieu Baerts if (!err || err == -EINPROGRESS) 2236cf7da0d6SPeter Krystad mptcp_copy_inaddrs(sock->sk, ssock->sk); 223741be81a8SPaolo Abeni else 223841be81a8SPaolo Abeni inet_sk_state_store(sock->sk, inet_sk_state_load(ssock->sk)); 22392303f994SPeter Krystad 22402303f994SPeter Krystad unlock: 22412303f994SPeter Krystad release_sock(sock->sk); 22422303f994SPeter Krystad return err; 22432303f994SPeter Krystad } 22442303f994SPeter Krystad 2245cf7da0d6SPeter Krystad static int mptcp_listen(struct socket *sock, int backlog) 2246cf7da0d6SPeter Krystad { 2247cf7da0d6SPeter Krystad struct mptcp_sock *msk = mptcp_sk(sock->sk); 2248cf7da0d6SPeter Krystad struct socket *ssock; 2249cf7da0d6SPeter Krystad int err; 2250cf7da0d6SPeter Krystad 2251cf7da0d6SPeter Krystad pr_debug("msk=%p", msk); 2252cf7da0d6SPeter Krystad 2253cf7da0d6SPeter Krystad lock_sock(sock->sk); 2254fa68018dSPaolo Abeni ssock = __mptcp_nmpc_socket(msk); 2255fa68018dSPaolo Abeni if (!ssock) { 2256fa68018dSPaolo Abeni err = -EINVAL; 2257cf7da0d6SPeter Krystad goto unlock; 2258cf7da0d6SPeter Krystad } 2259cf7da0d6SPeter Krystad 2260fa68018dSPaolo Abeni mptcp_token_destroy(msk); 2261fa68018dSPaolo Abeni inet_sk_state_store(sock->sk, TCP_LISTEN); 22625e20087dSFlorian Westphal sock_set_flag(sock->sk, SOCK_RCU_FREE); 22635e20087dSFlorian Westphal 2264cf7da0d6SPeter Krystad err = ssock->ops->listen(ssock, backlog); 2265cf7da0d6SPeter Krystad inet_sk_state_store(sock->sk, inet_sk_state_load(ssock->sk)); 2266cf7da0d6SPeter Krystad if (!err) 2267cf7da0d6SPeter Krystad mptcp_copy_inaddrs(sock->sk, ssock->sk); 2268cf7da0d6SPeter Krystad 2269cf7da0d6SPeter Krystad unlock: 2270cf7da0d6SPeter Krystad release_sock(sock->sk); 2271cf7da0d6SPeter Krystad return err; 2272cf7da0d6SPeter Krystad } 2273cf7da0d6SPeter Krystad 2274cf7da0d6SPeter Krystad static int mptcp_stream_accept(struct socket *sock, struct socket *newsock, 2275cf7da0d6SPeter Krystad int flags, bool kern) 2276cf7da0d6SPeter Krystad { 2277cf7da0d6SPeter Krystad struct mptcp_sock *msk = mptcp_sk(sock->sk); 2278cf7da0d6SPeter Krystad struct socket *ssock; 2279cf7da0d6SPeter Krystad int err; 2280cf7da0d6SPeter Krystad 2281cf7da0d6SPeter Krystad pr_debug("msk=%p", msk); 2282cf7da0d6SPeter Krystad 2283cf7da0d6SPeter Krystad lock_sock(sock->sk); 2284cf7da0d6SPeter Krystad if (sock->sk->sk_state != TCP_LISTEN) 2285cf7da0d6SPeter Krystad goto unlock_fail; 2286cf7da0d6SPeter Krystad 2287cf7da0d6SPeter Krystad ssock = __mptcp_nmpc_socket(msk); 2288cf7da0d6SPeter Krystad if (!ssock) 2289cf7da0d6SPeter Krystad goto unlock_fail; 2290cf7da0d6SPeter Krystad 22918a05661bSPaolo Abeni clear_bit(MPTCP_DATA_READY, &msk->flags); 2292cf7da0d6SPeter Krystad sock_hold(ssock->sk); 2293cf7da0d6SPeter Krystad release_sock(sock->sk); 2294cf7da0d6SPeter Krystad 2295cf7da0d6SPeter Krystad err = ssock->ops->accept(sock, newsock, flags, kern); 2296d2f77c53SPaolo Abeni if (err == 0 && !mptcp_is_tcpsk(newsock->sk)) { 2297cf7da0d6SPeter Krystad struct mptcp_sock *msk = mptcp_sk(newsock->sk); 2298cf7da0d6SPeter Krystad struct mptcp_subflow_context *subflow; 2299cf7da0d6SPeter Krystad 2300cf7da0d6SPeter Krystad /* set ssk->sk_socket of accept()ed flows to mptcp socket. 2301cf7da0d6SPeter Krystad * This is needed so NOSPACE flag can be set from tcp stack. 2302cf7da0d6SPeter Krystad */ 2303ec3edaa7SPeter Krystad __mptcp_flush_join_list(msk); 2304190f8b06SGeliang Tang mptcp_for_each_subflow(msk, subflow) { 2305cf7da0d6SPeter Krystad struct sock *ssk = mptcp_subflow_tcp_sock(subflow); 2306cf7da0d6SPeter Krystad 2307cf7da0d6SPeter Krystad if (!ssk->sk_socket) 2308cf7da0d6SPeter Krystad mptcp_sock_graft(ssk, newsock); 2309cf7da0d6SPeter Krystad } 2310cf7da0d6SPeter Krystad } 2311cf7da0d6SPeter Krystad 23128a05661bSPaolo Abeni if (inet_csk_listen_poll(ssock->sk)) 23138a05661bSPaolo Abeni set_bit(MPTCP_DATA_READY, &msk->flags); 2314cf7da0d6SPeter Krystad sock_put(ssock->sk); 2315cf7da0d6SPeter Krystad return err; 2316cf7da0d6SPeter Krystad 2317cf7da0d6SPeter Krystad unlock_fail: 2318cf7da0d6SPeter Krystad release_sock(sock->sk); 2319cf7da0d6SPeter Krystad return -EINVAL; 2320cf7da0d6SPeter Krystad } 2321cf7da0d6SPeter Krystad 23228a05661bSPaolo Abeni static __poll_t mptcp_check_readable(struct mptcp_sock *msk) 23238a05661bSPaolo Abeni { 23248a05661bSPaolo Abeni return test_bit(MPTCP_DATA_READY, &msk->flags) ? EPOLLIN | EPOLLRDNORM : 23258a05661bSPaolo Abeni 0; 23268a05661bSPaolo Abeni } 23278a05661bSPaolo Abeni 23282303f994SPeter Krystad static __poll_t mptcp_poll(struct file *file, struct socket *sock, 23292303f994SPeter Krystad struct poll_table_struct *wait) 23302303f994SPeter Krystad { 23311891c4a0SFlorian Westphal struct sock *sk = sock->sk; 23328ab183deSPaolo Abeni struct mptcp_sock *msk; 23332303f994SPeter Krystad __poll_t mask = 0; 23348a05661bSPaolo Abeni int state; 23352303f994SPeter Krystad 23361891c4a0SFlorian Westphal msk = mptcp_sk(sk); 23371891c4a0SFlorian Westphal sock_poll_wait(file, sock, wait); 23381891c4a0SFlorian Westphal 23398a05661bSPaolo Abeni state = inet_sk_state_load(sk); 23406719331cSPaolo Abeni pr_debug("msk=%p state=%d flags=%lx", msk, state, msk->flags); 23418a05661bSPaolo Abeni if (state == TCP_LISTEN) 23428a05661bSPaolo Abeni return mptcp_check_readable(msk); 23438a05661bSPaolo Abeni 23448a05661bSPaolo Abeni if (state != TCP_SYN_SENT && state != TCP_SYN_RECV) { 23458a05661bSPaolo Abeni mask |= mptcp_check_readable(msk); 234663561a40SPaolo Abeni if (test_bit(MPTCP_SEND_SPACE, &msk->flags)) 23471891c4a0SFlorian Westphal mask |= EPOLLOUT | EPOLLWRNORM; 23488a05661bSPaolo Abeni } 23491891c4a0SFlorian Westphal if (sk->sk_shutdown & RCV_SHUTDOWN) 23501891c4a0SFlorian Westphal mask |= EPOLLIN | EPOLLRDNORM | EPOLLRDHUP; 23511891c4a0SFlorian Westphal 23522303f994SPeter Krystad return mask; 23532303f994SPeter Krystad } 23542303f994SPeter Krystad 235521498490SPeter Krystad static int mptcp_shutdown(struct socket *sock, int how) 235621498490SPeter Krystad { 235721498490SPeter Krystad struct mptcp_sock *msk = mptcp_sk(sock->sk); 235821498490SPeter Krystad struct mptcp_subflow_context *subflow; 235921498490SPeter Krystad int ret = 0; 236021498490SPeter Krystad 236121498490SPeter Krystad pr_debug("sk=%p, how=%d", msk, how); 236221498490SPeter Krystad 236321498490SPeter Krystad lock_sock(sock->sk); 236421498490SPeter Krystad 236521498490SPeter Krystad how++; 236621498490SPeter Krystad if ((how & ~SHUTDOWN_MASK) || !how) { 236721498490SPeter Krystad ret = -EINVAL; 236821498490SPeter Krystad goto out_unlock; 236921498490SPeter Krystad } 237021498490SPeter Krystad 237121498490SPeter Krystad if (sock->state == SS_CONNECTING) { 237221498490SPeter Krystad if ((1 << sock->sk->sk_state) & 237321498490SPeter Krystad (TCPF_SYN_SENT | TCPF_SYN_RECV | TCPF_CLOSE)) 237421498490SPeter Krystad sock->state = SS_DISCONNECTING; 237521498490SPeter Krystad else 237621498490SPeter Krystad sock->state = SS_CONNECTED; 237721498490SPeter Krystad } 237821498490SPeter Krystad 237943b54c6eSMat Martineau /* If we've already sent a FIN, or it's a closed state, skip this. */ 238043b54c6eSMat Martineau if (__mptcp_check_fallback(msk)) { 238143b54c6eSMat Martineau if (how == SHUT_WR || how == SHUT_RDWR) 238243b54c6eSMat Martineau inet_sk_state_store(sock->sk, TCP_FIN_WAIT1); 23837279da61SMat Martineau 238421498490SPeter Krystad mptcp_for_each_subflow(msk, subflow) { 238521498490SPeter Krystad struct sock *tcp_sk = mptcp_subflow_tcp_sock(subflow); 238621498490SPeter Krystad 238743b54c6eSMat Martineau mptcp_subflow_shutdown(sock->sk, tcp_sk, how); 238843b54c6eSMat Martineau } 238943b54c6eSMat Martineau } else if ((how & SEND_SHUTDOWN) && 239043b54c6eSMat Martineau ((1 << sock->sk->sk_state) & 239143b54c6eSMat Martineau (TCPF_ESTABLISHED | TCPF_SYN_SENT | 239243b54c6eSMat Martineau TCPF_SYN_RECV | TCPF_CLOSE_WAIT)) && 239343b54c6eSMat Martineau mptcp_close_state(sock->sk)) { 239443b54c6eSMat Martineau __mptcp_flush_join_list(msk); 239543b54c6eSMat Martineau 239643b54c6eSMat Martineau WRITE_ONCE(msk->write_seq, msk->write_seq + 1); 239743b54c6eSMat Martineau WRITE_ONCE(msk->snd_data_fin_enable, 1); 239843b54c6eSMat Martineau 239943b54c6eSMat Martineau mptcp_for_each_subflow(msk, subflow) { 240043b54c6eSMat Martineau struct sock *tcp_sk = mptcp_subflow_tcp_sock(subflow); 240143b54c6eSMat Martineau 240243b54c6eSMat Martineau mptcp_subflow_shutdown(sock->sk, tcp_sk, how); 240343b54c6eSMat Martineau } 240421498490SPeter Krystad } 240521498490SPeter Krystad 2406e1ff9e82SDavide Caratti /* Wake up anyone sleeping in poll. */ 2407e1ff9e82SDavide Caratti sock->sk->sk_state_change(sock->sk); 2408e1ff9e82SDavide Caratti 240921498490SPeter Krystad out_unlock: 241021498490SPeter Krystad release_sock(sock->sk); 241121498490SPeter Krystad 241221498490SPeter Krystad return ret; 241321498490SPeter Krystad } 241421498490SPeter Krystad 2415e42f1ac6SFlorian Westphal static const struct proto_ops mptcp_stream_ops = { 2416e42f1ac6SFlorian Westphal .family = PF_INET, 2417e42f1ac6SFlorian Westphal .owner = THIS_MODULE, 2418e42f1ac6SFlorian Westphal .release = inet_release, 2419e42f1ac6SFlorian Westphal .bind = mptcp_bind, 2420e42f1ac6SFlorian Westphal .connect = mptcp_stream_connect, 2421e42f1ac6SFlorian Westphal .socketpair = sock_no_socketpair, 2422e42f1ac6SFlorian Westphal .accept = mptcp_stream_accept, 2423d2f77c53SPaolo Abeni .getname = inet_getname, 2424e42f1ac6SFlorian Westphal .poll = mptcp_poll, 2425e42f1ac6SFlorian Westphal .ioctl = inet_ioctl, 2426e42f1ac6SFlorian Westphal .gettstamp = sock_gettstamp, 2427e42f1ac6SFlorian Westphal .listen = mptcp_listen, 2428e42f1ac6SFlorian Westphal .shutdown = mptcp_shutdown, 2429e42f1ac6SFlorian Westphal .setsockopt = sock_common_setsockopt, 2430e42f1ac6SFlorian Westphal .getsockopt = sock_common_getsockopt, 2431e42f1ac6SFlorian Westphal .sendmsg = inet_sendmsg, 2432e42f1ac6SFlorian Westphal .recvmsg = inet_recvmsg, 2433e42f1ac6SFlorian Westphal .mmap = sock_no_mmap, 2434e42f1ac6SFlorian Westphal .sendpage = inet_sendpage, 2435e42f1ac6SFlorian Westphal }; 24362303f994SPeter Krystad 2437f870fa0bSMat Martineau static struct inet_protosw mptcp_protosw = { 2438f870fa0bSMat Martineau .type = SOCK_STREAM, 2439f870fa0bSMat Martineau .protocol = IPPROTO_MPTCP, 2440f870fa0bSMat Martineau .prot = &mptcp_prot, 24412303f994SPeter Krystad .ops = &mptcp_stream_ops, 24422303f994SPeter Krystad .flags = INET_PROTOSW_ICSK, 2443f870fa0bSMat Martineau }; 2444f870fa0bSMat Martineau 2445d39dcecaSPaolo Abeni void __init mptcp_proto_init(void) 2446f870fa0bSMat Martineau { 24472303f994SPeter Krystad mptcp_prot.h.hashinfo = tcp_prot.h.hashinfo; 24482303f994SPeter Krystad 2449d027236cSPaolo Abeni if (percpu_counter_init(&mptcp_sockets_allocated, 0, GFP_KERNEL)) 2450d027236cSPaolo Abeni panic("Failed to allocate MPTCP pcpu counter\n"); 2451d027236cSPaolo Abeni 24522303f994SPeter Krystad mptcp_subflow_init(); 24531b1c7a0eSPeter Krystad mptcp_pm_init(); 24542c5ebd00SPaolo Abeni mptcp_token_init(); 24552303f994SPeter Krystad 2456f870fa0bSMat Martineau if (proto_register(&mptcp_prot, 1) != 0) 2457f870fa0bSMat Martineau panic("Failed to register MPTCP proto.\n"); 2458f870fa0bSMat Martineau 2459f870fa0bSMat Martineau inet_register_protosw(&mptcp_protosw); 24606771bfd9SFlorian Westphal 24616771bfd9SFlorian Westphal BUILD_BUG_ON(sizeof(struct mptcp_skb_cb) > sizeof_field(struct sk_buff, cb)); 2462f870fa0bSMat Martineau } 2463f870fa0bSMat Martineau 2464f870fa0bSMat Martineau #if IS_ENABLED(CONFIG_MPTCP_IPV6) 2465e42f1ac6SFlorian Westphal static const struct proto_ops mptcp_v6_stream_ops = { 2466e42f1ac6SFlorian Westphal .family = PF_INET6, 2467e42f1ac6SFlorian Westphal .owner = THIS_MODULE, 2468e42f1ac6SFlorian Westphal .release = inet6_release, 2469e42f1ac6SFlorian Westphal .bind = mptcp_bind, 2470e42f1ac6SFlorian Westphal .connect = mptcp_stream_connect, 2471e42f1ac6SFlorian Westphal .socketpair = sock_no_socketpair, 2472e42f1ac6SFlorian Westphal .accept = mptcp_stream_accept, 2473d2f77c53SPaolo Abeni .getname = inet6_getname, 2474e42f1ac6SFlorian Westphal .poll = mptcp_poll, 2475e42f1ac6SFlorian Westphal .ioctl = inet6_ioctl, 2476e42f1ac6SFlorian Westphal .gettstamp = sock_gettstamp, 2477e42f1ac6SFlorian Westphal .listen = mptcp_listen, 2478e42f1ac6SFlorian Westphal .shutdown = mptcp_shutdown, 2479e42f1ac6SFlorian Westphal .setsockopt = sock_common_setsockopt, 2480e42f1ac6SFlorian Westphal .getsockopt = sock_common_getsockopt, 2481e42f1ac6SFlorian Westphal .sendmsg = inet6_sendmsg, 2482e42f1ac6SFlorian Westphal .recvmsg = inet6_recvmsg, 2483e42f1ac6SFlorian Westphal .mmap = sock_no_mmap, 2484e42f1ac6SFlorian Westphal .sendpage = inet_sendpage, 2485e42f1ac6SFlorian Westphal #ifdef CONFIG_COMPAT 24863986912fSChristoph Hellwig .compat_ioctl = inet6_compat_ioctl, 2487e42f1ac6SFlorian Westphal #endif 2488e42f1ac6SFlorian Westphal }; 2489e42f1ac6SFlorian Westphal 2490f870fa0bSMat Martineau static struct proto mptcp_v6_prot; 2491f870fa0bSMat Martineau 249279c0949eSPeter Krystad static void mptcp_v6_destroy(struct sock *sk) 249379c0949eSPeter Krystad { 249479c0949eSPeter Krystad mptcp_destroy(sk); 249579c0949eSPeter Krystad inet6_destroy_sock(sk); 249679c0949eSPeter Krystad } 249779c0949eSPeter Krystad 2498f870fa0bSMat Martineau static struct inet_protosw mptcp_v6_protosw = { 2499f870fa0bSMat Martineau .type = SOCK_STREAM, 2500f870fa0bSMat Martineau .protocol = IPPROTO_MPTCP, 2501f870fa0bSMat Martineau .prot = &mptcp_v6_prot, 25022303f994SPeter Krystad .ops = &mptcp_v6_stream_ops, 2503f870fa0bSMat Martineau .flags = INET_PROTOSW_ICSK, 2504f870fa0bSMat Martineau }; 2505f870fa0bSMat Martineau 2506d39dcecaSPaolo Abeni int __init mptcp_proto_v6_init(void) 2507f870fa0bSMat Martineau { 2508f870fa0bSMat Martineau int err; 2509f870fa0bSMat Martineau 2510f870fa0bSMat Martineau mptcp_v6_prot = mptcp_prot; 2511f870fa0bSMat Martineau strcpy(mptcp_v6_prot.name, "MPTCPv6"); 2512f870fa0bSMat Martineau mptcp_v6_prot.slab = NULL; 251379c0949eSPeter Krystad mptcp_v6_prot.destroy = mptcp_v6_destroy; 2514b0519de8SFlorian Westphal mptcp_v6_prot.obj_size = sizeof(struct mptcp6_sock); 2515f870fa0bSMat Martineau 2516f870fa0bSMat Martineau err = proto_register(&mptcp_v6_prot, 1); 2517f870fa0bSMat Martineau if (err) 2518f870fa0bSMat Martineau return err; 2519f870fa0bSMat Martineau 2520f870fa0bSMat Martineau err = inet6_register_protosw(&mptcp_v6_protosw); 2521f870fa0bSMat Martineau if (err) 2522f870fa0bSMat Martineau proto_unregister(&mptcp_v6_prot); 2523f870fa0bSMat Martineau 2524f870fa0bSMat Martineau return err; 2525f870fa0bSMat Martineau } 2526f870fa0bSMat Martineau #endif 2527