1f870fa0bSMat Martineau // SPDX-License-Identifier: GPL-2.0 2f870fa0bSMat Martineau /* Multipath TCP 3f870fa0bSMat Martineau * 4f870fa0bSMat Martineau * Copyright (c) 2017 - 2019, Intel Corporation. 5f870fa0bSMat Martineau */ 6f870fa0bSMat Martineau 7f870fa0bSMat Martineau #define pr_fmt(fmt) "MPTCP: " fmt 8f870fa0bSMat Martineau 9f870fa0bSMat Martineau #include <linux/kernel.h> 10f870fa0bSMat Martineau #include <linux/module.h> 11f870fa0bSMat Martineau #include <linux/netdevice.h> 127a6a6cbcSPaolo Abeni #include <linux/sched/signal.h> 137a6a6cbcSPaolo Abeni #include <linux/atomic.h> 14f870fa0bSMat Martineau #include <net/sock.h> 15f870fa0bSMat Martineau #include <net/inet_common.h> 16f870fa0bSMat Martineau #include <net/inet_hashtables.h> 17f870fa0bSMat Martineau #include <net/protocol.h> 18f870fa0bSMat Martineau #include <net/tcp.h> 19cf7da0d6SPeter Krystad #if IS_ENABLED(CONFIG_MPTCP_IPV6) 20cf7da0d6SPeter Krystad #include <net/transp_v6.h> 21cf7da0d6SPeter Krystad #endif 22f870fa0bSMat Martineau #include <net/mptcp.h> 23f870fa0bSMat Martineau #include "protocol.h" 24f870fa0bSMat Martineau 252303f994SPeter Krystad #define MPTCP_SAME_STATE TCP_MAX_STATES 262303f994SPeter Krystad 27b0519de8SFlorian Westphal #if IS_ENABLED(CONFIG_MPTCP_IPV6) 28b0519de8SFlorian Westphal struct mptcp6_sock { 29b0519de8SFlorian Westphal struct mptcp_sock msk; 30b0519de8SFlorian Westphal struct ipv6_pinfo np; 31b0519de8SFlorian Westphal }; 32b0519de8SFlorian Westphal #endif 33b0519de8SFlorian Westphal 346771bfd9SFlorian Westphal struct mptcp_skb_cb { 356771bfd9SFlorian Westphal u32 offset; 366771bfd9SFlorian Westphal }; 376771bfd9SFlorian Westphal 386771bfd9SFlorian Westphal #define MPTCP_SKB_CB(__skb) ((struct mptcp_skb_cb *)&((__skb)->cb[0])) 396771bfd9SFlorian Westphal 40*d027236cSPaolo Abeni static struct percpu_counter mptcp_sockets_allocated; 41*d027236cSPaolo Abeni 422303f994SPeter Krystad /* If msk has an initial subflow socket, and the MP_CAPABLE handshake has not 432303f994SPeter Krystad * completed yet or has failed, return the subflow socket. 442303f994SPeter Krystad * Otherwise return NULL. 452303f994SPeter Krystad */ 462303f994SPeter Krystad static struct socket *__mptcp_nmpc_socket(const struct mptcp_sock *msk) 472303f994SPeter Krystad { 48d22f4988SChristoph Paasch if (!msk->subflow || READ_ONCE(msk->can_ack)) 492303f994SPeter Krystad return NULL; 502303f994SPeter Krystad 512303f994SPeter Krystad return msk->subflow; 522303f994SPeter Krystad } 532303f994SPeter Krystad 548ab183deSPaolo Abeni static bool __mptcp_needs_tcp_fallback(const struct mptcp_sock *msk) 558ab183deSPaolo Abeni { 568ab183deSPaolo Abeni return msk->first && !sk_is_mptcp(msk->first); 578ab183deSPaolo Abeni } 588ab183deSPaolo Abeni 598ab183deSPaolo Abeni static struct socket *__mptcp_tcp_fallback(struct mptcp_sock *msk) 60cec37a6eSPeter Krystad { 61cec37a6eSPeter Krystad sock_owned_by_me((const struct sock *)msk); 62cec37a6eSPeter Krystad 638ab183deSPaolo Abeni if (likely(!__mptcp_needs_tcp_fallback(msk))) 64cec37a6eSPeter Krystad return NULL; 65cec37a6eSPeter Krystad 668ab183deSPaolo Abeni if (msk->subflow) { 672c22c06cSFlorian Westphal release_sock((struct sock *)msk); 682c22c06cSFlorian Westphal return msk->subflow; 698ab183deSPaolo Abeni } 708ab183deSPaolo Abeni 712c22c06cSFlorian Westphal return NULL; 72cec37a6eSPeter Krystad } 73cec37a6eSPeter Krystad 742303f994SPeter Krystad static bool __mptcp_can_create_subflow(const struct mptcp_sock *msk) 752303f994SPeter Krystad { 768ab183deSPaolo Abeni return !msk->first; 772303f994SPeter Krystad } 782303f994SPeter Krystad 792303f994SPeter Krystad static struct socket *__mptcp_socket_create(struct mptcp_sock *msk, int state) 802303f994SPeter Krystad { 812303f994SPeter Krystad struct mptcp_subflow_context *subflow; 822303f994SPeter Krystad struct sock *sk = (struct sock *)msk; 832303f994SPeter Krystad struct socket *ssock; 842303f994SPeter Krystad int err; 852303f994SPeter Krystad 862303f994SPeter Krystad ssock = __mptcp_nmpc_socket(msk); 872303f994SPeter Krystad if (ssock) 882303f994SPeter Krystad goto set_state; 892303f994SPeter Krystad 902303f994SPeter Krystad if (!__mptcp_can_create_subflow(msk)) 912303f994SPeter Krystad return ERR_PTR(-EINVAL); 922303f994SPeter Krystad 932303f994SPeter Krystad err = mptcp_subflow_create_socket(sk, &ssock); 942303f994SPeter Krystad if (err) 952303f994SPeter Krystad return ERR_PTR(err); 962303f994SPeter Krystad 978ab183deSPaolo Abeni msk->first = ssock->sk; 982303f994SPeter Krystad msk->subflow = ssock; 992303f994SPeter Krystad subflow = mptcp_subflow_ctx(ssock->sk); 100cec37a6eSPeter Krystad list_add(&subflow->node, &msk->conn_list); 1012303f994SPeter Krystad subflow->request_mptcp = 1; 1022303f994SPeter Krystad 1032303f994SPeter Krystad set_state: 1042303f994SPeter Krystad if (state != MPTCP_SAME_STATE) 1052303f994SPeter Krystad inet_sk_state_store(sk, state); 1062303f994SPeter Krystad return ssock; 1072303f994SPeter Krystad } 1082303f994SPeter Krystad 1096771bfd9SFlorian Westphal static void __mptcp_move_skb(struct mptcp_sock *msk, struct sock *ssk, 1106771bfd9SFlorian Westphal struct sk_buff *skb, 1116771bfd9SFlorian Westphal unsigned int offset, size_t copy_len) 1126771bfd9SFlorian Westphal { 1136771bfd9SFlorian Westphal struct sock *sk = (struct sock *)msk; 1146771bfd9SFlorian Westphal 1156771bfd9SFlorian Westphal __skb_unlink(skb, &ssk->sk_receive_queue); 116600911ffSFlorian Westphal skb_set_owner_r(skb, sk); 1176771bfd9SFlorian Westphal __skb_queue_tail(&sk->sk_receive_queue, skb); 1186771bfd9SFlorian Westphal 1196771bfd9SFlorian Westphal msk->ack_seq += copy_len; 1206771bfd9SFlorian Westphal MPTCP_SKB_CB(skb)->offset = offset; 1216771bfd9SFlorian Westphal } 1226771bfd9SFlorian Westphal 1236771bfd9SFlorian Westphal static bool __mptcp_move_skbs_from_subflow(struct mptcp_sock *msk, 1246771bfd9SFlorian Westphal struct sock *ssk, 1256771bfd9SFlorian Westphal unsigned int *bytes) 1266771bfd9SFlorian Westphal { 1276771bfd9SFlorian Westphal struct mptcp_subflow_context *subflow = mptcp_subflow_ctx(ssk); 128600911ffSFlorian Westphal struct sock *sk = (struct sock *)msk; 1296771bfd9SFlorian Westphal unsigned int moved = 0; 1306771bfd9SFlorian Westphal bool more_data_avail; 1316771bfd9SFlorian Westphal struct tcp_sock *tp; 1326771bfd9SFlorian Westphal bool done = false; 133600911ffSFlorian Westphal 134ec33916dSFlorian Westphal if (!(sk->sk_userlocks & SOCK_RCVBUF_LOCK)) { 135ec33916dSFlorian Westphal int rcvbuf = max(ssk->sk_rcvbuf, sk->sk_rcvbuf); 136ec33916dSFlorian Westphal 137600911ffSFlorian Westphal if (rcvbuf > sk->sk_rcvbuf) 138600911ffSFlorian Westphal sk->sk_rcvbuf = rcvbuf; 139ec33916dSFlorian Westphal } 1406771bfd9SFlorian Westphal 1416771bfd9SFlorian Westphal tp = tcp_sk(ssk); 1426771bfd9SFlorian Westphal do { 1436771bfd9SFlorian Westphal u32 map_remaining, offset; 1446771bfd9SFlorian Westphal u32 seq = tp->copied_seq; 1456771bfd9SFlorian Westphal struct sk_buff *skb; 1466771bfd9SFlorian Westphal bool fin; 1476771bfd9SFlorian Westphal 1486771bfd9SFlorian Westphal /* try to move as much data as available */ 1496771bfd9SFlorian Westphal map_remaining = subflow->map_data_len - 1506771bfd9SFlorian Westphal mptcp_subflow_get_map_offset(subflow); 1516771bfd9SFlorian Westphal 1526771bfd9SFlorian Westphal skb = skb_peek(&ssk->sk_receive_queue); 1536771bfd9SFlorian Westphal if (!skb) 1546771bfd9SFlorian Westphal break; 1556771bfd9SFlorian Westphal 1566771bfd9SFlorian Westphal offset = seq - TCP_SKB_CB(skb)->seq; 1576771bfd9SFlorian Westphal fin = TCP_SKB_CB(skb)->tcp_flags & TCPHDR_FIN; 1586771bfd9SFlorian Westphal if (fin) { 1596771bfd9SFlorian Westphal done = true; 1606771bfd9SFlorian Westphal seq++; 1616771bfd9SFlorian Westphal } 1626771bfd9SFlorian Westphal 1636771bfd9SFlorian Westphal if (offset < skb->len) { 1646771bfd9SFlorian Westphal size_t len = skb->len - offset; 1656771bfd9SFlorian Westphal 1666771bfd9SFlorian Westphal if (tp->urg_data) 1676771bfd9SFlorian Westphal done = true; 1686771bfd9SFlorian Westphal 1696771bfd9SFlorian Westphal __mptcp_move_skb(msk, ssk, skb, offset, len); 1706771bfd9SFlorian Westphal seq += len; 1716771bfd9SFlorian Westphal moved += len; 1726771bfd9SFlorian Westphal 1736771bfd9SFlorian Westphal if (WARN_ON_ONCE(map_remaining < len)) 1746771bfd9SFlorian Westphal break; 1756771bfd9SFlorian Westphal } else { 1766771bfd9SFlorian Westphal WARN_ON_ONCE(!fin); 1776771bfd9SFlorian Westphal sk_eat_skb(ssk, skb); 1786771bfd9SFlorian Westphal done = true; 1796771bfd9SFlorian Westphal } 1806771bfd9SFlorian Westphal 1816771bfd9SFlorian Westphal WRITE_ONCE(tp->copied_seq, seq); 1826771bfd9SFlorian Westphal more_data_avail = mptcp_subflow_data_available(ssk); 183600911ffSFlorian Westphal 184600911ffSFlorian Westphal if (atomic_read(&sk->sk_rmem_alloc) > READ_ONCE(sk->sk_rcvbuf)) { 185600911ffSFlorian Westphal done = true; 186600911ffSFlorian Westphal break; 187600911ffSFlorian Westphal } 1886771bfd9SFlorian Westphal } while (more_data_avail); 1896771bfd9SFlorian Westphal 1906771bfd9SFlorian Westphal *bytes = moved; 1916771bfd9SFlorian Westphal 1926771bfd9SFlorian Westphal return done; 1936771bfd9SFlorian Westphal } 1946771bfd9SFlorian Westphal 1952e52213cSFlorian Westphal /* In most cases we will be able to lock the mptcp socket. If its already 1962e52213cSFlorian Westphal * owned, we need to defer to the work queue to avoid ABBA deadlock. 1972e52213cSFlorian Westphal */ 1982e52213cSFlorian Westphal static bool move_skbs_to_msk(struct mptcp_sock *msk, struct sock *ssk) 1992e52213cSFlorian Westphal { 2002e52213cSFlorian Westphal struct sock *sk = (struct sock *)msk; 2012e52213cSFlorian Westphal unsigned int moved = 0; 2022e52213cSFlorian Westphal 2032e52213cSFlorian Westphal if (READ_ONCE(sk->sk_lock.owned)) 2042e52213cSFlorian Westphal return false; 2052e52213cSFlorian Westphal 2062e52213cSFlorian Westphal if (unlikely(!spin_trylock_bh(&sk->sk_lock.slock))) 2072e52213cSFlorian Westphal return false; 2082e52213cSFlorian Westphal 2092e52213cSFlorian Westphal /* must re-check after taking the lock */ 2102e52213cSFlorian Westphal if (!READ_ONCE(sk->sk_lock.owned)) 2112e52213cSFlorian Westphal __mptcp_move_skbs_from_subflow(msk, ssk, &moved); 2122e52213cSFlorian Westphal 2132e52213cSFlorian Westphal spin_unlock_bh(&sk->sk_lock.slock); 2142e52213cSFlorian Westphal 2152e52213cSFlorian Westphal return moved > 0; 2162e52213cSFlorian Westphal } 2172e52213cSFlorian Westphal 2182e52213cSFlorian Westphal void mptcp_data_ready(struct sock *sk, struct sock *ssk) 219101f6f85SFlorian Westphal { 220101f6f85SFlorian Westphal struct mptcp_sock *msk = mptcp_sk(sk); 221101f6f85SFlorian Westphal 222101f6f85SFlorian Westphal set_bit(MPTCP_DATA_READY, &msk->flags); 2236771bfd9SFlorian Westphal 2242e52213cSFlorian Westphal if (atomic_read(&sk->sk_rmem_alloc) < READ_ONCE(sk->sk_rcvbuf) && 2252e52213cSFlorian Westphal move_skbs_to_msk(msk, ssk)) 2262e52213cSFlorian Westphal goto wake; 2272e52213cSFlorian Westphal 228600911ffSFlorian Westphal /* don't schedule if mptcp sk is (still) over limit */ 229600911ffSFlorian Westphal if (atomic_read(&sk->sk_rmem_alloc) > READ_ONCE(sk->sk_rcvbuf)) 230600911ffSFlorian Westphal goto wake; 231600911ffSFlorian Westphal 23214c441b5SPaolo Abeni /* mptcp socket is owned, release_cb should retry */ 23314c441b5SPaolo Abeni if (!test_and_set_bit(TCP_DELACK_TIMER_DEFERRED, 23414c441b5SPaolo Abeni &sk->sk_tsq_flags)) { 23514c441b5SPaolo Abeni sock_hold(sk); 2366771bfd9SFlorian Westphal 23714c441b5SPaolo Abeni /* need to try again, its possible release_cb() has already 23814c441b5SPaolo Abeni * been called after the test_and_set_bit() above. 23914c441b5SPaolo Abeni */ 24014c441b5SPaolo Abeni move_skbs_to_msk(msk, ssk); 24114c441b5SPaolo Abeni } 242600911ffSFlorian Westphal wake: 243101f6f85SFlorian Westphal sk->sk_data_ready(sk); 244101f6f85SFlorian Westphal } 245101f6f85SFlorian Westphal 246ec3edaa7SPeter Krystad static void __mptcp_flush_join_list(struct mptcp_sock *msk) 247ec3edaa7SPeter Krystad { 248ec3edaa7SPeter Krystad if (likely(list_empty(&msk->join_list))) 249ec3edaa7SPeter Krystad return; 250ec3edaa7SPeter Krystad 251ec3edaa7SPeter Krystad spin_lock_bh(&msk->join_list_lock); 252ec3edaa7SPeter Krystad list_splice_tail_init(&msk->join_list, &msk->conn_list); 253ec3edaa7SPeter Krystad spin_unlock_bh(&msk->join_list_lock); 254ec3edaa7SPeter Krystad } 255ec3edaa7SPeter Krystad 256b51f9b80SPaolo Abeni static void mptcp_set_timeout(const struct sock *sk, const struct sock *ssk) 257b51f9b80SPaolo Abeni { 258b51f9b80SPaolo Abeni long tout = ssk && inet_csk(ssk)->icsk_pending ? 259b51f9b80SPaolo Abeni inet_csk(ssk)->icsk_timeout - jiffies : 0; 260b51f9b80SPaolo Abeni 261b51f9b80SPaolo Abeni if (tout <= 0) 262b51f9b80SPaolo Abeni tout = mptcp_sk(sk)->timer_ival; 263b51f9b80SPaolo Abeni mptcp_sk(sk)->timer_ival = tout > 0 ? tout : TCP_RTO_MIN; 264b51f9b80SPaolo Abeni } 265b51f9b80SPaolo Abeni 266b51f9b80SPaolo Abeni static bool mptcp_timer_pending(struct sock *sk) 267b51f9b80SPaolo Abeni { 268b51f9b80SPaolo Abeni return timer_pending(&inet_csk(sk)->icsk_retransmit_timer); 269b51f9b80SPaolo Abeni } 270b51f9b80SPaolo Abeni 271b51f9b80SPaolo Abeni static void mptcp_reset_timer(struct sock *sk) 272b51f9b80SPaolo Abeni { 273b51f9b80SPaolo Abeni struct inet_connection_sock *icsk = inet_csk(sk); 274b51f9b80SPaolo Abeni unsigned long tout; 275b51f9b80SPaolo Abeni 276b51f9b80SPaolo Abeni /* should never be called with mptcp level timer cleared */ 277b51f9b80SPaolo Abeni tout = READ_ONCE(mptcp_sk(sk)->timer_ival); 278b51f9b80SPaolo Abeni if (WARN_ON_ONCE(!tout)) 279b51f9b80SPaolo Abeni tout = TCP_RTO_MIN; 280b51f9b80SPaolo Abeni sk_reset_timer(sk, &icsk->icsk_retransmit_timer, jiffies + tout); 281b51f9b80SPaolo Abeni } 282b51f9b80SPaolo Abeni 283b51f9b80SPaolo Abeni void mptcp_data_acked(struct sock *sk) 284b51f9b80SPaolo Abeni { 285b51f9b80SPaolo Abeni mptcp_reset_timer(sk); 286b51f9b80SPaolo Abeni } 287b51f9b80SPaolo Abeni 288b51f9b80SPaolo Abeni static void mptcp_stop_timer(struct sock *sk) 289b51f9b80SPaolo Abeni { 290b51f9b80SPaolo Abeni struct inet_connection_sock *icsk = inet_csk(sk); 291b51f9b80SPaolo Abeni 292b51f9b80SPaolo Abeni sk_stop_timer(sk, &icsk->icsk_retransmit_timer); 293b51f9b80SPaolo Abeni mptcp_sk(sk)->timer_ival = 0; 294b51f9b80SPaolo Abeni } 295b51f9b80SPaolo Abeni 2966d0060f6SMat Martineau static bool mptcp_ext_cache_refill(struct mptcp_sock *msk) 2976d0060f6SMat Martineau { 2986d0060f6SMat Martineau if (!msk->cached_ext) 2996d0060f6SMat Martineau msk->cached_ext = __skb_ext_alloc(); 3006d0060f6SMat Martineau 3016d0060f6SMat Martineau return !!msk->cached_ext; 3026d0060f6SMat Martineau } 3036d0060f6SMat Martineau 3047a6a6cbcSPaolo Abeni static struct sock *mptcp_subflow_recv_lookup(const struct mptcp_sock *msk) 3057a6a6cbcSPaolo Abeni { 3067a6a6cbcSPaolo Abeni struct mptcp_subflow_context *subflow; 3077a6a6cbcSPaolo Abeni struct sock *sk = (struct sock *)msk; 3087a6a6cbcSPaolo Abeni 3097a6a6cbcSPaolo Abeni sock_owned_by_me(sk); 3107a6a6cbcSPaolo Abeni 3117a6a6cbcSPaolo Abeni mptcp_for_each_subflow(msk, subflow) { 3127a6a6cbcSPaolo Abeni if (subflow->data_avail) 3137a6a6cbcSPaolo Abeni return mptcp_subflow_tcp_sock(subflow); 3147a6a6cbcSPaolo Abeni } 3157a6a6cbcSPaolo Abeni 3167a6a6cbcSPaolo Abeni return NULL; 3177a6a6cbcSPaolo Abeni } 3187a6a6cbcSPaolo Abeni 31957040755SPaolo Abeni static inline bool mptcp_skb_can_collapse_to(const struct mptcp_sock *msk, 32057040755SPaolo Abeni const struct sk_buff *skb, 32157040755SPaolo Abeni const struct mptcp_ext *mpext) 3226d0060f6SMat Martineau { 32357040755SPaolo Abeni if (!tcp_skb_can_collapse_to(skb)) 32457040755SPaolo Abeni return false; 32557040755SPaolo Abeni 32657040755SPaolo Abeni /* can collapse only if MPTCP level sequence is in order */ 32757040755SPaolo Abeni return mpext && mpext->data_seq + mpext->data_len == msk->write_seq; 32857040755SPaolo Abeni } 32957040755SPaolo Abeni 33018b683bfSPaolo Abeni static bool mptcp_frag_can_collapse_to(const struct mptcp_sock *msk, 33118b683bfSPaolo Abeni const struct page_frag *pfrag, 33218b683bfSPaolo Abeni const struct mptcp_data_frag *df) 33318b683bfSPaolo Abeni { 33418b683bfSPaolo Abeni return df && pfrag->page == df->page && 33518b683bfSPaolo Abeni df->data_seq + df->data_len == msk->write_seq; 33618b683bfSPaolo Abeni } 33718b683bfSPaolo Abeni 338*d027236cSPaolo Abeni static void dfrag_uncharge(struct sock *sk, int len) 33918b683bfSPaolo Abeni { 340*d027236cSPaolo Abeni sk_mem_uncharge(sk, len); 341*d027236cSPaolo Abeni } 342*d027236cSPaolo Abeni 343*d027236cSPaolo Abeni static void dfrag_clear(struct sock *sk, struct mptcp_data_frag *dfrag) 344*d027236cSPaolo Abeni { 345*d027236cSPaolo Abeni int len = dfrag->data_len + dfrag->overhead; 346*d027236cSPaolo Abeni 34718b683bfSPaolo Abeni list_del(&dfrag->list); 348*d027236cSPaolo Abeni dfrag_uncharge(sk, len); 34918b683bfSPaolo Abeni put_page(dfrag->page); 35018b683bfSPaolo Abeni } 35118b683bfSPaolo Abeni 35218b683bfSPaolo Abeni static void mptcp_clean_una(struct sock *sk) 35318b683bfSPaolo Abeni { 35418b683bfSPaolo Abeni struct mptcp_sock *msk = mptcp_sk(sk); 35518b683bfSPaolo Abeni struct mptcp_data_frag *dtmp, *dfrag; 35618b683bfSPaolo Abeni u64 snd_una = atomic64_read(&msk->snd_una); 357*d027236cSPaolo Abeni bool cleaned = false; 35818b683bfSPaolo Abeni 35918b683bfSPaolo Abeni list_for_each_entry_safe(dfrag, dtmp, &msk->rtx_queue, list) { 36018b683bfSPaolo Abeni if (after64(dfrag->data_seq + dfrag->data_len, snd_una)) 36118b683bfSPaolo Abeni break; 36218b683bfSPaolo Abeni 363*d027236cSPaolo Abeni dfrag_clear(sk, dfrag); 364*d027236cSPaolo Abeni cleaned = true; 365*d027236cSPaolo Abeni } 366*d027236cSPaolo Abeni 367*d027236cSPaolo Abeni if (cleaned) { 368*d027236cSPaolo Abeni sk_mem_reclaim_partial(sk); 36918b683bfSPaolo Abeni } 37018b683bfSPaolo Abeni } 37118b683bfSPaolo Abeni 37218b683bfSPaolo Abeni /* ensure we get enough memory for the frag hdr, beyond some minimal amount of 37318b683bfSPaolo Abeni * data 37418b683bfSPaolo Abeni */ 37518b683bfSPaolo Abeni static bool mptcp_page_frag_refill(struct sock *sk, struct page_frag *pfrag) 37618b683bfSPaolo Abeni { 37718b683bfSPaolo Abeni if (likely(skb_page_frag_refill(32U + sizeof(struct mptcp_data_frag), 37818b683bfSPaolo Abeni pfrag, sk->sk_allocation))) 37918b683bfSPaolo Abeni return true; 38018b683bfSPaolo Abeni 38118b683bfSPaolo Abeni sk->sk_prot->enter_memory_pressure(sk); 38218b683bfSPaolo Abeni sk_stream_moderate_sndbuf(sk); 38318b683bfSPaolo Abeni return false; 38418b683bfSPaolo Abeni } 38518b683bfSPaolo Abeni 38618b683bfSPaolo Abeni static struct mptcp_data_frag * 38718b683bfSPaolo Abeni mptcp_carve_data_frag(const struct mptcp_sock *msk, struct page_frag *pfrag, 38818b683bfSPaolo Abeni int orig_offset) 38918b683bfSPaolo Abeni { 39018b683bfSPaolo Abeni int offset = ALIGN(orig_offset, sizeof(long)); 39118b683bfSPaolo Abeni struct mptcp_data_frag *dfrag; 39218b683bfSPaolo Abeni 39318b683bfSPaolo Abeni dfrag = (struct mptcp_data_frag *)(page_to_virt(pfrag->page) + offset); 39418b683bfSPaolo Abeni dfrag->data_len = 0; 39518b683bfSPaolo Abeni dfrag->data_seq = msk->write_seq; 39618b683bfSPaolo Abeni dfrag->overhead = offset - orig_offset + sizeof(struct mptcp_data_frag); 39718b683bfSPaolo Abeni dfrag->offset = offset + sizeof(struct mptcp_data_frag); 39818b683bfSPaolo Abeni dfrag->page = pfrag->page; 39918b683bfSPaolo Abeni 40018b683bfSPaolo Abeni return dfrag; 40118b683bfSPaolo Abeni } 40218b683bfSPaolo Abeni 40357040755SPaolo Abeni static int mptcp_sendmsg_frag(struct sock *sk, struct sock *ssk, 40457040755SPaolo Abeni struct msghdr *msg, long *timeo, int *pmss_now, 40557040755SPaolo Abeni int *ps_goal) 40657040755SPaolo Abeni { 40718b683bfSPaolo Abeni int mss_now, avail_size, size_goal, offset, ret, frag_truesize = 0; 40818b683bfSPaolo Abeni bool dfrag_collapsed, can_collapse = false; 4096d0060f6SMat Martineau struct mptcp_sock *msk = mptcp_sk(sk); 4106d0060f6SMat Martineau struct mptcp_ext *mpext = NULL; 41118b683bfSPaolo Abeni struct mptcp_data_frag *dfrag; 41257040755SPaolo Abeni struct sk_buff *skb, *tail; 4136d0060f6SMat Martineau struct page_frag *pfrag; 4146d0060f6SMat Martineau size_t psize; 4156d0060f6SMat Martineau 4166d0060f6SMat Martineau /* use the mptcp page cache so that we can easily move the data 4176d0060f6SMat Martineau * from one substream to another, but do per subflow memory accounting 4186d0060f6SMat Martineau */ 4196d0060f6SMat Martineau pfrag = sk_page_frag(sk); 42018b683bfSPaolo Abeni while (!mptcp_page_frag_refill(ssk, pfrag) || 4216d0060f6SMat Martineau !mptcp_ext_cache_refill(msk)) { 4226d0060f6SMat Martineau ret = sk_stream_wait_memory(ssk, timeo); 4236d0060f6SMat Martineau if (ret) 4246d0060f6SMat Martineau return ret; 42518b683bfSPaolo Abeni 42618b683bfSPaolo Abeni /* if sk_stream_wait_memory() sleeps snd_una can change 42718b683bfSPaolo Abeni * significantly, refresh the rtx queue 42818b683bfSPaolo Abeni */ 42918b683bfSPaolo Abeni mptcp_clean_una(sk); 43018b683bfSPaolo Abeni 4318ab183deSPaolo Abeni if (unlikely(__mptcp_needs_tcp_fallback(msk))) 4328ab183deSPaolo Abeni return 0; 4336d0060f6SMat Martineau } 4346d0060f6SMat Martineau 4356d0060f6SMat Martineau /* compute copy limit */ 4366d0060f6SMat Martineau mss_now = tcp_send_mss(ssk, &size_goal, msg->msg_flags); 43757040755SPaolo Abeni *pmss_now = mss_now; 43857040755SPaolo Abeni *ps_goal = size_goal; 43957040755SPaolo Abeni avail_size = size_goal; 44057040755SPaolo Abeni skb = tcp_write_queue_tail(ssk); 44157040755SPaolo Abeni if (skb) { 44257040755SPaolo Abeni mpext = skb_ext_find(skb, SKB_EXT_MPTCP); 4436d0060f6SMat Martineau 44457040755SPaolo Abeni /* Limit the write to the size available in the 44557040755SPaolo Abeni * current skb, if any, so that we create at most a new skb. 44657040755SPaolo Abeni * Explicitly tells TCP internals to avoid collapsing on later 44757040755SPaolo Abeni * queue management operation, to avoid breaking the ext <-> 44857040755SPaolo Abeni * SSN association set here 44957040755SPaolo Abeni */ 45057040755SPaolo Abeni can_collapse = (size_goal - skb->len > 0) && 45157040755SPaolo Abeni mptcp_skb_can_collapse_to(msk, skb, mpext); 45257040755SPaolo Abeni if (!can_collapse) 45357040755SPaolo Abeni TCP_SKB_CB(skb)->eor = 1; 45457040755SPaolo Abeni else 45557040755SPaolo Abeni avail_size = size_goal - skb->len; 45657040755SPaolo Abeni } 45718b683bfSPaolo Abeni 45818b683bfSPaolo Abeni /* reuse tail pfrag, if possible, or carve a new one from the page 45918b683bfSPaolo Abeni * allocator 46018b683bfSPaolo Abeni */ 46118b683bfSPaolo Abeni dfrag = mptcp_rtx_tail(sk); 46218b683bfSPaolo Abeni offset = pfrag->offset; 46318b683bfSPaolo Abeni dfrag_collapsed = mptcp_frag_can_collapse_to(msk, pfrag, dfrag); 46418b683bfSPaolo Abeni if (!dfrag_collapsed) { 46518b683bfSPaolo Abeni dfrag = mptcp_carve_data_frag(msk, pfrag, offset); 46618b683bfSPaolo Abeni offset = dfrag->offset; 46718b683bfSPaolo Abeni frag_truesize = dfrag->overhead; 46818b683bfSPaolo Abeni } 46918b683bfSPaolo Abeni psize = min_t(size_t, pfrag->size - offset, avail_size); 47057040755SPaolo Abeni 47157040755SPaolo Abeni /* Copy to page */ 4726d0060f6SMat Martineau pr_debug("left=%zu", msg_data_left(msg)); 47318b683bfSPaolo Abeni psize = copy_page_from_iter(pfrag->page, offset, 4746d0060f6SMat Martineau min_t(size_t, msg_data_left(msg), psize), 4756d0060f6SMat Martineau &msg->msg_iter); 4766d0060f6SMat Martineau pr_debug("left=%zu", msg_data_left(msg)); 4776d0060f6SMat Martineau if (!psize) 4786d0060f6SMat Martineau return -EINVAL; 4796d0060f6SMat Martineau 480*d027236cSPaolo Abeni if (!sk_wmem_schedule(sk, psize + dfrag->overhead)) 481*d027236cSPaolo Abeni return -ENOMEM; 482*d027236cSPaolo Abeni 48357040755SPaolo Abeni /* tell the TCP stack to delay the push so that we can safely 48457040755SPaolo Abeni * access the skb after the sendpages call 4856d0060f6SMat Martineau */ 48618b683bfSPaolo Abeni ret = do_tcp_sendpages(ssk, pfrag->page, offset, psize, 4876d0060f6SMat Martineau msg->msg_flags | MSG_SENDPAGE_NOTLAST); 4886d0060f6SMat Martineau if (ret <= 0) 4896d0060f6SMat Martineau return ret; 49018b683bfSPaolo Abeni 49118b683bfSPaolo Abeni frag_truesize += ret; 4926d0060f6SMat Martineau if (unlikely(ret < psize)) 4936d0060f6SMat Martineau iov_iter_revert(&msg->msg_iter, psize - ret); 4946d0060f6SMat Martineau 49518b683bfSPaolo Abeni /* send successful, keep track of sent data for mptcp-level 49618b683bfSPaolo Abeni * retransmission 49718b683bfSPaolo Abeni */ 49818b683bfSPaolo Abeni dfrag->data_len += ret; 49918b683bfSPaolo Abeni if (!dfrag_collapsed) { 50018b683bfSPaolo Abeni get_page(dfrag->page); 50118b683bfSPaolo Abeni list_add_tail(&dfrag->list, &msk->rtx_queue); 50218b683bfSPaolo Abeni } 50318b683bfSPaolo Abeni 504*d027236cSPaolo Abeni /* charge data on mptcp rtx queue to the master socket 505*d027236cSPaolo Abeni * Note: we charge such data both to sk and ssk 506*d027236cSPaolo Abeni */ 507*d027236cSPaolo Abeni sk->sk_forward_alloc -= frag_truesize; 508*d027236cSPaolo Abeni 50957040755SPaolo Abeni /* if the tail skb extension is still the cached one, collapsing 51057040755SPaolo Abeni * really happened. Note: we can't check for 'same skb' as the sk_buff 51157040755SPaolo Abeni * hdr on tail can be transmitted, freed and re-allocated by the 51257040755SPaolo Abeni * do_tcp_sendpages() call 51357040755SPaolo Abeni */ 51457040755SPaolo Abeni tail = tcp_write_queue_tail(ssk); 51557040755SPaolo Abeni if (mpext && tail && mpext == skb_ext_find(tail, SKB_EXT_MPTCP)) { 51657040755SPaolo Abeni WARN_ON_ONCE(!can_collapse); 51757040755SPaolo Abeni mpext->data_len += ret; 51857040755SPaolo Abeni goto out; 51957040755SPaolo Abeni } 52057040755SPaolo Abeni 5216d0060f6SMat Martineau skb = tcp_write_queue_tail(ssk); 5226d0060f6SMat Martineau mpext = __skb_ext_set(skb, SKB_EXT_MPTCP, msk->cached_ext); 5236d0060f6SMat Martineau msk->cached_ext = NULL; 5246d0060f6SMat Martineau 5256d0060f6SMat Martineau memset(mpext, 0, sizeof(*mpext)); 5266d0060f6SMat Martineau mpext->data_seq = msk->write_seq; 5276d0060f6SMat Martineau mpext->subflow_seq = mptcp_subflow_ctx(ssk)->rel_write_seq; 5286d0060f6SMat Martineau mpext->data_len = ret; 5296d0060f6SMat Martineau mpext->use_map = 1; 5306d0060f6SMat Martineau mpext->dsn64 = 1; 5316d0060f6SMat Martineau 5326d0060f6SMat Martineau pr_debug("data_seq=%llu subflow_seq=%u data_len=%u dsn64=%d", 5336d0060f6SMat Martineau mpext->data_seq, mpext->subflow_seq, mpext->data_len, 5346d0060f6SMat Martineau mpext->dsn64); 5356d0060f6SMat Martineau 53657040755SPaolo Abeni out: 53718b683bfSPaolo Abeni pfrag->offset += frag_truesize; 5386d0060f6SMat Martineau msk->write_seq += ret; 5396d0060f6SMat Martineau mptcp_subflow_ctx(ssk)->rel_write_seq += ret; 5406d0060f6SMat Martineau 5416d0060f6SMat Martineau return ret; 5426d0060f6SMat Martineau } 5436d0060f6SMat Martineau 544f296234cSPeter Krystad static struct sock *mptcp_subflow_get_send(struct mptcp_sock *msk) 545f296234cSPeter Krystad { 546f296234cSPeter Krystad struct mptcp_subflow_context *subflow; 547f296234cSPeter Krystad struct sock *backup = NULL; 548f296234cSPeter Krystad 549f296234cSPeter Krystad sock_owned_by_me((const struct sock *)msk); 550f296234cSPeter Krystad 551f296234cSPeter Krystad mptcp_for_each_subflow(msk, subflow) { 552f296234cSPeter Krystad struct sock *ssk = mptcp_subflow_tcp_sock(subflow); 553f296234cSPeter Krystad 554f296234cSPeter Krystad if (!sk_stream_memory_free(ssk)) { 555f296234cSPeter Krystad struct socket *sock = ssk->sk_socket; 556f296234cSPeter Krystad 557f296234cSPeter Krystad if (sock) { 558f296234cSPeter Krystad clear_bit(MPTCP_SEND_SPACE, &msk->flags); 559f296234cSPeter Krystad smp_mb__after_atomic(); 560f296234cSPeter Krystad 561f296234cSPeter Krystad /* enables sk->write_space() callbacks */ 562f296234cSPeter Krystad set_bit(SOCK_NOSPACE, &sock->flags); 563f296234cSPeter Krystad } 564f296234cSPeter Krystad 565f296234cSPeter Krystad return NULL; 566f296234cSPeter Krystad } 567f296234cSPeter Krystad 568f296234cSPeter Krystad if (subflow->backup) { 569f296234cSPeter Krystad if (!backup) 570f296234cSPeter Krystad backup = ssk; 571f296234cSPeter Krystad 572f296234cSPeter Krystad continue; 573f296234cSPeter Krystad } 574f296234cSPeter Krystad 575f296234cSPeter Krystad return ssk; 576f296234cSPeter Krystad } 577f296234cSPeter Krystad 578f296234cSPeter Krystad return backup; 579f296234cSPeter Krystad } 580f296234cSPeter Krystad 5811891c4a0SFlorian Westphal static void ssk_check_wmem(struct mptcp_sock *msk, struct sock *ssk) 5821891c4a0SFlorian Westphal { 5831891c4a0SFlorian Westphal struct socket *sock; 5841891c4a0SFlorian Westphal 5851891c4a0SFlorian Westphal if (likely(sk_stream_is_writeable(ssk))) 5861891c4a0SFlorian Westphal return; 5871891c4a0SFlorian Westphal 5881891c4a0SFlorian Westphal sock = READ_ONCE(ssk->sk_socket); 5891891c4a0SFlorian Westphal 5901891c4a0SFlorian Westphal if (sock) { 5911891c4a0SFlorian Westphal clear_bit(MPTCP_SEND_SPACE, &msk->flags); 5921891c4a0SFlorian Westphal smp_mb__after_atomic(); 5931891c4a0SFlorian Westphal /* set NOSPACE only after clearing SEND_SPACE flag */ 5941891c4a0SFlorian Westphal set_bit(SOCK_NOSPACE, &sock->flags); 5951891c4a0SFlorian Westphal } 5961891c4a0SFlorian Westphal } 5971891c4a0SFlorian Westphal 598f870fa0bSMat Martineau static int mptcp_sendmsg(struct sock *sk, struct msghdr *msg, size_t len) 599f870fa0bSMat Martineau { 60057040755SPaolo Abeni int mss_now = 0, size_goal = 0, ret = 0; 601f870fa0bSMat Martineau struct mptcp_sock *msk = mptcp_sk(sk); 602cec37a6eSPeter Krystad struct socket *ssock; 6036d0060f6SMat Martineau size_t copied = 0; 604cec37a6eSPeter Krystad struct sock *ssk; 6056d0060f6SMat Martineau long timeo; 606f870fa0bSMat Martineau 607f870fa0bSMat Martineau if (msg->msg_flags & ~(MSG_MORE | MSG_DONTWAIT | MSG_NOSIGNAL)) 608f870fa0bSMat Martineau return -EOPNOTSUPP; 609f870fa0bSMat Martineau 610cec37a6eSPeter Krystad lock_sock(sk); 6111954b860SMat Martineau 6121954b860SMat Martineau timeo = sock_sndtimeo(sk, msg->msg_flags & MSG_DONTWAIT); 6131954b860SMat Martineau 6141954b860SMat Martineau if ((1 << sk->sk_state) & ~(TCPF_ESTABLISHED | TCPF_CLOSE_WAIT)) { 6151954b860SMat Martineau ret = sk_stream_wait_connect(sk, &timeo); 6161954b860SMat Martineau if (ret) 6171954b860SMat Martineau goto out; 6181954b860SMat Martineau } 6191954b860SMat Martineau 620cec37a6eSPeter Krystad ssock = __mptcp_tcp_fallback(msk); 6218ab183deSPaolo Abeni if (unlikely(ssock)) { 6228ab183deSPaolo Abeni fallback: 623cec37a6eSPeter Krystad pr_debug("fallback passthrough"); 624cec37a6eSPeter Krystad ret = sock_sendmsg(ssock, msg); 6258ab183deSPaolo Abeni return ret >= 0 ? ret + copied : (copied ? copied : ret); 626cec37a6eSPeter Krystad } 627cec37a6eSPeter Krystad 62818b683bfSPaolo Abeni mptcp_clean_una(sk); 62918b683bfSPaolo Abeni 630ec3edaa7SPeter Krystad __mptcp_flush_join_list(msk); 631f296234cSPeter Krystad ssk = mptcp_subflow_get_send(msk); 632f296234cSPeter Krystad while (!sk_stream_memory_free(sk) || !ssk) { 633f296234cSPeter Krystad ret = sk_stream_wait_memory(sk, &timeo); 634f296234cSPeter Krystad if (ret) 635f296234cSPeter Krystad goto out; 636f296234cSPeter Krystad 63718b683bfSPaolo Abeni mptcp_clean_una(sk); 63818b683bfSPaolo Abeni 639f296234cSPeter Krystad ssk = mptcp_subflow_get_send(msk); 640f296234cSPeter Krystad if (list_empty(&msk->conn_list)) { 641f296234cSPeter Krystad ret = -ENOTCONN; 642f296234cSPeter Krystad goto out; 643f296234cSPeter Krystad } 644cec37a6eSPeter Krystad } 645cec37a6eSPeter Krystad 6466d0060f6SMat Martineau pr_debug("conn_list->subflow=%p", ssk); 647cec37a6eSPeter Krystad 6486d0060f6SMat Martineau lock_sock(ssk); 6496d0060f6SMat Martineau while (msg_data_left(msg)) { 65057040755SPaolo Abeni ret = mptcp_sendmsg_frag(sk, ssk, msg, &timeo, &mss_now, 65157040755SPaolo Abeni &size_goal); 6526d0060f6SMat Martineau if (ret < 0) 6536d0060f6SMat Martineau break; 6548ab183deSPaolo Abeni if (ret == 0 && unlikely(__mptcp_needs_tcp_fallback(msk))) { 6558ab183deSPaolo Abeni release_sock(ssk); 6568ab183deSPaolo Abeni ssock = __mptcp_tcp_fallback(msk); 6578ab183deSPaolo Abeni goto fallback; 6588ab183deSPaolo Abeni } 6596d0060f6SMat Martineau 6606d0060f6SMat Martineau copied += ret; 6616d0060f6SMat Martineau } 6626d0060f6SMat Martineau 663b51f9b80SPaolo Abeni mptcp_set_timeout(sk, ssk); 66457040755SPaolo Abeni if (copied) { 6656d0060f6SMat Martineau ret = copied; 66657040755SPaolo Abeni tcp_push(ssk, msg->msg_flags, mss_now, tcp_sk(ssk)->nonagle, 66757040755SPaolo Abeni size_goal); 668b51f9b80SPaolo Abeni 669b51f9b80SPaolo Abeni /* start the timer, if it's not pending */ 670b51f9b80SPaolo Abeni if (!mptcp_timer_pending(sk)) 671b51f9b80SPaolo Abeni mptcp_reset_timer(sk); 67257040755SPaolo Abeni } 6736d0060f6SMat Martineau 6741891c4a0SFlorian Westphal ssk_check_wmem(msk, ssk); 6756d0060f6SMat Martineau release_sock(ssk); 6761954b860SMat Martineau out: 677cec37a6eSPeter Krystad release_sock(sk); 678cec37a6eSPeter Krystad return ret; 679f870fa0bSMat Martineau } 680f870fa0bSMat Martineau 6817a6a6cbcSPaolo Abeni static void mptcp_wait_data(struct sock *sk, long *timeo) 6827a6a6cbcSPaolo Abeni { 6837a6a6cbcSPaolo Abeni DEFINE_WAIT_FUNC(wait, woken_wake_function); 6847a6a6cbcSPaolo Abeni struct mptcp_sock *msk = mptcp_sk(sk); 6857a6a6cbcSPaolo Abeni 6867a6a6cbcSPaolo Abeni add_wait_queue(sk_sleep(sk), &wait); 6877a6a6cbcSPaolo Abeni sk_set_bit(SOCKWQ_ASYNC_WAITDATA, sk); 6887a6a6cbcSPaolo Abeni 6897a6a6cbcSPaolo Abeni sk_wait_event(sk, timeo, 6907a6a6cbcSPaolo Abeni test_and_clear_bit(MPTCP_DATA_READY, &msk->flags), &wait); 6917a6a6cbcSPaolo Abeni 6927a6a6cbcSPaolo Abeni sk_clear_bit(SOCKWQ_ASYNC_WAITDATA, sk); 6937a6a6cbcSPaolo Abeni remove_wait_queue(sk_sleep(sk), &wait); 6947a6a6cbcSPaolo Abeni } 6957a6a6cbcSPaolo Abeni 6966771bfd9SFlorian Westphal static int __mptcp_recvmsg_mskq(struct mptcp_sock *msk, 6976771bfd9SFlorian Westphal struct msghdr *msg, 6986771bfd9SFlorian Westphal size_t len) 6996771bfd9SFlorian Westphal { 7006771bfd9SFlorian Westphal struct sock *sk = (struct sock *)msk; 7016771bfd9SFlorian Westphal struct sk_buff *skb; 7026771bfd9SFlorian Westphal int copied = 0; 7036771bfd9SFlorian Westphal 7046771bfd9SFlorian Westphal while ((skb = skb_peek(&sk->sk_receive_queue)) != NULL) { 7056771bfd9SFlorian Westphal u32 offset = MPTCP_SKB_CB(skb)->offset; 7066771bfd9SFlorian Westphal u32 data_len = skb->len - offset; 7076771bfd9SFlorian Westphal u32 count = min_t(size_t, len - copied, data_len); 7086771bfd9SFlorian Westphal int err; 7096771bfd9SFlorian Westphal 7106771bfd9SFlorian Westphal err = skb_copy_datagram_msg(skb, offset, msg, count); 7116771bfd9SFlorian Westphal if (unlikely(err < 0)) { 7126771bfd9SFlorian Westphal if (!copied) 7136771bfd9SFlorian Westphal return err; 7146771bfd9SFlorian Westphal break; 7156771bfd9SFlorian Westphal } 7166771bfd9SFlorian Westphal 7176771bfd9SFlorian Westphal copied += count; 7186771bfd9SFlorian Westphal 7196771bfd9SFlorian Westphal if (count < data_len) { 7206771bfd9SFlorian Westphal MPTCP_SKB_CB(skb)->offset += count; 7216771bfd9SFlorian Westphal break; 7226771bfd9SFlorian Westphal } 7236771bfd9SFlorian Westphal 7246771bfd9SFlorian Westphal __skb_unlink(skb, &sk->sk_receive_queue); 7256771bfd9SFlorian Westphal __kfree_skb(skb); 7266771bfd9SFlorian Westphal 7276771bfd9SFlorian Westphal if (copied >= len) 7286771bfd9SFlorian Westphal break; 7296771bfd9SFlorian Westphal } 7306771bfd9SFlorian Westphal 7316771bfd9SFlorian Westphal return copied; 7326771bfd9SFlorian Westphal } 7336771bfd9SFlorian Westphal 7346771bfd9SFlorian Westphal static bool __mptcp_move_skbs(struct mptcp_sock *msk) 7356771bfd9SFlorian Westphal { 7366771bfd9SFlorian Westphal unsigned int moved = 0; 7376771bfd9SFlorian Westphal bool done; 7386771bfd9SFlorian Westphal 7396771bfd9SFlorian Westphal do { 7406771bfd9SFlorian Westphal struct sock *ssk = mptcp_subflow_recv_lookup(msk); 7416771bfd9SFlorian Westphal 7426771bfd9SFlorian Westphal if (!ssk) 7436771bfd9SFlorian Westphal break; 7446771bfd9SFlorian Westphal 7456771bfd9SFlorian Westphal lock_sock(ssk); 7466771bfd9SFlorian Westphal done = __mptcp_move_skbs_from_subflow(msk, ssk, &moved); 7476771bfd9SFlorian Westphal release_sock(ssk); 7486771bfd9SFlorian Westphal } while (!done); 7496771bfd9SFlorian Westphal 7506771bfd9SFlorian Westphal return moved > 0; 7516771bfd9SFlorian Westphal } 7526771bfd9SFlorian Westphal 753f870fa0bSMat Martineau static int mptcp_recvmsg(struct sock *sk, struct msghdr *msg, size_t len, 754f870fa0bSMat Martineau int nonblock, int flags, int *addr_len) 755f870fa0bSMat Martineau { 756f870fa0bSMat Martineau struct mptcp_sock *msk = mptcp_sk(sk); 757cec37a6eSPeter Krystad struct socket *ssock; 758cec37a6eSPeter Krystad int copied = 0; 7597a6a6cbcSPaolo Abeni int target; 7607a6a6cbcSPaolo Abeni long timeo; 761f870fa0bSMat Martineau 762f870fa0bSMat Martineau if (msg->msg_flags & ~(MSG_WAITALL | MSG_DONTWAIT)) 763f870fa0bSMat Martineau return -EOPNOTSUPP; 764f870fa0bSMat Martineau 765cec37a6eSPeter Krystad lock_sock(sk); 766cec37a6eSPeter Krystad ssock = __mptcp_tcp_fallback(msk); 7678ab183deSPaolo Abeni if (unlikely(ssock)) { 7688ab183deSPaolo Abeni fallback: 769cec37a6eSPeter Krystad pr_debug("fallback-read subflow=%p", 770cec37a6eSPeter Krystad mptcp_subflow_ctx(ssock->sk)); 771cec37a6eSPeter Krystad copied = sock_recvmsg(ssock, msg, flags); 772cec37a6eSPeter Krystad return copied; 773cec37a6eSPeter Krystad } 774cec37a6eSPeter Krystad 7757a6a6cbcSPaolo Abeni timeo = sock_rcvtimeo(sk, nonblock); 7767a6a6cbcSPaolo Abeni 7777a6a6cbcSPaolo Abeni len = min_t(size_t, len, INT_MAX); 7787a6a6cbcSPaolo Abeni target = sock_rcvlowat(sk, flags & MSG_WAITALL, len); 779ec3edaa7SPeter Krystad __mptcp_flush_join_list(msk); 7807a6a6cbcSPaolo Abeni 7816771bfd9SFlorian Westphal while (len > (size_t)copied) { 7827a6a6cbcSPaolo Abeni int bytes_read; 7837a6a6cbcSPaolo Abeni 7846771bfd9SFlorian Westphal bytes_read = __mptcp_recvmsg_mskq(msk, msg, len - copied); 7856771bfd9SFlorian Westphal if (unlikely(bytes_read < 0)) { 7867a6a6cbcSPaolo Abeni if (!copied) 7877a6a6cbcSPaolo Abeni copied = bytes_read; 7886771bfd9SFlorian Westphal goto out_err; 789cec37a6eSPeter Krystad } 790cec37a6eSPeter Krystad 7917a6a6cbcSPaolo Abeni copied += bytes_read; 7927a6a6cbcSPaolo Abeni 7936771bfd9SFlorian Westphal if (skb_queue_empty(&sk->sk_receive_queue) && 7946771bfd9SFlorian Westphal __mptcp_move_skbs(msk)) 7956771bfd9SFlorian Westphal continue; 7967a6a6cbcSPaolo Abeni 7977a6a6cbcSPaolo Abeni /* only the master socket status is relevant here. The exit 7987a6a6cbcSPaolo Abeni * conditions mirror closely tcp_recvmsg() 7997a6a6cbcSPaolo Abeni */ 8007a6a6cbcSPaolo Abeni if (copied >= target) 8017a6a6cbcSPaolo Abeni break; 8027a6a6cbcSPaolo Abeni 8037a6a6cbcSPaolo Abeni if (copied) { 8047a6a6cbcSPaolo Abeni if (sk->sk_err || 8057a6a6cbcSPaolo Abeni sk->sk_state == TCP_CLOSE || 8067a6a6cbcSPaolo Abeni (sk->sk_shutdown & RCV_SHUTDOWN) || 8077a6a6cbcSPaolo Abeni !timeo || 8087a6a6cbcSPaolo Abeni signal_pending(current)) 8097a6a6cbcSPaolo Abeni break; 8107a6a6cbcSPaolo Abeni } else { 8117a6a6cbcSPaolo Abeni if (sk->sk_err) { 8127a6a6cbcSPaolo Abeni copied = sock_error(sk); 8137a6a6cbcSPaolo Abeni break; 8147a6a6cbcSPaolo Abeni } 8157a6a6cbcSPaolo Abeni 8167a6a6cbcSPaolo Abeni if (sk->sk_shutdown & RCV_SHUTDOWN) 8177a6a6cbcSPaolo Abeni break; 8187a6a6cbcSPaolo Abeni 8197a6a6cbcSPaolo Abeni if (sk->sk_state == TCP_CLOSE) { 8207a6a6cbcSPaolo Abeni copied = -ENOTCONN; 8217a6a6cbcSPaolo Abeni break; 8227a6a6cbcSPaolo Abeni } 8237a6a6cbcSPaolo Abeni 8247a6a6cbcSPaolo Abeni if (!timeo) { 8257a6a6cbcSPaolo Abeni copied = -EAGAIN; 8267a6a6cbcSPaolo Abeni break; 8277a6a6cbcSPaolo Abeni } 8287a6a6cbcSPaolo Abeni 8297a6a6cbcSPaolo Abeni if (signal_pending(current)) { 8307a6a6cbcSPaolo Abeni copied = sock_intr_errno(timeo); 8317a6a6cbcSPaolo Abeni break; 8327a6a6cbcSPaolo Abeni } 8337a6a6cbcSPaolo Abeni } 8347a6a6cbcSPaolo Abeni 8357a6a6cbcSPaolo Abeni pr_debug("block timeout %ld", timeo); 8367a6a6cbcSPaolo Abeni mptcp_wait_data(sk, &timeo); 8378ab183deSPaolo Abeni if (unlikely(__mptcp_tcp_fallback(msk))) 8388ab183deSPaolo Abeni goto fallback; 8397a6a6cbcSPaolo Abeni } 8407a6a6cbcSPaolo Abeni 8416771bfd9SFlorian Westphal if (skb_queue_empty(&sk->sk_receive_queue)) { 8426771bfd9SFlorian Westphal /* entire backlog drained, clear DATA_READY. */ 8437a6a6cbcSPaolo Abeni clear_bit(MPTCP_DATA_READY, &msk->flags); 8447a6a6cbcSPaolo Abeni 8456771bfd9SFlorian Westphal /* .. race-breaker: ssk might have gotten new data 8466771bfd9SFlorian Westphal * after last __mptcp_move_skbs() returned false. 8477a6a6cbcSPaolo Abeni */ 8486771bfd9SFlorian Westphal if (unlikely(__mptcp_move_skbs(msk))) 8496771bfd9SFlorian Westphal set_bit(MPTCP_DATA_READY, &msk->flags); 8506771bfd9SFlorian Westphal } else if (unlikely(!test_bit(MPTCP_DATA_READY, &msk->flags))) { 8516771bfd9SFlorian Westphal /* data to read but mptcp_wait_data() cleared DATA_READY */ 8527a6a6cbcSPaolo Abeni set_bit(MPTCP_DATA_READY, &msk->flags); 8537a6a6cbcSPaolo Abeni } 8546771bfd9SFlorian Westphal out_err: 855cec37a6eSPeter Krystad release_sock(sk); 856cec37a6eSPeter Krystad return copied; 857cec37a6eSPeter Krystad } 858cec37a6eSPeter Krystad 859b51f9b80SPaolo Abeni static void mptcp_retransmit_handler(struct sock *sk) 860b51f9b80SPaolo Abeni { 861b51f9b80SPaolo Abeni struct mptcp_sock *msk = mptcp_sk(sk); 862b51f9b80SPaolo Abeni 863b51f9b80SPaolo Abeni if (atomic64_read(&msk->snd_una) == msk->write_seq) 864b51f9b80SPaolo Abeni mptcp_stop_timer(sk); 865b51f9b80SPaolo Abeni else 866b51f9b80SPaolo Abeni mptcp_reset_timer(sk); 867b51f9b80SPaolo Abeni } 868b51f9b80SPaolo Abeni 869b51f9b80SPaolo Abeni static void mptcp_retransmit_timer(struct timer_list *t) 870b51f9b80SPaolo Abeni { 871b51f9b80SPaolo Abeni struct inet_connection_sock *icsk = from_timer(icsk, t, 872b51f9b80SPaolo Abeni icsk_retransmit_timer); 873b51f9b80SPaolo Abeni struct sock *sk = &icsk->icsk_inet.sk; 874b51f9b80SPaolo Abeni 875b51f9b80SPaolo Abeni bh_lock_sock(sk); 876b51f9b80SPaolo Abeni if (!sock_owned_by_user(sk)) { 877b51f9b80SPaolo Abeni mptcp_retransmit_handler(sk); 878b51f9b80SPaolo Abeni } else { 879b51f9b80SPaolo Abeni /* delegate our work to tcp_release_cb() */ 880b51f9b80SPaolo Abeni if (!test_and_set_bit(TCP_WRITE_TIMER_DEFERRED, 881b51f9b80SPaolo Abeni &sk->sk_tsq_flags)) 882b51f9b80SPaolo Abeni sock_hold(sk); 883b51f9b80SPaolo Abeni } 884b51f9b80SPaolo Abeni bh_unlock_sock(sk); 885b51f9b80SPaolo Abeni sock_put(sk); 886b51f9b80SPaolo Abeni } 887b51f9b80SPaolo Abeni 888cec37a6eSPeter Krystad /* subflow sockets can be either outgoing (connect) or incoming 889cec37a6eSPeter Krystad * (accept). 890cec37a6eSPeter Krystad * 891cec37a6eSPeter Krystad * Outgoing subflows use in-kernel sockets. 892cec37a6eSPeter Krystad * Incoming subflows do not have their own 'struct socket' allocated, 893cec37a6eSPeter Krystad * so we need to use tcp_close() after detaching them from the mptcp 894cec37a6eSPeter Krystad * parent socket. 895cec37a6eSPeter Krystad */ 896cec37a6eSPeter Krystad static void __mptcp_close_ssk(struct sock *sk, struct sock *ssk, 897cec37a6eSPeter Krystad struct mptcp_subflow_context *subflow, 898cec37a6eSPeter Krystad long timeout) 899cec37a6eSPeter Krystad { 900cec37a6eSPeter Krystad struct socket *sock = READ_ONCE(ssk->sk_socket); 901cec37a6eSPeter Krystad 902cec37a6eSPeter Krystad list_del(&subflow->node); 903cec37a6eSPeter Krystad 904cec37a6eSPeter Krystad if (sock && sock != sk->sk_socket) { 905cec37a6eSPeter Krystad /* outgoing subflow */ 906cec37a6eSPeter Krystad sock_release(sock); 907cec37a6eSPeter Krystad } else { 908cec37a6eSPeter Krystad /* incoming subflow */ 909cec37a6eSPeter Krystad tcp_close(ssk, timeout); 910cec37a6eSPeter Krystad } 911f870fa0bSMat Martineau } 912f870fa0bSMat Martineau 913dc24f8b4SPaolo Abeni static unsigned int mptcp_sync_mss(struct sock *sk, u32 pmtu) 914dc24f8b4SPaolo Abeni { 915dc24f8b4SPaolo Abeni return 0; 916dc24f8b4SPaolo Abeni } 917dc24f8b4SPaolo Abeni 91880992017SPaolo Abeni static void mptcp_worker(struct work_struct *work) 91980992017SPaolo Abeni { 92080992017SPaolo Abeni struct mptcp_sock *msk = container_of(work, struct mptcp_sock, work); 92180992017SPaolo Abeni struct sock *sk = &msk->sk.icsk_inet.sk; 92280992017SPaolo Abeni 92380992017SPaolo Abeni lock_sock(sk); 924ec3edaa7SPeter Krystad __mptcp_flush_join_list(msk); 9256771bfd9SFlorian Westphal __mptcp_move_skbs(msk); 92680992017SPaolo Abeni release_sock(sk); 92780992017SPaolo Abeni sock_put(sk); 92880992017SPaolo Abeni } 92980992017SPaolo Abeni 930784325e9SMatthieu Baerts static int __mptcp_init_sock(struct sock *sk) 931f870fa0bSMat Martineau { 932cec37a6eSPeter Krystad struct mptcp_sock *msk = mptcp_sk(sk); 933cec37a6eSPeter Krystad 934ec3edaa7SPeter Krystad spin_lock_init(&msk->join_list_lock); 935ec3edaa7SPeter Krystad 936cec37a6eSPeter Krystad INIT_LIST_HEAD(&msk->conn_list); 937ec3edaa7SPeter Krystad INIT_LIST_HEAD(&msk->join_list); 93818b683bfSPaolo Abeni INIT_LIST_HEAD(&msk->rtx_queue); 9391891c4a0SFlorian Westphal __set_bit(MPTCP_SEND_SPACE, &msk->flags); 94080992017SPaolo Abeni INIT_WORK(&msk->work, mptcp_worker); 941cec37a6eSPeter Krystad 9428ab183deSPaolo Abeni msk->first = NULL; 943dc24f8b4SPaolo Abeni inet_csk(sk)->icsk_sync_mss = mptcp_sync_mss; 9448ab183deSPaolo Abeni 9451b1c7a0eSPeter Krystad mptcp_pm_data_init(msk); 9461b1c7a0eSPeter Krystad 947b51f9b80SPaolo Abeni /* re-use the csk retrans timer for MPTCP-level retrans */ 948b51f9b80SPaolo Abeni timer_setup(&msk->sk.icsk_retransmit_timer, mptcp_retransmit_timer, 0); 949b51f9b80SPaolo Abeni 950f870fa0bSMat Martineau return 0; 951f870fa0bSMat Martineau } 952f870fa0bSMat Martineau 953784325e9SMatthieu Baerts static int mptcp_init_sock(struct sock *sk) 954784325e9SMatthieu Baerts { 95518b683bfSPaolo Abeni int ret = __mptcp_init_sock(sk); 95618b683bfSPaolo Abeni 95718b683bfSPaolo Abeni if (ret) 95818b683bfSPaolo Abeni return ret; 95918b683bfSPaolo Abeni 960*d027236cSPaolo Abeni sk_sockets_allocated_inc(sk); 961*d027236cSPaolo Abeni 962784325e9SMatthieu Baerts if (!mptcp_is_enabled(sock_net(sk))) 963784325e9SMatthieu Baerts return -ENOPROTOOPT; 964784325e9SMatthieu Baerts 96518b683bfSPaolo Abeni return 0; 96618b683bfSPaolo Abeni } 96718b683bfSPaolo Abeni 96818b683bfSPaolo Abeni static void __mptcp_clear_xmit(struct sock *sk) 96918b683bfSPaolo Abeni { 97018b683bfSPaolo Abeni struct mptcp_sock *msk = mptcp_sk(sk); 97118b683bfSPaolo Abeni struct mptcp_data_frag *dtmp, *dfrag; 97218b683bfSPaolo Abeni 973b51f9b80SPaolo Abeni sk_stop_timer(sk, &msk->sk.icsk_retransmit_timer); 974b51f9b80SPaolo Abeni 97518b683bfSPaolo Abeni list_for_each_entry_safe(dfrag, dtmp, &msk->rtx_queue, list) 976*d027236cSPaolo Abeni dfrag_clear(sk, dfrag); 977784325e9SMatthieu Baerts } 978784325e9SMatthieu Baerts 97980992017SPaolo Abeni static void mptcp_cancel_work(struct sock *sk) 98080992017SPaolo Abeni { 98180992017SPaolo Abeni struct mptcp_sock *msk = mptcp_sk(sk); 98280992017SPaolo Abeni 98380992017SPaolo Abeni if (cancel_work_sync(&msk->work)) 98480992017SPaolo Abeni sock_put(sk); 98580992017SPaolo Abeni } 98680992017SPaolo Abeni 98776c42a29SMat Martineau static void mptcp_subflow_shutdown(struct sock *ssk, int how, 98876c42a29SMat Martineau bool data_fin_tx_enable, u64 data_fin_tx_seq) 98921498490SPeter Krystad { 99021498490SPeter Krystad lock_sock(ssk); 99121498490SPeter Krystad 99221498490SPeter Krystad switch (ssk->sk_state) { 99321498490SPeter Krystad case TCP_LISTEN: 99421498490SPeter Krystad if (!(how & RCV_SHUTDOWN)) 99521498490SPeter Krystad break; 99621498490SPeter Krystad /* fall through */ 99721498490SPeter Krystad case TCP_SYN_SENT: 99821498490SPeter Krystad tcp_disconnect(ssk, O_NONBLOCK); 99921498490SPeter Krystad break; 100021498490SPeter Krystad default: 100176c42a29SMat Martineau if (data_fin_tx_enable) { 100276c42a29SMat Martineau struct mptcp_subflow_context *subflow; 100376c42a29SMat Martineau 100476c42a29SMat Martineau subflow = mptcp_subflow_ctx(ssk); 100576c42a29SMat Martineau subflow->data_fin_tx_seq = data_fin_tx_seq; 100676c42a29SMat Martineau subflow->data_fin_tx_enable = 1; 100776c42a29SMat Martineau } 100876c42a29SMat Martineau 100921498490SPeter Krystad ssk->sk_shutdown |= how; 101021498490SPeter Krystad tcp_shutdown(ssk, how); 101121498490SPeter Krystad break; 101221498490SPeter Krystad } 101321498490SPeter Krystad 101421498490SPeter Krystad /* Wake up anyone sleeping in poll. */ 101521498490SPeter Krystad ssk->sk_state_change(ssk); 101621498490SPeter Krystad release_sock(ssk); 101721498490SPeter Krystad } 101821498490SPeter Krystad 10198ab183deSPaolo Abeni /* Called with msk lock held, releases such lock before returning */ 10202c22c06cSFlorian Westphal static void mptcp_close(struct sock *sk, long timeout) 1021f870fa0bSMat Martineau { 1022cec37a6eSPeter Krystad struct mptcp_subflow_context *subflow, *tmp; 1023f870fa0bSMat Martineau struct mptcp_sock *msk = mptcp_sk(sk); 1024b2c5b614SFlorian Westphal LIST_HEAD(conn_list); 102576c42a29SMat Martineau u64 data_fin_tx_seq; 1026f870fa0bSMat Martineau 10272c22c06cSFlorian Westphal lock_sock(sk); 10282c22c06cSFlorian Westphal 102979c0949eSPeter Krystad mptcp_token_destroy(msk->token); 1030f870fa0bSMat Martineau inet_sk_state_store(sk, TCP_CLOSE); 1031f870fa0bSMat Martineau 1032ec3edaa7SPeter Krystad __mptcp_flush_join_list(msk); 1033ec3edaa7SPeter Krystad 1034b2c5b614SFlorian Westphal list_splice_init(&msk->conn_list, &conn_list); 1035b2c5b614SFlorian Westphal 103676c42a29SMat Martineau data_fin_tx_seq = msk->write_seq; 103776c42a29SMat Martineau 103818b683bfSPaolo Abeni __mptcp_clear_xmit(sk); 103918b683bfSPaolo Abeni 1040b2c5b614SFlorian Westphal release_sock(sk); 1041b2c5b614SFlorian Westphal 1042b2c5b614SFlorian Westphal list_for_each_entry_safe(subflow, tmp, &conn_list, node) { 1043cec37a6eSPeter Krystad struct sock *ssk = mptcp_subflow_tcp_sock(subflow); 1044cec37a6eSPeter Krystad 104576c42a29SMat Martineau subflow->data_fin_tx_seq = data_fin_tx_seq; 104676c42a29SMat Martineau subflow->data_fin_tx_enable = 1; 1047cec37a6eSPeter Krystad __mptcp_close_ssk(sk, ssk, subflow, timeout); 1048f870fa0bSMat Martineau } 1049f870fa0bSMat Martineau 105080992017SPaolo Abeni mptcp_cancel_work(sk); 1051926bdeabSPeter Krystad mptcp_pm_close(msk); 105280992017SPaolo Abeni 10536771bfd9SFlorian Westphal __skb_queue_purge(&sk->sk_receive_queue); 10546771bfd9SFlorian Westphal 1055cec37a6eSPeter Krystad sk_common_release(sk); 1056f870fa0bSMat Martineau } 1057f870fa0bSMat Martineau 1058cf7da0d6SPeter Krystad static void mptcp_copy_inaddrs(struct sock *msk, const struct sock *ssk) 1059cf7da0d6SPeter Krystad { 1060cf7da0d6SPeter Krystad #if IS_ENABLED(CONFIG_MPTCP_IPV6) 1061cf7da0d6SPeter Krystad const struct ipv6_pinfo *ssk6 = inet6_sk(ssk); 1062cf7da0d6SPeter Krystad struct ipv6_pinfo *msk6 = inet6_sk(msk); 1063cf7da0d6SPeter Krystad 1064cf7da0d6SPeter Krystad msk->sk_v6_daddr = ssk->sk_v6_daddr; 1065cf7da0d6SPeter Krystad msk->sk_v6_rcv_saddr = ssk->sk_v6_rcv_saddr; 1066cf7da0d6SPeter Krystad 1067cf7da0d6SPeter Krystad if (msk6 && ssk6) { 1068cf7da0d6SPeter Krystad msk6->saddr = ssk6->saddr; 1069cf7da0d6SPeter Krystad msk6->flow_label = ssk6->flow_label; 1070cf7da0d6SPeter Krystad } 1071cf7da0d6SPeter Krystad #endif 1072cf7da0d6SPeter Krystad 1073cf7da0d6SPeter Krystad inet_sk(msk)->inet_num = inet_sk(ssk)->inet_num; 1074cf7da0d6SPeter Krystad inet_sk(msk)->inet_dport = inet_sk(ssk)->inet_dport; 1075cf7da0d6SPeter Krystad inet_sk(msk)->inet_sport = inet_sk(ssk)->inet_sport; 1076cf7da0d6SPeter Krystad inet_sk(msk)->inet_daddr = inet_sk(ssk)->inet_daddr; 1077cf7da0d6SPeter Krystad inet_sk(msk)->inet_saddr = inet_sk(ssk)->inet_saddr; 1078cf7da0d6SPeter Krystad inet_sk(msk)->inet_rcv_saddr = inet_sk(ssk)->inet_rcv_saddr; 1079cf7da0d6SPeter Krystad } 1080cf7da0d6SPeter Krystad 108118b683bfSPaolo Abeni static int mptcp_disconnect(struct sock *sk, int flags) 108218b683bfSPaolo Abeni { 108318b683bfSPaolo Abeni lock_sock(sk); 108418b683bfSPaolo Abeni __mptcp_clear_xmit(sk); 108518b683bfSPaolo Abeni release_sock(sk); 108618b683bfSPaolo Abeni return tcp_disconnect(sk, flags); 108718b683bfSPaolo Abeni } 108818b683bfSPaolo Abeni 1089b0519de8SFlorian Westphal #if IS_ENABLED(CONFIG_MPTCP_IPV6) 1090b0519de8SFlorian Westphal static struct ipv6_pinfo *mptcp_inet6_sk(const struct sock *sk) 1091b0519de8SFlorian Westphal { 1092b0519de8SFlorian Westphal unsigned int offset = sizeof(struct mptcp6_sock) - sizeof(struct ipv6_pinfo); 1093b0519de8SFlorian Westphal 1094b0519de8SFlorian Westphal return (struct ipv6_pinfo *)(((u8 *)sk) + offset); 1095b0519de8SFlorian Westphal } 1096b0519de8SFlorian Westphal #endif 1097b0519de8SFlorian Westphal 109858b09919SPaolo Abeni struct sock *mptcp_sk_clone(const struct sock *sk, struct request_sock *req) 1099b0519de8SFlorian Westphal { 110058b09919SPaolo Abeni struct mptcp_subflow_request_sock *subflow_req = mptcp_subflow_rsk(req); 1101b0519de8SFlorian Westphal struct sock *nsk = sk_clone_lock(sk, GFP_ATOMIC); 110258b09919SPaolo Abeni struct mptcp_sock *msk; 110358b09919SPaolo Abeni u64 ack_seq; 1104b0519de8SFlorian Westphal 1105b0519de8SFlorian Westphal if (!nsk) 1106b0519de8SFlorian Westphal return NULL; 1107b0519de8SFlorian Westphal 1108b0519de8SFlorian Westphal #if IS_ENABLED(CONFIG_MPTCP_IPV6) 1109b0519de8SFlorian Westphal if (nsk->sk_family == AF_INET6) 1110b0519de8SFlorian Westphal inet_sk(nsk)->pinet6 = mptcp_inet6_sk(nsk); 1111b0519de8SFlorian Westphal #endif 1112b0519de8SFlorian Westphal 111358b09919SPaolo Abeni __mptcp_init_sock(nsk); 111458b09919SPaolo Abeni 111558b09919SPaolo Abeni msk = mptcp_sk(nsk); 111658b09919SPaolo Abeni msk->local_key = subflow_req->local_key; 111758b09919SPaolo Abeni msk->token = subflow_req->token; 111858b09919SPaolo Abeni msk->subflow = NULL; 111958b09919SPaolo Abeni 112058b09919SPaolo Abeni if (unlikely(mptcp_token_new_accept(subflow_req->token, nsk))) { 112158b09919SPaolo Abeni bh_unlock_sock(nsk); 112258b09919SPaolo Abeni 112358b09919SPaolo Abeni /* we can't call into mptcp_close() here - possible BH context 112458b09919SPaolo Abeni * free the sock directly 112558b09919SPaolo Abeni */ 112658b09919SPaolo Abeni nsk->sk_prot->destroy(nsk); 112758b09919SPaolo Abeni sk_free(nsk); 112858b09919SPaolo Abeni return NULL; 112958b09919SPaolo Abeni } 113058b09919SPaolo Abeni 113158b09919SPaolo Abeni msk->write_seq = subflow_req->idsn + 1; 1132cc9d2566SPaolo Abeni atomic64_set(&msk->snd_una, msk->write_seq); 113358b09919SPaolo Abeni if (subflow_req->remote_key_valid) { 113458b09919SPaolo Abeni msk->can_ack = true; 113558b09919SPaolo Abeni msk->remote_key = subflow_req->remote_key; 113658b09919SPaolo Abeni mptcp_crypto_key_sha(msk->remote_key, NULL, &ack_seq); 113758b09919SPaolo Abeni ack_seq++; 113858b09919SPaolo Abeni msk->ack_seq = ack_seq; 113958b09919SPaolo Abeni } 11407f20d5fcSPaolo Abeni 11417f20d5fcSPaolo Abeni /* will be fully established after successful MPC subflow creation */ 11427f20d5fcSPaolo Abeni inet_sk_state_store(nsk, TCP_SYN_RECV); 114358b09919SPaolo Abeni bh_unlock_sock(nsk); 114458b09919SPaolo Abeni 114558b09919SPaolo Abeni /* keep a single reference */ 114658b09919SPaolo Abeni __sock_put(nsk); 1147b0519de8SFlorian Westphal return nsk; 1148b0519de8SFlorian Westphal } 1149b0519de8SFlorian Westphal 1150cf7da0d6SPeter Krystad static struct sock *mptcp_accept(struct sock *sk, int flags, int *err, 1151cf7da0d6SPeter Krystad bool kern) 1152cf7da0d6SPeter Krystad { 1153cf7da0d6SPeter Krystad struct mptcp_sock *msk = mptcp_sk(sk); 1154cf7da0d6SPeter Krystad struct socket *listener; 1155cf7da0d6SPeter Krystad struct sock *newsk; 1156cf7da0d6SPeter Krystad 1157cf7da0d6SPeter Krystad listener = __mptcp_nmpc_socket(msk); 1158cf7da0d6SPeter Krystad if (WARN_ON_ONCE(!listener)) { 1159cf7da0d6SPeter Krystad *err = -EINVAL; 1160cf7da0d6SPeter Krystad return NULL; 1161cf7da0d6SPeter Krystad } 1162cf7da0d6SPeter Krystad 1163cf7da0d6SPeter Krystad pr_debug("msk=%p, listener=%p", msk, mptcp_subflow_ctx(listener->sk)); 1164cf7da0d6SPeter Krystad newsk = inet_csk_accept(listener->sk, flags, err, kern); 1165cf7da0d6SPeter Krystad if (!newsk) 1166cf7da0d6SPeter Krystad return NULL; 1167cf7da0d6SPeter Krystad 1168cf7da0d6SPeter Krystad pr_debug("msk=%p, subflow is mptcp=%d", msk, sk_is_mptcp(newsk)); 1169cf7da0d6SPeter Krystad 1170cf7da0d6SPeter Krystad if (sk_is_mptcp(newsk)) { 1171cf7da0d6SPeter Krystad struct mptcp_subflow_context *subflow; 1172cf7da0d6SPeter Krystad struct sock *new_mptcp_sock; 1173cf7da0d6SPeter Krystad struct sock *ssk = newsk; 1174cf7da0d6SPeter Krystad 1175cf7da0d6SPeter Krystad subflow = mptcp_subflow_ctx(newsk); 117658b09919SPaolo Abeni new_mptcp_sock = subflow->conn; 117758b09919SPaolo Abeni 117858b09919SPaolo Abeni /* is_mptcp should be false if subflow->conn is missing, see 117958b09919SPaolo Abeni * subflow_syn_recv_sock() 118058b09919SPaolo Abeni */ 118158b09919SPaolo Abeni if (WARN_ON_ONCE(!new_mptcp_sock)) { 118258b09919SPaolo Abeni tcp_sk(newsk)->is_mptcp = 0; 118358b09919SPaolo Abeni return newsk; 118458b09919SPaolo Abeni } 118558b09919SPaolo Abeni 118658b09919SPaolo Abeni /* acquire the 2nd reference for the owning socket */ 118758b09919SPaolo Abeni sock_hold(new_mptcp_sock); 1188cf7da0d6SPeter Krystad 1189cf7da0d6SPeter Krystad local_bh_disable(); 119058b09919SPaolo Abeni bh_lock_sock(new_mptcp_sock); 1191cf7da0d6SPeter Krystad msk = mptcp_sk(new_mptcp_sock); 11928ab183deSPaolo Abeni msk->first = newsk; 1193cf7da0d6SPeter Krystad 1194cf7da0d6SPeter Krystad newsk = new_mptcp_sock; 1195cf7da0d6SPeter Krystad mptcp_copy_inaddrs(newsk, ssk); 1196cf7da0d6SPeter Krystad list_add(&subflow->node, &msk->conn_list); 1197cf7da0d6SPeter Krystad 1198cf7da0d6SPeter Krystad bh_unlock_sock(new_mptcp_sock); 1199cf7da0d6SPeter Krystad local_bh_enable(); 1200cf7da0d6SPeter Krystad } 1201cf7da0d6SPeter Krystad 1202cf7da0d6SPeter Krystad return newsk; 1203cf7da0d6SPeter Krystad } 1204cf7da0d6SPeter Krystad 120579c0949eSPeter Krystad static void mptcp_destroy(struct sock *sk) 120679c0949eSPeter Krystad { 1207c9fd9c5fSFlorian Westphal struct mptcp_sock *msk = mptcp_sk(sk); 1208c9fd9c5fSFlorian Westphal 1209c9fd9c5fSFlorian Westphal if (msk->cached_ext) 1210c9fd9c5fSFlorian Westphal __skb_ext_put(msk->cached_ext); 1211*d027236cSPaolo Abeni 1212*d027236cSPaolo Abeni sk_sockets_allocated_dec(sk); 121379c0949eSPeter Krystad } 121479c0949eSPeter Krystad 1215717e79c8SPeter Krystad static int mptcp_setsockopt(struct sock *sk, int level, int optname, 121650e741bbSFlorian Westphal char __user *optval, unsigned int optlen) 1217717e79c8SPeter Krystad { 1218717e79c8SPeter Krystad struct mptcp_sock *msk = mptcp_sk(sk); 1219717e79c8SPeter Krystad struct socket *ssock; 1220717e79c8SPeter Krystad 1221717e79c8SPeter Krystad pr_debug("msk=%p", msk); 1222717e79c8SPeter Krystad 1223717e79c8SPeter Krystad /* @@ the meaning of setsockopt() when the socket is connected and 1224b6e4a1aeSMat Martineau * there are multiple subflows is not yet defined. It is up to the 1225b6e4a1aeSMat Martineau * MPTCP-level socket to configure the subflows until the subflow 1226b6e4a1aeSMat Martineau * is in TCP fallback, when TCP socket options are passed through 1227b6e4a1aeSMat Martineau * to the one remaining subflow. 1228717e79c8SPeter Krystad */ 1229717e79c8SPeter Krystad lock_sock(sk); 1230b6e4a1aeSMat Martineau ssock = __mptcp_tcp_fallback(msk); 1231b6e4a1aeSMat Martineau if (ssock) 1232b6e4a1aeSMat Martineau return tcp_setsockopt(ssock->sk, level, optname, optval, 1233b6e4a1aeSMat Martineau optlen); 123450e741bbSFlorian Westphal 123550e741bbSFlorian Westphal release_sock(sk); 123650e741bbSFlorian Westphal 1237b6e4a1aeSMat Martineau return -EOPNOTSUPP; 1238717e79c8SPeter Krystad } 1239717e79c8SPeter Krystad 1240717e79c8SPeter Krystad static int mptcp_getsockopt(struct sock *sk, int level, int optname, 124150e741bbSFlorian Westphal char __user *optval, int __user *option) 1242717e79c8SPeter Krystad { 1243717e79c8SPeter Krystad struct mptcp_sock *msk = mptcp_sk(sk); 1244717e79c8SPeter Krystad struct socket *ssock; 1245717e79c8SPeter Krystad 1246717e79c8SPeter Krystad pr_debug("msk=%p", msk); 1247717e79c8SPeter Krystad 1248b6e4a1aeSMat Martineau /* @@ the meaning of setsockopt() when the socket is connected and 1249b6e4a1aeSMat Martineau * there are multiple subflows is not yet defined. It is up to the 1250b6e4a1aeSMat Martineau * MPTCP-level socket to configure the subflows until the subflow 1251b6e4a1aeSMat Martineau * is in TCP fallback, when socket options are passed through 1252b6e4a1aeSMat Martineau * to the one remaining subflow. 1253717e79c8SPeter Krystad */ 1254717e79c8SPeter Krystad lock_sock(sk); 1255b6e4a1aeSMat Martineau ssock = __mptcp_tcp_fallback(msk); 1256b6e4a1aeSMat Martineau if (ssock) 1257b6e4a1aeSMat Martineau return tcp_getsockopt(ssock->sk, level, optname, optval, 1258b6e4a1aeSMat Martineau option); 125950e741bbSFlorian Westphal 126050e741bbSFlorian Westphal release_sock(sk); 126150e741bbSFlorian Westphal 1262b6e4a1aeSMat Martineau return -EOPNOTSUPP; 1263717e79c8SPeter Krystad } 1264717e79c8SPeter Krystad 1265b51f9b80SPaolo Abeni #define MPTCP_DEFERRED_ALL (TCPF_DELACK_TIMER_DEFERRED | \ 1266b51f9b80SPaolo Abeni TCPF_WRITE_TIMER_DEFERRED) 126714c441b5SPaolo Abeni 126814c441b5SPaolo Abeni /* this is very alike tcp_release_cb() but we must handle differently a 126914c441b5SPaolo Abeni * different set of events 127014c441b5SPaolo Abeni */ 127114c441b5SPaolo Abeni static void mptcp_release_cb(struct sock *sk) 127214c441b5SPaolo Abeni { 127314c441b5SPaolo Abeni unsigned long flags, nflags; 127414c441b5SPaolo Abeni 127514c441b5SPaolo Abeni do { 127614c441b5SPaolo Abeni flags = sk->sk_tsq_flags; 127714c441b5SPaolo Abeni if (!(flags & MPTCP_DEFERRED_ALL)) 127814c441b5SPaolo Abeni return; 127914c441b5SPaolo Abeni nflags = flags & ~MPTCP_DEFERRED_ALL; 128014c441b5SPaolo Abeni } while (cmpxchg(&sk->sk_tsq_flags, flags, nflags) != flags); 128114c441b5SPaolo Abeni 1282b51f9b80SPaolo Abeni sock_release_ownership(sk); 1283b51f9b80SPaolo Abeni 128414c441b5SPaolo Abeni if (flags & TCPF_DELACK_TIMER_DEFERRED) { 128514c441b5SPaolo Abeni struct mptcp_sock *msk = mptcp_sk(sk); 128614c441b5SPaolo Abeni struct sock *ssk; 128714c441b5SPaolo Abeni 128814c441b5SPaolo Abeni ssk = mptcp_subflow_recv_lookup(msk); 128914c441b5SPaolo Abeni if (!ssk || !schedule_work(&msk->work)) 129014c441b5SPaolo Abeni __sock_put(sk); 129114c441b5SPaolo Abeni } 1292b51f9b80SPaolo Abeni 1293b51f9b80SPaolo Abeni if (flags & TCPF_WRITE_TIMER_DEFERRED) { 1294b51f9b80SPaolo Abeni mptcp_retransmit_handler(sk); 1295b51f9b80SPaolo Abeni __sock_put(sk); 1296b51f9b80SPaolo Abeni } 129714c441b5SPaolo Abeni } 129814c441b5SPaolo Abeni 1299cec37a6eSPeter Krystad static int mptcp_get_port(struct sock *sk, unsigned short snum) 1300f870fa0bSMat Martineau { 1301f870fa0bSMat Martineau struct mptcp_sock *msk = mptcp_sk(sk); 1302cec37a6eSPeter Krystad struct socket *ssock; 1303f870fa0bSMat Martineau 1304cec37a6eSPeter Krystad ssock = __mptcp_nmpc_socket(msk); 1305cec37a6eSPeter Krystad pr_debug("msk=%p, subflow=%p", msk, ssock); 1306cec37a6eSPeter Krystad if (WARN_ON_ONCE(!ssock)) 1307cec37a6eSPeter Krystad return -EINVAL; 1308f870fa0bSMat Martineau 1309cec37a6eSPeter Krystad return inet_csk_get_port(ssock->sk, snum); 1310cec37a6eSPeter Krystad } 1311f870fa0bSMat Martineau 1312cec37a6eSPeter Krystad void mptcp_finish_connect(struct sock *ssk) 1313cec37a6eSPeter Krystad { 1314cec37a6eSPeter Krystad struct mptcp_subflow_context *subflow; 1315cec37a6eSPeter Krystad struct mptcp_sock *msk; 1316cec37a6eSPeter Krystad struct sock *sk; 13176d0060f6SMat Martineau u64 ack_seq; 1318f870fa0bSMat Martineau 1319cec37a6eSPeter Krystad subflow = mptcp_subflow_ctx(ssk); 1320f870fa0bSMat Martineau 1321cec37a6eSPeter Krystad if (!subflow->mp_capable) 1322cec37a6eSPeter Krystad return; 1323cec37a6eSPeter Krystad 1324cec37a6eSPeter Krystad sk = subflow->conn; 1325cec37a6eSPeter Krystad msk = mptcp_sk(sk); 1326cec37a6eSPeter Krystad 1327648ef4b8SMat Martineau pr_debug("msk=%p, token=%u", sk, subflow->token); 1328648ef4b8SMat Martineau 13296d0060f6SMat Martineau mptcp_crypto_key_sha(subflow->remote_key, NULL, &ack_seq); 13306d0060f6SMat Martineau ack_seq++; 1331648ef4b8SMat Martineau subflow->map_seq = ack_seq; 1332648ef4b8SMat Martineau subflow->map_subflow_seq = 1; 13336d0060f6SMat Martineau subflow->rel_write_seq = 1; 13346d0060f6SMat Martineau 1335cec37a6eSPeter Krystad /* the socket is not connected yet, no msk/subflow ops can access/race 1336cec37a6eSPeter Krystad * accessing the field below 1337cec37a6eSPeter Krystad */ 1338cec37a6eSPeter Krystad WRITE_ONCE(msk->remote_key, subflow->remote_key); 1339cec37a6eSPeter Krystad WRITE_ONCE(msk->local_key, subflow->local_key); 134079c0949eSPeter Krystad WRITE_ONCE(msk->token, subflow->token); 13416d0060f6SMat Martineau WRITE_ONCE(msk->write_seq, subflow->idsn + 1); 13426d0060f6SMat Martineau WRITE_ONCE(msk->ack_seq, ack_seq); 1343d22f4988SChristoph Paasch WRITE_ONCE(msk->can_ack, 1); 1344cc9d2566SPaolo Abeni atomic64_set(&msk->snd_una, msk->write_seq); 13451b1c7a0eSPeter Krystad 13461b1c7a0eSPeter Krystad mptcp_pm_new_connection(msk, 0); 1347f870fa0bSMat Martineau } 1348f870fa0bSMat Martineau 1349cf7da0d6SPeter Krystad static void mptcp_sock_graft(struct sock *sk, struct socket *parent) 1350cf7da0d6SPeter Krystad { 1351cf7da0d6SPeter Krystad write_lock_bh(&sk->sk_callback_lock); 1352cf7da0d6SPeter Krystad rcu_assign_pointer(sk->sk_wq, &parent->wq); 1353cf7da0d6SPeter Krystad sk_set_socket(sk, parent); 1354cf7da0d6SPeter Krystad sk->sk_uid = SOCK_INODE(parent)->i_uid; 1355cf7da0d6SPeter Krystad write_unlock_bh(&sk->sk_callback_lock); 1356cf7da0d6SPeter Krystad } 1357cf7da0d6SPeter Krystad 1358f296234cSPeter Krystad bool mptcp_finish_join(struct sock *sk) 1359f296234cSPeter Krystad { 1360f296234cSPeter Krystad struct mptcp_subflow_context *subflow = mptcp_subflow_ctx(sk); 1361f296234cSPeter Krystad struct mptcp_sock *msk = mptcp_sk(subflow->conn); 1362f296234cSPeter Krystad struct sock *parent = (void *)msk; 1363f296234cSPeter Krystad struct socket *parent_sock; 1364ec3edaa7SPeter Krystad bool ret; 1365f296234cSPeter Krystad 1366f296234cSPeter Krystad pr_debug("msk=%p, subflow=%p", msk, subflow); 1367f296234cSPeter Krystad 1368f296234cSPeter Krystad /* mptcp socket already closing? */ 1369f296234cSPeter Krystad if (inet_sk_state_load(parent) != TCP_ESTABLISHED) 1370f296234cSPeter Krystad return false; 1371f296234cSPeter Krystad 1372f296234cSPeter Krystad if (!msk->pm.server_side) 1373f296234cSPeter Krystad return true; 1374f296234cSPeter Krystad 1375f296234cSPeter Krystad /* passive connection, attach to msk socket */ 1376f296234cSPeter Krystad parent_sock = READ_ONCE(parent->sk_socket); 1377f296234cSPeter Krystad if (parent_sock && !sk->sk_socket) 1378f296234cSPeter Krystad mptcp_sock_graft(sk, parent_sock); 1379f296234cSPeter Krystad 1380ec3edaa7SPeter Krystad ret = mptcp_pm_allow_new_subflow(msk); 1381ec3edaa7SPeter Krystad if (ret) { 1382ec3edaa7SPeter Krystad /* active connections are already on conn_list */ 1383ec3edaa7SPeter Krystad spin_lock_bh(&msk->join_list_lock); 1384ec3edaa7SPeter Krystad if (!WARN_ON_ONCE(!list_empty(&subflow->node))) 1385ec3edaa7SPeter Krystad list_add_tail(&subflow->node, &msk->join_list); 1386ec3edaa7SPeter Krystad spin_unlock_bh(&msk->join_list_lock); 1387ec3edaa7SPeter Krystad } 1388ec3edaa7SPeter Krystad return ret; 1389f296234cSPeter Krystad } 1390f296234cSPeter Krystad 1391f296234cSPeter Krystad bool mptcp_sk_is_subflow(const struct sock *sk) 1392f296234cSPeter Krystad { 1393f296234cSPeter Krystad struct mptcp_subflow_context *subflow = mptcp_subflow_ctx(sk); 1394f296234cSPeter Krystad 1395f296234cSPeter Krystad return subflow->mp_join == 1; 1396f296234cSPeter Krystad } 1397f296234cSPeter Krystad 13981891c4a0SFlorian Westphal static bool mptcp_memory_free(const struct sock *sk, int wake) 13991891c4a0SFlorian Westphal { 14001891c4a0SFlorian Westphal struct mptcp_sock *msk = mptcp_sk(sk); 14011891c4a0SFlorian Westphal 14021891c4a0SFlorian Westphal return wake ? test_bit(MPTCP_SEND_SPACE, &msk->flags) : true; 14031891c4a0SFlorian Westphal } 14041891c4a0SFlorian Westphal 1405f870fa0bSMat Martineau static struct proto mptcp_prot = { 1406f870fa0bSMat Martineau .name = "MPTCP", 1407f870fa0bSMat Martineau .owner = THIS_MODULE, 1408f870fa0bSMat Martineau .init = mptcp_init_sock, 140918b683bfSPaolo Abeni .disconnect = mptcp_disconnect, 1410f870fa0bSMat Martineau .close = mptcp_close, 1411cf7da0d6SPeter Krystad .accept = mptcp_accept, 1412717e79c8SPeter Krystad .setsockopt = mptcp_setsockopt, 1413717e79c8SPeter Krystad .getsockopt = mptcp_getsockopt, 1414f870fa0bSMat Martineau .shutdown = tcp_shutdown, 141579c0949eSPeter Krystad .destroy = mptcp_destroy, 1416f870fa0bSMat Martineau .sendmsg = mptcp_sendmsg, 1417f870fa0bSMat Martineau .recvmsg = mptcp_recvmsg, 141814c441b5SPaolo Abeni .release_cb = mptcp_release_cb, 1419f870fa0bSMat Martineau .hash = inet_hash, 1420f870fa0bSMat Martineau .unhash = inet_unhash, 1421cec37a6eSPeter Krystad .get_port = mptcp_get_port, 1422*d027236cSPaolo Abeni .sockets_allocated = &mptcp_sockets_allocated, 1423*d027236cSPaolo Abeni .memory_allocated = &tcp_memory_allocated, 1424*d027236cSPaolo Abeni .memory_pressure = &tcp_memory_pressure, 14251891c4a0SFlorian Westphal .stream_memory_free = mptcp_memory_free, 1426*d027236cSPaolo Abeni .sysctl_wmem_offset = offsetof(struct net, ipv4.sysctl_tcp_wmem), 1427*d027236cSPaolo Abeni .sysctl_mem = sysctl_tcp_mem, 1428f870fa0bSMat Martineau .obj_size = sizeof(struct mptcp_sock), 1429f870fa0bSMat Martineau .no_autobind = true, 1430f870fa0bSMat Martineau }; 1431f870fa0bSMat Martineau 14322303f994SPeter Krystad static int mptcp_bind(struct socket *sock, struct sockaddr *uaddr, int addr_len) 14332303f994SPeter Krystad { 14342303f994SPeter Krystad struct mptcp_sock *msk = mptcp_sk(sock->sk); 14352303f994SPeter Krystad struct socket *ssock; 1436cf7da0d6SPeter Krystad int err; 14372303f994SPeter Krystad 14382303f994SPeter Krystad lock_sock(sock->sk); 14392303f994SPeter Krystad ssock = __mptcp_socket_create(msk, MPTCP_SAME_STATE); 14402303f994SPeter Krystad if (IS_ERR(ssock)) { 14412303f994SPeter Krystad err = PTR_ERR(ssock); 14422303f994SPeter Krystad goto unlock; 14432303f994SPeter Krystad } 14442303f994SPeter Krystad 14452303f994SPeter Krystad err = ssock->ops->bind(ssock, uaddr, addr_len); 1446cf7da0d6SPeter Krystad if (!err) 1447cf7da0d6SPeter Krystad mptcp_copy_inaddrs(sock->sk, ssock->sk); 14482303f994SPeter Krystad 14492303f994SPeter Krystad unlock: 14502303f994SPeter Krystad release_sock(sock->sk); 14512303f994SPeter Krystad return err; 14522303f994SPeter Krystad } 14532303f994SPeter Krystad 14542303f994SPeter Krystad static int mptcp_stream_connect(struct socket *sock, struct sockaddr *uaddr, 14552303f994SPeter Krystad int addr_len, int flags) 14562303f994SPeter Krystad { 14572303f994SPeter Krystad struct mptcp_sock *msk = mptcp_sk(sock->sk); 14582303f994SPeter Krystad struct socket *ssock; 14592303f994SPeter Krystad int err; 14602303f994SPeter Krystad 14612303f994SPeter Krystad lock_sock(sock->sk); 14622303f994SPeter Krystad ssock = __mptcp_socket_create(msk, TCP_SYN_SENT); 14632303f994SPeter Krystad if (IS_ERR(ssock)) { 14642303f994SPeter Krystad err = PTR_ERR(ssock); 14652303f994SPeter Krystad goto unlock; 14662303f994SPeter Krystad } 14672303f994SPeter Krystad 1468cf7da0d6SPeter Krystad #ifdef CONFIG_TCP_MD5SIG 1469cf7da0d6SPeter Krystad /* no MPTCP if MD5SIG is enabled on this socket or we may run out of 1470cf7da0d6SPeter Krystad * TCP option space. 1471cf7da0d6SPeter Krystad */ 1472cf7da0d6SPeter Krystad if (rcu_access_pointer(tcp_sk(ssock->sk)->md5sig_info)) 1473cf7da0d6SPeter Krystad mptcp_subflow_ctx(ssock->sk)->request_mptcp = 0; 1474cf7da0d6SPeter Krystad #endif 1475cf7da0d6SPeter Krystad 14762303f994SPeter Krystad err = ssock->ops->connect(ssock, uaddr, addr_len, flags); 14772303f994SPeter Krystad inet_sk_state_store(sock->sk, inet_sk_state_load(ssock->sk)); 1478cf7da0d6SPeter Krystad mptcp_copy_inaddrs(sock->sk, ssock->sk); 14792303f994SPeter Krystad 14802303f994SPeter Krystad unlock: 14812303f994SPeter Krystad release_sock(sock->sk); 14822303f994SPeter Krystad return err; 14832303f994SPeter Krystad } 14842303f994SPeter Krystad 1485cf7da0d6SPeter Krystad static int mptcp_v4_getname(struct socket *sock, struct sockaddr *uaddr, 1486cf7da0d6SPeter Krystad int peer) 1487cf7da0d6SPeter Krystad { 1488cf7da0d6SPeter Krystad if (sock->sk->sk_prot == &tcp_prot) { 1489cf7da0d6SPeter Krystad /* we are being invoked from __sys_accept4, after 1490cf7da0d6SPeter Krystad * mptcp_accept() has just accepted a non-mp-capable 1491cf7da0d6SPeter Krystad * flow: sk is a tcp_sk, not an mptcp one. 1492cf7da0d6SPeter Krystad * 1493cf7da0d6SPeter Krystad * Hand the socket over to tcp so all further socket ops 1494cf7da0d6SPeter Krystad * bypass mptcp. 1495cf7da0d6SPeter Krystad */ 1496cf7da0d6SPeter Krystad sock->ops = &inet_stream_ops; 1497cf7da0d6SPeter Krystad } 1498cf7da0d6SPeter Krystad 1499cf7da0d6SPeter Krystad return inet_getname(sock, uaddr, peer); 1500cf7da0d6SPeter Krystad } 1501cf7da0d6SPeter Krystad 1502cf7da0d6SPeter Krystad #if IS_ENABLED(CONFIG_MPTCP_IPV6) 1503cf7da0d6SPeter Krystad static int mptcp_v6_getname(struct socket *sock, struct sockaddr *uaddr, 1504cf7da0d6SPeter Krystad int peer) 1505cf7da0d6SPeter Krystad { 1506cf7da0d6SPeter Krystad if (sock->sk->sk_prot == &tcpv6_prot) { 1507cf7da0d6SPeter Krystad /* we are being invoked from __sys_accept4 after 1508cf7da0d6SPeter Krystad * mptcp_accept() has accepted a non-mp-capable 1509cf7da0d6SPeter Krystad * subflow: sk is a tcp_sk, not mptcp. 1510cf7da0d6SPeter Krystad * 1511cf7da0d6SPeter Krystad * Hand the socket over to tcp so all further 1512cf7da0d6SPeter Krystad * socket ops bypass mptcp. 1513cf7da0d6SPeter Krystad */ 1514cf7da0d6SPeter Krystad sock->ops = &inet6_stream_ops; 1515cf7da0d6SPeter Krystad } 1516cf7da0d6SPeter Krystad 1517cf7da0d6SPeter Krystad return inet6_getname(sock, uaddr, peer); 1518cf7da0d6SPeter Krystad } 1519cf7da0d6SPeter Krystad #endif 1520cf7da0d6SPeter Krystad 1521cf7da0d6SPeter Krystad static int mptcp_listen(struct socket *sock, int backlog) 1522cf7da0d6SPeter Krystad { 1523cf7da0d6SPeter Krystad struct mptcp_sock *msk = mptcp_sk(sock->sk); 1524cf7da0d6SPeter Krystad struct socket *ssock; 1525cf7da0d6SPeter Krystad int err; 1526cf7da0d6SPeter Krystad 1527cf7da0d6SPeter Krystad pr_debug("msk=%p", msk); 1528cf7da0d6SPeter Krystad 1529cf7da0d6SPeter Krystad lock_sock(sock->sk); 1530cf7da0d6SPeter Krystad ssock = __mptcp_socket_create(msk, TCP_LISTEN); 1531cf7da0d6SPeter Krystad if (IS_ERR(ssock)) { 1532cf7da0d6SPeter Krystad err = PTR_ERR(ssock); 1533cf7da0d6SPeter Krystad goto unlock; 1534cf7da0d6SPeter Krystad } 1535cf7da0d6SPeter Krystad 1536cf7da0d6SPeter Krystad err = ssock->ops->listen(ssock, backlog); 1537cf7da0d6SPeter Krystad inet_sk_state_store(sock->sk, inet_sk_state_load(ssock->sk)); 1538cf7da0d6SPeter Krystad if (!err) 1539cf7da0d6SPeter Krystad mptcp_copy_inaddrs(sock->sk, ssock->sk); 1540cf7da0d6SPeter Krystad 1541cf7da0d6SPeter Krystad unlock: 1542cf7da0d6SPeter Krystad release_sock(sock->sk); 1543cf7da0d6SPeter Krystad return err; 1544cf7da0d6SPeter Krystad } 1545cf7da0d6SPeter Krystad 1546cf7da0d6SPeter Krystad static bool is_tcp_proto(const struct proto *p) 1547cf7da0d6SPeter Krystad { 1548cf7da0d6SPeter Krystad #if IS_ENABLED(CONFIG_MPTCP_IPV6) 1549cf7da0d6SPeter Krystad return p == &tcp_prot || p == &tcpv6_prot; 1550cf7da0d6SPeter Krystad #else 1551cf7da0d6SPeter Krystad return p == &tcp_prot; 1552cf7da0d6SPeter Krystad #endif 1553cf7da0d6SPeter Krystad } 1554cf7da0d6SPeter Krystad 1555cf7da0d6SPeter Krystad static int mptcp_stream_accept(struct socket *sock, struct socket *newsock, 1556cf7da0d6SPeter Krystad int flags, bool kern) 1557cf7da0d6SPeter Krystad { 1558cf7da0d6SPeter Krystad struct mptcp_sock *msk = mptcp_sk(sock->sk); 1559cf7da0d6SPeter Krystad struct socket *ssock; 1560cf7da0d6SPeter Krystad int err; 1561cf7da0d6SPeter Krystad 1562cf7da0d6SPeter Krystad pr_debug("msk=%p", msk); 1563cf7da0d6SPeter Krystad 1564cf7da0d6SPeter Krystad lock_sock(sock->sk); 1565cf7da0d6SPeter Krystad if (sock->sk->sk_state != TCP_LISTEN) 1566cf7da0d6SPeter Krystad goto unlock_fail; 1567cf7da0d6SPeter Krystad 1568cf7da0d6SPeter Krystad ssock = __mptcp_nmpc_socket(msk); 1569cf7da0d6SPeter Krystad if (!ssock) 1570cf7da0d6SPeter Krystad goto unlock_fail; 1571cf7da0d6SPeter Krystad 1572cf7da0d6SPeter Krystad sock_hold(ssock->sk); 1573cf7da0d6SPeter Krystad release_sock(sock->sk); 1574cf7da0d6SPeter Krystad 1575cf7da0d6SPeter Krystad err = ssock->ops->accept(sock, newsock, flags, kern); 1576cf7da0d6SPeter Krystad if (err == 0 && !is_tcp_proto(newsock->sk->sk_prot)) { 1577cf7da0d6SPeter Krystad struct mptcp_sock *msk = mptcp_sk(newsock->sk); 1578cf7da0d6SPeter Krystad struct mptcp_subflow_context *subflow; 1579cf7da0d6SPeter Krystad 1580cf7da0d6SPeter Krystad /* set ssk->sk_socket of accept()ed flows to mptcp socket. 1581cf7da0d6SPeter Krystad * This is needed so NOSPACE flag can be set from tcp stack. 1582cf7da0d6SPeter Krystad */ 1583ec3edaa7SPeter Krystad __mptcp_flush_join_list(msk); 1584cf7da0d6SPeter Krystad list_for_each_entry(subflow, &msk->conn_list, node) { 1585cf7da0d6SPeter Krystad struct sock *ssk = mptcp_subflow_tcp_sock(subflow); 1586cf7da0d6SPeter Krystad 1587cf7da0d6SPeter Krystad if (!ssk->sk_socket) 1588cf7da0d6SPeter Krystad mptcp_sock_graft(ssk, newsock); 1589cf7da0d6SPeter Krystad } 1590cf7da0d6SPeter Krystad } 1591cf7da0d6SPeter Krystad 1592cf7da0d6SPeter Krystad sock_put(ssock->sk); 1593cf7da0d6SPeter Krystad return err; 1594cf7da0d6SPeter Krystad 1595cf7da0d6SPeter Krystad unlock_fail: 1596cf7da0d6SPeter Krystad release_sock(sock->sk); 1597cf7da0d6SPeter Krystad return -EINVAL; 1598cf7da0d6SPeter Krystad } 1599cf7da0d6SPeter Krystad 16002303f994SPeter Krystad static __poll_t mptcp_poll(struct file *file, struct socket *sock, 16012303f994SPeter Krystad struct poll_table_struct *wait) 16022303f994SPeter Krystad { 16031891c4a0SFlorian Westphal struct sock *sk = sock->sk; 16048ab183deSPaolo Abeni struct mptcp_sock *msk; 16051891c4a0SFlorian Westphal struct socket *ssock; 16062303f994SPeter Krystad __poll_t mask = 0; 16072303f994SPeter Krystad 16081891c4a0SFlorian Westphal msk = mptcp_sk(sk); 16091891c4a0SFlorian Westphal lock_sock(sk); 16101891c4a0SFlorian Westphal ssock = __mptcp_nmpc_socket(msk); 16111891c4a0SFlorian Westphal if (ssock) { 16121891c4a0SFlorian Westphal mask = ssock->ops->poll(file, ssock, wait); 16131891c4a0SFlorian Westphal release_sock(sk); 16141891c4a0SFlorian Westphal return mask; 16151891c4a0SFlorian Westphal } 16161891c4a0SFlorian Westphal 16171891c4a0SFlorian Westphal release_sock(sk); 16181891c4a0SFlorian Westphal sock_poll_wait(file, sock, wait); 16191891c4a0SFlorian Westphal lock_sock(sk); 16208ab183deSPaolo Abeni ssock = __mptcp_tcp_fallback(msk); 16218ab183deSPaolo Abeni if (unlikely(ssock)) 16228ab183deSPaolo Abeni return ssock->ops->poll(file, ssock, NULL); 16231891c4a0SFlorian Westphal 16241891c4a0SFlorian Westphal if (test_bit(MPTCP_DATA_READY, &msk->flags)) 16251891c4a0SFlorian Westphal mask = EPOLLIN | EPOLLRDNORM; 16261891c4a0SFlorian Westphal if (sk_stream_is_writeable(sk) && 16271891c4a0SFlorian Westphal test_bit(MPTCP_SEND_SPACE, &msk->flags)) 16281891c4a0SFlorian Westphal mask |= EPOLLOUT | EPOLLWRNORM; 16291891c4a0SFlorian Westphal if (sk->sk_shutdown & RCV_SHUTDOWN) 16301891c4a0SFlorian Westphal mask |= EPOLLIN | EPOLLRDNORM | EPOLLRDHUP; 16311891c4a0SFlorian Westphal 16321891c4a0SFlorian Westphal release_sock(sk); 16331891c4a0SFlorian Westphal 16342303f994SPeter Krystad return mask; 16352303f994SPeter Krystad } 16362303f994SPeter Krystad 163721498490SPeter Krystad static int mptcp_shutdown(struct socket *sock, int how) 163821498490SPeter Krystad { 163921498490SPeter Krystad struct mptcp_sock *msk = mptcp_sk(sock->sk); 164021498490SPeter Krystad struct mptcp_subflow_context *subflow; 164121498490SPeter Krystad int ret = 0; 164221498490SPeter Krystad 164321498490SPeter Krystad pr_debug("sk=%p, how=%d", msk, how); 164421498490SPeter Krystad 164521498490SPeter Krystad lock_sock(sock->sk); 164621498490SPeter Krystad 164721498490SPeter Krystad if (how == SHUT_WR || how == SHUT_RDWR) 164821498490SPeter Krystad inet_sk_state_store(sock->sk, TCP_FIN_WAIT1); 164921498490SPeter Krystad 165021498490SPeter Krystad how++; 165121498490SPeter Krystad 165221498490SPeter Krystad if ((how & ~SHUTDOWN_MASK) || !how) { 165321498490SPeter Krystad ret = -EINVAL; 165421498490SPeter Krystad goto out_unlock; 165521498490SPeter Krystad } 165621498490SPeter Krystad 165721498490SPeter Krystad if (sock->state == SS_CONNECTING) { 165821498490SPeter Krystad if ((1 << sock->sk->sk_state) & 165921498490SPeter Krystad (TCPF_SYN_SENT | TCPF_SYN_RECV | TCPF_CLOSE)) 166021498490SPeter Krystad sock->state = SS_DISCONNECTING; 166121498490SPeter Krystad else 166221498490SPeter Krystad sock->state = SS_CONNECTED; 166321498490SPeter Krystad } 166421498490SPeter Krystad 1665ec3edaa7SPeter Krystad __mptcp_flush_join_list(msk); 166621498490SPeter Krystad mptcp_for_each_subflow(msk, subflow) { 166721498490SPeter Krystad struct sock *tcp_sk = mptcp_subflow_tcp_sock(subflow); 166821498490SPeter Krystad 166976c42a29SMat Martineau mptcp_subflow_shutdown(tcp_sk, how, 1, msk->write_seq); 167021498490SPeter Krystad } 167121498490SPeter Krystad 167221498490SPeter Krystad out_unlock: 167321498490SPeter Krystad release_sock(sock->sk); 167421498490SPeter Krystad 167521498490SPeter Krystad return ret; 167621498490SPeter Krystad } 167721498490SPeter Krystad 1678e42f1ac6SFlorian Westphal static const struct proto_ops mptcp_stream_ops = { 1679e42f1ac6SFlorian Westphal .family = PF_INET, 1680e42f1ac6SFlorian Westphal .owner = THIS_MODULE, 1681e42f1ac6SFlorian Westphal .release = inet_release, 1682e42f1ac6SFlorian Westphal .bind = mptcp_bind, 1683e42f1ac6SFlorian Westphal .connect = mptcp_stream_connect, 1684e42f1ac6SFlorian Westphal .socketpair = sock_no_socketpair, 1685e42f1ac6SFlorian Westphal .accept = mptcp_stream_accept, 1686e42f1ac6SFlorian Westphal .getname = mptcp_v4_getname, 1687e42f1ac6SFlorian Westphal .poll = mptcp_poll, 1688e42f1ac6SFlorian Westphal .ioctl = inet_ioctl, 1689e42f1ac6SFlorian Westphal .gettstamp = sock_gettstamp, 1690e42f1ac6SFlorian Westphal .listen = mptcp_listen, 1691e42f1ac6SFlorian Westphal .shutdown = mptcp_shutdown, 1692e42f1ac6SFlorian Westphal .setsockopt = sock_common_setsockopt, 1693e42f1ac6SFlorian Westphal .getsockopt = sock_common_getsockopt, 1694e42f1ac6SFlorian Westphal .sendmsg = inet_sendmsg, 1695e42f1ac6SFlorian Westphal .recvmsg = inet_recvmsg, 1696e42f1ac6SFlorian Westphal .mmap = sock_no_mmap, 1697e42f1ac6SFlorian Westphal .sendpage = inet_sendpage, 1698e42f1ac6SFlorian Westphal #ifdef CONFIG_COMPAT 1699e42f1ac6SFlorian Westphal .compat_setsockopt = compat_sock_common_setsockopt, 1700e42f1ac6SFlorian Westphal .compat_getsockopt = compat_sock_common_getsockopt, 1701e42f1ac6SFlorian Westphal #endif 1702e42f1ac6SFlorian Westphal }; 17032303f994SPeter Krystad 1704f870fa0bSMat Martineau static struct inet_protosw mptcp_protosw = { 1705f870fa0bSMat Martineau .type = SOCK_STREAM, 1706f870fa0bSMat Martineau .protocol = IPPROTO_MPTCP, 1707f870fa0bSMat Martineau .prot = &mptcp_prot, 17082303f994SPeter Krystad .ops = &mptcp_stream_ops, 17092303f994SPeter Krystad .flags = INET_PROTOSW_ICSK, 1710f870fa0bSMat Martineau }; 1711f870fa0bSMat Martineau 1712784325e9SMatthieu Baerts void mptcp_proto_init(void) 1713f870fa0bSMat Martineau { 17142303f994SPeter Krystad mptcp_prot.h.hashinfo = tcp_prot.h.hashinfo; 17152303f994SPeter Krystad 1716*d027236cSPaolo Abeni if (percpu_counter_init(&mptcp_sockets_allocated, 0, GFP_KERNEL)) 1717*d027236cSPaolo Abeni panic("Failed to allocate MPTCP pcpu counter\n"); 1718*d027236cSPaolo Abeni 17192303f994SPeter Krystad mptcp_subflow_init(); 17201b1c7a0eSPeter Krystad mptcp_pm_init(); 17212303f994SPeter Krystad 1722f870fa0bSMat Martineau if (proto_register(&mptcp_prot, 1) != 0) 1723f870fa0bSMat Martineau panic("Failed to register MPTCP proto.\n"); 1724f870fa0bSMat Martineau 1725f870fa0bSMat Martineau inet_register_protosw(&mptcp_protosw); 17266771bfd9SFlorian Westphal 17276771bfd9SFlorian Westphal BUILD_BUG_ON(sizeof(struct mptcp_skb_cb) > sizeof_field(struct sk_buff, cb)); 1728f870fa0bSMat Martineau } 1729f870fa0bSMat Martineau 1730f870fa0bSMat Martineau #if IS_ENABLED(CONFIG_MPTCP_IPV6) 1731e42f1ac6SFlorian Westphal static const struct proto_ops mptcp_v6_stream_ops = { 1732e42f1ac6SFlorian Westphal .family = PF_INET6, 1733e42f1ac6SFlorian Westphal .owner = THIS_MODULE, 1734e42f1ac6SFlorian Westphal .release = inet6_release, 1735e42f1ac6SFlorian Westphal .bind = mptcp_bind, 1736e42f1ac6SFlorian Westphal .connect = mptcp_stream_connect, 1737e42f1ac6SFlorian Westphal .socketpair = sock_no_socketpair, 1738e42f1ac6SFlorian Westphal .accept = mptcp_stream_accept, 1739e42f1ac6SFlorian Westphal .getname = mptcp_v6_getname, 1740e42f1ac6SFlorian Westphal .poll = mptcp_poll, 1741e42f1ac6SFlorian Westphal .ioctl = inet6_ioctl, 1742e42f1ac6SFlorian Westphal .gettstamp = sock_gettstamp, 1743e42f1ac6SFlorian Westphal .listen = mptcp_listen, 1744e42f1ac6SFlorian Westphal .shutdown = mptcp_shutdown, 1745e42f1ac6SFlorian Westphal .setsockopt = sock_common_setsockopt, 1746e42f1ac6SFlorian Westphal .getsockopt = sock_common_getsockopt, 1747e42f1ac6SFlorian Westphal .sendmsg = inet6_sendmsg, 1748e42f1ac6SFlorian Westphal .recvmsg = inet6_recvmsg, 1749e42f1ac6SFlorian Westphal .mmap = sock_no_mmap, 1750e42f1ac6SFlorian Westphal .sendpage = inet_sendpage, 1751e42f1ac6SFlorian Westphal #ifdef CONFIG_COMPAT 1752e42f1ac6SFlorian Westphal .compat_setsockopt = compat_sock_common_setsockopt, 1753e42f1ac6SFlorian Westphal .compat_getsockopt = compat_sock_common_getsockopt, 1754e42f1ac6SFlorian Westphal #endif 1755e42f1ac6SFlorian Westphal }; 1756e42f1ac6SFlorian Westphal 1757f870fa0bSMat Martineau static struct proto mptcp_v6_prot; 1758f870fa0bSMat Martineau 175979c0949eSPeter Krystad static void mptcp_v6_destroy(struct sock *sk) 176079c0949eSPeter Krystad { 176179c0949eSPeter Krystad mptcp_destroy(sk); 176279c0949eSPeter Krystad inet6_destroy_sock(sk); 176379c0949eSPeter Krystad } 176479c0949eSPeter Krystad 1765f870fa0bSMat Martineau static struct inet_protosw mptcp_v6_protosw = { 1766f870fa0bSMat Martineau .type = SOCK_STREAM, 1767f870fa0bSMat Martineau .protocol = IPPROTO_MPTCP, 1768f870fa0bSMat Martineau .prot = &mptcp_v6_prot, 17692303f994SPeter Krystad .ops = &mptcp_v6_stream_ops, 1770f870fa0bSMat Martineau .flags = INET_PROTOSW_ICSK, 1771f870fa0bSMat Martineau }; 1772f870fa0bSMat Martineau 1773784325e9SMatthieu Baerts int mptcp_proto_v6_init(void) 1774f870fa0bSMat Martineau { 1775f870fa0bSMat Martineau int err; 1776f870fa0bSMat Martineau 1777f870fa0bSMat Martineau mptcp_v6_prot = mptcp_prot; 1778f870fa0bSMat Martineau strcpy(mptcp_v6_prot.name, "MPTCPv6"); 1779f870fa0bSMat Martineau mptcp_v6_prot.slab = NULL; 178079c0949eSPeter Krystad mptcp_v6_prot.destroy = mptcp_v6_destroy; 1781b0519de8SFlorian Westphal mptcp_v6_prot.obj_size = sizeof(struct mptcp6_sock); 1782f870fa0bSMat Martineau 1783f870fa0bSMat Martineau err = proto_register(&mptcp_v6_prot, 1); 1784f870fa0bSMat Martineau if (err) 1785f870fa0bSMat Martineau return err; 1786f870fa0bSMat Martineau 1787f870fa0bSMat Martineau err = inet6_register_protosw(&mptcp_v6_protosw); 1788f870fa0bSMat Martineau if (err) 1789f870fa0bSMat Martineau proto_unregister(&mptcp_v6_prot); 1790f870fa0bSMat Martineau 1791f870fa0bSMat Martineau return err; 1792f870fa0bSMat Martineau } 1793f870fa0bSMat Martineau #endif 1794