1 // SPDX-License-Identifier: GPL-2.0 2 3 #include "vmlinux.h" 4 5 #include <bpf/bpf_helpers.h> 6 #include <bpf/bpf_tracing.h> 7 8 char _license[] SEC("license") = "GPL"; 9 10 #define USEC_PER_SEC 1000000UL 11 12 #define min(a, b) ((a) < (b) ? (a) : (b)) 13 14 static inline struct tcp_sock *tcp_sk(const struct sock *sk) 15 { 16 return (struct tcp_sock *)sk; 17 } 18 19 SEC("struct_ops/write_sk_pacing_init") 20 void BPF_PROG(write_sk_pacing_init, struct sock *sk) 21 { 22 #ifdef ENABLE_ATOMICS_TESTS 23 __sync_bool_compare_and_swap(&sk->sk_pacing_status, SK_PACING_NONE, 24 SK_PACING_NEEDED); 25 #else 26 sk->sk_pacing_status = SK_PACING_NEEDED; 27 #endif 28 } 29 30 SEC("struct_ops/write_sk_pacing_cong_control") 31 void BPF_PROG(write_sk_pacing_cong_control, struct sock *sk, 32 const struct rate_sample *rs) 33 { 34 const struct tcp_sock *tp = tcp_sk(sk); 35 unsigned long rate = 36 ((tp->snd_cwnd * tp->mss_cache * USEC_PER_SEC) << 3) / 37 (tp->srtt_us ?: 1U << 3); 38 sk->sk_pacing_rate = min(rate, sk->sk_max_pacing_rate); 39 } 40 41 SEC("struct_ops/write_sk_pacing_ssthresh") 42 __u32 BPF_PROG(write_sk_pacing_ssthresh, struct sock *sk) 43 { 44 return tcp_sk(sk)->snd_ssthresh; 45 } 46 47 SEC("struct_ops/write_sk_pacing_undo_cwnd") 48 __u32 BPF_PROG(write_sk_pacing_undo_cwnd, struct sock *sk) 49 { 50 return tcp_sk(sk)->snd_cwnd; 51 } 52 53 SEC(".struct_ops") 54 struct tcp_congestion_ops write_sk_pacing = { 55 .init = (void *)write_sk_pacing_init, 56 .cong_control = (void *)write_sk_pacing_cong_control, 57 .ssthresh = (void *)write_sk_pacing_ssthresh, 58 .undo_cwnd = (void *)write_sk_pacing_undo_cwnd, 59 .name = "bpf_w_sk_pacing", 60 }; 61