1 // SPDX-License-Identifier: GPL-2.0
2 
3 #include "vmlinux.h"
4 
5 #include <bpf/bpf_helpers.h>
6 #include <bpf/bpf_tracing.h>
7 
8 char _license[] SEC("license") = "GPL";
9 
10 #define USEC_PER_SEC 1000000UL
11 
12 #define min(a, b) ((a) < (b) ? (a) : (b))
13 
14 static inline struct tcp_sock *tcp_sk(const struct sock *sk)
15 {
16 	return (struct tcp_sock *)sk;
17 }
18 
19 SEC("struct_ops/write_sk_pacing_init")
20 void BPF_PROG(write_sk_pacing_init, struct sock *sk)
21 {
22 #ifdef ENABLE_ATOMICS_TESTS
23 	__sync_bool_compare_and_swap(&sk->sk_pacing_status, SK_PACING_NONE,
24 				     SK_PACING_NEEDED);
25 #else
26 	sk->sk_pacing_status = SK_PACING_NEEDED;
27 #endif
28 }
29 
30 SEC("struct_ops/write_sk_pacing_cong_control")
31 void BPF_PROG(write_sk_pacing_cong_control, struct sock *sk,
32 	      const struct rate_sample *rs)
33 {
34 	const struct tcp_sock *tp = tcp_sk(sk);
35 	unsigned long rate =
36 		((tp->snd_cwnd * tp->mss_cache * USEC_PER_SEC) << 3) /
37 		(tp->srtt_us ?: 1U << 3);
38 	sk->sk_pacing_rate = min(rate, sk->sk_max_pacing_rate);
39 }
40 
41 SEC("struct_ops/write_sk_pacing_ssthresh")
42 __u32 BPF_PROG(write_sk_pacing_ssthresh, struct sock *sk)
43 {
44 	return tcp_sk(sk)->snd_ssthresh;
45 }
46 
47 SEC("struct_ops/write_sk_pacing_undo_cwnd")
48 __u32 BPF_PROG(write_sk_pacing_undo_cwnd, struct sock *sk)
49 {
50 	return tcp_sk(sk)->snd_cwnd;
51 }
52 
53 SEC(".struct_ops")
54 struct tcp_congestion_ops write_sk_pacing = {
55 	.init = (void *)write_sk_pacing_init,
56 	.cong_control = (void *)write_sk_pacing_cong_control,
57 	.ssthresh = (void *)write_sk_pacing_ssthresh,
58 	.undo_cwnd = (void *)write_sk_pacing_undo_cwnd,
59 	.name = "bpf_w_sk_pacing",
60 };
61