1 /*-
2 * Copyright (c) 2015-2017 Patrick Kelsey
3 * All rights reserved.
4 *
5 * Redistribution and use in source and binary forms, with or without
6 * modification, are permitted provided that the following conditions
7 * are met:
8 * 1. Redistributions of source code must retain the above copyright
9 * notice, this list of conditions and the following disclaimer.
10 * 2. Redistributions in binary form must reproduce the above copyright
11 * notice, this list of conditions and the following disclaimer in the
12 * documentation and/or other materials provided with the distribution.
13 *
14 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
15 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
16 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
17 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
18 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
19 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
20 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
21 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
22 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
23 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
24 * SUCH DAMAGE.
25 */
26
27 /*
28 * This is an implementation of TCP Fast Open (TFO) [RFC7413]. To include
29 * this code, add the following line to your kernel config:
30 *
31 * options TCP_RFC7413
32 *
33 *
34 * The generated TFO cookies are the 64-bit output of
35 * SipHash24(key=<16-byte-key>, msg=<client-ip>). Multiple concurrent valid
36 * keys are supported so that time-based rolling cookie invalidation
37 * policies can be implemented in the system. The default number of
38 * concurrent keys is 2. This can be adjusted in the kernel config as
39 * follows:
40 *
41 * options TCP_RFC7413_MAX_KEYS=<num-keys>
42 *
43 *
44 * In addition to the facilities defined in RFC7413, this implementation
45 * supports a pre-shared key (PSK) mode of operation in which the TFO server
46 * requires the client to be in possession of a shared secret in order for
47 * the client to be able to successfully open TFO connections with the
48 * server. This is useful, for example, in environments where TFO servers
49 * are exposed to both internal and external clients and only wish to allow
50 * TFO connections from internal clients.
51 *
52 * In the PSK mode of operation, the server generates and sends TFO cookies
53 * to requesting clients as usual. However, when validating cookies
54 * received in TFO SYNs from clients, the server requires the
55 * client-supplied cookie to equal SipHash24(key=<16-byte-psk>,
56 * msg=<cookie-sent-to-client>).
57 *
58 * Multiple concurrent valid pre-shared keys are supported so that
59 * time-based rolling PSK invalidation policies can be implemented in the
60 * system. The default number of concurrent pre-shared keys is 2. This can
61 * be adjusted in the kernel config as follows:
62 *
63 * options TCP_RFC7413_MAX_PSKS=<num-psks>
64 *
65 *
66 * The following TFO-specific sysctls are defined:
67 *
68 * net.inet.tcp.fastopen.acceptany (RW, default 0)
69 * When non-zero, all client-supplied TFO cookies will be considered to
70 * be valid.
71 *
72 * net.inet.tcp.fastopen.autokey (RW, default 120)
73 * When this and net.inet.tcp.fastopen.server_enable are non-zero, a new
74 * key will be automatically generated after this many seconds.
75 *
76 * net.inet.tcp.fastopen.ccache_bucket_limit
77 * (RWTUN, default TCP_FASTOPEN_CCACHE_BUCKET_LIMIT_DEFAULT)
78 * The maximum number of entries in a client cookie cache bucket.
79 *
80 * net.inet.tcp.fastopen.ccache_buckets
81 * (RDTUN, default TCP_FASTOPEN_CCACHE_BUCKETS_DEFAULT)
82 * The number of client cookie cache buckets.
83 *
84 * net.inet.tcp.fastopen.ccache_list (RO)
85 * Print the client cookie cache.
86 *
87 * net.inet.tcp.fastopen.client_enable (RW, default 0)
88 * When zero, no new active (i.e., client) TFO connections can be
89 * created. On the transition from enabled to disabled, the client
90 * cookie cache is cleared and disabled. The transition from enabled to
91 * disabled does not affect any active TFO connections in progress; it
92 * only prevents new ones from being made.
93 *
94 * net.inet.tcp.fastopen.keylen (RD)
95 * The key length in bytes.
96 *
97 * net.inet.tcp.fastopen.maxkeys (RD)
98 * The maximum number of keys supported.
99 *
100 * net.inet.tcp.fastopen.maxpsks (RD)
101 * The maximum number of pre-shared keys supported.
102 *
103 * net.inet.tcp.fastopen.numkeys (RD)
104 * The current number of keys installed.
105 *
106 * net.inet.tcp.fastopen.numpsks (RD)
107 * The current number of pre-shared keys installed.
108 *
109 * net.inet.tcp.fastopen.path_disable_time
110 * (RW, default TCP_FASTOPEN_PATH_DISABLE_TIME_DEFAULT)
111 * When a failure occurs while trying to create a new active (i.e.,
112 * client) TFO connection, new active connections on the same path, as
113 * determined by the tuple {client_ip, server_ip, server_port}, will be
114 * forced to be non-TFO for this many seconds. Note that the path
115 * disable mechanism relies on state stored in client cookie cache
116 * entries, so it is possible for the disable time for a given path to
117 * be reduced if the corresponding client cookie cache entry is reused
118 * due to resource pressure before the disable period has elapsed.
119 *
120 * net.inet.tcp.fastopen.psk_enable (RW, default 0)
121 * When non-zero, pre-shared key (PSK) mode is enabled for all TFO
122 * servers. On the transition from enabled to disabled, all installed
123 * pre-shared keys are removed.
124 *
125 * net.inet.tcp.fastopen.server_enable (RW, default 0)
126 * When zero, no new passive (i.e., server) TFO connections can be
127 * created. On the transition from enabled to disabled, all installed
128 * keys and pre-shared keys are removed. On the transition from
129 * disabled to enabled, if net.inet.tcp.fastopen.autokey is non-zero and
130 * there are no keys installed, a new key will be generated immediately.
131 * The transition from enabled to disabled does not affect any passive
132 * TFO connections in progress; it only prevents new ones from being
133 * made.
134 *
135 * net.inet.tcp.fastopen.setkey (WR)
136 * Install a new key by writing net.inet.tcp.fastopen.keylen bytes to
137 * this sysctl.
138 *
139 * net.inet.tcp.fastopen.setpsk (WR)
140 * Install a new pre-shared key by writing net.inet.tcp.fastopen.keylen
141 * bytes to this sysctl.
142 *
143 * In order for TFO connections to be created via a listen socket, that
144 * socket must have the TCP_FASTOPEN socket option set on it. This option
145 * can be set on the socket either before or after the listen() is invoked.
146 * Clearing this option on a listen socket after it has been set has no
147 * effect on existing TFO connections or TFO connections in progress; it
148 * only prevents new TFO connections from being made.
149 *
150 * For passively-created sockets, the TCP_FASTOPEN socket option can be
151 * queried to determine whether the connection was established using TFO.
152 * Note that connections that are established via a TFO SYN, but that fall
153 * back to using a non-TFO SYN|ACK will have the TCP_FASTOPEN socket option
154 * set.
155 *
156 * Per the RFC, this implementation limits the number of TFO connections
157 * that can be in the SYN_RECEIVED state on a per listen-socket basis.
158 * Whenever this limit is exceeded, requests for new TFO connections are
159 * serviced as non-TFO requests. Without such a limit, given a valid TFO
160 * cookie, an attacker could keep the listen queue in an overflow condition
161 * using a TFO SYN flood. This implementation sets the limit at half the
162 * configured listen backlog.
163 *
164 */
165
166 #include <sys/cdefs.h>
167 #include "opt_inet.h"
168
169 #include <sys/param.h>
170 #include <sys/jail.h>
171 #include <sys/kernel.h>
172 #include <sys/hash.h>
173 #include <sys/limits.h>
174 #include <sys/lock.h>
175 #include <sys/proc.h>
176 #include <sys/rmlock.h>
177 #include <sys/sbuf.h>
178 #include <sys/socket.h>
179 #include <sys/socketvar.h>
180 #include <sys/sysctl.h>
181 #include <sys/systm.h>
182
183 #include <crypto/siphash/siphash.h>
184
185 #include <net/vnet.h>
186
187 #include <netinet/in.h>
188 #include <netinet/in_pcb.h>
189 #include <netinet/tcp_var.h>
190 #include <netinet/tcp_fastopen.h>
191
192 #define TCP_FASTOPEN_KEY_LEN SIPHASH_KEY_LENGTH
193
194 #if TCP_FASTOPEN_PSK_LEN != TCP_FASTOPEN_KEY_LEN
195 #error TCP_FASTOPEN_PSK_LEN must be equal to TCP_FASTOPEN_KEY_LEN
196 #endif
197
198 /*
199 * Because a PSK-mode setsockopt() uses tcpcb.t_tfo_cookie.client to hold
200 * the PSK until the connect occurs.
201 */
202 #if TCP_FASTOPEN_MAX_COOKIE_LEN < TCP_FASTOPEN_PSK_LEN
203 #error TCP_FASTOPEN_MAX_COOKIE_LEN must be >= TCP_FASTOPEN_PSK_LEN
204 #endif
205
206 #define TCP_FASTOPEN_CCACHE_BUCKET_LIMIT_DEFAULT 16
207 #define TCP_FASTOPEN_CCACHE_BUCKETS_DEFAULT 2048 /* must be power of 2 */
208
209 #define TCP_FASTOPEN_PATH_DISABLE_TIME_DEFAULT 900 /* seconds */
210
211 #if !defined(TCP_RFC7413_MAX_KEYS) || (TCP_RFC7413_MAX_KEYS < 1)
212 #define TCP_FASTOPEN_MAX_KEYS 2
213 #else
214 #define TCP_FASTOPEN_MAX_KEYS TCP_RFC7413_MAX_KEYS
215 #endif
216
217 #if TCP_FASTOPEN_MAX_KEYS > 10
218 #undef TCP_FASTOPEN_MAX_KEYS
219 #define TCP_FASTOPEN_MAX_KEYS 10
220 #endif
221
222 #if !defined(TCP_RFC7413_MAX_PSKS) || (TCP_RFC7413_MAX_PSKS < 1)
223 #define TCP_FASTOPEN_MAX_PSKS 2
224 #else
225 #define TCP_FASTOPEN_MAX_PSKS TCP_RFC7413_MAX_PSKS
226 #endif
227
228 #if TCP_FASTOPEN_MAX_PSKS > 10
229 #undef TCP_FASTOPEN_MAX_PSKS
230 #define TCP_FASTOPEN_MAX_PSKS 10
231 #endif
232
233 struct tcp_fastopen_keylist {
234 unsigned int newest;
235 unsigned int newest_psk;
236 uint8_t key[TCP_FASTOPEN_MAX_KEYS][TCP_FASTOPEN_KEY_LEN];
237 uint8_t psk[TCP_FASTOPEN_MAX_PSKS][TCP_FASTOPEN_KEY_LEN];
238 };
239
240 struct tcp_fastopen_callout {
241 struct callout c;
242 struct vnet *v;
243 };
244
245 static struct tcp_fastopen_ccache_entry *tcp_fastopen_ccache_lookup(
246 struct in_conninfo *, struct tcp_fastopen_ccache_bucket **);
247 static struct tcp_fastopen_ccache_entry *tcp_fastopen_ccache_create(
248 struct tcp_fastopen_ccache_bucket *, struct in_conninfo *, uint16_t, uint8_t,
249 uint8_t *);
250 static void tcp_fastopen_ccache_bucket_trim(struct tcp_fastopen_ccache_bucket *,
251 unsigned int);
252 static void tcp_fastopen_ccache_entry_drop(struct tcp_fastopen_ccache_entry *,
253 struct tcp_fastopen_ccache_bucket *);
254
255 SYSCTL_NODE(_net_inet_tcp, OID_AUTO, fastopen, CTLFLAG_RW | CTLFLAG_MPSAFE, 0,
256 "TCP Fast Open");
257
258 VNET_DEFINE_STATIC(int, tcp_fastopen_acceptany) = 0;
259 #define V_tcp_fastopen_acceptany VNET(tcp_fastopen_acceptany)
260 SYSCTL_INT(_net_inet_tcp_fastopen, OID_AUTO, acceptany,
261 CTLFLAG_VNET | CTLFLAG_RW, &VNET_NAME(tcp_fastopen_acceptany), 0,
262 "Accept any non-empty cookie");
263
264 VNET_DEFINE_STATIC(unsigned int, tcp_fastopen_autokey) = 120;
265 #define V_tcp_fastopen_autokey VNET(tcp_fastopen_autokey)
266 static int sysctl_net_inet_tcp_fastopen_autokey(SYSCTL_HANDLER_ARGS);
267 SYSCTL_PROC(_net_inet_tcp_fastopen, OID_AUTO, autokey,
268 CTLFLAG_VNET | CTLTYPE_UINT | CTLFLAG_RW | CTLFLAG_MPSAFE,
269 NULL, 0, &sysctl_net_inet_tcp_fastopen_autokey, "IU",
270 "Number of seconds between auto-generation of a new key; zero disables");
271
272 static int sysctl_net_inet_tcp_fastopen_ccache_bucket_limit(SYSCTL_HANDLER_ARGS);
273 SYSCTL_PROC(_net_inet_tcp_fastopen, OID_AUTO, ccache_bucket_limit,
274 CTLFLAG_VNET | CTLTYPE_UINT | CTLFLAG_RWTUN | CTLFLAG_NOFETCH | CTLFLAG_NEEDGIANT,
275 NULL, 0, &sysctl_net_inet_tcp_fastopen_ccache_bucket_limit, "IU",
276 "Max entries per bucket in client cookie cache");
277
278 VNET_DEFINE_STATIC(unsigned int, tcp_fastopen_ccache_buckets) =
279 TCP_FASTOPEN_CCACHE_BUCKETS_DEFAULT;
280 #define V_tcp_fastopen_ccache_buckets VNET(tcp_fastopen_ccache_buckets)
281 SYSCTL_UINT(_net_inet_tcp_fastopen, OID_AUTO, ccache_buckets,
282 CTLFLAG_VNET | CTLFLAG_RDTUN, &VNET_NAME(tcp_fastopen_ccache_buckets), 0,
283 "Client cookie cache number of buckets (power of 2)");
284
285 VNET_DEFINE(unsigned int, tcp_fastopen_client_enable) = 1;
286 static int sysctl_net_inet_tcp_fastopen_client_enable(SYSCTL_HANDLER_ARGS);
287 SYSCTL_PROC(_net_inet_tcp_fastopen, OID_AUTO, client_enable,
288 CTLFLAG_VNET | CTLTYPE_UINT | CTLFLAG_RW | CTLFLAG_NEEDGIANT,
289 NULL, 0, &sysctl_net_inet_tcp_fastopen_client_enable, "IU",
290 "Enable/disable TCP Fast Open client functionality");
291
292 SYSCTL_INT(_net_inet_tcp_fastopen, OID_AUTO, keylen,
293 CTLFLAG_RD, SYSCTL_NULL_INT_PTR, TCP_FASTOPEN_KEY_LEN,
294 "Key length in bytes");
295
296 SYSCTL_INT(_net_inet_tcp_fastopen, OID_AUTO, maxkeys,
297 CTLFLAG_RD, SYSCTL_NULL_INT_PTR, TCP_FASTOPEN_MAX_KEYS,
298 "Maximum number of keys supported");
299
300 SYSCTL_INT(_net_inet_tcp_fastopen, OID_AUTO, maxpsks,
301 CTLFLAG_RD, SYSCTL_NULL_INT_PTR, TCP_FASTOPEN_MAX_PSKS,
302 "Maximum number of pre-shared keys supported");
303
304 VNET_DEFINE_STATIC(unsigned int, tcp_fastopen_numkeys) = 0;
305 #define V_tcp_fastopen_numkeys VNET(tcp_fastopen_numkeys)
306 SYSCTL_UINT(_net_inet_tcp_fastopen, OID_AUTO, numkeys,
307 CTLFLAG_VNET | CTLFLAG_RD, &VNET_NAME(tcp_fastopen_numkeys), 0,
308 "Number of keys installed");
309
310 VNET_DEFINE_STATIC(unsigned int, tcp_fastopen_numpsks) = 0;
311 #define V_tcp_fastopen_numpsks VNET(tcp_fastopen_numpsks)
312 SYSCTL_UINT(_net_inet_tcp_fastopen, OID_AUTO, numpsks,
313 CTLFLAG_VNET | CTLFLAG_RD, &VNET_NAME(tcp_fastopen_numpsks), 0,
314 "Number of pre-shared keys installed");
315
316 VNET_DEFINE_STATIC(unsigned int, tcp_fastopen_path_disable_time) =
317 TCP_FASTOPEN_PATH_DISABLE_TIME_DEFAULT;
318 #define V_tcp_fastopen_path_disable_time VNET(tcp_fastopen_path_disable_time)
319 SYSCTL_UINT(_net_inet_tcp_fastopen, OID_AUTO, path_disable_time,
320 CTLFLAG_VNET | CTLFLAG_RW, &VNET_NAME(tcp_fastopen_path_disable_time), 0,
321 "Seconds a TFO failure disables a {client_ip, server_ip, server_port} path");
322
323 VNET_DEFINE_STATIC(unsigned int, tcp_fastopen_psk_enable) = 0;
324 #define V_tcp_fastopen_psk_enable VNET(tcp_fastopen_psk_enable)
325 static int sysctl_net_inet_tcp_fastopen_psk_enable(SYSCTL_HANDLER_ARGS);
326 SYSCTL_PROC(_net_inet_tcp_fastopen, OID_AUTO, psk_enable,
327 CTLFLAG_VNET | CTLTYPE_UINT | CTLFLAG_RW | CTLFLAG_MPSAFE,
328 NULL, 0, &sysctl_net_inet_tcp_fastopen_psk_enable, "IU",
329 "Enable/disable TCP Fast Open server pre-shared key mode");
330
331 VNET_DEFINE(unsigned int, tcp_fastopen_server_enable) = 0;
332 static int sysctl_net_inet_tcp_fastopen_server_enable(SYSCTL_HANDLER_ARGS);
333 SYSCTL_PROC(_net_inet_tcp_fastopen, OID_AUTO, server_enable,
334 CTLFLAG_VNET | CTLTYPE_UINT | CTLFLAG_RW | CTLFLAG_MPSAFE,
335 NULL, 0, &sysctl_net_inet_tcp_fastopen_server_enable, "IU",
336 "Enable/disable TCP Fast Open server functionality");
337
338 static int sysctl_net_inet_tcp_fastopen_setkey(SYSCTL_HANDLER_ARGS);
339 SYSCTL_PROC(_net_inet_tcp_fastopen, OID_AUTO, setkey,
340 CTLFLAG_VNET | CTLTYPE_OPAQUE | CTLFLAG_WR | CTLFLAG_MPSAFE,
341 NULL, 0, &sysctl_net_inet_tcp_fastopen_setkey, "",
342 "Install a new key");
343
344 static int sysctl_net_inet_tcp_fastopen_setpsk(SYSCTL_HANDLER_ARGS);
345 SYSCTL_PROC(_net_inet_tcp_fastopen, OID_AUTO, setpsk,
346 CTLFLAG_VNET | CTLTYPE_OPAQUE | CTLFLAG_WR | CTLFLAG_MPSAFE,
347 NULL, 0, &sysctl_net_inet_tcp_fastopen_setpsk, "",
348 "Install a new pre-shared key");
349
350 static int sysctl_net_inet_tcp_fastopen_ccache_list(SYSCTL_HANDLER_ARGS);
351 SYSCTL_PROC(_net_inet_tcp_fastopen, OID_AUTO, ccache_list,
352 CTLFLAG_VNET | CTLTYPE_STRING | CTLFLAG_RD | CTLFLAG_SKIP | CTLFLAG_MPSAFE,
353 NULL, 0, sysctl_net_inet_tcp_fastopen_ccache_list, "A",
354 "List of all client cookie cache entries");
355
356 VNET_DEFINE_STATIC(struct rmlock, tcp_fastopen_keylock);
357 #define V_tcp_fastopen_keylock VNET(tcp_fastopen_keylock)
358
359 #define TCP_FASTOPEN_KEYS_RLOCK(t) rm_rlock(&V_tcp_fastopen_keylock, (t))
360 #define TCP_FASTOPEN_KEYS_RUNLOCK(t) rm_runlock(&V_tcp_fastopen_keylock, (t))
361 #define TCP_FASTOPEN_KEYS_WLOCK() rm_wlock(&V_tcp_fastopen_keylock)
362 #define TCP_FASTOPEN_KEYS_WUNLOCK() rm_wunlock(&V_tcp_fastopen_keylock)
363
364 VNET_DEFINE_STATIC(struct tcp_fastopen_keylist, tcp_fastopen_keys);
365 #define V_tcp_fastopen_keys VNET(tcp_fastopen_keys)
366
367 VNET_DEFINE_STATIC(struct tcp_fastopen_callout, tcp_fastopen_autokey_ctx);
368 #define V_tcp_fastopen_autokey_ctx VNET(tcp_fastopen_autokey_ctx)
369
370 VNET_DEFINE_STATIC(uma_zone_t, counter_zone);
371 #define V_counter_zone VNET(counter_zone)
372
373 static MALLOC_DEFINE(M_TCP_FASTOPEN_CCACHE, "tfo_ccache", "TFO client cookie cache buckets");
374
375 VNET_DEFINE_STATIC(struct tcp_fastopen_ccache, tcp_fastopen_ccache);
376 #define V_tcp_fastopen_ccache VNET(tcp_fastopen_ccache)
377
378 #define CCB_LOCK(ccb) mtx_lock(&(ccb)->ccb_mtx)
379 #define CCB_UNLOCK(ccb) mtx_unlock(&(ccb)->ccb_mtx)
380 #define CCB_LOCK_ASSERT(ccb) mtx_assert(&(ccb)->ccb_mtx, MA_OWNED)
381
382 void
tcp_fastopen_init(void)383 tcp_fastopen_init(void)
384 {
385 unsigned int i;
386
387 V_counter_zone = uma_zcreate("tfo", sizeof(unsigned int),
388 NULL, NULL, NULL, NULL, UMA_ALIGN_PTR, 0);
389 rm_init(&V_tcp_fastopen_keylock, "tfo_keylock");
390 callout_init_rm(&V_tcp_fastopen_autokey_ctx.c,
391 &V_tcp_fastopen_keylock, 0);
392 V_tcp_fastopen_autokey_ctx.v = curvnet;
393 V_tcp_fastopen_keys.newest = TCP_FASTOPEN_MAX_KEYS - 1;
394 V_tcp_fastopen_keys.newest_psk = TCP_FASTOPEN_MAX_PSKS - 1;
395
396 TUNABLE_INT_FETCH("net.inet.tcp.fastopen.ccache_bucket_limit",
397 &V_tcp_fastopen_ccache.bucket_limit);
398 if (V_tcp_fastopen_ccache.bucket_limit == 0)
399 V_tcp_fastopen_ccache.bucket_limit =
400 TCP_FASTOPEN_CCACHE_BUCKET_LIMIT_DEFAULT;
401
402 /* May already be non-zero if kernel tunable was set */
403 if ((V_tcp_fastopen_ccache_buckets == 0) ||
404 !powerof2(V_tcp_fastopen_ccache_buckets))
405 V_tcp_fastopen_ccache.buckets =
406 TCP_FASTOPEN_CCACHE_BUCKETS_DEFAULT;
407 else
408 V_tcp_fastopen_ccache.buckets = V_tcp_fastopen_ccache_buckets;
409
410 V_tcp_fastopen_ccache.mask = V_tcp_fastopen_ccache.buckets - 1;
411 V_tcp_fastopen_ccache.secret = arc4random();
412
413 V_tcp_fastopen_ccache.base = malloc(V_tcp_fastopen_ccache.buckets *
414 sizeof(struct tcp_fastopen_ccache_bucket), M_TCP_FASTOPEN_CCACHE,
415 M_WAITOK | M_ZERO);
416
417 for (i = 0; i < V_tcp_fastopen_ccache.buckets; i++) {
418 TAILQ_INIT(&V_tcp_fastopen_ccache.base[i].ccb_entries);
419 mtx_init(&V_tcp_fastopen_ccache.base[i].ccb_mtx, "tfo_ccache_bucket",
420 NULL, MTX_DEF);
421 if (V_tcp_fastopen_client_enable) {
422 /* enable bucket */
423 V_tcp_fastopen_ccache.base[i].ccb_num_entries = 0;
424 } else {
425 /* disable bucket */
426 V_tcp_fastopen_ccache.base[i].ccb_num_entries = -1;
427 }
428 V_tcp_fastopen_ccache.base[i].ccb_ccache = &V_tcp_fastopen_ccache;
429 }
430
431 /*
432 * Note that while the total number of entries in the cookie cache
433 * is limited by the table management logic to
434 * V_tcp_fastopen_ccache.buckets *
435 * V_tcp_fastopen_ccache.bucket_limit, the total number of items in
436 * this zone can exceed that amount by the number of CPUs in the
437 * system times the maximum number of unallocated items that can be
438 * present in each UMA per-CPU cache for this zone.
439 */
440 V_tcp_fastopen_ccache.zone = uma_zcreate("tfo_ccache_entries",
441 sizeof(struct tcp_fastopen_ccache_entry), NULL, NULL, NULL, NULL,
442 UMA_ALIGN_CACHE, 0);
443 }
444
445 void
tcp_fastopen_destroy(void)446 tcp_fastopen_destroy(void)
447 {
448 struct tcp_fastopen_ccache_bucket *ccb;
449 unsigned int i;
450
451 for (i = 0; i < V_tcp_fastopen_ccache.buckets; i++) {
452 ccb = &V_tcp_fastopen_ccache.base[i];
453 tcp_fastopen_ccache_bucket_trim(ccb, 0);
454 mtx_destroy(&ccb->ccb_mtx);
455 }
456
457 KASSERT(uma_zone_get_cur(V_tcp_fastopen_ccache.zone) == 0,
458 ("%s: TFO ccache zone allocation count not 0", __func__));
459 uma_zdestroy(V_tcp_fastopen_ccache.zone);
460 free(V_tcp_fastopen_ccache.base, M_TCP_FASTOPEN_CCACHE);
461
462 callout_drain(&V_tcp_fastopen_autokey_ctx.c);
463 rm_destroy(&V_tcp_fastopen_keylock);
464 uma_zdestroy(V_counter_zone);
465 }
466
467 unsigned int *
tcp_fastopen_alloc_counter(void)468 tcp_fastopen_alloc_counter(void)
469 {
470 unsigned int *counter;
471 counter = uma_zalloc(V_counter_zone, M_NOWAIT);
472 if (counter)
473 *counter = 1;
474 return (counter);
475 }
476
477 void
tcp_fastopen_decrement_counter(unsigned int * counter)478 tcp_fastopen_decrement_counter(unsigned int *counter)
479 {
480 if (*counter == 1)
481 uma_zfree(V_counter_zone, counter);
482 else
483 atomic_subtract_int(counter, 1);
484 }
485
486 static void
tcp_fastopen_addkey_locked(uint8_t * key)487 tcp_fastopen_addkey_locked(uint8_t *key)
488 {
489
490 V_tcp_fastopen_keys.newest++;
491 if (V_tcp_fastopen_keys.newest == TCP_FASTOPEN_MAX_KEYS)
492 V_tcp_fastopen_keys.newest = 0;
493 memcpy(V_tcp_fastopen_keys.key[V_tcp_fastopen_keys.newest], key,
494 TCP_FASTOPEN_KEY_LEN);
495 if (V_tcp_fastopen_numkeys < TCP_FASTOPEN_MAX_KEYS)
496 V_tcp_fastopen_numkeys++;
497 }
498
499 static void
tcp_fastopen_addpsk_locked(uint8_t * psk)500 tcp_fastopen_addpsk_locked(uint8_t *psk)
501 {
502
503 V_tcp_fastopen_keys.newest_psk++;
504 if (V_tcp_fastopen_keys.newest_psk == TCP_FASTOPEN_MAX_PSKS)
505 V_tcp_fastopen_keys.newest_psk = 0;
506 memcpy(V_tcp_fastopen_keys.psk[V_tcp_fastopen_keys.newest_psk], psk,
507 TCP_FASTOPEN_KEY_LEN);
508 if (V_tcp_fastopen_numpsks < TCP_FASTOPEN_MAX_PSKS)
509 V_tcp_fastopen_numpsks++;
510 }
511
512 static void
tcp_fastopen_autokey_locked(void)513 tcp_fastopen_autokey_locked(void)
514 {
515 uint8_t newkey[TCP_FASTOPEN_KEY_LEN];
516
517 arc4rand(newkey, TCP_FASTOPEN_KEY_LEN, 0);
518 tcp_fastopen_addkey_locked(newkey);
519 }
520
521 static void
tcp_fastopen_autokey_callout(void * arg)522 tcp_fastopen_autokey_callout(void *arg)
523 {
524 struct tcp_fastopen_callout *ctx = arg;
525
526 CURVNET_SET(ctx->v);
527 tcp_fastopen_autokey_locked();
528 callout_reset(&ctx->c, V_tcp_fastopen_autokey * hz,
529 tcp_fastopen_autokey_callout, ctx);
530 CURVNET_RESTORE();
531 }
532
533 static uint64_t
tcp_fastopen_make_cookie(uint8_t key[SIPHASH_KEY_LENGTH],struct in_conninfo * inc)534 tcp_fastopen_make_cookie(uint8_t key[SIPHASH_KEY_LENGTH], struct in_conninfo *inc)
535 {
536 SIPHASH_CTX ctx;
537 uint64_t siphash;
538
539 SipHash24_Init(&ctx);
540 SipHash_SetKey(&ctx, key);
541 switch (inc->inc_flags & INC_ISIPV6) {
542 #ifdef INET
543 case 0:
544 SipHash_Update(&ctx, &inc->inc_faddr, sizeof(inc->inc_faddr));
545 break;
546 #endif
547 #ifdef INET6
548 case INC_ISIPV6:
549 SipHash_Update(&ctx, &inc->inc6_faddr, sizeof(inc->inc6_faddr));
550 break;
551 #endif
552 }
553 SipHash_Final((u_int8_t *)&siphash, &ctx);
554
555 return (siphash);
556 }
557
558 static uint64_t
tcp_fastopen_make_psk_cookie(uint8_t * psk,uint8_t * cookie,uint8_t cookie_len)559 tcp_fastopen_make_psk_cookie(uint8_t *psk, uint8_t *cookie, uint8_t cookie_len)
560 {
561 SIPHASH_CTX ctx;
562 uint64_t psk_cookie;
563
564 SipHash24_Init(&ctx);
565 SipHash_SetKey(&ctx, psk);
566 SipHash_Update(&ctx, cookie, cookie_len);
567 SipHash_Final((u_int8_t *)&psk_cookie, &ctx);
568
569 return (psk_cookie);
570 }
571
572 static int
tcp_fastopen_find_cookie_match_locked(uint8_t * wire_cookie,uint64_t * cur_cookie)573 tcp_fastopen_find_cookie_match_locked(uint8_t *wire_cookie, uint64_t *cur_cookie)
574 {
575 unsigned int i, psk_index;
576 uint64_t psk_cookie;
577
578 if (V_tcp_fastopen_psk_enable) {
579 psk_index = V_tcp_fastopen_keys.newest_psk;
580 for (i = 0; i < V_tcp_fastopen_numpsks; i++) {
581 psk_cookie =
582 tcp_fastopen_make_psk_cookie(
583 V_tcp_fastopen_keys.psk[psk_index],
584 (uint8_t *)cur_cookie,
585 TCP_FASTOPEN_COOKIE_LEN);
586
587 if (memcmp(wire_cookie, &psk_cookie,
588 TCP_FASTOPEN_COOKIE_LEN) == 0)
589 return (1);
590
591 if (psk_index == 0)
592 psk_index = TCP_FASTOPEN_MAX_PSKS - 1;
593 else
594 psk_index--;
595 }
596 } else if (memcmp(wire_cookie, cur_cookie, TCP_FASTOPEN_COOKIE_LEN) == 0)
597 return (1);
598
599 return (0);
600 }
601
602 /*
603 * Return values:
604 * -1 the cookie is invalid and no valid cookie is available
605 * 0 the cookie is invalid and the latest cookie has been returned
606 * 1 the cookie is valid and the latest cookie has been returned
607 */
608 int
tcp_fastopen_check_cookie(struct in_conninfo * inc,uint8_t * cookie,unsigned int len,uint64_t * latest_cookie)609 tcp_fastopen_check_cookie(struct in_conninfo *inc, uint8_t *cookie,
610 unsigned int len, uint64_t *latest_cookie)
611 {
612 struct rm_priotracker tracker;
613 unsigned int i, key_index;
614 int rv;
615 uint64_t cur_cookie;
616
617 if (V_tcp_fastopen_acceptany) {
618 *latest_cookie = 0;
619 return (1);
620 }
621
622 TCP_FASTOPEN_KEYS_RLOCK(&tracker);
623 if (len != TCP_FASTOPEN_COOKIE_LEN) {
624 if (V_tcp_fastopen_numkeys > 0) {
625 *latest_cookie =
626 tcp_fastopen_make_cookie(
627 V_tcp_fastopen_keys.key[V_tcp_fastopen_keys.newest],
628 inc);
629 rv = 0;
630 } else
631 rv = -1;
632 goto out;
633 }
634
635 /*
636 * Check against each available key, from newest to oldest.
637 */
638 key_index = V_tcp_fastopen_keys.newest;
639 for (i = 0; i < V_tcp_fastopen_numkeys; i++) {
640 cur_cookie =
641 tcp_fastopen_make_cookie(V_tcp_fastopen_keys.key[key_index],
642 inc);
643 if (i == 0)
644 *latest_cookie = cur_cookie;
645 rv = tcp_fastopen_find_cookie_match_locked(cookie, &cur_cookie);
646 if (rv)
647 goto out;
648 if (key_index == 0)
649 key_index = TCP_FASTOPEN_MAX_KEYS - 1;
650 else
651 key_index--;
652 }
653 rv = 0;
654
655 out:
656 TCP_FASTOPEN_KEYS_RUNLOCK(&tracker);
657 return (rv);
658 }
659
660 static int
sysctl_net_inet_tcp_fastopen_autokey(SYSCTL_HANDLER_ARGS)661 sysctl_net_inet_tcp_fastopen_autokey(SYSCTL_HANDLER_ARGS)
662 {
663 int error;
664 unsigned int new;
665
666 new = V_tcp_fastopen_autokey;
667 error = sysctl_handle_int(oidp, &new, 0, req);
668 if (error == 0 && req->newptr) {
669 if (new > (INT_MAX / hz))
670 return (EINVAL);
671
672 TCP_FASTOPEN_KEYS_WLOCK();
673 if (V_tcp_fastopen_server_enable) {
674 if (V_tcp_fastopen_autokey && !new)
675 callout_stop(&V_tcp_fastopen_autokey_ctx.c);
676 else if (new)
677 callout_reset(&V_tcp_fastopen_autokey_ctx.c,
678 new * hz, tcp_fastopen_autokey_callout,
679 &V_tcp_fastopen_autokey_ctx);
680 }
681 V_tcp_fastopen_autokey = new;
682 TCP_FASTOPEN_KEYS_WUNLOCK();
683 }
684
685 return (error);
686 }
687
688 static int
sysctl_net_inet_tcp_fastopen_psk_enable(SYSCTL_HANDLER_ARGS)689 sysctl_net_inet_tcp_fastopen_psk_enable(SYSCTL_HANDLER_ARGS)
690 {
691 int error;
692 unsigned int new;
693
694 new = V_tcp_fastopen_psk_enable;
695 error = sysctl_handle_int(oidp, &new, 0, req);
696 if (error == 0 && req->newptr) {
697 if (V_tcp_fastopen_psk_enable && !new) {
698 /* enabled -> disabled */
699 TCP_FASTOPEN_KEYS_WLOCK();
700 V_tcp_fastopen_numpsks = 0;
701 V_tcp_fastopen_keys.newest_psk =
702 TCP_FASTOPEN_MAX_PSKS - 1;
703 V_tcp_fastopen_psk_enable = 0;
704 TCP_FASTOPEN_KEYS_WUNLOCK();
705 } else if (!V_tcp_fastopen_psk_enable && new) {
706 /* disabled -> enabled */
707 TCP_FASTOPEN_KEYS_WLOCK();
708 V_tcp_fastopen_psk_enable = 1;
709 TCP_FASTOPEN_KEYS_WUNLOCK();
710 }
711 }
712 return (error);
713 }
714
715 static int
sysctl_net_inet_tcp_fastopen_server_enable(SYSCTL_HANDLER_ARGS)716 sysctl_net_inet_tcp_fastopen_server_enable(SYSCTL_HANDLER_ARGS)
717 {
718 int error;
719 unsigned int new;
720
721 new = V_tcp_fastopen_server_enable;
722 error = sysctl_handle_int(oidp, &new, 0, req);
723 if (error == 0 && req->newptr) {
724 if (V_tcp_fastopen_server_enable && !new) {
725 /* enabled -> disabled */
726 TCP_FASTOPEN_KEYS_WLOCK();
727 V_tcp_fastopen_numkeys = 0;
728 V_tcp_fastopen_keys.newest = TCP_FASTOPEN_MAX_KEYS - 1;
729 if (V_tcp_fastopen_autokey)
730 callout_stop(&V_tcp_fastopen_autokey_ctx.c);
731 V_tcp_fastopen_numpsks = 0;
732 V_tcp_fastopen_keys.newest_psk =
733 TCP_FASTOPEN_MAX_PSKS - 1;
734 V_tcp_fastopen_server_enable = 0;
735 TCP_FASTOPEN_KEYS_WUNLOCK();
736 } else if (!V_tcp_fastopen_server_enable && new) {
737 /* disabled -> enabled */
738 TCP_FASTOPEN_KEYS_WLOCK();
739 if (V_tcp_fastopen_autokey &&
740 (V_tcp_fastopen_numkeys == 0)) {
741 tcp_fastopen_autokey_locked();
742 callout_reset(&V_tcp_fastopen_autokey_ctx.c,
743 V_tcp_fastopen_autokey * hz,
744 tcp_fastopen_autokey_callout,
745 &V_tcp_fastopen_autokey_ctx);
746 }
747 V_tcp_fastopen_server_enable = 1;
748 TCP_FASTOPEN_KEYS_WUNLOCK();
749 }
750 }
751 return (error);
752 }
753
754 static int
sysctl_net_inet_tcp_fastopen_setkey(SYSCTL_HANDLER_ARGS)755 sysctl_net_inet_tcp_fastopen_setkey(SYSCTL_HANDLER_ARGS)
756 {
757 int error;
758 uint8_t newkey[TCP_FASTOPEN_KEY_LEN];
759
760 if (req->oldptr != NULL || req->oldlen != 0)
761 return (EINVAL);
762 if (req->newptr == NULL)
763 return (EPERM);
764 if (req->newlen != sizeof(newkey))
765 return (EINVAL);
766 error = SYSCTL_IN(req, newkey, sizeof(newkey));
767 if (error)
768 return (error);
769
770 TCP_FASTOPEN_KEYS_WLOCK();
771 tcp_fastopen_addkey_locked(newkey);
772 TCP_FASTOPEN_KEYS_WUNLOCK();
773
774 return (0);
775 }
776
777 static int
sysctl_net_inet_tcp_fastopen_setpsk(SYSCTL_HANDLER_ARGS)778 sysctl_net_inet_tcp_fastopen_setpsk(SYSCTL_HANDLER_ARGS)
779 {
780 int error;
781 uint8_t newpsk[TCP_FASTOPEN_KEY_LEN];
782
783 if (req->oldptr != NULL || req->oldlen != 0)
784 return (EINVAL);
785 if (req->newptr == NULL)
786 return (EPERM);
787 if (req->newlen != sizeof(newpsk))
788 return (EINVAL);
789 error = SYSCTL_IN(req, newpsk, sizeof(newpsk));
790 if (error)
791 return (error);
792
793 TCP_FASTOPEN_KEYS_WLOCK();
794 tcp_fastopen_addpsk_locked(newpsk);
795 TCP_FASTOPEN_KEYS_WUNLOCK();
796
797 return (0);
798 }
799
800 static int
sysctl_net_inet_tcp_fastopen_ccache_bucket_limit(SYSCTL_HANDLER_ARGS)801 sysctl_net_inet_tcp_fastopen_ccache_bucket_limit(SYSCTL_HANDLER_ARGS)
802 {
803 struct tcp_fastopen_ccache_bucket *ccb;
804 int error;
805 unsigned int new;
806 unsigned int i;
807
808 new = V_tcp_fastopen_ccache.bucket_limit;
809 error = sysctl_handle_int(oidp, &new, 0, req);
810 if (error == 0 && req->newptr) {
811 if ((new == 0) || (new > INT_MAX))
812 error = EINVAL;
813 else {
814 if (new < V_tcp_fastopen_ccache.bucket_limit) {
815 for (i = 0; i < V_tcp_fastopen_ccache.buckets;
816 i++) {
817 ccb = &V_tcp_fastopen_ccache.base[i];
818 tcp_fastopen_ccache_bucket_trim(ccb, new);
819 }
820 }
821 V_tcp_fastopen_ccache.bucket_limit = new;
822 }
823 }
824 return (error);
825 }
826
827 static int
sysctl_net_inet_tcp_fastopen_client_enable(SYSCTL_HANDLER_ARGS)828 sysctl_net_inet_tcp_fastopen_client_enable(SYSCTL_HANDLER_ARGS)
829 {
830 struct tcp_fastopen_ccache_bucket *ccb;
831 int error;
832 unsigned int new, i;
833
834 new = V_tcp_fastopen_client_enable;
835 error = sysctl_handle_int(oidp, &new, 0, req);
836 if (error == 0 && req->newptr) {
837 if (V_tcp_fastopen_client_enable && !new) {
838 /* enabled -> disabled */
839 for (i = 0; i < V_tcp_fastopen_ccache.buckets; i++) {
840 ccb = &V_tcp_fastopen_ccache.base[i];
841 KASSERT(ccb->ccb_num_entries > -1,
842 ("%s: ccb->ccb_num_entries %d is negative",
843 __func__, ccb->ccb_num_entries));
844 tcp_fastopen_ccache_bucket_trim(ccb, 0);
845 }
846 V_tcp_fastopen_client_enable = 0;
847 } else if (!V_tcp_fastopen_client_enable && new) {
848 /* disabled -> enabled */
849 for (i = 0; i < V_tcp_fastopen_ccache.buckets; i++) {
850 ccb = &V_tcp_fastopen_ccache.base[i];
851 CCB_LOCK(ccb);
852 KASSERT(TAILQ_EMPTY(&ccb->ccb_entries),
853 ("%s: ccb->ccb_entries not empty", __func__));
854 KASSERT(ccb->ccb_num_entries == -1,
855 ("%s: ccb->ccb_num_entries %d not -1", __func__,
856 ccb->ccb_num_entries));
857 ccb->ccb_num_entries = 0; /* enable bucket */
858 CCB_UNLOCK(ccb);
859 }
860 V_tcp_fastopen_client_enable = 1;
861 }
862 }
863 return (error);
864 }
865
866 void
tcp_fastopen_connect(struct tcpcb * tp)867 tcp_fastopen_connect(struct tcpcb *tp)
868 {
869 struct inpcb *inp = tptoinpcb(tp);
870 struct tcp_fastopen_ccache_bucket *ccb;
871 struct tcp_fastopen_ccache_entry *cce;
872 sbintime_t now;
873 uint16_t server_mss;
874 uint64_t psk_cookie;
875
876 psk_cookie = 0;
877 cce = tcp_fastopen_ccache_lookup(&inp->inp_inc, &ccb);
878 if (cce) {
879 if (cce->disable_time == 0) {
880 if ((cce->cookie_len > 0) &&
881 (tp->t_tfo_client_cookie_len ==
882 TCP_FASTOPEN_PSK_LEN)) {
883 psk_cookie =
884 tcp_fastopen_make_psk_cookie(
885 tp->t_tfo_cookie.client,
886 cce->cookie, cce->cookie_len);
887 } else {
888 tp->t_tfo_client_cookie_len = cce->cookie_len;
889 memcpy(tp->t_tfo_cookie.client, cce->cookie,
890 cce->cookie_len);
891 }
892 server_mss = cce->server_mss;
893 CCB_UNLOCK(ccb);
894 if (tp->t_tfo_client_cookie_len ==
895 TCP_FASTOPEN_PSK_LEN && psk_cookie) {
896 tp->t_tfo_client_cookie_len =
897 TCP_FASTOPEN_COOKIE_LEN;
898 memcpy(tp->t_tfo_cookie.client, &psk_cookie,
899 TCP_FASTOPEN_COOKIE_LEN);
900 }
901 tcp_mss(tp, server_mss ? server_mss : -1);
902 tp->snd_wnd = tp->t_maxseg;
903 } else {
904 /*
905 * The path is disabled. Check the time and
906 * possibly re-enable.
907 */
908 now = getsbinuptime();
909 if (now - cce->disable_time >
910 ((sbintime_t)V_tcp_fastopen_path_disable_time << 32)) {
911 /*
912 * Re-enable path. Force a TFO cookie
913 * request. Forget the old MSS as it may be
914 * bogus now, and we will rediscover it in
915 * the SYN|ACK.
916 */
917 cce->disable_time = 0;
918 cce->server_mss = 0;
919 cce->cookie_len = 0;
920 /*
921 * tp->t_tfo... cookie details are already
922 * zero from the tcpcb init.
923 */
924 } else {
925 /*
926 * Path is disabled, so disable TFO on this
927 * connection.
928 */
929 tp->t_flags &= ~TF_FASTOPEN;
930 }
931 CCB_UNLOCK(ccb);
932 tcp_mss(tp, -1);
933 /*
934 * snd_wnd is irrelevant since we are either forcing
935 * a TFO cookie request or disabling TFO - either
936 * way, no data with the SYN.
937 */
938 }
939 } else {
940 /*
941 * A new entry for this path will be created when a SYN|ACK
942 * comes back, or the attempt otherwise fails.
943 */
944 CCB_UNLOCK(ccb);
945 tcp_mss(tp, -1);
946 /*
947 * snd_wnd is irrelevant since we are forcing a TFO cookie
948 * request.
949 */
950 }
951 }
952
953 void
tcp_fastopen_disable_path(struct tcpcb * tp)954 tcp_fastopen_disable_path(struct tcpcb *tp)
955 {
956 struct in_conninfo *inc = &tptoinpcb(tp)->inp_inc;
957 struct tcp_fastopen_ccache_bucket *ccb;
958 struct tcp_fastopen_ccache_entry *cce;
959
960 cce = tcp_fastopen_ccache_lookup(inc, &ccb);
961 if (cce) {
962 cce->server_mss = 0;
963 cce->cookie_len = 0;
964 /*
965 * Preserve the existing disable time if it is already
966 * disabled.
967 */
968 if (cce->disable_time == 0)
969 cce->disable_time = getsbinuptime();
970 } else /* use invalid cookie len to create disabled entry */
971 tcp_fastopen_ccache_create(ccb, inc, 0,
972 TCP_FASTOPEN_MAX_COOKIE_LEN + 1, NULL);
973
974 CCB_UNLOCK(ccb);
975 tp->t_flags &= ~TF_FASTOPEN;
976 }
977
978 void
tcp_fastopen_update_cache(struct tcpcb * tp,uint16_t mss,uint8_t cookie_len,uint8_t * cookie)979 tcp_fastopen_update_cache(struct tcpcb *tp, uint16_t mss,
980 uint8_t cookie_len, uint8_t *cookie)
981 {
982 struct in_conninfo *inc = &tptoinpcb(tp)->inp_inc;
983 struct tcp_fastopen_ccache_bucket *ccb;
984 struct tcp_fastopen_ccache_entry *cce;
985
986 cce = tcp_fastopen_ccache_lookup(inc, &ccb);
987 if (cce) {
988 if ((cookie_len >= TCP_FASTOPEN_MIN_COOKIE_LEN) &&
989 (cookie_len <= TCP_FASTOPEN_MAX_COOKIE_LEN) &&
990 ((cookie_len & 0x1) == 0)) {
991 cce->server_mss = mss;
992 cce->cookie_len = cookie_len;
993 memcpy(cce->cookie, cookie, cookie_len);
994 cce->disable_time = 0;
995 } else {
996 /* invalid cookie length, disable entry */
997 cce->server_mss = 0;
998 cce->cookie_len = 0;
999 /*
1000 * Preserve the existing disable time if it is
1001 * already disabled.
1002 */
1003 if (cce->disable_time == 0)
1004 cce->disable_time = getsbinuptime();
1005 }
1006 } else
1007 tcp_fastopen_ccache_create(ccb, inc, mss, cookie_len, cookie);
1008
1009 CCB_UNLOCK(ccb);
1010 }
1011
1012 static struct tcp_fastopen_ccache_entry *
tcp_fastopen_ccache_lookup(struct in_conninfo * inc,struct tcp_fastopen_ccache_bucket ** ccbp)1013 tcp_fastopen_ccache_lookup(struct in_conninfo *inc,
1014 struct tcp_fastopen_ccache_bucket **ccbp)
1015 {
1016 struct tcp_fastopen_ccache_bucket *ccb;
1017 struct tcp_fastopen_ccache_entry *cce;
1018 uint32_t last_word;
1019 uint32_t hash;
1020
1021 hash = jenkins_hash32((uint32_t *)&inc->inc_ie.ie_dependladdr, 4,
1022 V_tcp_fastopen_ccache.secret);
1023 hash = jenkins_hash32((uint32_t *)&inc->inc_ie.ie_dependfaddr, 4,
1024 hash);
1025 last_word = inc->inc_fport;
1026 hash = jenkins_hash32(&last_word, 1, hash);
1027 ccb = &V_tcp_fastopen_ccache.base[hash & V_tcp_fastopen_ccache.mask];
1028 *ccbp = ccb;
1029 CCB_LOCK(ccb);
1030
1031 /*
1032 * Always returns with locked bucket.
1033 */
1034 TAILQ_FOREACH(cce, &ccb->ccb_entries, cce_link)
1035 if ((!(cce->af == AF_INET6) == !(inc->inc_flags & INC_ISIPV6)) &&
1036 (cce->server_port == inc->inc_ie.ie_fport) &&
1037 (((cce->af == AF_INET) &&
1038 (cce->cce_client_ip.v4.s_addr == inc->inc_laddr.s_addr) &&
1039 (cce->cce_server_ip.v4.s_addr == inc->inc_faddr.s_addr)) ||
1040 ((cce->af == AF_INET6) &&
1041 IN6_ARE_ADDR_EQUAL(&cce->cce_client_ip.v6, &inc->inc6_laddr) &&
1042 IN6_ARE_ADDR_EQUAL(&cce->cce_server_ip.v6, &inc->inc6_faddr))))
1043 break;
1044
1045 return (cce);
1046 }
1047
1048 static struct tcp_fastopen_ccache_entry *
tcp_fastopen_ccache_create(struct tcp_fastopen_ccache_bucket * ccb,struct in_conninfo * inc,uint16_t mss,uint8_t cookie_len,uint8_t * cookie)1049 tcp_fastopen_ccache_create(struct tcp_fastopen_ccache_bucket *ccb,
1050 struct in_conninfo *inc, uint16_t mss, uint8_t cookie_len, uint8_t *cookie)
1051 {
1052 struct tcp_fastopen_ccache_entry *cce;
1053
1054 /*
1055 * 1. Create a new entry, or
1056 * 2. Reclaim an existing entry, or
1057 * 3. Fail
1058 */
1059
1060 CCB_LOCK_ASSERT(ccb);
1061
1062 cce = NULL;
1063 if (ccb->ccb_num_entries < V_tcp_fastopen_ccache.bucket_limit)
1064 cce = uma_zalloc(V_tcp_fastopen_ccache.zone, M_NOWAIT);
1065
1066 if (cce == NULL) {
1067 /*
1068 * At bucket limit, or out of memory - reclaim last
1069 * entry in bucket.
1070 */
1071 cce = TAILQ_LAST(&ccb->ccb_entries, bucket_entries);
1072 if (cce == NULL) {
1073 /* XXX count this event */
1074 return (NULL);
1075 }
1076
1077 TAILQ_REMOVE(&ccb->ccb_entries, cce, cce_link);
1078 } else
1079 ccb->ccb_num_entries++;
1080
1081 TAILQ_INSERT_HEAD(&ccb->ccb_entries, cce, cce_link);
1082 cce->af = (inc->inc_flags & INC_ISIPV6) ? AF_INET6 : AF_INET;
1083 if (cce->af == AF_INET) {
1084 cce->cce_client_ip.v4 = inc->inc_laddr;
1085 cce->cce_server_ip.v4 = inc->inc_faddr;
1086 } else {
1087 cce->cce_client_ip.v6 = inc->inc6_laddr;
1088 cce->cce_server_ip.v6 = inc->inc6_faddr;
1089 }
1090 cce->server_port = inc->inc_fport;
1091 if ((cookie_len >= TCP_FASTOPEN_MIN_COOKIE_LEN) &&
1092 (cookie_len <= TCP_FASTOPEN_MAX_COOKIE_LEN) &&
1093 ((cookie_len & 0x1) == 0)) {
1094 cce->server_mss = mss;
1095 cce->cookie_len = cookie_len;
1096 memcpy(cce->cookie, cookie, cookie_len);
1097 cce->disable_time = 0;
1098 } else {
1099 /* invalid cookie length, disable cce */
1100 cce->server_mss = 0;
1101 cce->cookie_len = 0;
1102 cce->disable_time = getsbinuptime();
1103 }
1104
1105 return (cce);
1106 }
1107
1108 static void
tcp_fastopen_ccache_bucket_trim(struct tcp_fastopen_ccache_bucket * ccb,unsigned int limit)1109 tcp_fastopen_ccache_bucket_trim(struct tcp_fastopen_ccache_bucket *ccb,
1110 unsigned int limit)
1111 {
1112 struct tcp_fastopen_ccache_entry *cce, *cce_tmp;
1113 unsigned int entries;
1114
1115 CCB_LOCK(ccb);
1116 entries = 0;
1117 TAILQ_FOREACH_SAFE(cce, &ccb->ccb_entries, cce_link, cce_tmp) {
1118 entries++;
1119 if (entries > limit)
1120 tcp_fastopen_ccache_entry_drop(cce, ccb);
1121 }
1122 KASSERT(ccb->ccb_num_entries <= (int)limit,
1123 ("%s: ccb->ccb_num_entries %d exceeds limit %d", __func__,
1124 ccb->ccb_num_entries, limit));
1125 if (limit == 0) {
1126 KASSERT(TAILQ_EMPTY(&ccb->ccb_entries),
1127 ("%s: ccb->ccb_entries not empty", __func__));
1128 ccb->ccb_num_entries = -1; /* disable bucket */
1129 }
1130 CCB_UNLOCK(ccb);
1131 }
1132
1133 static void
tcp_fastopen_ccache_entry_drop(struct tcp_fastopen_ccache_entry * cce,struct tcp_fastopen_ccache_bucket * ccb)1134 tcp_fastopen_ccache_entry_drop(struct tcp_fastopen_ccache_entry *cce,
1135 struct tcp_fastopen_ccache_bucket *ccb)
1136 {
1137
1138 CCB_LOCK_ASSERT(ccb);
1139
1140 TAILQ_REMOVE(&ccb->ccb_entries, cce, cce_link);
1141 ccb->ccb_num_entries--;
1142 uma_zfree(V_tcp_fastopen_ccache.zone, cce);
1143 }
1144
1145 static int
sysctl_net_inet_tcp_fastopen_ccache_list(SYSCTL_HANDLER_ARGS)1146 sysctl_net_inet_tcp_fastopen_ccache_list(SYSCTL_HANDLER_ARGS)
1147 {
1148 struct sbuf sb;
1149 struct tcp_fastopen_ccache_bucket *ccb;
1150 struct tcp_fastopen_ccache_entry *cce;
1151 sbintime_t now, duration, limit;
1152 const int linesize = 128;
1153 int i, error, num_entries;
1154 unsigned int j;
1155 #ifdef INET6
1156 char clt_buf[INET6_ADDRSTRLEN], srv_buf[INET6_ADDRSTRLEN];
1157 #else
1158 char clt_buf[INET_ADDRSTRLEN], srv_buf[INET_ADDRSTRLEN];
1159 #endif
1160
1161 if (jailed_without_vnet(curthread->td_ucred) != 0)
1162 return (EPERM);
1163
1164 /* Only allow root to read the client cookie cache */
1165 if (curthread->td_ucred->cr_uid != 0)
1166 return (EPERM);
1167
1168 num_entries = 0;
1169 for (i = 0; i < V_tcp_fastopen_ccache.buckets; i++) {
1170 ccb = &V_tcp_fastopen_ccache.base[i];
1171 CCB_LOCK(ccb);
1172 if (ccb->ccb_num_entries > 0)
1173 num_entries += ccb->ccb_num_entries;
1174 CCB_UNLOCK(ccb);
1175 }
1176 sbuf_new(&sb, NULL, linesize * (num_entries + 1), SBUF_INCLUDENUL);
1177
1178 sbuf_printf(&sb,
1179 "\nLocal IP address Remote IP address Port MSS"
1180 " Disabled Cookie\n");
1181
1182 now = getsbinuptime();
1183 limit = (sbintime_t)V_tcp_fastopen_path_disable_time << 32;
1184 for (i = 0; i < V_tcp_fastopen_ccache.buckets; i++) {
1185 ccb = &V_tcp_fastopen_ccache.base[i];
1186 CCB_LOCK(ccb);
1187 TAILQ_FOREACH(cce, &ccb->ccb_entries, cce_link) {
1188 if (cce->disable_time != 0) {
1189 duration = now - cce->disable_time;
1190 if (limit >= duration)
1191 duration = limit - duration;
1192 else
1193 duration = 0;
1194 } else
1195 duration = 0;
1196 sbuf_printf(&sb,
1197 "%-20s %-20s %5u %5u ",
1198 inet_ntop(cce->af, &cce->cce_client_ip,
1199 clt_buf, sizeof(clt_buf)),
1200 inet_ntop(cce->af, &cce->cce_server_ip,
1201 srv_buf, sizeof(srv_buf)),
1202 ntohs(cce->server_port),
1203 cce->server_mss);
1204 if (duration > 0)
1205 sbuf_printf(&sb, "%7ds ", sbintime_getsec(duration));
1206 else
1207 sbuf_printf(&sb, "%8s ", "No");
1208 for (j = 0; j < cce->cookie_len; j++)
1209 sbuf_printf(&sb, "%02x", cce->cookie[j]);
1210 sbuf_putc(&sb, '\n');
1211 }
1212 CCB_UNLOCK(ccb);
1213 }
1214 error = sbuf_finish(&sb);
1215 if (error == 0)
1216 error = SYSCTL_OUT(req, sbuf_data(&sb), sbuf_len(&sb));
1217 sbuf_delete(&sb);
1218 return (error);
1219 }
1220