1 /*- 2 * Copyright (c) 2015-2017 Patrick Kelsey 3 * All rights reserved. 4 * 5 * Redistribution and use in source and binary forms, with or without 6 * modification, are permitted provided that the following conditions 7 * are met: 8 * 1. Redistributions of source code must retain the above copyright 9 * notice, this list of conditions and the following disclaimer. 10 * 2. Redistributions in binary form must reproduce the above copyright 11 * notice, this list of conditions and the following disclaimer in the 12 * documentation and/or other materials provided with the distribution. 13 * 14 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 15 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 16 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 17 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 18 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 19 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 20 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 21 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 22 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 23 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 24 * SUCH DAMAGE. 25 */ 26 27 /* 28 * This is an implementation of TCP Fast Open (TFO) [RFC7413]. To include 29 * this code, add the following line to your kernel config: 30 * 31 * options TCP_RFC7413 32 * 33 * 34 * The generated TFO cookies are the 64-bit output of 35 * SipHash24(key=<16-byte-key>, msg=<client-ip>). Multiple concurrent valid 36 * keys are supported so that time-based rolling cookie invalidation 37 * policies can be implemented in the system. The default number of 38 * concurrent keys is 2. This can be adjusted in the kernel config as 39 * follows: 40 * 41 * options TCP_RFC7413_MAX_KEYS=<num-keys> 42 * 43 * 44 * In addition to the facilities defined in RFC7413, this implementation 45 * supports a pre-shared key (PSK) mode of operation in which the TFO server 46 * requires the client to be in posession of a shared secret in order for 47 * the client to be able to successfully open TFO connections with the 48 * server. This is useful, for example, in environments where TFO servers 49 * are exposed to both internal and external clients and only wish to allow 50 * TFO connections from internal clients. 51 * 52 * In the PSK mode of operation, the server generates and sends TFO cookies 53 * to requesting clients as usual. However, when validating cookies 54 * received in TFO SYNs from clients, the server requires the 55 * client-supplied cookie to equal SipHash24(key=<16-byte-psk>, 56 * msg=<cookie-sent-to-client>). 57 * 58 * Multiple concurrent valid pre-shared keys are supported so that 59 * time-based rolling PSK invalidation policies can be implemented in the 60 * system. The default number of concurrent pre-shared keys is 2. This can 61 * be adjusted in the kernel config as follows: 62 * 63 * options TCP_RFC7413_MAX_PSKS=<num-psks> 64 * 65 * 66 * The following TFO-specific sysctls are defined: 67 * 68 * net.inet.tcp.fastopen.acceptany (RW, default 0) 69 * When non-zero, all client-supplied TFO cookies will be considered to 70 * be valid. 71 * 72 * net.inet.tcp.fastopen.autokey (RW, default 120) 73 * When this and net.inet.tcp.fastopen.server_enable are non-zero, a new 74 * key will be automatically generated after this many seconds. 75 * 76 * net.inet.tcp.fastopen.ccache_bucket_limit 77 * (RWTUN, default TCP_FASTOPEN_CCACHE_BUCKET_LIMIT_DEFAULT) 78 * The maximum number of entries in a client cookie cache bucket. 79 * 80 * net.inet.tcp.fastopen.ccache_buckets 81 * (RDTUN, default TCP_FASTOPEN_CCACHE_BUCKETS_DEFAULT) 82 * The number of client cookie cache buckets. 83 * 84 * net.inet.tcp.fastopen.ccache_list (RO) 85 * Print the client cookie cache. 86 * 87 * net.inet.tcp.fastopen.client_enable (RW, default 0) 88 * When zero, no new active (i.e., client) TFO connections can be 89 * created. On the transition from enabled to disabled, the client 90 * cookie cache is cleared and disabled. The transition from enabled to 91 * disabled does not affect any active TFO connections in progress; it 92 * only prevents new ones from being made. 93 * 94 * net.inet.tcp.fastopen.keylen (RD) 95 * The key length in bytes. 96 * 97 * net.inet.tcp.fastopen.maxkeys (RD) 98 * The maximum number of keys supported. 99 * 100 * net.inet.tcp.fastopen.maxpsks (RD) 101 * The maximum number of pre-shared keys supported. 102 * 103 * net.inet.tcp.fastopen.numkeys (RD) 104 * The current number of keys installed. 105 * 106 * net.inet.tcp.fastopen.numpsks (RD) 107 * The current number of pre-shared keys installed. 108 * 109 * net.inet.tcp.fastopen.path_disable_time 110 * (RW, default TCP_FASTOPEN_PATH_DISABLE_TIME_DEFAULT) 111 * When a failure occurs while trying to create a new active (i.e., 112 * client) TFO connection, new active connections on the same path, as 113 * determined by the tuple {client_ip, server_ip, server_port}, will be 114 * forced to be non-TFO for this many seconds. Note that the path 115 * disable mechanism relies on state stored in client cookie cache 116 * entries, so it is possible for the disable time for a given path to 117 * be reduced if the corresponding client cookie cache entry is reused 118 * due to resource pressure before the disable period has elapsed. 119 * 120 * net.inet.tcp.fastopen.psk_enable (RW, default 0) 121 * When non-zero, pre-shared key (PSK) mode is enabled for all TFO 122 * servers. On the transition from enabled to disabled, all installed 123 * pre-shared keys are removed. 124 * 125 * net.inet.tcp.fastopen.server_enable (RW, default 0) 126 * When zero, no new passive (i.e., server) TFO connections can be 127 * created. On the transition from enabled to disabled, all installed 128 * keys and pre-shared keys are removed. On the transition from 129 * disabled to enabled, if net.inet.tcp.fastopen.autokey is non-zero and 130 * there are no keys installed, a new key will be generated immediately. 131 * The transition from enabled to disabled does not affect any passive 132 * TFO connections in progress; it only prevents new ones from being 133 * made. 134 * 135 * net.inet.tcp.fastopen.setkey (WR) 136 * Install a new key by writing net.inet.tcp.fastopen.keylen bytes to 137 * this sysctl. 138 * 139 * net.inet.tcp.fastopen.setpsk (WR) 140 * Install a new pre-shared key by writing net.inet.tcp.fastopen.keylen 141 * bytes to this sysctl. 142 * 143 * In order for TFO connections to be created via a listen socket, that 144 * socket must have the TCP_FASTOPEN socket option set on it. This option 145 * can be set on the socket either before or after the listen() is invoked. 146 * Clearing this option on a listen socket after it has been set has no 147 * effect on existing TFO connections or TFO connections in progress; it 148 * only prevents new TFO connections from being made. 149 * 150 * For passively-created sockets, the TCP_FASTOPEN socket option can be 151 * queried to determine whether the connection was established using TFO. 152 * Note that connections that are established via a TFO SYN, but that fall 153 * back to using a non-TFO SYN|ACK will have the TCP_FASTOPEN socket option 154 * set. 155 * 156 * Per the RFC, this implementation limits the number of TFO connections 157 * that can be in the SYN_RECEIVED state on a per listen-socket basis. 158 * Whenever this limit is exceeded, requests for new TFO connections are 159 * serviced as non-TFO requests. Without such a limit, given a valid TFO 160 * cookie, an attacker could keep the listen queue in an overflow condition 161 * using a TFO SYN flood. This implementation sets the limit at half the 162 * configured listen backlog. 163 * 164 */ 165 166 #include <sys/cdefs.h> 167 __FBSDID("$FreeBSD$"); 168 169 #include "opt_inet.h" 170 171 #include <sys/param.h> 172 #include <sys/jail.h> 173 #include <sys/kernel.h> 174 #include <sys/hash.h> 175 #include <sys/limits.h> 176 #include <sys/lock.h> 177 #include <sys/proc.h> 178 #include <sys/rmlock.h> 179 #include <sys/sbuf.h> 180 #include <sys/socket.h> 181 #include <sys/socketvar.h> 182 #include <sys/sysctl.h> 183 #include <sys/systm.h> 184 185 #include <crypto/siphash/siphash.h> 186 187 #include <net/vnet.h> 188 189 #include <netinet/in.h> 190 #include <netinet/in_pcb.h> 191 #include <netinet/tcp_var.h> 192 #include <netinet/tcp_fastopen.h> 193 194 195 #define TCP_FASTOPEN_KEY_LEN SIPHASH_KEY_LENGTH 196 197 #if TCP_FASTOPEN_PSK_LEN != TCP_FASTOPEN_KEY_LEN 198 #error TCP_FASTOPEN_PSK_LEN must be equal to TCP_FASTOPEN_KEY_LEN 199 #endif 200 201 /* 202 * Because a PSK-mode setsockopt() uses tcpcb.t_tfo_cookie.client to hold 203 * the PSK until the connect occurs. 204 */ 205 #if TCP_FASTOPEN_MAX_COOKIE_LEN < TCP_FASTOPEN_PSK_LEN 206 #error TCP_FASTOPEN_MAX_COOKIE_LEN must be >= TCP_FASTOPEN_PSK_LEN 207 #endif 208 209 #define TCP_FASTOPEN_CCACHE_BUCKET_LIMIT_DEFAULT 16 210 #define TCP_FASTOPEN_CCACHE_BUCKETS_DEFAULT 2048 /* must be power of 2 */ 211 212 #define TCP_FASTOPEN_PATH_DISABLE_TIME_DEFAULT 900 /* seconds */ 213 214 #if !defined(TCP_RFC7413_MAX_KEYS) || (TCP_RFC7413_MAX_KEYS < 1) 215 #define TCP_FASTOPEN_MAX_KEYS 2 216 #else 217 #define TCP_FASTOPEN_MAX_KEYS TCP_RFC7413_MAX_KEYS 218 #endif 219 220 #if TCP_FASTOPEN_MAX_KEYS > 10 221 #undef TCP_FASTOPEN_MAX_KEYS 222 #define TCP_FASTOPEN_MAX_KEYS 10 223 #endif 224 225 #if !defined(TCP_RFC7413_MAX_PSKS) || (TCP_RFC7413_MAX_PSKS < 1) 226 #define TCP_FASTOPEN_MAX_PSKS 2 227 #else 228 #define TCP_FASTOPEN_MAX_PSKS TCP_RFC7413_MAX_PSKS 229 #endif 230 231 #if TCP_FASTOPEN_MAX_PSKS > 10 232 #undef TCP_FASTOPEN_MAX_PSKS 233 #define TCP_FASTOPEN_MAX_PSKS 10 234 #endif 235 236 struct tcp_fastopen_keylist { 237 unsigned int newest; 238 unsigned int newest_psk; 239 uint8_t key[TCP_FASTOPEN_MAX_KEYS][TCP_FASTOPEN_KEY_LEN]; 240 uint8_t psk[TCP_FASTOPEN_MAX_PSKS][TCP_FASTOPEN_KEY_LEN]; 241 }; 242 243 struct tcp_fastopen_callout { 244 struct callout c; 245 struct vnet *v; 246 }; 247 248 static struct tcp_fastopen_ccache_entry *tcp_fastopen_ccache_lookup( 249 struct in_conninfo *, struct tcp_fastopen_ccache_bucket **); 250 static struct tcp_fastopen_ccache_entry *tcp_fastopen_ccache_create( 251 struct tcp_fastopen_ccache_bucket *, struct in_conninfo *, uint16_t, uint8_t, 252 uint8_t *); 253 static void tcp_fastopen_ccache_bucket_trim(struct tcp_fastopen_ccache_bucket *, 254 unsigned int); 255 static void tcp_fastopen_ccache_entry_drop(struct tcp_fastopen_ccache_entry *, 256 struct tcp_fastopen_ccache_bucket *); 257 258 SYSCTL_NODE(_net_inet_tcp, OID_AUTO, fastopen, CTLFLAG_RW | CTLFLAG_MPSAFE, 0, 259 "TCP Fast Open"); 260 261 VNET_DEFINE_STATIC(int, tcp_fastopen_acceptany) = 0; 262 #define V_tcp_fastopen_acceptany VNET(tcp_fastopen_acceptany) 263 SYSCTL_INT(_net_inet_tcp_fastopen, OID_AUTO, acceptany, 264 CTLFLAG_VNET | CTLFLAG_RW, &VNET_NAME(tcp_fastopen_acceptany), 0, 265 "Accept any non-empty cookie"); 266 267 VNET_DEFINE_STATIC(unsigned int, tcp_fastopen_autokey) = 120; 268 #define V_tcp_fastopen_autokey VNET(tcp_fastopen_autokey) 269 static int sysctl_net_inet_tcp_fastopen_autokey(SYSCTL_HANDLER_ARGS); 270 SYSCTL_PROC(_net_inet_tcp_fastopen, OID_AUTO, autokey, 271 CTLFLAG_VNET | CTLTYPE_UINT | CTLFLAG_RW | CTLFLAG_MPSAFE, 272 NULL, 0, &sysctl_net_inet_tcp_fastopen_autokey, "IU", 273 "Number of seconds between auto-generation of a new key; zero disables"); 274 275 static int sysctl_net_inet_tcp_fastopen_ccache_bucket_limit(SYSCTL_HANDLER_ARGS); 276 SYSCTL_PROC(_net_inet_tcp_fastopen, OID_AUTO, ccache_bucket_limit, 277 CTLFLAG_VNET | CTLTYPE_UINT | CTLFLAG_RWTUN | CTLFLAG_NEEDGIANT, 278 NULL, 0, &sysctl_net_inet_tcp_fastopen_ccache_bucket_limit, "IU", 279 "Max entries per bucket in client cookie cache"); 280 281 VNET_DEFINE_STATIC(unsigned int, tcp_fastopen_ccache_buckets) = 282 TCP_FASTOPEN_CCACHE_BUCKETS_DEFAULT; 283 #define V_tcp_fastopen_ccache_buckets VNET(tcp_fastopen_ccache_buckets) 284 SYSCTL_UINT(_net_inet_tcp_fastopen, OID_AUTO, ccache_buckets, 285 CTLFLAG_VNET | CTLFLAG_RDTUN, &VNET_NAME(tcp_fastopen_ccache_buckets), 0, 286 "Client cookie cache number of buckets (power of 2)"); 287 288 VNET_DEFINE(unsigned int, tcp_fastopen_client_enable) = 1; 289 static int sysctl_net_inet_tcp_fastopen_client_enable(SYSCTL_HANDLER_ARGS); 290 SYSCTL_PROC(_net_inet_tcp_fastopen, OID_AUTO, client_enable, 291 CTLFLAG_VNET | CTLTYPE_UINT | CTLFLAG_RW | CTLFLAG_NEEDGIANT, 292 NULL, 0, &sysctl_net_inet_tcp_fastopen_client_enable, "IU", 293 "Enable/disable TCP Fast Open client functionality"); 294 295 SYSCTL_INT(_net_inet_tcp_fastopen, OID_AUTO, keylen, 296 CTLFLAG_RD, SYSCTL_NULL_INT_PTR, TCP_FASTOPEN_KEY_LEN, 297 "Key length in bytes"); 298 299 SYSCTL_INT(_net_inet_tcp_fastopen, OID_AUTO, maxkeys, 300 CTLFLAG_RD, SYSCTL_NULL_INT_PTR, TCP_FASTOPEN_MAX_KEYS, 301 "Maximum number of keys supported"); 302 303 SYSCTL_INT(_net_inet_tcp_fastopen, OID_AUTO, maxpsks, 304 CTLFLAG_RD, SYSCTL_NULL_INT_PTR, TCP_FASTOPEN_MAX_PSKS, 305 "Maximum number of pre-shared keys supported"); 306 307 VNET_DEFINE_STATIC(unsigned int, tcp_fastopen_numkeys) = 0; 308 #define V_tcp_fastopen_numkeys VNET(tcp_fastopen_numkeys) 309 SYSCTL_UINT(_net_inet_tcp_fastopen, OID_AUTO, numkeys, 310 CTLFLAG_VNET | CTLFLAG_RD, &VNET_NAME(tcp_fastopen_numkeys), 0, 311 "Number of keys installed"); 312 313 VNET_DEFINE_STATIC(unsigned int, tcp_fastopen_numpsks) = 0; 314 #define V_tcp_fastopen_numpsks VNET(tcp_fastopen_numpsks) 315 SYSCTL_UINT(_net_inet_tcp_fastopen, OID_AUTO, numpsks, 316 CTLFLAG_VNET | CTLFLAG_RD, &VNET_NAME(tcp_fastopen_numpsks), 0, 317 "Number of pre-shared keys installed"); 318 319 VNET_DEFINE_STATIC(unsigned int, tcp_fastopen_path_disable_time) = 320 TCP_FASTOPEN_PATH_DISABLE_TIME_DEFAULT; 321 #define V_tcp_fastopen_path_disable_time VNET(tcp_fastopen_path_disable_time) 322 SYSCTL_UINT(_net_inet_tcp_fastopen, OID_AUTO, path_disable_time, 323 CTLFLAG_VNET | CTLFLAG_RW, &VNET_NAME(tcp_fastopen_path_disable_time), 0, 324 "Seconds a TFO failure disables a {client_ip, server_ip, server_port} path"); 325 326 VNET_DEFINE_STATIC(unsigned int, tcp_fastopen_psk_enable) = 0; 327 #define V_tcp_fastopen_psk_enable VNET(tcp_fastopen_psk_enable) 328 static int sysctl_net_inet_tcp_fastopen_psk_enable(SYSCTL_HANDLER_ARGS); 329 SYSCTL_PROC(_net_inet_tcp_fastopen, OID_AUTO, psk_enable, 330 CTLFLAG_VNET | CTLTYPE_UINT | CTLFLAG_RW | CTLFLAG_MPSAFE, 331 NULL, 0, &sysctl_net_inet_tcp_fastopen_psk_enable, "IU", 332 "Enable/disable TCP Fast Open server pre-shared key mode"); 333 334 VNET_DEFINE(unsigned int, tcp_fastopen_server_enable) = 0; 335 static int sysctl_net_inet_tcp_fastopen_server_enable(SYSCTL_HANDLER_ARGS); 336 SYSCTL_PROC(_net_inet_tcp_fastopen, OID_AUTO, server_enable, 337 CTLFLAG_VNET | CTLTYPE_UINT | CTLFLAG_RW | CTLFLAG_MPSAFE, 338 NULL, 0, &sysctl_net_inet_tcp_fastopen_server_enable, "IU", 339 "Enable/disable TCP Fast Open server functionality"); 340 341 static int sysctl_net_inet_tcp_fastopen_setkey(SYSCTL_HANDLER_ARGS); 342 SYSCTL_PROC(_net_inet_tcp_fastopen, OID_AUTO, setkey, 343 CTLFLAG_VNET | CTLTYPE_OPAQUE | CTLFLAG_WR | CTLFLAG_MPSAFE, 344 NULL, 0, &sysctl_net_inet_tcp_fastopen_setkey, "", 345 "Install a new key"); 346 347 static int sysctl_net_inet_tcp_fastopen_setpsk(SYSCTL_HANDLER_ARGS); 348 SYSCTL_PROC(_net_inet_tcp_fastopen, OID_AUTO, setpsk, 349 CTLFLAG_VNET | CTLTYPE_OPAQUE | CTLFLAG_WR | CTLFLAG_MPSAFE, 350 NULL, 0, &sysctl_net_inet_tcp_fastopen_setpsk, "", 351 "Install a new pre-shared key"); 352 353 static int sysctl_net_inet_tcp_fastopen_ccache_list(SYSCTL_HANDLER_ARGS); 354 SYSCTL_PROC(_net_inet_tcp_fastopen, OID_AUTO, ccache_list, 355 CTLFLAG_VNET | CTLTYPE_STRING | CTLFLAG_RD | CTLFLAG_SKIP | CTLFLAG_MPSAFE, 356 NULL, 0, sysctl_net_inet_tcp_fastopen_ccache_list, "A", 357 "List of all client cookie cache entries"); 358 359 VNET_DEFINE_STATIC(struct rmlock, tcp_fastopen_keylock); 360 #define V_tcp_fastopen_keylock VNET(tcp_fastopen_keylock) 361 362 #define TCP_FASTOPEN_KEYS_RLOCK(t) rm_rlock(&V_tcp_fastopen_keylock, (t)) 363 #define TCP_FASTOPEN_KEYS_RUNLOCK(t) rm_runlock(&V_tcp_fastopen_keylock, (t)) 364 #define TCP_FASTOPEN_KEYS_WLOCK() rm_wlock(&V_tcp_fastopen_keylock) 365 #define TCP_FASTOPEN_KEYS_WUNLOCK() rm_wunlock(&V_tcp_fastopen_keylock) 366 367 VNET_DEFINE_STATIC(struct tcp_fastopen_keylist, tcp_fastopen_keys); 368 #define V_tcp_fastopen_keys VNET(tcp_fastopen_keys) 369 370 VNET_DEFINE_STATIC(struct tcp_fastopen_callout, tcp_fastopen_autokey_ctx); 371 #define V_tcp_fastopen_autokey_ctx VNET(tcp_fastopen_autokey_ctx) 372 373 VNET_DEFINE_STATIC(uma_zone_t, counter_zone); 374 #define V_counter_zone VNET(counter_zone) 375 376 static MALLOC_DEFINE(M_TCP_FASTOPEN_CCACHE, "tfo_ccache", "TFO client cookie cache buckets"); 377 378 VNET_DEFINE_STATIC(struct tcp_fastopen_ccache, tcp_fastopen_ccache); 379 #define V_tcp_fastopen_ccache VNET(tcp_fastopen_ccache) 380 381 #define CCB_LOCK(ccb) mtx_lock(&(ccb)->ccb_mtx) 382 #define CCB_UNLOCK(ccb) mtx_unlock(&(ccb)->ccb_mtx) 383 #define CCB_LOCK_ASSERT(ccb) mtx_assert(&(ccb)->ccb_mtx, MA_OWNED) 384 385 386 void 387 tcp_fastopen_init(void) 388 { 389 unsigned int i; 390 391 V_counter_zone = uma_zcreate("tfo", sizeof(unsigned int), 392 NULL, NULL, NULL, NULL, UMA_ALIGN_PTR, 0); 393 rm_init(&V_tcp_fastopen_keylock, "tfo_keylock"); 394 callout_init_rm(&V_tcp_fastopen_autokey_ctx.c, 395 &V_tcp_fastopen_keylock, 0); 396 V_tcp_fastopen_autokey_ctx.v = curvnet; 397 V_tcp_fastopen_keys.newest = TCP_FASTOPEN_MAX_KEYS - 1; 398 V_tcp_fastopen_keys.newest_psk = TCP_FASTOPEN_MAX_PSKS - 1; 399 400 /* May already be non-zero if kernel tunable was set */ 401 if (V_tcp_fastopen_ccache.bucket_limit == 0) 402 V_tcp_fastopen_ccache.bucket_limit = 403 TCP_FASTOPEN_CCACHE_BUCKET_LIMIT_DEFAULT; 404 405 /* May already be non-zero if kernel tunable was set */ 406 if ((V_tcp_fastopen_ccache_buckets == 0) || 407 !powerof2(V_tcp_fastopen_ccache_buckets)) 408 V_tcp_fastopen_ccache.buckets = 409 TCP_FASTOPEN_CCACHE_BUCKETS_DEFAULT; 410 else 411 V_tcp_fastopen_ccache.buckets = V_tcp_fastopen_ccache_buckets; 412 413 V_tcp_fastopen_ccache.mask = V_tcp_fastopen_ccache.buckets - 1; 414 V_tcp_fastopen_ccache.secret = arc4random(); 415 416 V_tcp_fastopen_ccache.base = malloc(V_tcp_fastopen_ccache.buckets * 417 sizeof(struct tcp_fastopen_ccache_bucket), M_TCP_FASTOPEN_CCACHE, 418 M_WAITOK | M_ZERO); 419 420 for (i = 0; i < V_tcp_fastopen_ccache.buckets; i++) { 421 TAILQ_INIT(&V_tcp_fastopen_ccache.base[i].ccb_entries); 422 mtx_init(&V_tcp_fastopen_ccache.base[i].ccb_mtx, "tfo_ccache_bucket", 423 NULL, MTX_DEF); 424 if (V_tcp_fastopen_client_enable) { 425 /* enable bucket */ 426 V_tcp_fastopen_ccache.base[i].ccb_num_entries = 0; 427 } else { 428 /* disable bucket */ 429 V_tcp_fastopen_ccache.base[i].ccb_num_entries = -1; 430 } 431 V_tcp_fastopen_ccache.base[i].ccb_ccache = &V_tcp_fastopen_ccache; 432 } 433 434 /* 435 * Note that while the total number of entries in the cookie cache 436 * is limited by the table management logic to 437 * V_tcp_fastopen_ccache.buckets * 438 * V_tcp_fastopen_ccache.bucket_limit, the total number of items in 439 * this zone can exceed that amount by the number of CPUs in the 440 * system times the maximum number of unallocated items that can be 441 * present in each UMA per-CPU cache for this zone. 442 */ 443 V_tcp_fastopen_ccache.zone = uma_zcreate("tfo_ccache_entries", 444 sizeof(struct tcp_fastopen_ccache_entry), NULL, NULL, NULL, NULL, 445 UMA_ALIGN_CACHE, 0); 446 } 447 448 void 449 tcp_fastopen_destroy(void) 450 { 451 struct tcp_fastopen_ccache_bucket *ccb; 452 unsigned int i; 453 454 for (i = 0; i < V_tcp_fastopen_ccache.buckets; i++) { 455 ccb = &V_tcp_fastopen_ccache.base[i]; 456 tcp_fastopen_ccache_bucket_trim(ccb, 0); 457 mtx_destroy(&ccb->ccb_mtx); 458 } 459 460 KASSERT(uma_zone_get_cur(V_tcp_fastopen_ccache.zone) == 0, 461 ("%s: TFO ccache zone allocation count not 0", __func__)); 462 uma_zdestroy(V_tcp_fastopen_ccache.zone); 463 free(V_tcp_fastopen_ccache.base, M_TCP_FASTOPEN_CCACHE); 464 465 callout_drain(&V_tcp_fastopen_autokey_ctx.c); 466 rm_destroy(&V_tcp_fastopen_keylock); 467 uma_zdestroy(V_counter_zone); 468 } 469 470 unsigned int * 471 tcp_fastopen_alloc_counter(void) 472 { 473 unsigned int *counter; 474 counter = uma_zalloc(V_counter_zone, M_NOWAIT); 475 if (counter) 476 *counter = 1; 477 return (counter); 478 } 479 480 void 481 tcp_fastopen_decrement_counter(unsigned int *counter) 482 { 483 if (*counter == 1) 484 uma_zfree(V_counter_zone, counter); 485 else 486 atomic_subtract_int(counter, 1); 487 } 488 489 static void 490 tcp_fastopen_addkey_locked(uint8_t *key) 491 { 492 493 V_tcp_fastopen_keys.newest++; 494 if (V_tcp_fastopen_keys.newest == TCP_FASTOPEN_MAX_KEYS) 495 V_tcp_fastopen_keys.newest = 0; 496 memcpy(V_tcp_fastopen_keys.key[V_tcp_fastopen_keys.newest], key, 497 TCP_FASTOPEN_KEY_LEN); 498 if (V_tcp_fastopen_numkeys < TCP_FASTOPEN_MAX_KEYS) 499 V_tcp_fastopen_numkeys++; 500 } 501 502 static void 503 tcp_fastopen_addpsk_locked(uint8_t *psk) 504 { 505 506 V_tcp_fastopen_keys.newest_psk++; 507 if (V_tcp_fastopen_keys.newest_psk == TCP_FASTOPEN_MAX_PSKS) 508 V_tcp_fastopen_keys.newest_psk = 0; 509 memcpy(V_tcp_fastopen_keys.psk[V_tcp_fastopen_keys.newest_psk], psk, 510 TCP_FASTOPEN_KEY_LEN); 511 if (V_tcp_fastopen_numpsks < TCP_FASTOPEN_MAX_PSKS) 512 V_tcp_fastopen_numpsks++; 513 } 514 515 static void 516 tcp_fastopen_autokey_locked(void) 517 { 518 uint8_t newkey[TCP_FASTOPEN_KEY_LEN]; 519 520 arc4rand(newkey, TCP_FASTOPEN_KEY_LEN, 0); 521 tcp_fastopen_addkey_locked(newkey); 522 } 523 524 static void 525 tcp_fastopen_autokey_callout(void *arg) 526 { 527 struct tcp_fastopen_callout *ctx = arg; 528 529 CURVNET_SET(ctx->v); 530 tcp_fastopen_autokey_locked(); 531 callout_reset(&ctx->c, V_tcp_fastopen_autokey * hz, 532 tcp_fastopen_autokey_callout, ctx); 533 CURVNET_RESTORE(); 534 } 535 536 537 static uint64_t 538 tcp_fastopen_make_cookie(uint8_t key[SIPHASH_KEY_LENGTH], struct in_conninfo *inc) 539 { 540 SIPHASH_CTX ctx; 541 uint64_t siphash; 542 543 SipHash24_Init(&ctx); 544 SipHash_SetKey(&ctx, key); 545 switch (inc->inc_flags & INC_ISIPV6) { 546 #ifdef INET 547 case 0: 548 SipHash_Update(&ctx, &inc->inc_faddr, sizeof(inc->inc_faddr)); 549 break; 550 #endif 551 #ifdef INET6 552 case INC_ISIPV6: 553 SipHash_Update(&ctx, &inc->inc6_faddr, sizeof(inc->inc6_faddr)); 554 break; 555 #endif 556 } 557 SipHash_Final((u_int8_t *)&siphash, &ctx); 558 559 return (siphash); 560 } 561 562 static uint64_t 563 tcp_fastopen_make_psk_cookie(uint8_t *psk, uint8_t *cookie, uint8_t cookie_len) 564 { 565 SIPHASH_CTX ctx; 566 uint64_t psk_cookie; 567 568 SipHash24_Init(&ctx); 569 SipHash_SetKey(&ctx, psk); 570 SipHash_Update(&ctx, cookie, cookie_len); 571 SipHash_Final((u_int8_t *)&psk_cookie, &ctx); 572 573 return (psk_cookie); 574 } 575 576 static int 577 tcp_fastopen_find_cookie_match_locked(uint8_t *wire_cookie, uint64_t *cur_cookie) 578 { 579 unsigned int i, psk_index; 580 uint64_t psk_cookie; 581 582 if (V_tcp_fastopen_psk_enable) { 583 psk_index = V_tcp_fastopen_keys.newest_psk; 584 for (i = 0; i < V_tcp_fastopen_numpsks; i++) { 585 psk_cookie = 586 tcp_fastopen_make_psk_cookie( 587 V_tcp_fastopen_keys.psk[psk_index], 588 (uint8_t *)cur_cookie, 589 TCP_FASTOPEN_COOKIE_LEN); 590 591 if (memcmp(wire_cookie, &psk_cookie, 592 TCP_FASTOPEN_COOKIE_LEN) == 0) 593 return (1); 594 595 if (psk_index == 0) 596 psk_index = TCP_FASTOPEN_MAX_PSKS - 1; 597 else 598 psk_index--; 599 } 600 } else if (memcmp(wire_cookie, cur_cookie, TCP_FASTOPEN_COOKIE_LEN) == 0) 601 return (1); 602 603 return (0); 604 } 605 606 /* 607 * Return values: 608 * -1 the cookie is invalid and no valid cookie is available 609 * 0 the cookie is invalid and the latest cookie has been returned 610 * 1 the cookie is valid and the latest cookie has been returned 611 */ 612 int 613 tcp_fastopen_check_cookie(struct in_conninfo *inc, uint8_t *cookie, 614 unsigned int len, uint64_t *latest_cookie) 615 { 616 struct rm_priotracker tracker; 617 unsigned int i, key_index; 618 int rv; 619 uint64_t cur_cookie; 620 621 if (V_tcp_fastopen_acceptany) { 622 *latest_cookie = 0; 623 return (1); 624 } 625 626 TCP_FASTOPEN_KEYS_RLOCK(&tracker); 627 if (len != TCP_FASTOPEN_COOKIE_LEN) { 628 if (V_tcp_fastopen_numkeys > 0) { 629 *latest_cookie = 630 tcp_fastopen_make_cookie( 631 V_tcp_fastopen_keys.key[V_tcp_fastopen_keys.newest], 632 inc); 633 rv = 0; 634 } else 635 rv = -1; 636 goto out; 637 } 638 639 /* 640 * Check against each available key, from newest to oldest. 641 */ 642 key_index = V_tcp_fastopen_keys.newest; 643 for (i = 0; i < V_tcp_fastopen_numkeys; i++) { 644 cur_cookie = 645 tcp_fastopen_make_cookie(V_tcp_fastopen_keys.key[key_index], 646 inc); 647 if (i == 0) 648 *latest_cookie = cur_cookie; 649 rv = tcp_fastopen_find_cookie_match_locked(cookie, &cur_cookie); 650 if (rv) 651 goto out; 652 if (key_index == 0) 653 key_index = TCP_FASTOPEN_MAX_KEYS - 1; 654 else 655 key_index--; 656 } 657 rv = 0; 658 659 out: 660 TCP_FASTOPEN_KEYS_RUNLOCK(&tracker); 661 return (rv); 662 } 663 664 static int 665 sysctl_net_inet_tcp_fastopen_autokey(SYSCTL_HANDLER_ARGS) 666 { 667 int error; 668 unsigned int new; 669 670 new = V_tcp_fastopen_autokey; 671 error = sysctl_handle_int(oidp, &new, 0, req); 672 if (error == 0 && req->newptr) { 673 if (new > (INT_MAX / hz)) 674 return (EINVAL); 675 676 TCP_FASTOPEN_KEYS_WLOCK(); 677 if (V_tcp_fastopen_server_enable) { 678 if (V_tcp_fastopen_autokey && !new) 679 callout_stop(&V_tcp_fastopen_autokey_ctx.c); 680 else if (new) 681 callout_reset(&V_tcp_fastopen_autokey_ctx.c, 682 new * hz, tcp_fastopen_autokey_callout, 683 &V_tcp_fastopen_autokey_ctx); 684 } 685 V_tcp_fastopen_autokey = new; 686 TCP_FASTOPEN_KEYS_WUNLOCK(); 687 } 688 689 return (error); 690 } 691 692 static int 693 sysctl_net_inet_tcp_fastopen_psk_enable(SYSCTL_HANDLER_ARGS) 694 { 695 int error; 696 unsigned int new; 697 698 new = V_tcp_fastopen_psk_enable; 699 error = sysctl_handle_int(oidp, &new, 0, req); 700 if (error == 0 && req->newptr) { 701 if (V_tcp_fastopen_psk_enable && !new) { 702 /* enabled -> disabled */ 703 TCP_FASTOPEN_KEYS_WLOCK(); 704 V_tcp_fastopen_numpsks = 0; 705 V_tcp_fastopen_keys.newest_psk = 706 TCP_FASTOPEN_MAX_PSKS - 1; 707 V_tcp_fastopen_psk_enable = 0; 708 TCP_FASTOPEN_KEYS_WUNLOCK(); 709 } else if (!V_tcp_fastopen_psk_enable && new) { 710 /* disabled -> enabled */ 711 TCP_FASTOPEN_KEYS_WLOCK(); 712 V_tcp_fastopen_psk_enable = 1; 713 TCP_FASTOPEN_KEYS_WUNLOCK(); 714 } 715 } 716 return (error); 717 } 718 719 static int 720 sysctl_net_inet_tcp_fastopen_server_enable(SYSCTL_HANDLER_ARGS) 721 { 722 int error; 723 unsigned int new; 724 725 new = V_tcp_fastopen_server_enable; 726 error = sysctl_handle_int(oidp, &new, 0, req); 727 if (error == 0 && req->newptr) { 728 if (V_tcp_fastopen_server_enable && !new) { 729 /* enabled -> disabled */ 730 TCP_FASTOPEN_KEYS_WLOCK(); 731 V_tcp_fastopen_numkeys = 0; 732 V_tcp_fastopen_keys.newest = TCP_FASTOPEN_MAX_KEYS - 1; 733 if (V_tcp_fastopen_autokey) 734 callout_stop(&V_tcp_fastopen_autokey_ctx.c); 735 V_tcp_fastopen_numpsks = 0; 736 V_tcp_fastopen_keys.newest_psk = 737 TCP_FASTOPEN_MAX_PSKS - 1; 738 V_tcp_fastopen_server_enable = 0; 739 TCP_FASTOPEN_KEYS_WUNLOCK(); 740 } else if (!V_tcp_fastopen_server_enable && new) { 741 /* disabled -> enabled */ 742 TCP_FASTOPEN_KEYS_WLOCK(); 743 if (V_tcp_fastopen_autokey && 744 (V_tcp_fastopen_numkeys == 0)) { 745 tcp_fastopen_autokey_locked(); 746 callout_reset(&V_tcp_fastopen_autokey_ctx.c, 747 V_tcp_fastopen_autokey * hz, 748 tcp_fastopen_autokey_callout, 749 &V_tcp_fastopen_autokey_ctx); 750 } 751 V_tcp_fastopen_server_enable = 1; 752 TCP_FASTOPEN_KEYS_WUNLOCK(); 753 } 754 } 755 return (error); 756 } 757 758 static int 759 sysctl_net_inet_tcp_fastopen_setkey(SYSCTL_HANDLER_ARGS) 760 { 761 int error; 762 uint8_t newkey[TCP_FASTOPEN_KEY_LEN]; 763 764 if (req->oldptr != NULL || req->oldlen != 0) 765 return (EINVAL); 766 if (req->newptr == NULL) 767 return (EPERM); 768 if (req->newlen != sizeof(newkey)) 769 return (EINVAL); 770 error = SYSCTL_IN(req, newkey, sizeof(newkey)); 771 if (error) 772 return (error); 773 774 TCP_FASTOPEN_KEYS_WLOCK(); 775 tcp_fastopen_addkey_locked(newkey); 776 TCP_FASTOPEN_KEYS_WUNLOCK(); 777 778 return (0); 779 } 780 781 static int 782 sysctl_net_inet_tcp_fastopen_setpsk(SYSCTL_HANDLER_ARGS) 783 { 784 int error; 785 uint8_t newpsk[TCP_FASTOPEN_KEY_LEN]; 786 787 if (req->oldptr != NULL || req->oldlen != 0) 788 return (EINVAL); 789 if (req->newptr == NULL) 790 return (EPERM); 791 if (req->newlen != sizeof(newpsk)) 792 return (EINVAL); 793 error = SYSCTL_IN(req, newpsk, sizeof(newpsk)); 794 if (error) 795 return (error); 796 797 TCP_FASTOPEN_KEYS_WLOCK(); 798 tcp_fastopen_addpsk_locked(newpsk); 799 TCP_FASTOPEN_KEYS_WUNLOCK(); 800 801 return (0); 802 } 803 804 static int 805 sysctl_net_inet_tcp_fastopen_ccache_bucket_limit(SYSCTL_HANDLER_ARGS) 806 { 807 struct tcp_fastopen_ccache_bucket *ccb; 808 int error; 809 unsigned int new; 810 unsigned int i; 811 812 new = V_tcp_fastopen_ccache.bucket_limit; 813 error = sysctl_handle_int(oidp, &new, 0, req); 814 if (error == 0 && req->newptr) { 815 if ((new == 0) || (new > INT_MAX)) 816 error = EINVAL; 817 else { 818 if (new < V_tcp_fastopen_ccache.bucket_limit) { 819 for (i = 0; i < V_tcp_fastopen_ccache.buckets; 820 i++) { 821 ccb = &V_tcp_fastopen_ccache.base[i]; 822 tcp_fastopen_ccache_bucket_trim(ccb, new); 823 } 824 } 825 V_tcp_fastopen_ccache.bucket_limit = new; 826 } 827 828 } 829 return (error); 830 } 831 832 static int 833 sysctl_net_inet_tcp_fastopen_client_enable(SYSCTL_HANDLER_ARGS) 834 { 835 struct tcp_fastopen_ccache_bucket *ccb; 836 int error; 837 unsigned int new, i; 838 839 new = V_tcp_fastopen_client_enable; 840 error = sysctl_handle_int(oidp, &new, 0, req); 841 if (error == 0 && req->newptr) { 842 if (V_tcp_fastopen_client_enable && !new) { 843 /* enabled -> disabled */ 844 for (i = 0; i < V_tcp_fastopen_ccache.buckets; i++) { 845 ccb = &V_tcp_fastopen_ccache.base[i]; 846 KASSERT(ccb->ccb_num_entries > -1, 847 ("%s: ccb->ccb_num_entries %d is negative", 848 __func__, ccb->ccb_num_entries)); 849 tcp_fastopen_ccache_bucket_trim(ccb, 0); 850 } 851 V_tcp_fastopen_client_enable = 0; 852 } else if (!V_tcp_fastopen_client_enable && new) { 853 /* disabled -> enabled */ 854 for (i = 0; i < V_tcp_fastopen_ccache.buckets; i++) { 855 ccb = &V_tcp_fastopen_ccache.base[i]; 856 CCB_LOCK(ccb); 857 KASSERT(TAILQ_EMPTY(&ccb->ccb_entries), 858 ("%s: ccb->ccb_entries not empty", __func__)); 859 KASSERT(ccb->ccb_num_entries == -1, 860 ("%s: ccb->ccb_num_entries %d not -1", __func__, 861 ccb->ccb_num_entries)); 862 ccb->ccb_num_entries = 0; /* enable bucket */ 863 CCB_UNLOCK(ccb); 864 } 865 V_tcp_fastopen_client_enable = 1; 866 } 867 } 868 return (error); 869 } 870 871 void 872 tcp_fastopen_connect(struct tcpcb *tp) 873 { 874 struct inpcb *inp; 875 struct tcp_fastopen_ccache_bucket *ccb; 876 struct tcp_fastopen_ccache_entry *cce; 877 sbintime_t now; 878 uint16_t server_mss; 879 uint64_t psk_cookie; 880 881 psk_cookie = 0; 882 inp = tp->t_inpcb; 883 cce = tcp_fastopen_ccache_lookup(&inp->inp_inc, &ccb); 884 if (cce) { 885 if (cce->disable_time == 0) { 886 if ((cce->cookie_len > 0) && 887 (tp->t_tfo_client_cookie_len == 888 TCP_FASTOPEN_PSK_LEN)) { 889 psk_cookie = 890 tcp_fastopen_make_psk_cookie( 891 tp->t_tfo_cookie.client, 892 cce->cookie, cce->cookie_len); 893 } else { 894 tp->t_tfo_client_cookie_len = cce->cookie_len; 895 memcpy(tp->t_tfo_cookie.client, cce->cookie, 896 cce->cookie_len); 897 } 898 server_mss = cce->server_mss; 899 CCB_UNLOCK(ccb); 900 if (tp->t_tfo_client_cookie_len == 901 TCP_FASTOPEN_PSK_LEN && psk_cookie) { 902 tp->t_tfo_client_cookie_len = 903 TCP_FASTOPEN_COOKIE_LEN; 904 memcpy(tp->t_tfo_cookie.client, &psk_cookie, 905 TCP_FASTOPEN_COOKIE_LEN); 906 } 907 tcp_mss(tp, server_mss ? server_mss : -1); 908 tp->snd_wnd = tp->t_maxseg; 909 } else { 910 /* 911 * The path is disabled. Check the time and 912 * possibly re-enable. 913 */ 914 now = getsbinuptime(); 915 if (now - cce->disable_time > 916 ((sbintime_t)V_tcp_fastopen_path_disable_time << 32)) { 917 /* 918 * Re-enable path. Force a TFO cookie 919 * request. Forget the old MSS as it may be 920 * bogus now, and we will rediscover it in 921 * the SYN|ACK. 922 */ 923 cce->disable_time = 0; 924 cce->server_mss = 0; 925 cce->cookie_len = 0; 926 /* 927 * tp->t_tfo... cookie details are already 928 * zero from the tcpcb init. 929 */ 930 } else { 931 /* 932 * Path is disabled, so disable TFO on this 933 * connection. 934 */ 935 tp->t_flags &= ~TF_FASTOPEN; 936 } 937 CCB_UNLOCK(ccb); 938 tcp_mss(tp, -1); 939 /* 940 * snd_wnd is irrelevant since we are either forcing 941 * a TFO cookie request or disabling TFO - either 942 * way, no data with the SYN. 943 */ 944 } 945 } else { 946 /* 947 * A new entry for this path will be created when a SYN|ACK 948 * comes back, or the attempt otherwise fails. 949 */ 950 CCB_UNLOCK(ccb); 951 tcp_mss(tp, -1); 952 /* 953 * snd_wnd is irrelevant since we are forcing a TFO cookie 954 * request. 955 */ 956 } 957 } 958 959 void 960 tcp_fastopen_disable_path(struct tcpcb *tp) 961 { 962 struct in_conninfo *inc = &tp->t_inpcb->inp_inc; 963 struct tcp_fastopen_ccache_bucket *ccb; 964 struct tcp_fastopen_ccache_entry *cce; 965 966 cce = tcp_fastopen_ccache_lookup(inc, &ccb); 967 if (cce) { 968 cce->server_mss = 0; 969 cce->cookie_len = 0; 970 /* 971 * Preserve the existing disable time if it is already 972 * disabled. 973 */ 974 if (cce->disable_time == 0) 975 cce->disable_time = getsbinuptime(); 976 } else /* use invalid cookie len to create disabled entry */ 977 tcp_fastopen_ccache_create(ccb, inc, 0, 978 TCP_FASTOPEN_MAX_COOKIE_LEN + 1, NULL); 979 980 CCB_UNLOCK(ccb); 981 tp->t_flags &= ~TF_FASTOPEN; 982 } 983 984 void 985 tcp_fastopen_update_cache(struct tcpcb *tp, uint16_t mss, 986 uint8_t cookie_len, uint8_t *cookie) 987 { 988 struct in_conninfo *inc = &tp->t_inpcb->inp_inc; 989 struct tcp_fastopen_ccache_bucket *ccb; 990 struct tcp_fastopen_ccache_entry *cce; 991 992 cce = tcp_fastopen_ccache_lookup(inc, &ccb); 993 if (cce) { 994 if ((cookie_len >= TCP_FASTOPEN_MIN_COOKIE_LEN) && 995 (cookie_len <= TCP_FASTOPEN_MAX_COOKIE_LEN) && 996 ((cookie_len & 0x1) == 0)) { 997 cce->server_mss = mss; 998 cce->cookie_len = cookie_len; 999 memcpy(cce->cookie, cookie, cookie_len); 1000 cce->disable_time = 0; 1001 } else { 1002 /* invalid cookie length, disable entry */ 1003 cce->server_mss = 0; 1004 cce->cookie_len = 0; 1005 /* 1006 * Preserve the existing disable time if it is 1007 * already disabled. 1008 */ 1009 if (cce->disable_time == 0) 1010 cce->disable_time = getsbinuptime(); 1011 } 1012 } else 1013 tcp_fastopen_ccache_create(ccb, inc, mss, cookie_len, cookie); 1014 1015 CCB_UNLOCK(ccb); 1016 } 1017 1018 static struct tcp_fastopen_ccache_entry * 1019 tcp_fastopen_ccache_lookup(struct in_conninfo *inc, 1020 struct tcp_fastopen_ccache_bucket **ccbp) 1021 { 1022 struct tcp_fastopen_ccache_bucket *ccb; 1023 struct tcp_fastopen_ccache_entry *cce; 1024 uint32_t last_word; 1025 uint32_t hash; 1026 1027 hash = jenkins_hash32((uint32_t *)&inc->inc_ie.ie_dependladdr, 4, 1028 V_tcp_fastopen_ccache.secret); 1029 hash = jenkins_hash32((uint32_t *)&inc->inc_ie.ie_dependfaddr, 4, 1030 hash); 1031 last_word = inc->inc_fport; 1032 hash = jenkins_hash32(&last_word, 1, hash); 1033 ccb = &V_tcp_fastopen_ccache.base[hash & V_tcp_fastopen_ccache.mask]; 1034 *ccbp = ccb; 1035 CCB_LOCK(ccb); 1036 1037 /* 1038 * Always returns with locked bucket. 1039 */ 1040 TAILQ_FOREACH(cce, &ccb->ccb_entries, cce_link) 1041 if ((!(cce->af == AF_INET6) == !(inc->inc_flags & INC_ISIPV6)) && 1042 (cce->server_port == inc->inc_ie.ie_fport) && 1043 (((cce->af == AF_INET) && 1044 (cce->cce_client_ip.v4.s_addr == inc->inc_laddr.s_addr) && 1045 (cce->cce_server_ip.v4.s_addr == inc->inc_faddr.s_addr)) || 1046 ((cce->af == AF_INET6) && 1047 IN6_ARE_ADDR_EQUAL(&cce->cce_client_ip.v6, &inc->inc6_laddr) && 1048 IN6_ARE_ADDR_EQUAL(&cce->cce_server_ip.v6, &inc->inc6_faddr)))) 1049 break; 1050 1051 return (cce); 1052 } 1053 1054 static struct tcp_fastopen_ccache_entry * 1055 tcp_fastopen_ccache_create(struct tcp_fastopen_ccache_bucket *ccb, 1056 struct in_conninfo *inc, uint16_t mss, uint8_t cookie_len, uint8_t *cookie) 1057 { 1058 struct tcp_fastopen_ccache_entry *cce; 1059 1060 /* 1061 * 1. Create a new entry, or 1062 * 2. Reclaim an existing entry, or 1063 * 3. Fail 1064 */ 1065 1066 CCB_LOCK_ASSERT(ccb); 1067 1068 cce = NULL; 1069 if (ccb->ccb_num_entries < V_tcp_fastopen_ccache.bucket_limit) 1070 cce = uma_zalloc(V_tcp_fastopen_ccache.zone, M_NOWAIT); 1071 1072 if (cce == NULL) { 1073 /* 1074 * At bucket limit, or out of memory - reclaim last 1075 * entry in bucket. 1076 */ 1077 cce = TAILQ_LAST(&ccb->ccb_entries, bucket_entries); 1078 if (cce == NULL) { 1079 /* XXX count this event */ 1080 return (NULL); 1081 } 1082 1083 TAILQ_REMOVE(&ccb->ccb_entries, cce, cce_link); 1084 } else 1085 ccb->ccb_num_entries++; 1086 1087 TAILQ_INSERT_HEAD(&ccb->ccb_entries, cce, cce_link); 1088 cce->af = (inc->inc_flags & INC_ISIPV6) ? AF_INET6 : AF_INET; 1089 if (cce->af == AF_INET) { 1090 cce->cce_client_ip.v4 = inc->inc_laddr; 1091 cce->cce_server_ip.v4 = inc->inc_faddr; 1092 } else { 1093 cce->cce_client_ip.v6 = inc->inc6_laddr; 1094 cce->cce_server_ip.v6 = inc->inc6_faddr; 1095 } 1096 cce->server_port = inc->inc_fport; 1097 if ((cookie_len >= TCP_FASTOPEN_MIN_COOKIE_LEN) && 1098 (cookie_len <= TCP_FASTOPEN_MAX_COOKIE_LEN) && 1099 ((cookie_len & 0x1) == 0)) { 1100 cce->server_mss = mss; 1101 cce->cookie_len = cookie_len; 1102 memcpy(cce->cookie, cookie, cookie_len); 1103 cce->disable_time = 0; 1104 } else { 1105 /* invalid cookie length, disable cce */ 1106 cce->server_mss = 0; 1107 cce->cookie_len = 0; 1108 cce->disable_time = getsbinuptime(); 1109 } 1110 1111 return (cce); 1112 } 1113 1114 static void 1115 tcp_fastopen_ccache_bucket_trim(struct tcp_fastopen_ccache_bucket *ccb, 1116 unsigned int limit) 1117 { 1118 struct tcp_fastopen_ccache_entry *cce, *cce_tmp; 1119 unsigned int entries; 1120 1121 CCB_LOCK(ccb); 1122 entries = 0; 1123 TAILQ_FOREACH_SAFE(cce, &ccb->ccb_entries, cce_link, cce_tmp) { 1124 entries++; 1125 if (entries > limit) 1126 tcp_fastopen_ccache_entry_drop(cce, ccb); 1127 } 1128 KASSERT(ccb->ccb_num_entries <= (int)limit, 1129 ("%s: ccb->ccb_num_entries %d exceeds limit %d", __func__, 1130 ccb->ccb_num_entries, limit)); 1131 if (limit == 0) { 1132 KASSERT(TAILQ_EMPTY(&ccb->ccb_entries), 1133 ("%s: ccb->ccb_entries not empty", __func__)); 1134 ccb->ccb_num_entries = -1; /* disable bucket */ 1135 } 1136 CCB_UNLOCK(ccb); 1137 } 1138 1139 static void 1140 tcp_fastopen_ccache_entry_drop(struct tcp_fastopen_ccache_entry *cce, 1141 struct tcp_fastopen_ccache_bucket *ccb) 1142 { 1143 1144 CCB_LOCK_ASSERT(ccb); 1145 1146 TAILQ_REMOVE(&ccb->ccb_entries, cce, cce_link); 1147 ccb->ccb_num_entries--; 1148 uma_zfree(V_tcp_fastopen_ccache.zone, cce); 1149 } 1150 1151 static int 1152 sysctl_net_inet_tcp_fastopen_ccache_list(SYSCTL_HANDLER_ARGS) 1153 { 1154 struct sbuf sb; 1155 struct tcp_fastopen_ccache_bucket *ccb; 1156 struct tcp_fastopen_ccache_entry *cce; 1157 sbintime_t now, duration, limit; 1158 const int linesize = 128; 1159 int i, error, num_entries; 1160 unsigned int j; 1161 #ifdef INET6 1162 char clt_buf[INET6_ADDRSTRLEN], srv_buf[INET6_ADDRSTRLEN]; 1163 #else 1164 char clt_buf[INET_ADDRSTRLEN], srv_buf[INET_ADDRSTRLEN]; 1165 #endif 1166 1167 if (jailed_without_vnet(curthread->td_ucred) != 0) 1168 return (EPERM); 1169 1170 /* Only allow root to read the client cookie cache */ 1171 if (curthread->td_ucred->cr_uid != 0) 1172 return (EPERM); 1173 1174 num_entries = 0; 1175 for (i = 0; i < V_tcp_fastopen_ccache.buckets; i++) { 1176 ccb = &V_tcp_fastopen_ccache.base[i]; 1177 CCB_LOCK(ccb); 1178 if (ccb->ccb_num_entries > 0) 1179 num_entries += ccb->ccb_num_entries; 1180 CCB_UNLOCK(ccb); 1181 } 1182 sbuf_new(&sb, NULL, linesize * (num_entries + 1), SBUF_INCLUDENUL); 1183 1184 sbuf_printf(&sb, 1185 "\nLocal IP address Remote IP address Port MSS" 1186 " Disabled Cookie\n"); 1187 1188 now = getsbinuptime(); 1189 limit = (sbintime_t)V_tcp_fastopen_path_disable_time << 32; 1190 for (i = 0; i < V_tcp_fastopen_ccache.buckets; i++) { 1191 ccb = &V_tcp_fastopen_ccache.base[i]; 1192 CCB_LOCK(ccb); 1193 TAILQ_FOREACH(cce, &ccb->ccb_entries, cce_link) { 1194 if (cce->disable_time != 0) { 1195 duration = now - cce->disable_time; 1196 if (limit >= duration) 1197 duration = limit - duration; 1198 else 1199 duration = 0; 1200 } else 1201 duration = 0; 1202 sbuf_printf(&sb, 1203 "%-20s %-20s %5u %5u ", 1204 inet_ntop(cce->af, &cce->cce_client_ip, 1205 clt_buf, sizeof(clt_buf)), 1206 inet_ntop(cce->af, &cce->cce_server_ip, 1207 srv_buf, sizeof(srv_buf)), 1208 ntohs(cce->server_port), 1209 cce->server_mss); 1210 if (duration > 0) 1211 sbuf_printf(&sb, "%7ds ", sbintime_getsec(duration)); 1212 else 1213 sbuf_printf(&sb, "%8s ", "No"); 1214 for (j = 0; j < cce->cookie_len; j++) 1215 sbuf_printf(&sb, "%02x", cce->cookie[j]); 1216 sbuf_putc(&sb, '\n'); 1217 } 1218 CCB_UNLOCK(ccb); 1219 } 1220 error = sbuf_finish(&sb); 1221 if (error == 0) 1222 error = SYSCTL_OUT(req, sbuf_data(&sb), sbuf_len(&sb)); 1223 sbuf_delete(&sb); 1224 return (error); 1225 } 1226 1227