1 /*- 2 * Copyright (c) 2010-2011 Juniper Networks, Inc. 3 * All rights reserved. 4 * 5 * This software was developed by Robert N. M. Watson under contract 6 * to Juniper Networks, Inc. 7 * 8 * Redistribution and use in source and binary forms, with or without 9 * modification, are permitted provided that the following conditions 10 * are met: 11 * 1. Redistributions of source code must retain the above copyright 12 * notice, this list of conditions and the following disclaimer. 13 * 2. Redistributions in binary form must reproduce the above copyright 14 * notice, this list of conditions and the following disclaimer in the 15 * documentation and/or other materials provided with the distribution. 16 * 17 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 18 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 19 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 20 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 21 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 22 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 23 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 24 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 25 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 26 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 27 * SUCH DAMAGE. 28 */ 29 30 #include <sys/cdefs.h> 31 32 __FBSDID("$FreeBSD$"); 33 34 #include "opt_inet6.h" 35 #include "opt_pcbgroup.h" 36 37 #ifndef PCBGROUP 38 #error "options RSS depends on options PCBGROUP" 39 #endif 40 41 #include <sys/param.h> 42 #include <sys/mbuf.h> 43 #include <sys/socket.h> 44 #include <sys/priv.h> 45 #include <sys/kernel.h> 46 #include <sys/smp.h> 47 #include <sys/sysctl.h> 48 #include <sys/sbuf.h> 49 50 #include <net/if.h> 51 #include <net/if_var.h> 52 #include <net/netisr.h> 53 #include <net/rss_config.h> 54 #include <net/toeplitz.h> 55 56 #if 0 57 #include <netinet/in.h> 58 #include <netinet/in_pcb.h> 59 #include <netinet/in_rss.h> 60 #include <netinet/in_var.h> 61 62 /* for software rss hash support */ 63 #include <netinet/ip.h> 64 #include <netinet/tcp.h> 65 #include <netinet/udp.h> 66 #endif 67 68 /*- 69 * Operating system parts of receiver-side scaling (RSS), which allows 70 * network cards to direct flows to particular receive queues based on hashes 71 * of header tuples. This implementation aligns RSS buckets with connection 72 * groups at the TCP/IP layer, so each bucket is associated with exactly one 73 * group. As a result, the group lookup structures (and lock) should have an 74 * effective affinity with exactly one CPU. 75 * 76 * Network device drivers needing to configure RSS will query this framework 77 * for parameters, such as the current RSS key, hashing policies, number of 78 * bits, and indirection table mapping hashes to buckets and CPUs. They may 79 * provide their own supplementary information, such as queue<->CPU bindings. 80 * It is the responsibility of the network device driver to inject packets 81 * into the stack on as close to the right CPU as possible, if playing by RSS 82 * rules. 83 * 84 * TODO: 85 * 86 * - Synchronization for rss_key and other future-configurable parameters. 87 * - Event handler drivers can register to pick up RSS configuration changes. 88 * - Should we allow rss_basecpu to be configured? 89 * - Randomize key on boot. 90 * - IPv6 support. 91 * - Statistics on how often there's a misalignment between hardware 92 * placement and pcbgroup expectations. 93 */ 94 95 SYSCTL_DECL(_net_inet); 96 SYSCTL_NODE(_net_inet, OID_AUTO, rss, CTLFLAG_RW, 0, "Receive-side steering"); 97 98 /* 99 * Toeplitz is the only required hash function in the RSS spec, so use it by 100 * default. 101 */ 102 static u_int rss_hashalgo = RSS_HASH_TOEPLITZ; 103 SYSCTL_INT(_net_inet_rss, OID_AUTO, hashalgo, CTLFLAG_RDTUN, &rss_hashalgo, 0, 104 "RSS hash algorithm"); 105 106 /* 107 * Size of the indirection table; at most 128 entries per the RSS spec. We 108 * size it to at least 2 times the number of CPUs by default to allow useful 109 * rebalancing. If not set explicitly with a loader tunable, we tune based 110 * on the number of CPUs present. 111 * 112 * XXXRW: buckets might be better to use for the tunable than bits. 113 */ 114 static u_int rss_bits; 115 SYSCTL_INT(_net_inet_rss, OID_AUTO, bits, CTLFLAG_RDTUN, &rss_bits, 0, 116 "RSS bits"); 117 118 static u_int rss_mask; 119 SYSCTL_INT(_net_inet_rss, OID_AUTO, mask, CTLFLAG_RD, &rss_mask, 0, 120 "RSS mask"); 121 122 static const u_int rss_maxbits = RSS_MAXBITS; 123 SYSCTL_INT(_net_inet_rss, OID_AUTO, maxbits, CTLFLAG_RD, 124 __DECONST(int *, &rss_maxbits), 0, "RSS maximum bits"); 125 126 /* 127 * RSS's own count of the number of CPUs it could be using for processing. 128 * Bounded to 64 by RSS constants. 129 */ 130 static u_int rss_ncpus; 131 SYSCTL_INT(_net_inet_rss, OID_AUTO, ncpus, CTLFLAG_RD, &rss_ncpus, 0, 132 "Number of CPUs available to RSS"); 133 134 #define RSS_MAXCPUS (1 << (RSS_MAXBITS - 1)) 135 static const u_int rss_maxcpus = RSS_MAXCPUS; 136 SYSCTL_INT(_net_inet_rss, OID_AUTO, maxcpus, CTLFLAG_RD, 137 __DECONST(int *, &rss_maxcpus), 0, "RSS maximum CPUs that can be used"); 138 139 /* 140 * Variable exists just for reporting rss_bits in a user-friendly way. 141 */ 142 static u_int rss_buckets; 143 SYSCTL_INT(_net_inet_rss, OID_AUTO, buckets, CTLFLAG_RD, &rss_buckets, 0, 144 "RSS buckets"); 145 146 /* 147 * Base CPU number; devices will add this to all CPU numbers returned by the 148 * RSS indirection table. Currently unmodifable in FreeBSD. 149 */ 150 static const u_int rss_basecpu; 151 SYSCTL_INT(_net_inet_rss, OID_AUTO, basecpu, CTLFLAG_RD, 152 __DECONST(int *, &rss_basecpu), 0, "RSS base CPU"); 153 154 /* 155 * RSS secret key, intended to prevent attacks on load-balancing. Its 156 * effectiveness may be limited by algorithm choice and available entropy 157 * during the boot. 158 * 159 * XXXRW: And that we don't randomize it yet! 160 * 161 * This is the default Microsoft RSS specification key which is also 162 * the Chelsio T5 firmware default key. 163 */ 164 static uint8_t rss_key[RSS_KEYSIZE] = { 165 0x6d, 0x5a, 0x56, 0xda, 0x25, 0x5b, 0x0e, 0xc2, 166 0x41, 0x67, 0x25, 0x3d, 0x43, 0xa3, 0x8f, 0xb0, 167 0xd0, 0xca, 0x2b, 0xcb, 0xae, 0x7b, 0x30, 0xb4, 168 0x77, 0xcb, 0x2d, 0xa3, 0x80, 0x30, 0xf2, 0x0c, 169 0x6a, 0x42, 0xb7, 0x3b, 0xbe, 0xac, 0x01, 0xfa, 170 }; 171 172 /* 173 * RSS hash->CPU table, which maps hashed packet headers to particular CPUs. 174 * Drivers may supplement this table with a seperate CPU<->queue table when 175 * programming devices. 176 */ 177 struct rss_table_entry { 178 uint8_t rte_cpu; /* CPU affinity of bucket. */ 179 }; 180 static struct rss_table_entry rss_table[RSS_TABLE_MAXLEN]; 181 182 static void 183 rss_init(__unused void *arg) 184 { 185 u_int i; 186 u_int cpuid; 187 188 /* 189 * Validate tunables, coerce to sensible values. 190 */ 191 switch (rss_hashalgo) { 192 case RSS_HASH_TOEPLITZ: 193 case RSS_HASH_NAIVE: 194 break; 195 196 default: 197 printf("%s: invalid RSS hashalgo %u, coercing to %u", 198 __func__, rss_hashalgo, RSS_HASH_TOEPLITZ); 199 rss_hashalgo = RSS_HASH_TOEPLITZ; 200 } 201 202 /* 203 * Count available CPUs. 204 * 205 * XXXRW: Note incorrect assumptions regarding contiguity of this set 206 * elsewhere. 207 */ 208 rss_ncpus = 0; 209 for (i = 0; i <= mp_maxid; i++) { 210 if (CPU_ABSENT(i)) 211 continue; 212 rss_ncpus++; 213 } 214 if (rss_ncpus > RSS_MAXCPUS) 215 rss_ncpus = RSS_MAXCPUS; 216 217 /* 218 * Tune RSS table entries to be no less than 2x the number of CPUs 219 * -- unless we're running uniprocessor, in which case there's not 220 * much point in having buckets to rearrange for load-balancing! 221 */ 222 if (rss_ncpus > 1) { 223 if (rss_bits == 0) 224 rss_bits = fls(rss_ncpus - 1) + 1; 225 226 /* 227 * Microsoft limits RSS table entries to 128, so apply that 228 * limit to both auto-detected CPU counts and user-configured 229 * ones. 230 */ 231 if (rss_bits == 0 || rss_bits > RSS_MAXBITS) { 232 printf("%s: RSS bits %u not valid, coercing to %u", 233 __func__, rss_bits, RSS_MAXBITS); 234 rss_bits = RSS_MAXBITS; 235 } 236 237 /* 238 * Figure out how many buckets to use; warn if less than the 239 * number of configured CPUs, although this is not a fatal 240 * problem. 241 */ 242 rss_buckets = (1 << rss_bits); 243 if (rss_buckets < rss_ncpus) 244 printf("%s: WARNING: rss_buckets (%u) less than " 245 "rss_ncpus (%u)\n", __func__, rss_buckets, 246 rss_ncpus); 247 rss_mask = rss_buckets - 1; 248 } else { 249 rss_bits = 0; 250 rss_buckets = 1; 251 rss_mask = 0; 252 } 253 254 /* 255 * Set up initial CPU assignments: round-robin by default. 256 */ 257 cpuid = CPU_FIRST(); 258 for (i = 0; i < rss_buckets; i++) { 259 rss_table[i].rte_cpu = cpuid; 260 cpuid = CPU_NEXT(cpuid); 261 } 262 263 /* 264 * Randomize rrs_key. 265 * 266 * XXXRW: Not yet. If nothing else, will require an rss_isbadkey() 267 * loop to check for "bad" RSS keys. 268 */ 269 } 270 SYSINIT(rss_init, SI_SUB_SOFTINTR, SI_ORDER_SECOND, rss_init, NULL); 271 272 static uint32_t 273 rss_naive_hash(u_int keylen, const uint8_t *key, u_int datalen, 274 const uint8_t *data) 275 { 276 uint32_t v; 277 u_int i; 278 279 v = 0; 280 for (i = 0; i < keylen; i++) 281 v += key[i]; 282 for (i = 0; i < datalen; i++) 283 v += data[i]; 284 return (v); 285 } 286 287 uint32_t 288 rss_hash(u_int datalen, const uint8_t *data) 289 { 290 291 switch (rss_hashalgo) { 292 case RSS_HASH_TOEPLITZ: 293 return (toeplitz_hash(sizeof(rss_key), rss_key, datalen, 294 data)); 295 296 case RSS_HASH_NAIVE: 297 return (rss_naive_hash(sizeof(rss_key), rss_key, datalen, 298 data)); 299 300 default: 301 panic("%s: unsupported/unknown hashalgo %d", __func__, 302 rss_hashalgo); 303 } 304 } 305 306 /* 307 * Query the number of RSS bits in use. 308 */ 309 u_int 310 rss_getbits(void) 311 { 312 313 return (rss_bits); 314 } 315 316 /* 317 * Query the RSS bucket associated with an RSS hash. 318 */ 319 u_int 320 rss_getbucket(u_int hash) 321 { 322 323 return (hash & rss_mask); 324 } 325 326 /* 327 * Query the RSS layer bucket associated with the given 328 * entry in the RSS hash space. 329 * 330 * The RSS indirection table is 0 .. rss_buckets-1, 331 * covering the low 'rss_bits' of the total 128 slot 332 * RSS indirection table. So just mask off rss_bits and 333 * return that. 334 * 335 * NIC drivers can then iterate over the 128 slot RSS 336 * indirection table and fetch which RSS bucket to 337 * map it to. This will typically be a CPU queue 338 */ 339 u_int 340 rss_get_indirection_to_bucket(u_int index) 341 { 342 343 return (index & rss_mask); 344 } 345 346 /* 347 * Query the RSS CPU associated with an RSS bucket. 348 */ 349 u_int 350 rss_getcpu(u_int bucket) 351 { 352 353 return (rss_table[bucket].rte_cpu); 354 } 355 356 /* 357 * netisr CPU affinity lookup given just the hash and hashtype. 358 */ 359 u_int 360 rss_hash2cpuid(uint32_t hash_val, uint32_t hash_type) 361 { 362 363 switch (hash_type) { 364 case M_HASHTYPE_RSS_IPV4: 365 case M_HASHTYPE_RSS_TCP_IPV4: 366 case M_HASHTYPE_RSS_UDP_IPV4: 367 case M_HASHTYPE_RSS_IPV6: 368 case M_HASHTYPE_RSS_TCP_IPV6: 369 case M_HASHTYPE_RSS_UDP_IPV6: 370 return (rss_getcpu(rss_getbucket(hash_val))); 371 default: 372 return (NETISR_CPUID_NONE); 373 } 374 } 375 376 /* 377 * Query the RSS bucket associated with the given hash value and 378 * type. 379 */ 380 int 381 rss_hash2bucket(uint32_t hash_val, uint32_t hash_type, uint32_t *bucket_id) 382 { 383 384 switch (hash_type) { 385 case M_HASHTYPE_RSS_IPV4: 386 case M_HASHTYPE_RSS_TCP_IPV4: 387 case M_HASHTYPE_RSS_UDP_IPV4: 388 case M_HASHTYPE_RSS_IPV6: 389 case M_HASHTYPE_RSS_TCP_IPV6: 390 case M_HASHTYPE_RSS_UDP_IPV6: 391 *bucket_id = rss_getbucket(hash_val); 392 return (0); 393 default: 394 return (-1); 395 } 396 } 397 398 /* 399 * netisr CPU affinity lookup routine for use by protocols. 400 */ 401 struct mbuf * 402 rss_m2cpuid(struct mbuf *m, uintptr_t source, u_int *cpuid) 403 { 404 405 M_ASSERTPKTHDR(m); 406 *cpuid = rss_hash2cpuid(m->m_pkthdr.flowid, M_HASHTYPE_GET(m)); 407 return (m); 408 } 409 410 int 411 rss_m2bucket(struct mbuf *m, uint32_t *bucket_id) 412 { 413 414 M_ASSERTPKTHDR(m); 415 416 return(rss_hash2bucket(m->m_pkthdr.flowid, M_HASHTYPE_GET(m), 417 bucket_id)); 418 } 419 420 /* 421 * Query the RSS hash algorithm. 422 */ 423 u_int 424 rss_gethashalgo(void) 425 { 426 427 return (rss_hashalgo); 428 } 429 430 /* 431 * Query the current RSS key; likely to be used by device drivers when 432 * configuring hardware RSS. Caller must pass an array of size RSS_KEYSIZE. 433 * 434 * XXXRW: Perhaps we should do the accept-a-length-and-truncate thing? 435 */ 436 void 437 rss_getkey(uint8_t *key) 438 { 439 440 bcopy(rss_key, key, sizeof(rss_key)); 441 } 442 443 /* 444 * Query the number of buckets; this may be used by both network device 445 * drivers, which will need to populate hardware shadows of the software 446 * indirection table, and the network stack itself (such as when deciding how 447 * many connection groups to allocate). 448 */ 449 u_int 450 rss_getnumbuckets(void) 451 { 452 453 return (rss_buckets); 454 } 455 456 /* 457 * Query the number of CPUs in use by RSS; may be useful to device drivers 458 * trying to figure out how to map a larger number of CPUs into a smaller 459 * number of receive queues. 460 */ 461 u_int 462 rss_getnumcpus(void) 463 { 464 465 return (rss_ncpus); 466 } 467 468 /* 469 * Return the supported RSS hash configuration. 470 * 471 * NICs should query this to determine what to configure in their redirection 472 * matching table. 473 */ 474 inline u_int 475 rss_gethashconfig(void) 476 { 477 478 /* Return 4-tuple for TCP; 2-tuple for others */ 479 /* 480 * UDP may fragment more often than TCP and thus we'll end up with 481 * NICs returning 2-tuple fragments. 482 * udp_init() and udplite_init() both currently initialise things 483 * as 2-tuple. 484 * So for now disable UDP 4-tuple hashing until all of the other 485 * pieces are in place. 486 */ 487 return ( 488 RSS_HASHTYPE_RSS_IPV4 489 | RSS_HASHTYPE_RSS_TCP_IPV4 490 | RSS_HASHTYPE_RSS_IPV6 491 | RSS_HASHTYPE_RSS_TCP_IPV6 492 | RSS_HASHTYPE_RSS_IPV6_EX 493 | RSS_HASHTYPE_RSS_TCP_IPV6_EX 494 #if 0 495 | RSS_HASHTYPE_RSS_UDP_IPV4 496 | RSS_HASHTYPE_RSS_UDP_IPV4_EX 497 | RSS_HASHTYPE_RSS_UDP_IPV6 498 | RSS_HASHTYPE_RSS_UDP_IPV6_EX 499 #endif 500 ); 501 } 502 503 /* 504 * XXXRW: Confirm that sysctl -a won't dump this keying material, don't want 505 * it appearing in debugging output unnecessarily. 506 */ 507 static int 508 sysctl_rss_key(SYSCTL_HANDLER_ARGS) 509 { 510 uint8_t temp_rss_key[RSS_KEYSIZE]; 511 int error; 512 513 error = priv_check(req->td, PRIV_NETINET_HASHKEY); 514 if (error) 515 return (error); 516 517 bcopy(rss_key, temp_rss_key, sizeof(temp_rss_key)); 518 error = sysctl_handle_opaque(oidp, temp_rss_key, 519 sizeof(temp_rss_key), req); 520 if (error) 521 return (error); 522 if (req->newptr != NULL) { 523 /* XXXRW: Not yet. */ 524 return (EINVAL); 525 } 526 return (0); 527 } 528 SYSCTL_PROC(_net_inet_rss, OID_AUTO, key, 529 CTLTYPE_OPAQUE | CTLFLAG_RD | CTLFLAG_MPSAFE, NULL, 0, sysctl_rss_key, 530 "", "RSS keying material"); 531 532 static int 533 sysctl_rss_bucket_mapping(SYSCTL_HANDLER_ARGS) 534 { 535 struct sbuf *sb; 536 int error; 537 int i; 538 539 error = 0; 540 error = sysctl_wire_old_buffer(req, 0); 541 if (error != 0) 542 return (error); 543 sb = sbuf_new_for_sysctl(NULL, NULL, 512, req); 544 if (sb == NULL) 545 return (ENOMEM); 546 for (i = 0; i < rss_buckets; i++) { 547 sbuf_printf(sb, "%s%d:%d", i == 0 ? "" : " ", 548 i, 549 rss_getcpu(i)); 550 } 551 error = sbuf_finish(sb); 552 sbuf_delete(sb); 553 554 return (error); 555 } 556 SYSCTL_PROC(_net_inet_rss, OID_AUTO, bucket_mapping, 557 CTLTYPE_STRING | CTLFLAG_RD, NULL, 0, 558 sysctl_rss_bucket_mapping, "", "RSS bucket -> CPU mapping"); 559