1da14cebeSEric Cheng /* 2da14cebeSEric Cheng * CDDL HEADER START 3da14cebeSEric Cheng * 4da14cebeSEric Cheng * The contents of this file are subject to the terms of the 5da14cebeSEric Cheng * Common Development and Distribution License (the "License"). 6da14cebeSEric Cheng * You may not use this file except in compliance with the License. 7da14cebeSEric Cheng * 8da14cebeSEric Cheng * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE 9da14cebeSEric Cheng * or http://www.opensolaris.org/os/licensing. 10da14cebeSEric Cheng * See the License for the specific language governing permissions 11da14cebeSEric Cheng * and limitations under the License. 12da14cebeSEric Cheng * 13da14cebeSEric Cheng * When distributing Covered Code, include this CDDL HEADER in each 14da14cebeSEric Cheng * file and include the License file at usr/src/OPENSOLARIS.LICENSE. 15da14cebeSEric Cheng * If applicable, add the following below this CDDL HEADER, with the 16da14cebeSEric Cheng * fields enclosed by brackets "[]" replaced with your own identifying 17da14cebeSEric Cheng * information: Portions Copyright [yyyy] [name of copyright owner] 18da14cebeSEric Cheng * 19da14cebeSEric Cheng * CDDL HEADER END 20da14cebeSEric Cheng */ 21da14cebeSEric Cheng /* 22*ae6aa22aSVenugopal Iyer * Copyright 2009 Sun Microsystems, Inc. All rights reserved. 23da14cebeSEric Cheng * Use is subject to license terms. 24da14cebeSEric Cheng */ 25da14cebeSEric Cheng 26da14cebeSEric Cheng #include <sys/types.h> 27da14cebeSEric Cheng #include <sys/callb.h> 28da14cebeSEric Cheng #include <sys/sdt.h> 29da14cebeSEric Cheng #include <sys/strsubr.h> 30da14cebeSEric Cheng #include <sys/strsun.h> 31da14cebeSEric Cheng #include <sys/vlan.h> 32da14cebeSEric Cheng #include <inet/ipsec_impl.h> 33da14cebeSEric Cheng #include <inet/ip_impl.h> 34da14cebeSEric Cheng #include <inet/sadb.h> 35da14cebeSEric Cheng #include <inet/ipsecesp.h> 36da14cebeSEric Cheng #include <inet/ipsecah.h> 37da14cebeSEric Cheng 38da14cebeSEric Cheng #include <sys/mac_impl.h> 39da14cebeSEric Cheng #include <sys/mac_client_impl.h> 40da14cebeSEric Cheng #include <sys/mac_client_priv.h> 41da14cebeSEric Cheng #include <sys/mac_soft_ring.h> 42da14cebeSEric Cheng #include <sys/mac_flow_impl.h> 43da14cebeSEric Cheng 44da14cebeSEric Cheng static void mac_srs_soft_rings_signal(mac_soft_ring_set_t *, uint_t); 45da14cebeSEric Cheng static void mac_srs_update_fanout_list(mac_soft_ring_set_t *); 46da14cebeSEric Cheng static void mac_srs_poll_unbind(mac_soft_ring_set_t *); 47da14cebeSEric Cheng static void mac_srs_worker_unbind(mac_soft_ring_set_t *); 48da14cebeSEric Cheng static void mac_srs_soft_rings_quiesce(mac_soft_ring_set_t *, uint_t); 49da14cebeSEric Cheng 50da14cebeSEric Cheng static int mac_srs_cpu_setup(cpu_setup_t, int, void *); 51da14cebeSEric Cheng static void mac_srs_worker_bind(mac_soft_ring_set_t *, processorid_t); 52da14cebeSEric Cheng static void mac_srs_poll_bind(mac_soft_ring_set_t *, processorid_t); 53da14cebeSEric Cheng static void mac_srs_threads_unbind(mac_soft_ring_set_t *); 54da14cebeSEric Cheng static void mac_srs_add_glist(mac_soft_ring_set_t *); 55da14cebeSEric Cheng static void mac_srs_remove_glist(mac_soft_ring_set_t *); 56da14cebeSEric Cheng static void mac_srs_fanout_list_free(mac_soft_ring_set_t *); 57da14cebeSEric Cheng static void mac_soft_ring_remove(mac_soft_ring_set_t *, mac_soft_ring_t *); 58da14cebeSEric Cheng 59da14cebeSEric Cheng static int mac_compute_soft_ring_count(flow_entry_t *, int); 60da14cebeSEric Cheng static void mac_walk_srs_and_bind(int); 61da14cebeSEric Cheng static void mac_walk_srs_and_unbind(int); 62da14cebeSEric Cheng 63da14cebeSEric Cheng extern mac_group_t *mac_reserve_rx_group(mac_client_impl_t *, uint8_t *, 64da14cebeSEric Cheng mac_rx_group_reserve_type_t); 65da14cebeSEric Cheng extern void mac_release_rx_group(mac_client_impl_t *, mac_group_t *); 66da14cebeSEric Cheng 67da14cebeSEric Cheng extern boolean_t mac_latency_optimize; 68da14cebeSEric Cheng 69da14cebeSEric Cheng static kmem_cache_t *mac_srs_cache; 70da14cebeSEric Cheng kmem_cache_t *mac_soft_ring_cache; 71da14cebeSEric Cheng 72da14cebeSEric Cheng /* 73da14cebeSEric Cheng * The duration in msec we wait before signalling the soft ring 74da14cebeSEric Cheng * worker thread in case packets get queued. 75da14cebeSEric Cheng */ 76*ae6aa22aSVenugopal Iyer uint32_t mac_soft_ring_worker_wait = 0; 77*ae6aa22aSVenugopal Iyer 78*ae6aa22aSVenugopal Iyer /* 79*ae6aa22aSVenugopal Iyer * A global tunable for turning polling on/off. By default, dynamic 80*ae6aa22aSVenugopal Iyer * polling is always on and is always very beneficial. It should be 81*ae6aa22aSVenugopal Iyer * turned off with absolute care and for the rare workload (very 82*ae6aa22aSVenugopal Iyer * low latency sensitive traffic). 83*ae6aa22aSVenugopal Iyer */ 84*ae6aa22aSVenugopal Iyer int mac_poll_enable = B_TRUE; 85da14cebeSEric Cheng 86da14cebeSEric Cheng /* 87da14cebeSEric Cheng * Need to set mac_soft_ring_max_q_cnt based on bandwidth and perhaps latency. 88da14cebeSEric Cheng * Large values could end up in consuming lot of system memory and cause 89da14cebeSEric Cheng * system hang. 90da14cebeSEric Cheng */ 91*ae6aa22aSVenugopal Iyer int mac_soft_ring_max_q_cnt = 1024; 92*ae6aa22aSVenugopal Iyer int mac_soft_ring_min_q_cnt = 256; 93*ae6aa22aSVenugopal Iyer int mac_soft_ring_poll_thres = 16; 94da14cebeSEric Cheng 95da14cebeSEric Cheng /* 96da14cebeSEric Cheng * Default value of number of TX rings to be assigned to a MAC client. 97da14cebeSEric Cheng * If less than 'mac_tx_ring_count' worth of Tx rings is available, then 98da14cebeSEric Cheng * as many as is available will be assigned to the newly created MAC client. 99da14cebeSEric Cheng * If no TX rings are available, then MAC client(s) will be assigned the 100da14cebeSEric Cheng * default Tx ring. Default Tx ring can be shared among multiple MAC clients. 101da14cebeSEric Cheng */ 102*ae6aa22aSVenugopal Iyer uint32_t mac_tx_ring_count = 8; 103*ae6aa22aSVenugopal Iyer boolean_t mac_tx_serialize = B_FALSE; 104da14cebeSEric Cheng 105da14cebeSEric Cheng /* 106da14cebeSEric Cheng * mac_tx_srs_hiwat is the queue depth threshold at which callers of 107da14cebeSEric Cheng * mac_tx() will be notified of flow control condition. 108da14cebeSEric Cheng * 109da14cebeSEric Cheng * TCP does not honour flow control condition sent up by mac_tx(). 110da14cebeSEric Cheng * Thus provision is made for TCP to allow more packets to be queued 111da14cebeSEric Cheng * in SRS upto a maximum of mac_tx_srs_max_q_cnt. 112da14cebeSEric Cheng * 113da14cebeSEric Cheng * Note that mac_tx_srs_hiwat is always be lesser than 114da14cebeSEric Cheng * mac_tx_srs_max_q_cnt. 115da14cebeSEric Cheng */ 116*ae6aa22aSVenugopal Iyer uint32_t mac_tx_srs_max_q_cnt = 100000; 117*ae6aa22aSVenugopal Iyer uint32_t mac_tx_srs_hiwat = 1000; 118da14cebeSEric Cheng 119da14cebeSEric Cheng /* 120da14cebeSEric Cheng * mac_rx_soft_ring_count, mac_soft_ring_10gig_count: 121da14cebeSEric Cheng * 122da14cebeSEric Cheng * Global tunables that determines the number of soft rings to be used for 123da14cebeSEric Cheng * fanning out incoming traffic on a link. These count will be used only 124da14cebeSEric Cheng * when no explicit set of CPUs was assigned to the data-links. 125da14cebeSEric Cheng * 126da14cebeSEric Cheng * mac_rx_soft_ring_count tunable will come into effect only if 127da14cebeSEric Cheng * mac_soft_ring_enable is set. mac_soft_ring_enable is turned on by 128da14cebeSEric Cheng * default only for sun4v platforms. 129da14cebeSEric Cheng * 130da14cebeSEric Cheng * mac_rx_soft_ring_10gig_count will come into effect if you are running on a 131da14cebeSEric Cheng * 10Gbps link and is not dependent upon mac_soft_ring_enable. 132da14cebeSEric Cheng * 133da14cebeSEric Cheng * The number of soft rings for fanout for a link or a flow is determined 134da14cebeSEric Cheng * by mac_compute_soft_ring_count() routine. This routine will take into 135da14cebeSEric Cheng * account mac_soft_ring_enable, mac_rx_soft_ring_count and 136da14cebeSEric Cheng * mac_rx_soft_ring_10gig_count to determine the soft ring count for a link. 137da14cebeSEric Cheng * 138da14cebeSEric Cheng * If a bandwidth is specified, the determination of the number of soft 139da14cebeSEric Cheng * rings is based on specified bandwidth, CPU speed and number of CPUs in 140da14cebeSEric Cheng * the system. 141da14cebeSEric Cheng */ 142*ae6aa22aSVenugopal Iyer uint_t mac_rx_soft_ring_count = 8; 143*ae6aa22aSVenugopal Iyer uint_t mac_rx_soft_ring_10gig_count = 8; 144da14cebeSEric Cheng 145da14cebeSEric Cheng /* 146da14cebeSEric Cheng * Every Tx and Rx mac_soft_ring_set_t (mac_srs) created gets added 147da14cebeSEric Cheng * to mac_srs_g_list and mac_srs_g_lock protects mac_srs_g_list. The 148da14cebeSEric Cheng * list is used to walk the list of all MAC threads when a CPU is 149da14cebeSEric Cheng * coming online or going offline. 150da14cebeSEric Cheng */ 151da14cebeSEric Cheng static mac_soft_ring_set_t *mac_srs_g_list = NULL; 152da14cebeSEric Cheng static krwlock_t mac_srs_g_lock; 153da14cebeSEric Cheng 154da14cebeSEric Cheng /* 155da14cebeSEric Cheng * Whether the SRS threads should be bound, or not. 156da14cebeSEric Cheng */ 157*ae6aa22aSVenugopal Iyer boolean_t mac_srs_thread_bind = B_TRUE; 158da14cebeSEric Cheng 159da14cebeSEric Cheng /* 160da14cebeSEric Cheng * CPU to fallback to, used by mac_next_bind_cpu(). 161da14cebeSEric Cheng */ 162*ae6aa22aSVenugopal Iyer processorid_t srs_bind_cpu = 0; 163da14cebeSEric Cheng 164da14cebeSEric Cheng /* 165da14cebeSEric Cheng * If cpu bindings are specified by user, then Tx SRS and its soft 166da14cebeSEric Cheng * rings should also be bound to the CPUs specified by user. The 167da14cebeSEric Cheng * CPUs for Tx bindings are at the end of the cpu list provided by 168da14cebeSEric Cheng * the user. If enough CPUs are not available (for Tx and Rx 169da14cebeSEric Cheng * SRSes), then the CPUs are shared by both Tx and Rx SRSes. 170da14cebeSEric Cheng */ 171da14cebeSEric Cheng #define BIND_TX_SRS_AND_SOFT_RINGS(mac_tx_srs, mrp) { \ 172da14cebeSEric Cheng processorid_t cpuid; \ 173da14cebeSEric Cheng int i, j; \ 174da14cebeSEric Cheng mac_soft_ring_t *softring; \ 175da14cebeSEric Cheng \ 176da14cebeSEric Cheng cpuid = mrp->mrp_cpu[mrp->mrp_ncpus - 1]; \ 177da14cebeSEric Cheng mac_srs_worker_bind(mac_tx_srs, cpuid); \ 178da14cebeSEric Cheng if (TX_MULTI_RING_MODE(mac_tx_srs)) { \ 179da14cebeSEric Cheng j = mrp->mrp_ncpus - 1; \ 180da14cebeSEric Cheng for (i = 0; \ 181da14cebeSEric Cheng i < mac_tx_srs->srs_oth_ring_count; i++, j--) { \ 182da14cebeSEric Cheng if (j < 0) \ 183da14cebeSEric Cheng j = mrp->mrp_ncpus - 1; \ 184da14cebeSEric Cheng cpuid = mrp->mrp_cpu[j]; \ 185da14cebeSEric Cheng softring = mac_tx_srs->srs_oth_soft_rings[i]; \ 186da14cebeSEric Cheng (void) mac_soft_ring_bind(softring, cpuid); \ 187da14cebeSEric Cheng } \ 188da14cebeSEric Cheng } \ 189da14cebeSEric Cheng } 190da14cebeSEric Cheng 191da14cebeSEric Cheng /* INIT and FINI ROUTINES */ 192da14cebeSEric Cheng 193da14cebeSEric Cheng void 194da14cebeSEric Cheng mac_soft_ring_init(void) 195da14cebeSEric Cheng { 196da14cebeSEric Cheng mac_soft_ring_cache = kmem_cache_create("mac_soft_ring_cache", 197da14cebeSEric Cheng sizeof (mac_soft_ring_t), 64, NULL, NULL, NULL, NULL, NULL, 0); 198da14cebeSEric Cheng 199da14cebeSEric Cheng mac_srs_cache = kmem_cache_create("mac_srs_cache", 200da14cebeSEric Cheng sizeof (mac_soft_ring_set_t), 201da14cebeSEric Cheng 64, NULL, NULL, NULL, NULL, NULL, 0); 202da14cebeSEric Cheng 203da14cebeSEric Cheng rw_init(&mac_srs_g_lock, NULL, RW_DEFAULT, NULL); 204da14cebeSEric Cheng mutex_enter(&cpu_lock); 205da14cebeSEric Cheng register_cpu_setup_func(mac_srs_cpu_setup, NULL); 206da14cebeSEric Cheng mutex_exit(&cpu_lock); 207da14cebeSEric Cheng } 208da14cebeSEric Cheng 209da14cebeSEric Cheng void 210da14cebeSEric Cheng mac_soft_ring_finish(void) 211da14cebeSEric Cheng { 212da14cebeSEric Cheng mutex_enter(&cpu_lock); 213da14cebeSEric Cheng unregister_cpu_setup_func(mac_srs_cpu_setup, NULL); 214da14cebeSEric Cheng mutex_exit(&cpu_lock); 215da14cebeSEric Cheng rw_destroy(&mac_srs_g_lock); 216da14cebeSEric Cheng kmem_cache_destroy(mac_soft_ring_cache); 217da14cebeSEric Cheng kmem_cache_destroy(mac_srs_cache); 218da14cebeSEric Cheng } 219da14cebeSEric Cheng 220da14cebeSEric Cheng static void 221da14cebeSEric Cheng mac_srs_soft_rings_free(mac_soft_ring_set_t *mac_srs, boolean_t release_tx_ring) 222da14cebeSEric Cheng { 223da14cebeSEric Cheng mac_soft_ring_t *softring, *next, *head; 224da14cebeSEric Cheng 225da14cebeSEric Cheng /* 226da14cebeSEric Cheng * Synchronize with mac_walk_srs_bind/unbind which are callbacks from 227da14cebeSEric Cheng * DR. The callbacks from DR are called with cpu_lock held, and hence 228da14cebeSEric Cheng * can't wait to grab the mac perimeter. The soft ring list is hence 229da14cebeSEric Cheng * protected for read access by srs_lock. Changing the soft ring list 230da14cebeSEric Cheng * needs the mac perimeter and the srs_lock. 231da14cebeSEric Cheng */ 232da14cebeSEric Cheng mutex_enter(&mac_srs->srs_lock); 233da14cebeSEric Cheng 234da14cebeSEric Cheng head = mac_srs->srs_soft_ring_head; 235da14cebeSEric Cheng mac_srs->srs_soft_ring_head = NULL; 236da14cebeSEric Cheng mac_srs->srs_soft_ring_tail = NULL; 237da14cebeSEric Cheng mac_srs->srs_soft_ring_count = 0; 238da14cebeSEric Cheng 239da14cebeSEric Cheng mutex_exit(&mac_srs->srs_lock); 240da14cebeSEric Cheng 241da14cebeSEric Cheng for (softring = head; softring != NULL; softring = next) { 242da14cebeSEric Cheng next = softring->s_ring_next; 243da14cebeSEric Cheng mac_soft_ring_free(softring, release_tx_ring); 244da14cebeSEric Cheng } 245da14cebeSEric Cheng } 246da14cebeSEric Cheng 247da14cebeSEric Cheng static void 248da14cebeSEric Cheng mac_srs_add_glist(mac_soft_ring_set_t *mac_srs) 249da14cebeSEric Cheng { 250da14cebeSEric Cheng ASSERT(mac_srs->srs_next == NULL && mac_srs->srs_prev == NULL); 251da14cebeSEric Cheng ASSERT(MAC_PERIM_HELD((mac_handle_t)mac_srs->srs_mcip->mci_mip)); 252da14cebeSEric Cheng 253da14cebeSEric Cheng rw_enter(&mac_srs_g_lock, RW_WRITER); 254da14cebeSEric Cheng mutex_enter(&mac_srs->srs_lock); 255da14cebeSEric Cheng 256da14cebeSEric Cheng ASSERT((mac_srs->srs_state & SRS_IN_GLIST) == 0); 257da14cebeSEric Cheng 258da14cebeSEric Cheng if (mac_srs_g_list == NULL) { 259da14cebeSEric Cheng mac_srs_g_list = mac_srs; 260da14cebeSEric Cheng } else { 261da14cebeSEric Cheng mac_srs->srs_next = mac_srs_g_list; 262da14cebeSEric Cheng mac_srs_g_list->srs_prev = mac_srs; 263da14cebeSEric Cheng mac_srs->srs_prev = NULL; 264da14cebeSEric Cheng mac_srs_g_list = mac_srs; 265da14cebeSEric Cheng } 266da14cebeSEric Cheng mac_srs->srs_state |= SRS_IN_GLIST; 267da14cebeSEric Cheng 268da14cebeSEric Cheng mutex_exit(&mac_srs->srs_lock); 269da14cebeSEric Cheng rw_exit(&mac_srs_g_lock); 270da14cebeSEric Cheng } 271da14cebeSEric Cheng 272da14cebeSEric Cheng static void 273da14cebeSEric Cheng mac_srs_remove_glist(mac_soft_ring_set_t *mac_srs) 274da14cebeSEric Cheng { 275da14cebeSEric Cheng ASSERT(MAC_PERIM_HELD((mac_handle_t)mac_srs->srs_mcip->mci_mip)); 276da14cebeSEric Cheng 277da14cebeSEric Cheng rw_enter(&mac_srs_g_lock, RW_WRITER); 278da14cebeSEric Cheng mutex_enter(&mac_srs->srs_lock); 279da14cebeSEric Cheng 280da14cebeSEric Cheng ASSERT((mac_srs->srs_state & SRS_IN_GLIST) != 0); 281da14cebeSEric Cheng 282da14cebeSEric Cheng if (mac_srs == mac_srs_g_list) { 283da14cebeSEric Cheng mac_srs_g_list = mac_srs->srs_next; 284da14cebeSEric Cheng if (mac_srs_g_list != NULL) 285da14cebeSEric Cheng mac_srs_g_list->srs_prev = NULL; 286da14cebeSEric Cheng } else { 287da14cebeSEric Cheng mac_srs->srs_prev->srs_next = mac_srs->srs_next; 288da14cebeSEric Cheng if (mac_srs->srs_next != NULL) 289da14cebeSEric Cheng mac_srs->srs_next->srs_prev = mac_srs->srs_prev; 290da14cebeSEric Cheng } 291da14cebeSEric Cheng mac_srs->srs_state &= ~SRS_IN_GLIST; 292da14cebeSEric Cheng 293da14cebeSEric Cheng mutex_exit(&mac_srs->srs_lock); 294da14cebeSEric Cheng rw_exit(&mac_srs_g_lock); 295da14cebeSEric Cheng } 296da14cebeSEric Cheng 297da14cebeSEric Cheng /* POLLING SETUP AND TEAR DOWN ROUTINES */ 298da14cebeSEric Cheng 299da14cebeSEric Cheng /* 300da14cebeSEric Cheng * mac_srs_client_poll_quiesce and mac_srs_client_poll_restart 301da14cebeSEric Cheng * 302da14cebeSEric Cheng * These routines are used to call back into the upper layer 303da14cebeSEric Cheng * (primarily TCP squeue) to stop polling the soft rings or 304da14cebeSEric Cheng * restart polling. 305da14cebeSEric Cheng */ 306da14cebeSEric Cheng void 307da14cebeSEric Cheng mac_srs_client_poll_quiesce(mac_client_impl_t *mcip, 308da14cebeSEric Cheng mac_soft_ring_set_t *mac_srs) 309da14cebeSEric Cheng { 310da14cebeSEric Cheng mac_soft_ring_t *softring; 311da14cebeSEric Cheng 312da14cebeSEric Cheng ASSERT(MAC_PERIM_HELD((mac_handle_t)mcip->mci_mip)); 313da14cebeSEric Cheng 314da14cebeSEric Cheng if (!(mac_srs->srs_type & SRST_CLIENT_POLL_ENABLED)) { 315da14cebeSEric Cheng ASSERT(!(mac_srs->srs_type & SRST_DLS_BYPASS)); 316da14cebeSEric Cheng return; 317da14cebeSEric Cheng } 318da14cebeSEric Cheng 319da14cebeSEric Cheng for (softring = mac_srs->srs_soft_ring_head; 320da14cebeSEric Cheng softring != NULL; softring = softring->s_ring_next) { 321da14cebeSEric Cheng if ((softring->s_ring_type & ST_RING_TCP) && 322da14cebeSEric Cheng (softring->s_ring_rx_arg2 != NULL)) { 323da14cebeSEric Cheng mcip->mci_resource_quiesce(mcip->mci_resource_arg, 324da14cebeSEric Cheng softring->s_ring_rx_arg2); 325da14cebeSEric Cheng } 326da14cebeSEric Cheng } 327da14cebeSEric Cheng } 328da14cebeSEric Cheng 329da14cebeSEric Cheng void 330da14cebeSEric Cheng mac_srs_client_poll_restart(mac_client_impl_t *mcip, 331da14cebeSEric Cheng mac_soft_ring_set_t *mac_srs) 332da14cebeSEric Cheng { 333da14cebeSEric Cheng mac_soft_ring_t *softring; 334da14cebeSEric Cheng 335da14cebeSEric Cheng ASSERT(MAC_PERIM_HELD((mac_handle_t)mcip->mci_mip)); 336da14cebeSEric Cheng 337da14cebeSEric Cheng if (!(mac_srs->srs_type & SRST_CLIENT_POLL_ENABLED)) { 338da14cebeSEric Cheng ASSERT(!(mac_srs->srs_type & SRST_DLS_BYPASS)); 339da14cebeSEric Cheng return; 340da14cebeSEric Cheng } 341da14cebeSEric Cheng 342da14cebeSEric Cheng for (softring = mac_srs->srs_soft_ring_head; 343da14cebeSEric Cheng softring != NULL; softring = softring->s_ring_next) { 344da14cebeSEric Cheng if ((softring->s_ring_type & ST_RING_TCP) && 345da14cebeSEric Cheng (softring->s_ring_rx_arg2 != NULL)) { 346da14cebeSEric Cheng mcip->mci_resource_restart(mcip->mci_resource_arg, 347da14cebeSEric Cheng softring->s_ring_rx_arg2); 348da14cebeSEric Cheng } 349da14cebeSEric Cheng } 350da14cebeSEric Cheng } 351da14cebeSEric Cheng 352da14cebeSEric Cheng /* 353da14cebeSEric Cheng * Register the given SRS and associated soft rings with the consumer and 354da14cebeSEric Cheng * enable the polling interface used by the consumer.(i.e IP) over this 355da14cebeSEric Cheng * SRS and associated soft rings. 356da14cebeSEric Cheng */ 357da14cebeSEric Cheng void 358da14cebeSEric Cheng mac_srs_client_poll_enable(mac_client_impl_t *mcip, 359da14cebeSEric Cheng mac_soft_ring_set_t *mac_srs) 360da14cebeSEric Cheng { 361da14cebeSEric Cheng mac_rx_fifo_t mrf; 362da14cebeSEric Cheng mac_soft_ring_t *softring; 363da14cebeSEric Cheng 364da14cebeSEric Cheng ASSERT(mac_srs->srs_mcip == mcip); 365da14cebeSEric Cheng ASSERT(MAC_PERIM_HELD((mac_handle_t)mcip->mci_mip)); 366da14cebeSEric Cheng 367da14cebeSEric Cheng if (!(mcip->mci_state_flags & MCIS_CLIENT_POLL_CAPABLE)) 368da14cebeSEric Cheng return; 369da14cebeSEric Cheng 370da14cebeSEric Cheng bzero(&mrf, sizeof (mac_rx_fifo_t)); 371da14cebeSEric Cheng mrf.mrf_type = MAC_RX_FIFO; 372da14cebeSEric Cheng 373da14cebeSEric Cheng /* 374da14cebeSEric Cheng * A SRS is capable of acting as a soft ring for cases 375da14cebeSEric Cheng * where no fanout is needed. This is the case for userland 376da14cebeSEric Cheng * flows. 377da14cebeSEric Cheng */ 378da14cebeSEric Cheng if (mac_srs->srs_type & SRST_NO_SOFT_RINGS) 379da14cebeSEric Cheng return; 380da14cebeSEric Cheng 381da14cebeSEric Cheng mrf.mrf_receive = (mac_receive_t)mac_soft_ring_poll; 382da14cebeSEric Cheng mrf.mrf_intr_enable = (mac_intr_enable_t)mac_soft_ring_intr_enable; 383da14cebeSEric Cheng mrf.mrf_intr_disable = (mac_intr_disable_t)mac_soft_ring_intr_disable; 384da14cebeSEric Cheng mac_srs->srs_type |= SRST_CLIENT_POLL_ENABLED; 385da14cebeSEric Cheng 386da14cebeSEric Cheng softring = mac_srs->srs_soft_ring_head; 387da14cebeSEric Cheng while (softring != NULL) { 388da14cebeSEric Cheng if (softring->s_ring_type & (ST_RING_TCP | ST_RING_UDP)) { 389da14cebeSEric Cheng /* 390da14cebeSEric Cheng * TCP and UDP support DLS bypass. Squeue polling 391da14cebeSEric Cheng * support implies DLS bypass since the squeue poll 392da14cebeSEric Cheng * path does not have DLS processing. 393da14cebeSEric Cheng */ 394da14cebeSEric Cheng mac_soft_ring_dls_bypass(softring, 395da14cebeSEric Cheng mcip->mci_direct_rx_fn, mcip->mci_direct_rx_arg); 396da14cebeSEric Cheng } 397da14cebeSEric Cheng /* 398da14cebeSEric Cheng * Non-TCP protocols don't support squeues. Hence we don't 399da14cebeSEric Cheng * make any ring addition callbacks for non-TCP rings 400da14cebeSEric Cheng */ 401da14cebeSEric Cheng if (!(softring->s_ring_type & ST_RING_TCP)) { 402da14cebeSEric Cheng softring->s_ring_rx_arg2 = NULL; 403da14cebeSEric Cheng softring = softring->s_ring_next; 404da14cebeSEric Cheng continue; 405da14cebeSEric Cheng } 406da14cebeSEric Cheng mrf.mrf_rx_arg = softring; 407da14cebeSEric Cheng mrf.mrf_intr_handle = (mac_intr_handle_t)softring; 408da14cebeSEric Cheng mrf.mrf_cpu_id = softring->s_ring_cpuid; 409da14cebeSEric Cheng mrf.mrf_flow_priority = mac_srs->srs_pri; 410da14cebeSEric Cheng 411da14cebeSEric Cheng softring->s_ring_rx_arg2 = mcip->mci_resource_add( 412da14cebeSEric Cheng mcip->mci_resource_arg, (mac_resource_t *)&mrf); 413da14cebeSEric Cheng 414da14cebeSEric Cheng softring = softring->s_ring_next; 415da14cebeSEric Cheng } 416da14cebeSEric Cheng } 417da14cebeSEric Cheng 418da14cebeSEric Cheng /* 419da14cebeSEric Cheng * Unregister the given SRS and associated soft rings with the consumer and 420da14cebeSEric Cheng * disable the polling interface used by the consumer.(i.e IP) over this 421da14cebeSEric Cheng * SRS and associated soft rings. 422da14cebeSEric Cheng */ 423da14cebeSEric Cheng void 424da14cebeSEric Cheng mac_srs_client_poll_disable(mac_client_impl_t *mcip, 425da14cebeSEric Cheng mac_soft_ring_set_t *mac_srs) 426da14cebeSEric Cheng { 427da14cebeSEric Cheng mac_soft_ring_t *softring; 428da14cebeSEric Cheng 429da14cebeSEric Cheng ASSERT(MAC_PERIM_HELD((mac_handle_t)mcip->mci_mip)); 430da14cebeSEric Cheng 431da14cebeSEric Cheng /* 432da14cebeSEric Cheng * A SRS is capable of acting as a soft ring for cases 433da14cebeSEric Cheng * where no protocol fanout is needed. This is the case 434da14cebeSEric Cheng * for userland flows. Nothing to do here. 435da14cebeSEric Cheng */ 436da14cebeSEric Cheng if (mac_srs->srs_type & SRST_NO_SOFT_RINGS) 437da14cebeSEric Cheng return; 438da14cebeSEric Cheng 439da14cebeSEric Cheng mutex_enter(&mac_srs->srs_lock); 440da14cebeSEric Cheng if (!(mac_srs->srs_type & SRST_CLIENT_POLL_ENABLED)) { 441da14cebeSEric Cheng ASSERT(!(mac_srs->srs_type & SRST_DLS_BYPASS)); 442da14cebeSEric Cheng mutex_exit(&mac_srs->srs_lock); 443da14cebeSEric Cheng return; 444da14cebeSEric Cheng } 445da14cebeSEric Cheng mac_srs->srs_type &= ~(SRST_CLIENT_POLL_ENABLED | SRST_DLS_BYPASS); 446da14cebeSEric Cheng mutex_exit(&mac_srs->srs_lock); 447da14cebeSEric Cheng 448da14cebeSEric Cheng /* 449da14cebeSEric Cheng * DLS bypass is now disabled in the case of both TCP and UDP. 450da14cebeSEric Cheng * Reset the soft ring callbacks to the standard 'mac_rx_deliver' 451da14cebeSEric Cheng * callback. In addition, in the case of TCP, invoke IP's callback 452da14cebeSEric Cheng * for ring removal. 453da14cebeSEric Cheng */ 454da14cebeSEric Cheng for (softring = mac_srs->srs_soft_ring_head; 455da14cebeSEric Cheng softring != NULL; softring = softring->s_ring_next) { 456da14cebeSEric Cheng if (!(softring->s_ring_type & (ST_RING_UDP | ST_RING_TCP))) 457da14cebeSEric Cheng continue; 458da14cebeSEric Cheng 459da14cebeSEric Cheng if ((softring->s_ring_type & ST_RING_TCP) && 460da14cebeSEric Cheng softring->s_ring_rx_arg2 != NULL) { 461da14cebeSEric Cheng mcip->mci_resource_remove(mcip->mci_resource_arg, 462da14cebeSEric Cheng softring->s_ring_rx_arg2); 463da14cebeSEric Cheng } 464da14cebeSEric Cheng 465da14cebeSEric Cheng mutex_enter(&softring->s_ring_lock); 466da14cebeSEric Cheng while (softring->s_ring_state & S_RING_PROC) { 467da14cebeSEric Cheng softring->s_ring_state |= S_RING_CLIENT_WAIT; 468da14cebeSEric Cheng cv_wait(&softring->s_ring_client_cv, 469da14cebeSEric Cheng &softring->s_ring_lock); 470da14cebeSEric Cheng } 471da14cebeSEric Cheng softring->s_ring_state &= ~S_RING_CLIENT_WAIT; 472da14cebeSEric Cheng softring->s_ring_rx_arg2 = NULL; 473da14cebeSEric Cheng softring->s_ring_rx_func = mac_rx_deliver; 474da14cebeSEric Cheng softring->s_ring_rx_arg1 = mcip; 475da14cebeSEric Cheng mutex_exit(&softring->s_ring_lock); 476da14cebeSEric Cheng } 477da14cebeSEric Cheng } 478da14cebeSEric Cheng 479da14cebeSEric Cheng /* 480da14cebeSEric Cheng * Enable or disable poll capability of the SRS on the underlying Rx ring. 481da14cebeSEric Cheng * 482da14cebeSEric Cheng * There is a need to enable or disable the poll capability of an SRS over an 483da14cebeSEric Cheng * Rx ring depending on the number of mac clients sharing the ring and also 484da14cebeSEric Cheng * whether user flows are configured on it. However the poll state is actively 485da14cebeSEric Cheng * manipulated by the SRS worker and poll threads and uncoordinated changes by 486da14cebeSEric Cheng * yet another thread to the underlying capability can surprise them leading 487da14cebeSEric Cheng * to assert failures. Instead we quiesce the SRS, make the changes and then 488da14cebeSEric Cheng * restart the SRS. 489da14cebeSEric Cheng */ 490da14cebeSEric Cheng static void 491da14cebeSEric Cheng mac_srs_poll_state_change(mac_soft_ring_set_t *mac_srs, 492da14cebeSEric Cheng boolean_t turn_off_poll_capab, mac_rx_func_t rx_func) 493da14cebeSEric Cheng { 494da14cebeSEric Cheng boolean_t need_restart = B_FALSE; 495da14cebeSEric Cheng mac_srs_rx_t *srs_rx = &mac_srs->srs_rx; 496da14cebeSEric Cheng mac_ring_t *ring; 497da14cebeSEric Cheng 498da14cebeSEric Cheng if (!SRS_QUIESCED(mac_srs)) { 499da14cebeSEric Cheng mac_rx_srs_quiesce(mac_srs, SRS_QUIESCE); 500da14cebeSEric Cheng need_restart = B_TRUE; 501da14cebeSEric Cheng } 502da14cebeSEric Cheng 503da14cebeSEric Cheng ring = mac_srs->srs_ring; 504da14cebeSEric Cheng if ((ring != NULL) && 505da14cebeSEric Cheng (ring->mr_classify_type == MAC_HW_CLASSIFIER)) { 506da14cebeSEric Cheng if (turn_off_poll_capab) 507da14cebeSEric Cheng mac_srs->srs_state &= ~SRS_POLLING_CAPAB; 508*ae6aa22aSVenugopal Iyer else if (mac_poll_enable) 509da14cebeSEric Cheng mac_srs->srs_state |= SRS_POLLING_CAPAB; 510da14cebeSEric Cheng } 511da14cebeSEric Cheng srs_rx->sr_lower_proc = rx_func; 512da14cebeSEric Cheng 513da14cebeSEric Cheng if (need_restart) 514da14cebeSEric Cheng mac_rx_srs_restart(mac_srs); 515da14cebeSEric Cheng } 516da14cebeSEric Cheng 517da14cebeSEric Cheng /* CPU RECONFIGURATION AND FANOUT COMPUTATION ROUTINES */ 518da14cebeSEric Cheng 519da14cebeSEric Cheng /* 520da14cebeSEric Cheng * Return the next CPU to be used to bind a MAC kernel thread. 521da14cebeSEric Cheng */ 522da14cebeSEric Cheng static processorid_t 523da14cebeSEric Cheng mac_next_bind_cpu(void) 524da14cebeSEric Cheng { 525da14cebeSEric Cheng static processorid_t srs_curr_cpu = -1; 526da14cebeSEric Cheng cpu_t *cp; 527da14cebeSEric Cheng 528da14cebeSEric Cheng ASSERT(MUTEX_HELD(&cpu_lock)); 529da14cebeSEric Cheng 530da14cebeSEric Cheng srs_curr_cpu++; 531da14cebeSEric Cheng cp = cpu_get(srs_curr_cpu); 532da14cebeSEric Cheng if (cp == NULL || !cpu_is_online(cp)) 533da14cebeSEric Cheng srs_curr_cpu = srs_bind_cpu; 534da14cebeSEric Cheng 535da14cebeSEric Cheng return (srs_curr_cpu); 536da14cebeSEric Cheng } 537da14cebeSEric Cheng 538da14cebeSEric Cheng /* ARGSUSED */ 539da14cebeSEric Cheng static int 540da14cebeSEric Cheng mac_srs_cpu_setup(cpu_setup_t what, int id, void *arg) 541da14cebeSEric Cheng { 542da14cebeSEric Cheng ASSERT(MUTEX_HELD(&cpu_lock)); 543da14cebeSEric Cheng switch (what) { 544da14cebeSEric Cheng case CPU_CONFIG: 545da14cebeSEric Cheng case CPU_ON: 546da14cebeSEric Cheng case CPU_CPUPART_IN: 547da14cebeSEric Cheng mac_walk_srs_and_bind(id); 548da14cebeSEric Cheng break; 549da14cebeSEric Cheng 550da14cebeSEric Cheng case CPU_UNCONFIG: 551da14cebeSEric Cheng case CPU_OFF: 552da14cebeSEric Cheng case CPU_CPUPART_OUT: 553da14cebeSEric Cheng mac_walk_srs_and_unbind(id); 554da14cebeSEric Cheng break; 555da14cebeSEric Cheng 556da14cebeSEric Cheng default: 557da14cebeSEric Cheng break; 558da14cebeSEric Cheng } 559da14cebeSEric Cheng return (0); 560da14cebeSEric Cheng } 561da14cebeSEric Cheng 562da14cebeSEric Cheng /* 563da14cebeSEric Cheng * mac_compute_soft_ring_count(): 564da14cebeSEric Cheng * 565da14cebeSEric Cheng * This routine computes the number of soft rings needed to handle incoming 566da14cebeSEric Cheng * load given a flow_entry. 567da14cebeSEric Cheng * 568da14cebeSEric Cheng * The routine does the following: 569da14cebeSEric Cheng * 1) soft rings will be created if mac_soft_ring_enable is set. 570da14cebeSEric Cheng * 2) If the underlying link is a 10Gbps link, then soft rings will be 571da14cebeSEric Cheng * created even if mac_soft_ring_enable is not set. The number of soft 572da14cebeSEric Cheng * rings, so created, will equal mac_rx_soft_ring_10gig_count. 573da14cebeSEric Cheng * 3) On a sun4v platform (i.e., mac_soft_ring_enable is set), 2 times the 574da14cebeSEric Cheng * mac_rx_soft_ring_10gig_count number of soft rings will be created for a 575da14cebeSEric Cheng * 10Gbps link. 576da14cebeSEric Cheng * 577da14cebeSEric Cheng * If a bandwidth limit is specified, the number that gets computed is 578da14cebeSEric Cheng * dependent upon CPU speed, the number of Rx rings configured, and 579da14cebeSEric Cheng * the bandwidth limit. 580da14cebeSEric Cheng * If more Rx rings are available, less number of soft rings is needed. 581da14cebeSEric Cheng * 582da14cebeSEric Cheng * mac_use_bw_heuristic is another "hidden" variable that can be used to 583da14cebeSEric Cheng * override the default use of soft ring count computation. Depending upon 584da14cebeSEric Cheng * the usefulness of it, mac_use_bw_heuristic can later be made into a 585da14cebeSEric Cheng * data-link property or removed altogether. 586da14cebeSEric Cheng * 587da14cebeSEric Cheng * TODO: Cleanup and tighten some of the assumptions. 588da14cebeSEric Cheng */ 589da14cebeSEric Cheng boolean_t mac_use_bw_heuristic = B_TRUE; 590da14cebeSEric Cheng static int 591da14cebeSEric Cheng mac_compute_soft_ring_count(flow_entry_t *flent, int rx_srs_cnt) 592da14cebeSEric Cheng { 593da14cebeSEric Cheng uint64_t cpu_speed, bw = 0; 594da14cebeSEric Cheng int srings = 0; 595da14cebeSEric Cheng boolean_t bw_enabled = B_FALSE; 596da14cebeSEric Cheng 597da14cebeSEric Cheng ASSERT(!(flent->fe_type & FLOW_USER)); 598da14cebeSEric Cheng if (flent->fe_resource_props.mrp_mask & MRP_MAXBW && 599da14cebeSEric Cheng mac_use_bw_heuristic) { 600da14cebeSEric Cheng /* bandwidth enabled */ 601da14cebeSEric Cheng bw_enabled = B_TRUE; 602da14cebeSEric Cheng bw = flent->fe_resource_props.mrp_maxbw; 603da14cebeSEric Cheng } 604da14cebeSEric Cheng if (!bw_enabled) { 605da14cebeSEric Cheng /* No bandwidth enabled */ 606da14cebeSEric Cheng if (mac_soft_ring_enable) 607da14cebeSEric Cheng srings = mac_rx_soft_ring_count; 608da14cebeSEric Cheng 609da14cebeSEric Cheng /* Is this a 10Gig link? */ 610da14cebeSEric Cheng flent->fe_nic_speed = mac_client_stat_get(flent->fe_mcip, 611da14cebeSEric Cheng MAC_STAT_IFSPEED); 612da14cebeSEric Cheng /* convert to Mbps */ 613da14cebeSEric Cheng if (((flent->fe_nic_speed)/1000000) > 1000 && 614da14cebeSEric Cheng mac_rx_soft_ring_10gig_count > 0) { 615da14cebeSEric Cheng /* This is a 10Gig link */ 616da14cebeSEric Cheng srings = mac_rx_soft_ring_10gig_count; 617da14cebeSEric Cheng /* 618da14cebeSEric Cheng * Use 2 times mac_rx_soft_ring_10gig_count for 619da14cebeSEric Cheng * sun4v systems. 620da14cebeSEric Cheng */ 621da14cebeSEric Cheng if (mac_soft_ring_enable) 622da14cebeSEric Cheng srings = srings * 2; 623da14cebeSEric Cheng } 624da14cebeSEric Cheng } else { 625da14cebeSEric Cheng /* 626da14cebeSEric Cheng * Soft ring computation using CPU speed and specified 627da14cebeSEric Cheng * bandwidth limit. 628da14cebeSEric Cheng */ 629da14cebeSEric Cheng /* Assumption: all CPUs have the same frequency */ 630da14cebeSEric Cheng cpu_speed = (uint64_t)CPU->cpu_type_info.pi_clock; 631da14cebeSEric Cheng 632da14cebeSEric Cheng /* cpu_speed is in MHz; make bw in units of Mbps. */ 633da14cebeSEric Cheng bw = bw/1000000; 634da14cebeSEric Cheng 635da14cebeSEric Cheng if (bw >= 1000) { 636da14cebeSEric Cheng /* 637da14cebeSEric Cheng * bw is greater than or equal to 1Gbps. 638da14cebeSEric Cheng * The number of soft rings required is a function 639da14cebeSEric Cheng * of bandwidth and CPU speed. To keep this simple, 640da14cebeSEric Cheng * let's use this rule: 1GHz CPU can handle 1Gbps. 641da14cebeSEric Cheng * If bw is less than 1 Gbps, then there is no need 642da14cebeSEric Cheng * for soft rings. Assumption is that CPU speeds 643da14cebeSEric Cheng * (on modern systems) are at least 1GHz. 644da14cebeSEric Cheng */ 645da14cebeSEric Cheng srings = bw/cpu_speed; 646da14cebeSEric Cheng if (srings <= 1 && mac_soft_ring_enable) { 647da14cebeSEric Cheng /* 648da14cebeSEric Cheng * Give at least 2 soft rings 649da14cebeSEric Cheng * for sun4v systems 650da14cebeSEric Cheng */ 651da14cebeSEric Cheng srings = 2; 652da14cebeSEric Cheng } 653da14cebeSEric Cheng } 654da14cebeSEric Cheng } 655da14cebeSEric Cheng /* 656da14cebeSEric Cheng * If the flent has multiple Rx SRSs, then each SRS need not 657da14cebeSEric Cheng * have that many soft rings on top of it. The number of 658da14cebeSEric Cheng * soft rings for each Rx SRS is found by dividing srings by 659da14cebeSEric Cheng * rx_srs_cnt. 660da14cebeSEric Cheng */ 661da14cebeSEric Cheng if (rx_srs_cnt > 1) { 662da14cebeSEric Cheng int remainder; 663da14cebeSEric Cheng 664da14cebeSEric Cheng remainder = srings%rx_srs_cnt; 665da14cebeSEric Cheng srings = srings/rx_srs_cnt; 666da14cebeSEric Cheng if (remainder != 0) 667da14cebeSEric Cheng srings++; 668da14cebeSEric Cheng /* 669da14cebeSEric Cheng * Fanning out to 1 soft ring is not very useful. 670da14cebeSEric Cheng * Set it as well to 0 and mac_srs_fanout_init() 671da14cebeSEric Cheng * will take care of creating a single soft ring 672da14cebeSEric Cheng * for proto fanout. 673da14cebeSEric Cheng */ 674da14cebeSEric Cheng if (srings == 1) 675da14cebeSEric Cheng srings = 0; 676da14cebeSEric Cheng } 677da14cebeSEric Cheng /* Do some more massaging */ 678da14cebeSEric Cheng srings = min(srings, ncpus); 679da14cebeSEric Cheng srings = min(srings, MAX_SR_FANOUT); 680da14cebeSEric Cheng return (srings); 681da14cebeSEric Cheng } 682da14cebeSEric Cheng 683da14cebeSEric Cheng /* 684da14cebeSEric Cheng * Assignment of user specified CPUs to a link. 685da14cebeSEric Cheng * 686da14cebeSEric Cheng * Minimum CPUs required to get an optimal assignmet: 687da14cebeSEric Cheng * For each Rx SRS, atleast two CPUs are needed if mac_latency_optimize 688da14cebeSEric Cheng * flag is set -- one for polling, one for fanout soft ring. 689da14cebeSEric Cheng * If mac_latency_optimize is not set, then 3 CPUs are needed -- one 690da14cebeSEric Cheng * for polling, one for SRS worker thread and one for fanout soft ring. 691da14cebeSEric Cheng * 692da14cebeSEric Cheng * The CPUs needed for Tx side is equal to the number of Tx rings 693da14cebeSEric Cheng * the link is using. 694da14cebeSEric Cheng * 695da14cebeSEric Cheng * mac_flow_user_cpu_init() categorizes the CPU assignment depending 696da14cebeSEric Cheng * upon the number of CPUs in 3 different buckets. 697da14cebeSEric Cheng * 698da14cebeSEric Cheng * In the first bucket, the most optimal case is handled. The user has 699da14cebeSEric Cheng * passed enough number of CPUs and every thread gets its own CPU. 700da14cebeSEric Cheng * 701da14cebeSEric Cheng * The second and third are the sub-optimal cases. Enough CPUs are not 702da14cebeSEric Cheng * available. 703da14cebeSEric Cheng * 704da14cebeSEric Cheng * The second bucket handles the case where atleast one distinct CPU is 705da14cebeSEric Cheng * is available for each of the Rx rings (Rx SRSes) and Tx rings (Tx 706da14cebeSEric Cheng * SRS or soft rings). 707da14cebeSEric Cheng * 708da14cebeSEric Cheng * In the third case (worst case scenario), specified CPU count is less 709da14cebeSEric Cheng * than the Rx rings configured for the link. In this case, we round 710da14cebeSEric Cheng * robin the CPUs among the Rx SRSes and Tx SRS/soft rings. 711da14cebeSEric Cheng */ 712da14cebeSEric Cheng static void 713da14cebeSEric Cheng mac_flow_user_cpu_init(flow_entry_t *flent, mac_resource_props_t *mrp) 714da14cebeSEric Cheng { 715da14cebeSEric Cheng mac_soft_ring_set_t *rx_srs, *tx_srs; 716da14cebeSEric Cheng int i, srs_cnt; 717da14cebeSEric Cheng mac_cpus_t *srs_cpu; 718da14cebeSEric Cheng int no_of_cpus, cpu_cnt; 719da14cebeSEric Cheng int rx_srs_cnt, reqd_rx_cpu_cnt; 720da14cebeSEric Cheng int fanout_cpu_cnt, reqd_tx_cpu_cnt; 721da14cebeSEric Cheng int reqd_poll_worker_cnt, fanout_cnt_per_srs; 722da14cebeSEric Cheng 723da14cebeSEric Cheng ASSERT(mrp->mrp_fanout_mode == MCM_CPUS); 724da14cebeSEric Cheng /* 725da14cebeSEric Cheng * The check for nbc_ncpus to be within limits for 726da14cebeSEric Cheng * the user specified case was done earlier and if 727da14cebeSEric Cheng * not within limits, an error would have been 728da14cebeSEric Cheng * returned to the user. 729da14cebeSEric Cheng */ 730da14cebeSEric Cheng ASSERT(mrp->mrp_ncpus > 0 && mrp->mrp_ncpus <= MAX_SR_FANOUT); 731da14cebeSEric Cheng 732da14cebeSEric Cheng no_of_cpus = mrp->mrp_ncpus; 733da14cebeSEric Cheng 734da14cebeSEric Cheng if (mrp->mrp_intr_cpu != -1) { 735da14cebeSEric Cheng /* 736da14cebeSEric Cheng * interrupt has been re-targetted. Poll 737da14cebeSEric Cheng * thread needs to be bound to interrupt 738da14cebeSEric Cheng * CPU. Presently only fixed interrupts 739da14cebeSEric Cheng * are re-targetted, MSI-x aren't. 740da14cebeSEric Cheng * 741da14cebeSEric Cheng * Find where in the list is the intr 742da14cebeSEric Cheng * CPU and swap it with the first one. 743da14cebeSEric Cheng * We will be using the first CPU in the 744da14cebeSEric Cheng * list for poll. 745da14cebeSEric Cheng */ 746da14cebeSEric Cheng for (i = 0; i < no_of_cpus; i++) { 747da14cebeSEric Cheng if (mrp->mrp_cpu[i] == mrp->mrp_intr_cpu) 748da14cebeSEric Cheng break; 749da14cebeSEric Cheng } 750da14cebeSEric Cheng mrp->mrp_cpu[i] = mrp->mrp_cpu[0]; 751da14cebeSEric Cheng mrp->mrp_cpu[0] = mrp->mrp_intr_cpu; 752da14cebeSEric Cheng } 753da14cebeSEric Cheng 754da14cebeSEric Cheng /* 755da14cebeSEric Cheng * Requirements: 756da14cebeSEric Cheng * The number of CPUs that each Rx ring needs is dependent 757da14cebeSEric Cheng * upon mac_latency_optimize flag. 758da14cebeSEric Cheng * 1) If set, atleast 2 CPUs are needed -- one for 759da14cebeSEric Cheng * polling, one for fanout soft ring. 760da14cebeSEric Cheng * 2) If not set, then atleast 3 CPUs are needed -- one 761da14cebeSEric Cheng * for polling, one for srs worker thread, and one for 762da14cebeSEric Cheng * fanout soft ring. 763da14cebeSEric Cheng */ 764da14cebeSEric Cheng rx_srs_cnt = (flent->fe_rx_srs_cnt > 1) ? 765da14cebeSEric Cheng (flent->fe_rx_srs_cnt - 1) : flent->fe_rx_srs_cnt; 766da14cebeSEric Cheng reqd_rx_cpu_cnt = mac_latency_optimize ? 767da14cebeSEric Cheng (rx_srs_cnt * 2) : (rx_srs_cnt * 3); 768da14cebeSEric Cheng 769da14cebeSEric Cheng /* How many CPUs are needed for Tx side? */ 770da14cebeSEric Cheng tx_srs = flent->fe_tx_srs; 771da14cebeSEric Cheng reqd_tx_cpu_cnt = TX_MULTI_RING_MODE(tx_srs) ? 772da14cebeSEric Cheng tx_srs->srs_oth_ring_count : 1; 773da14cebeSEric Cheng 774da14cebeSEric Cheng /* CPUs needed for Rx SRSes poll and worker threads */ 775da14cebeSEric Cheng reqd_poll_worker_cnt = mac_latency_optimize ? 776da14cebeSEric Cheng rx_srs_cnt : rx_srs_cnt * 2; 777da14cebeSEric Cheng 778da14cebeSEric Cheng /* Has the user provided enough CPUs? */ 779da14cebeSEric Cheng if (no_of_cpus >= (reqd_rx_cpu_cnt + reqd_tx_cpu_cnt)) { 780da14cebeSEric Cheng /* 781da14cebeSEric Cheng * Best case scenario. There is enough CPUs. All 782da14cebeSEric Cheng * Rx rings will get their own set of CPUs plus 783da14cebeSEric Cheng * Tx soft rings will get their own. 784da14cebeSEric Cheng */ 785da14cebeSEric Cheng /* 786da14cebeSEric Cheng * fanout_cpu_cnt is the number of CPUs available 787da14cebeSEric Cheng * for Rx side fanout soft rings. 788da14cebeSEric Cheng */ 789da14cebeSEric Cheng fanout_cpu_cnt = no_of_cpus - 790da14cebeSEric Cheng reqd_poll_worker_cnt - reqd_tx_cpu_cnt; 791da14cebeSEric Cheng 792da14cebeSEric Cheng /* 793da14cebeSEric Cheng * Divide fanout_cpu_cnt by rx_srs_cnt to find 794da14cebeSEric Cheng * out how many fanout soft rings each Rx SRS 795da14cebeSEric Cheng * can have. 796da14cebeSEric Cheng */ 797da14cebeSEric Cheng fanout_cnt_per_srs = fanout_cpu_cnt/rx_srs_cnt; 798da14cebeSEric Cheng 799da14cebeSEric Cheng /* Do the assignment for the default Rx ring */ 800da14cebeSEric Cheng cpu_cnt = 0; 801da14cebeSEric Cheng rx_srs = flent->fe_rx_srs[0]; 802da14cebeSEric Cheng ASSERT(rx_srs->srs_ring == NULL); 803da14cebeSEric Cheng if (rx_srs->srs_fanout_state == SRS_FANOUT_INIT) 804da14cebeSEric Cheng rx_srs->srs_fanout_state = SRS_FANOUT_REINIT; 805da14cebeSEric Cheng srs_cpu = &rx_srs->srs_cpu; 806da14cebeSEric Cheng srs_cpu->mc_ncpus = no_of_cpus; 807da14cebeSEric Cheng bcopy(mrp->mrp_cpu, 808da14cebeSEric Cheng srs_cpu->mc_cpus, sizeof (srs_cpu->mc_cpus)); 809da14cebeSEric Cheng srs_cpu->mc_fanout_cnt = fanout_cnt_per_srs; 810da14cebeSEric Cheng srs_cpu->mc_pollid = mrp->mrp_cpu[cpu_cnt++]; 811da14cebeSEric Cheng srs_cpu->mc_intr_cpu = mrp->mrp_intr_cpu; 812da14cebeSEric Cheng srs_cpu->mc_workerid = srs_cpu->mc_pollid; 813da14cebeSEric Cheng if (!mac_latency_optimize) 814da14cebeSEric Cheng srs_cpu->mc_workerid = mrp->mrp_cpu[cpu_cnt++]; 815da14cebeSEric Cheng for (i = 0; i < fanout_cnt_per_srs; i++) 816da14cebeSEric Cheng srs_cpu->mc_fanout_cpus[i] = mrp->mrp_cpu[cpu_cnt++]; 817da14cebeSEric Cheng 818da14cebeSEric Cheng /* Do the assignment for h/w Rx SRSes */ 819da14cebeSEric Cheng if (flent->fe_rx_srs_cnt > 1) { 820da14cebeSEric Cheng cpu_cnt = 0; 821da14cebeSEric Cheng for (srs_cnt = 1; 822da14cebeSEric Cheng srs_cnt < flent->fe_rx_srs_cnt; srs_cnt++) { 823da14cebeSEric Cheng rx_srs = flent->fe_rx_srs[srs_cnt]; 824da14cebeSEric Cheng ASSERT(rx_srs->srs_ring != NULL); 825da14cebeSEric Cheng if (rx_srs->srs_fanout_state == 826da14cebeSEric Cheng SRS_FANOUT_INIT) { 827da14cebeSEric Cheng rx_srs->srs_fanout_state = 828da14cebeSEric Cheng SRS_FANOUT_REINIT; 829da14cebeSEric Cheng } 830da14cebeSEric Cheng srs_cpu = &rx_srs->srs_cpu; 831da14cebeSEric Cheng srs_cpu->mc_ncpus = no_of_cpus; 832da14cebeSEric Cheng bcopy(mrp->mrp_cpu, srs_cpu->mc_cpus, 833da14cebeSEric Cheng sizeof (srs_cpu->mc_cpus)); 834da14cebeSEric Cheng srs_cpu->mc_fanout_cnt = fanout_cnt_per_srs; 835da14cebeSEric Cheng /* The first CPU in the list is the intr CPU */ 836da14cebeSEric Cheng srs_cpu->mc_pollid = mrp->mrp_cpu[cpu_cnt++]; 837da14cebeSEric Cheng srs_cpu->mc_intr_cpu = mrp->mrp_intr_cpu; 838da14cebeSEric Cheng srs_cpu->mc_workerid = srs_cpu->mc_pollid; 839da14cebeSEric Cheng if (!mac_latency_optimize) { 840da14cebeSEric Cheng srs_cpu->mc_workerid = 841da14cebeSEric Cheng mrp->mrp_cpu[cpu_cnt++]; 842da14cebeSEric Cheng } 843da14cebeSEric Cheng for (i = 0; i < fanout_cnt_per_srs; i++) { 844da14cebeSEric Cheng srs_cpu->mc_fanout_cpus[i] = 845da14cebeSEric Cheng mrp->mrp_cpu[cpu_cnt++]; 846da14cebeSEric Cheng } 847da14cebeSEric Cheng ASSERT(cpu_cnt <= no_of_cpus); 848da14cebeSEric Cheng } 849da14cebeSEric Cheng } 850da14cebeSEric Cheng return; 851da14cebeSEric Cheng } 852da14cebeSEric Cheng 853da14cebeSEric Cheng /* 854da14cebeSEric Cheng * Sub-optimal case. 855da14cebeSEric Cheng * We have the following information: 856da14cebeSEric Cheng * no_of_cpus - no. of cpus that user passed. 857da14cebeSEric Cheng * rx_srs_cnt - no. of rx rings. 858da14cebeSEric Cheng * reqd_rx_cpu_cnt = mac_latency_optimize?rx_srs_cnt*2:rx_srs_cnt*3 859da14cebeSEric Cheng * reqd_tx_cpu_cnt - no. of cpus reqd. for Tx side. 860da14cebeSEric Cheng * reqd_poll_worker_cnt = mac_latency_optimize?rx_srs_cnt:rx_srs_cnt*2 861da14cebeSEric Cheng */ 862da14cebeSEric Cheng /* 863da14cebeSEric Cheng * If we bind the Rx fanout soft rings to the same CPUs 864da14cebeSEric Cheng * as poll/worker, would that be enough? 865da14cebeSEric Cheng */ 866da14cebeSEric Cheng if (no_of_cpus >= (rx_srs_cnt + reqd_tx_cpu_cnt)) { 867da14cebeSEric Cheng boolean_t worker_assign = B_FALSE; 868da14cebeSEric Cheng 869da14cebeSEric Cheng /* 870da14cebeSEric Cheng * If mac_latency_optimize is not set, are there 871da14cebeSEric Cheng * enough CPUs to assign a CPU for worker also? 872da14cebeSEric Cheng */ 873da14cebeSEric Cheng if (no_of_cpus >= (reqd_poll_worker_cnt + reqd_tx_cpu_cnt)) 874da14cebeSEric Cheng worker_assign = B_TRUE; 875da14cebeSEric Cheng /* 876da14cebeSEric Cheng * Zero'th Rx SRS is the default Rx ring. It is not 877da14cebeSEric Cheng * associated with h/w Rx ring. 878da14cebeSEric Cheng */ 879da14cebeSEric Cheng rx_srs = flent->fe_rx_srs[0]; 880da14cebeSEric Cheng ASSERT(rx_srs->srs_ring == NULL); 881da14cebeSEric Cheng if (rx_srs->srs_fanout_state == SRS_FANOUT_INIT) 882da14cebeSEric Cheng rx_srs->srs_fanout_state = SRS_FANOUT_REINIT; 883da14cebeSEric Cheng cpu_cnt = 0; 884da14cebeSEric Cheng srs_cpu = &rx_srs->srs_cpu; 885da14cebeSEric Cheng srs_cpu->mc_ncpus = no_of_cpus; 886da14cebeSEric Cheng bcopy(mrp->mrp_cpu, 887da14cebeSEric Cheng srs_cpu->mc_cpus, sizeof (srs_cpu->mc_cpus)); 888da14cebeSEric Cheng srs_cpu->mc_fanout_cnt = 1; 889da14cebeSEric Cheng srs_cpu->mc_pollid = mrp->mrp_cpu[cpu_cnt++]; 890da14cebeSEric Cheng srs_cpu->mc_intr_cpu = mrp->mrp_intr_cpu; 891da14cebeSEric Cheng srs_cpu->mc_workerid = srs_cpu->mc_pollid; 892da14cebeSEric Cheng if (!mac_latency_optimize && worker_assign) 893da14cebeSEric Cheng srs_cpu->mc_workerid = mrp->mrp_cpu[cpu_cnt++]; 894da14cebeSEric Cheng srs_cpu->mc_fanout_cpus[0] = mrp->mrp_cpu[cpu_cnt]; 895da14cebeSEric Cheng 896da14cebeSEric Cheng /* Do CPU bindings for SRSes having h/w Rx rings */ 897da14cebeSEric Cheng if (flent->fe_rx_srs_cnt > 1) { 898da14cebeSEric Cheng cpu_cnt = 0; 899da14cebeSEric Cheng for (srs_cnt = 1; 900da14cebeSEric Cheng srs_cnt < flent->fe_rx_srs_cnt; srs_cnt++) { 901da14cebeSEric Cheng rx_srs = flent->fe_rx_srs[srs_cnt]; 902da14cebeSEric Cheng ASSERT(rx_srs->srs_ring != NULL); 903da14cebeSEric Cheng if (rx_srs->srs_fanout_state == 904da14cebeSEric Cheng SRS_FANOUT_INIT) { 905da14cebeSEric Cheng rx_srs->srs_fanout_state = 906da14cebeSEric Cheng SRS_FANOUT_REINIT; 907da14cebeSEric Cheng } 908da14cebeSEric Cheng srs_cpu = &rx_srs->srs_cpu; 909da14cebeSEric Cheng srs_cpu->mc_ncpus = no_of_cpus; 910da14cebeSEric Cheng bcopy(mrp->mrp_cpu, srs_cpu->mc_cpus, 911da14cebeSEric Cheng sizeof (srs_cpu->mc_cpus)); 912da14cebeSEric Cheng srs_cpu->mc_pollid = 913da14cebeSEric Cheng mrp->mrp_cpu[cpu_cnt]; 914da14cebeSEric Cheng srs_cpu->mc_intr_cpu = mrp->mrp_intr_cpu; 915da14cebeSEric Cheng srs_cpu->mc_workerid = srs_cpu->mc_pollid; 916da14cebeSEric Cheng if (!mac_latency_optimize && worker_assign) { 917da14cebeSEric Cheng srs_cpu->mc_workerid = 918da14cebeSEric Cheng mrp->mrp_cpu[++cpu_cnt]; 919da14cebeSEric Cheng } 920da14cebeSEric Cheng srs_cpu->mc_fanout_cnt = 1; 921da14cebeSEric Cheng srs_cpu->mc_fanout_cpus[0] = 922da14cebeSEric Cheng mrp->mrp_cpu[cpu_cnt]; 923da14cebeSEric Cheng cpu_cnt++; 924da14cebeSEric Cheng ASSERT(cpu_cnt <= no_of_cpus); 925da14cebeSEric Cheng } 926da14cebeSEric Cheng } 927da14cebeSEric Cheng return; 928da14cebeSEric Cheng } 929da14cebeSEric Cheng 930da14cebeSEric Cheng /* 931da14cebeSEric Cheng * Real sub-optimal case. Not enough CPUs for poll and 932da14cebeSEric Cheng * Tx soft rings. Do a round robin assignment where 933da14cebeSEric Cheng * each Rx SRS will get the same CPU for poll, worker 934da14cebeSEric Cheng * and fanout soft ring. 935da14cebeSEric Cheng */ 936da14cebeSEric Cheng cpu_cnt = 0; 937da14cebeSEric Cheng for (srs_cnt = 0; srs_cnt < flent->fe_rx_srs_cnt; srs_cnt++) { 938da14cebeSEric Cheng rx_srs = flent->fe_rx_srs[srs_cnt]; 939da14cebeSEric Cheng srs_cpu = &rx_srs->srs_cpu; 940da14cebeSEric Cheng if (rx_srs->srs_fanout_state == SRS_FANOUT_INIT) 941da14cebeSEric Cheng rx_srs->srs_fanout_state = SRS_FANOUT_REINIT; 942da14cebeSEric Cheng srs_cpu->mc_ncpus = no_of_cpus; 943da14cebeSEric Cheng bcopy(mrp->mrp_cpu, 944da14cebeSEric Cheng srs_cpu->mc_cpus, sizeof (srs_cpu->mc_cpus)); 945da14cebeSEric Cheng srs_cpu->mc_fanout_cnt = 1; 946da14cebeSEric Cheng srs_cpu->mc_pollid = mrp->mrp_cpu[cpu_cnt]; 947da14cebeSEric Cheng srs_cpu->mc_intr_cpu = mrp->mrp_intr_cpu; 948da14cebeSEric Cheng srs_cpu->mc_workerid = mrp->mrp_cpu[cpu_cnt]; 949da14cebeSEric Cheng srs_cpu->mc_fanout_cpus[0] = mrp->mrp_cpu[cpu_cnt]; 950da14cebeSEric Cheng if (++cpu_cnt >= no_of_cpus) 951da14cebeSEric Cheng cpu_cnt = 0; 952da14cebeSEric Cheng } 953da14cebeSEric Cheng } 954da14cebeSEric Cheng 955da14cebeSEric Cheng /* 956da14cebeSEric Cheng * mac_flow_cpu_init(): 957da14cebeSEric Cheng * 958da14cebeSEric Cheng * Each SRS has a mac_cpu_t structure, srs_cpu. This routine fills in 959da14cebeSEric Cheng * the CPU binding information in srs_cpu for all Rx SRSes associated 960da14cebeSEric Cheng * with a flent. 961da14cebeSEric Cheng */ 962da14cebeSEric Cheng static void 963da14cebeSEric Cheng mac_flow_cpu_init(flow_entry_t *flent, mac_resource_props_t *mrp) 964da14cebeSEric Cheng { 965da14cebeSEric Cheng mac_soft_ring_set_t *rx_srs; 966da14cebeSEric Cheng processorid_t cpuid; 967da14cebeSEric Cheng int j, srs_cnt, soft_ring_cnt = 0; 968da14cebeSEric Cheng mac_cpus_t *srs_cpu; 969da14cebeSEric Cheng 970da14cebeSEric Cheng if (mrp->mrp_mask & MRP_CPUS_USERSPEC) { 971da14cebeSEric Cheng mac_flow_user_cpu_init(flent, mrp); 972da14cebeSEric Cheng } else { 973da14cebeSEric Cheng /* 974da14cebeSEric Cheng * Compute the number of soft rings needed on top for each Rx 975da14cebeSEric Cheng * SRS. "rx_srs_cnt-1" indicates the number of Rx SRS 976da14cebeSEric Cheng * associated with h/w Rx rings. Soft ring count needed for 977da14cebeSEric Cheng * each h/w Rx SRS is computed and the same is applied to 978da14cebeSEric Cheng * software classified Rx SRS. The first Rx SRS in fe_rx_srs[] 979da14cebeSEric Cheng * is the software classified Rx SRS. 980da14cebeSEric Cheng */ 981da14cebeSEric Cheng soft_ring_cnt = mac_compute_soft_ring_count(flent, 982da14cebeSEric Cheng flent->fe_rx_srs_cnt - 1); 983da14cebeSEric Cheng if (soft_ring_cnt == 0) { 984da14cebeSEric Cheng /* 985da14cebeSEric Cheng * Even when soft_ring_cnt is 0, we still need 986da14cebeSEric Cheng * to create a soft ring for TCP, UDP and 987da14cebeSEric Cheng * OTHER. So set it to 1. 988da14cebeSEric Cheng */ 989da14cebeSEric Cheng soft_ring_cnt = 1; 990da14cebeSEric Cheng } 991da14cebeSEric Cheng for (srs_cnt = 0; srs_cnt < flent->fe_rx_srs_cnt; srs_cnt++) { 992da14cebeSEric Cheng rx_srs = flent->fe_rx_srs[srs_cnt]; 993da14cebeSEric Cheng srs_cpu = &rx_srs->srs_cpu; 994da14cebeSEric Cheng if (rx_srs->srs_fanout_state == SRS_FANOUT_INIT) { 995da14cebeSEric Cheng if (soft_ring_cnt == srs_cpu->mc_fanout_cnt) 996da14cebeSEric Cheng continue; 997da14cebeSEric Cheng rx_srs->srs_fanout_state = SRS_FANOUT_REINIT; 998da14cebeSEric Cheng } 999da14cebeSEric Cheng srs_cpu->mc_ncpus = soft_ring_cnt; 1000da14cebeSEric Cheng srs_cpu->mc_fanout_cnt = soft_ring_cnt; 1001da14cebeSEric Cheng mutex_enter(&cpu_lock); 1002da14cebeSEric Cheng for (j = 0; j < soft_ring_cnt; j++) { 1003da14cebeSEric Cheng cpuid = mac_next_bind_cpu(); 1004da14cebeSEric Cheng srs_cpu->mc_cpus[j] = cpuid; 1005da14cebeSEric Cheng srs_cpu->mc_fanout_cpus[j] = cpuid; 1006da14cebeSEric Cheng } 1007da14cebeSEric Cheng cpuid = mac_next_bind_cpu(); 1008da14cebeSEric Cheng srs_cpu->mc_pollid = cpuid; 1009da14cebeSEric Cheng /* increment ncpus to account for polling cpu */ 1010da14cebeSEric Cheng srs_cpu->mc_ncpus++; 1011da14cebeSEric Cheng srs_cpu->mc_cpus[j++] = cpuid; 1012da14cebeSEric Cheng if (!mac_latency_optimize) { 1013da14cebeSEric Cheng cpuid = mac_next_bind_cpu(); 1014da14cebeSEric Cheng srs_cpu->mc_ncpus++; 1015da14cebeSEric Cheng srs_cpu->mc_cpus[j++] = cpuid; 1016da14cebeSEric Cheng } 1017da14cebeSEric Cheng srs_cpu->mc_workerid = cpuid; 1018da14cebeSEric Cheng mutex_exit(&cpu_lock); 1019da14cebeSEric Cheng } 1020da14cebeSEric Cheng } 1021da14cebeSEric Cheng } 1022da14cebeSEric Cheng 1023da14cebeSEric Cheng /* 1024da14cebeSEric Cheng * DATAPATH SETUP ROUTINES 1025da14cebeSEric Cheng * (setup SRS and set/update FANOUT, B/W and PRIORITY) 1026da14cebeSEric Cheng */ 1027da14cebeSEric Cheng 1028da14cebeSEric Cheng static void 1029da14cebeSEric Cheng mac_srs_fanout_list_alloc(mac_soft_ring_set_t *mac_srs) 1030da14cebeSEric Cheng { 1031da14cebeSEric Cheng mac_srs->srs_tcp_soft_rings = (mac_soft_ring_t **) 1032da14cebeSEric Cheng kmem_zalloc(sizeof (mac_soft_ring_t *) * MAX_SR_FANOUT, KM_SLEEP); 1033da14cebeSEric Cheng mac_srs->srs_udp_soft_rings = (mac_soft_ring_t **) 1034da14cebeSEric Cheng kmem_zalloc(sizeof (mac_soft_ring_t *) * MAX_SR_FANOUT, KM_SLEEP); 1035da14cebeSEric Cheng mac_srs->srs_oth_soft_rings = (mac_soft_ring_t **) 1036da14cebeSEric Cheng kmem_zalloc(sizeof (mac_soft_ring_t *) * MAX_SR_FANOUT, KM_SLEEP); 1037da14cebeSEric Cheng } 1038da14cebeSEric Cheng 1039da14cebeSEric Cheng static void 1040da14cebeSEric Cheng mac_srs_worker_bind(mac_soft_ring_set_t *mac_srs, processorid_t cpuid) 1041da14cebeSEric Cheng { 1042da14cebeSEric Cheng cpu_t *cp; 1043da14cebeSEric Cheng boolean_t clear = B_FALSE; 1044da14cebeSEric Cheng 1045da14cebeSEric Cheng ASSERT(MUTEX_HELD(&cpu_lock)); 1046da14cebeSEric Cheng 1047da14cebeSEric Cheng if (!mac_srs_thread_bind) 1048da14cebeSEric Cheng return; 1049da14cebeSEric Cheng 1050da14cebeSEric Cheng cp = cpu_get(cpuid); 1051da14cebeSEric Cheng if (cp == NULL || !cpu_is_online(cp)) 1052da14cebeSEric Cheng return; 1053da14cebeSEric Cheng 1054da14cebeSEric Cheng mutex_enter(&mac_srs->srs_lock); 1055da14cebeSEric Cheng mac_srs->srs_state |= SRS_WORKER_BOUND; 1056da14cebeSEric Cheng if (mac_srs->srs_worker_cpuid != -1) 1057da14cebeSEric Cheng clear = B_TRUE; 1058da14cebeSEric Cheng mac_srs->srs_worker_cpuid = cpuid; 1059da14cebeSEric Cheng mutex_exit(&mac_srs->srs_lock); 1060da14cebeSEric Cheng 1061da14cebeSEric Cheng if (clear) 1062da14cebeSEric Cheng thread_affinity_clear(mac_srs->srs_worker); 1063da14cebeSEric Cheng 1064da14cebeSEric Cheng thread_affinity_set(mac_srs->srs_worker, cpuid); 1065da14cebeSEric Cheng DTRACE_PROBE1(worker__CPU, processorid_t, cpuid); 1066da14cebeSEric Cheng } 1067da14cebeSEric Cheng 1068da14cebeSEric Cheng static void 1069da14cebeSEric Cheng mac_srs_poll_bind(mac_soft_ring_set_t *mac_srs, processorid_t cpuid) 1070da14cebeSEric Cheng { 1071da14cebeSEric Cheng cpu_t *cp; 1072da14cebeSEric Cheng boolean_t clear = B_FALSE; 1073da14cebeSEric Cheng 1074da14cebeSEric Cheng ASSERT(MUTEX_HELD(&cpu_lock)); 1075da14cebeSEric Cheng 1076da14cebeSEric Cheng if (!mac_srs_thread_bind || mac_srs->srs_poll_thr == NULL) 1077da14cebeSEric Cheng return; 1078da14cebeSEric Cheng 1079da14cebeSEric Cheng cp = cpu_get(cpuid); 1080da14cebeSEric Cheng if (cp == NULL || !cpu_is_online(cp)) 1081da14cebeSEric Cheng return; 1082da14cebeSEric Cheng 1083da14cebeSEric Cheng mutex_enter(&mac_srs->srs_lock); 1084da14cebeSEric Cheng mac_srs->srs_state |= SRS_POLL_BOUND; 1085da14cebeSEric Cheng if (mac_srs->srs_poll_cpuid != -1) 1086da14cebeSEric Cheng clear = B_TRUE; 1087da14cebeSEric Cheng mac_srs->srs_poll_cpuid = cpuid; 1088da14cebeSEric Cheng mutex_exit(&mac_srs->srs_lock); 1089da14cebeSEric Cheng 1090da14cebeSEric Cheng if (clear) 1091da14cebeSEric Cheng thread_affinity_clear(mac_srs->srs_poll_thr); 1092da14cebeSEric Cheng 1093da14cebeSEric Cheng thread_affinity_set(mac_srs->srs_poll_thr, cpuid); 1094da14cebeSEric Cheng DTRACE_PROBE1(poll__CPU, processorid_t, cpuid); 1095da14cebeSEric Cheng } 1096da14cebeSEric Cheng 1097da14cebeSEric Cheng /* 1098da14cebeSEric Cheng * When a CPU comes back online, bind the MAC kernel threads which 1099da14cebeSEric Cheng * were previously bound to that CPU, and had to be unbound because 1100da14cebeSEric Cheng * the CPU was going away. 1101da14cebeSEric Cheng * 1102da14cebeSEric Cheng * These functions are called with cpu_lock held and hence we can't 1103da14cebeSEric Cheng * cv_wait to grab the mac perimeter. Since these functions walk the soft 1104da14cebeSEric Cheng * ring list of an SRS without being in the perimeter, the list itself 1105da14cebeSEric Cheng * is protected by the SRS lock. 1106da14cebeSEric Cheng */ 1107da14cebeSEric Cheng static void 1108da14cebeSEric Cheng mac_walk_srs_and_bind(int cpuid) 1109da14cebeSEric Cheng { 1110da14cebeSEric Cheng mac_soft_ring_set_t *mac_srs; 1111da14cebeSEric Cheng mac_soft_ring_t *soft_ring; 1112da14cebeSEric Cheng 1113da14cebeSEric Cheng rw_enter(&mac_srs_g_lock, RW_READER); 1114da14cebeSEric Cheng 1115da14cebeSEric Cheng if ((mac_srs = mac_srs_g_list) == NULL) 1116da14cebeSEric Cheng goto done; 1117da14cebeSEric Cheng 1118da14cebeSEric Cheng for (; mac_srs != NULL; mac_srs = mac_srs->srs_next) { 1119da14cebeSEric Cheng if (mac_srs->srs_worker_cpuid == -1 && 1120da14cebeSEric Cheng mac_srs->srs_worker_cpuid_save == cpuid) { 1121da14cebeSEric Cheng mac_srs->srs_worker_cpuid_save = -1; 1122da14cebeSEric Cheng mac_srs_worker_bind(mac_srs, cpuid); 1123da14cebeSEric Cheng } 1124da14cebeSEric Cheng 1125da14cebeSEric Cheng if (!(mac_srs->srs_type & SRST_TX)) { 1126da14cebeSEric Cheng if (mac_srs->srs_poll_cpuid == -1 && 1127da14cebeSEric Cheng mac_srs->srs_poll_cpuid_save == cpuid) { 1128da14cebeSEric Cheng mac_srs->srs_poll_cpuid_save = -1; 1129da14cebeSEric Cheng mac_srs_poll_bind(mac_srs, cpuid); 1130da14cebeSEric Cheng } 1131da14cebeSEric Cheng } 1132da14cebeSEric Cheng 1133da14cebeSEric Cheng /* Next tackle the soft rings associated with the srs */ 1134da14cebeSEric Cheng mutex_enter(&mac_srs->srs_lock); 1135da14cebeSEric Cheng for (soft_ring = mac_srs->srs_soft_ring_head; soft_ring != NULL; 1136da14cebeSEric Cheng soft_ring = soft_ring->s_ring_next) { 1137da14cebeSEric Cheng if (soft_ring->s_ring_cpuid == -1 && 1138da14cebeSEric Cheng soft_ring->s_ring_cpuid_save == cpuid) { 1139da14cebeSEric Cheng soft_ring->s_ring_cpuid_save = -1; 1140da14cebeSEric Cheng (void) mac_soft_ring_bind(soft_ring, cpuid); 1141da14cebeSEric Cheng } 1142da14cebeSEric Cheng } 1143da14cebeSEric Cheng mutex_exit(&mac_srs->srs_lock); 1144da14cebeSEric Cheng } 1145da14cebeSEric Cheng done: 1146da14cebeSEric Cheng rw_exit(&mac_srs_g_lock); 1147da14cebeSEric Cheng } 1148da14cebeSEric Cheng 1149da14cebeSEric Cheng /* 1150da14cebeSEric Cheng * Change the priority of the SRS's poll and worker thread. Additionally, 1151da14cebeSEric Cheng * update the priority of the worker threads for the SRS's soft rings. 1152da14cebeSEric Cheng * Need to modify any associated squeue threads. 1153da14cebeSEric Cheng */ 1154da14cebeSEric Cheng void 1155da14cebeSEric Cheng mac_update_srs_priority(mac_soft_ring_set_t *mac_srs, pri_t prival) 1156da14cebeSEric Cheng { 1157da14cebeSEric Cheng mac_soft_ring_t *ringp; 1158da14cebeSEric Cheng 1159da14cebeSEric Cheng mac_srs->srs_pri = prival; 1160da14cebeSEric Cheng thread_lock(mac_srs->srs_worker); 1161da14cebeSEric Cheng (void) thread_change_pri(mac_srs->srs_worker, mac_srs->srs_pri, 0); 1162da14cebeSEric Cheng thread_unlock(mac_srs->srs_worker); 1163da14cebeSEric Cheng if (mac_srs->srs_poll_thr != NULL) { 1164da14cebeSEric Cheng thread_lock(mac_srs->srs_poll_thr); 1165da14cebeSEric Cheng (void) thread_change_pri(mac_srs->srs_poll_thr, 1166da14cebeSEric Cheng mac_srs->srs_pri, 0); 1167da14cebeSEric Cheng thread_unlock(mac_srs->srs_poll_thr); 1168da14cebeSEric Cheng } 1169da14cebeSEric Cheng if ((ringp = mac_srs->srs_soft_ring_head) == NULL) 1170da14cebeSEric Cheng return; 1171da14cebeSEric Cheng while (ringp != mac_srs->srs_soft_ring_tail) { 1172da14cebeSEric Cheng thread_lock(ringp->s_ring_worker); 1173da14cebeSEric Cheng (void) thread_change_pri(ringp->s_ring_worker, 1174da14cebeSEric Cheng mac_srs->srs_pri, 0); 1175da14cebeSEric Cheng thread_unlock(ringp->s_ring_worker); 1176da14cebeSEric Cheng ringp = ringp->s_ring_next; 1177da14cebeSEric Cheng } 1178da14cebeSEric Cheng ASSERT(ringp == mac_srs->srs_soft_ring_tail); 1179da14cebeSEric Cheng thread_lock(ringp->s_ring_worker); 1180da14cebeSEric Cheng (void) thread_change_pri(ringp->s_ring_worker, mac_srs->srs_pri, 0); 1181da14cebeSEric Cheng thread_unlock(ringp->s_ring_worker); 1182da14cebeSEric Cheng } 1183da14cebeSEric Cheng 1184da14cebeSEric Cheng /* 1185da14cebeSEric Cheng * Change the receive bandwidth limit. 1186da14cebeSEric Cheng */ 1187da14cebeSEric Cheng static void 1188da14cebeSEric Cheng mac_rx_srs_update_bwlimit(mac_soft_ring_set_t *srs, mac_resource_props_t *mrp) 1189da14cebeSEric Cheng { 1190da14cebeSEric Cheng mac_soft_ring_t *softring; 1191da14cebeSEric Cheng 1192da14cebeSEric Cheng mutex_enter(&srs->srs_lock); 1193da14cebeSEric Cheng mutex_enter(&srs->srs_bw->mac_bw_lock); 1194da14cebeSEric Cheng 1195da14cebeSEric Cheng if (mrp->mrp_maxbw == MRP_MAXBW_RESETVAL) { 1196da14cebeSEric Cheng /* Reset bandwidth limit */ 1197da14cebeSEric Cheng if (srs->srs_type & SRST_BW_CONTROL) { 1198da14cebeSEric Cheng softring = srs->srs_soft_ring_head; 1199da14cebeSEric Cheng while (softring != NULL) { 1200da14cebeSEric Cheng softring->s_ring_type &= ~ST_RING_BW_CTL; 1201da14cebeSEric Cheng softring = softring->s_ring_next; 1202da14cebeSEric Cheng } 1203da14cebeSEric Cheng srs->srs_type &= ~SRST_BW_CONTROL; 1204da14cebeSEric Cheng srs->srs_drain_func = mac_rx_srs_drain; 1205da14cebeSEric Cheng } 1206da14cebeSEric Cheng } else { 1207da14cebeSEric Cheng /* Set/Modify bandwidth limit */ 1208da14cebeSEric Cheng srs->srs_bw->mac_bw_limit = FLOW_BYTES_PER_TICK(mrp->mrp_maxbw); 1209da14cebeSEric Cheng /* 1210da14cebeSEric Cheng * Give twice the queuing capability before 1211da14cebeSEric Cheng * dropping packets. The unit is bytes/tick. 1212da14cebeSEric Cheng */ 1213da14cebeSEric Cheng srs->srs_bw->mac_bw_drop_threshold = 1214da14cebeSEric Cheng srs->srs_bw->mac_bw_limit << 1; 1215da14cebeSEric Cheng if (!(srs->srs_type & SRST_BW_CONTROL)) { 1216da14cebeSEric Cheng softring = srs->srs_soft_ring_head; 1217da14cebeSEric Cheng while (softring != NULL) { 1218da14cebeSEric Cheng softring->s_ring_type |= ST_RING_BW_CTL; 1219da14cebeSEric Cheng softring = softring->s_ring_next; 1220da14cebeSEric Cheng } 1221da14cebeSEric Cheng srs->srs_type |= SRST_BW_CONTROL; 1222da14cebeSEric Cheng srs->srs_drain_func = mac_rx_srs_drain_bw; 1223da14cebeSEric Cheng } 1224da14cebeSEric Cheng } 1225da14cebeSEric Cheng done: 1226da14cebeSEric Cheng mutex_exit(&srs->srs_bw->mac_bw_lock); 1227da14cebeSEric Cheng mutex_exit(&srs->srs_lock); 1228da14cebeSEric Cheng } 1229da14cebeSEric Cheng 1230da14cebeSEric Cheng /* Change the transmit bandwidth limit */ 1231da14cebeSEric Cheng static void 1232da14cebeSEric Cheng mac_tx_srs_update_bwlimit(mac_soft_ring_set_t *srs, mac_resource_props_t *mrp) 1233da14cebeSEric Cheng { 1234da14cebeSEric Cheng mac_srs_tx_t *srs_tx = &srs->srs_tx; 1235da14cebeSEric Cheng uint32_t tx_mode; 1236da14cebeSEric Cheng mac_impl_t *mip = srs->srs_mcip->mci_mip; 1237da14cebeSEric Cheng 1238da14cebeSEric Cheng mutex_enter(&srs->srs_lock); 1239da14cebeSEric Cheng mutex_enter(&srs->srs_bw->mac_bw_lock); 1240da14cebeSEric Cheng 1241da14cebeSEric Cheng tx_mode = srs_tx->st_mode; 1242da14cebeSEric Cheng 1243da14cebeSEric Cheng if (mrp->mrp_maxbw == MRP_MAXBW_RESETVAL) { 1244da14cebeSEric Cheng /* Reset bandwidth limit */ 1245da14cebeSEric Cheng if (tx_mode == SRS_TX_BW) { 1246da14cebeSEric Cheng if (mac_tx_serialize || 1247da14cebeSEric Cheng (mip->mi_v12n_level & MAC_VIRT_SERIALIZE)) { 1248da14cebeSEric Cheng srs_tx->st_mode = SRS_TX_SERIALIZE; 1249da14cebeSEric Cheng } else { 1250da14cebeSEric Cheng srs_tx->st_mode = SRS_TX_DEFAULT; 1251da14cebeSEric Cheng } 1252da14cebeSEric Cheng } else if (tx_mode == SRS_TX_BW_FANOUT) { 1253da14cebeSEric Cheng srs_tx->st_mode = SRS_TX_FANOUT; 1254da14cebeSEric Cheng } 1255da14cebeSEric Cheng srs->srs_type &= ~SRST_BW_CONTROL; 1256da14cebeSEric Cheng } else { 1257da14cebeSEric Cheng /* Set/Modify bandwidth limit */ 1258da14cebeSEric Cheng srs->srs_bw->mac_bw_limit = FLOW_BYTES_PER_TICK(mrp->mrp_maxbw); 1259da14cebeSEric Cheng /* 1260da14cebeSEric Cheng * Give twice the queuing capability before 1261da14cebeSEric Cheng * dropping packets. The unit is bytes/tick. 1262da14cebeSEric Cheng */ 1263da14cebeSEric Cheng srs->srs_bw->mac_bw_drop_threshold = 1264da14cebeSEric Cheng srs->srs_bw->mac_bw_limit << 1; 1265da14cebeSEric Cheng srs->srs_type |= SRST_BW_CONTROL; 1266da14cebeSEric Cheng if (tx_mode != SRS_TX_BW && 1267da14cebeSEric Cheng tx_mode != SRS_TX_BW_FANOUT) { 1268da14cebeSEric Cheng if (tx_mode == SRS_TX_SERIALIZE || 1269da14cebeSEric Cheng tx_mode == SRS_TX_DEFAULT) { 1270da14cebeSEric Cheng srs_tx->st_mode = SRS_TX_BW; 1271da14cebeSEric Cheng } else if (tx_mode == SRS_TX_FANOUT) { 1272da14cebeSEric Cheng srs_tx->st_mode = SRS_TX_BW_FANOUT; 1273da14cebeSEric Cheng } else { 1274da14cebeSEric Cheng ASSERT(0); 1275da14cebeSEric Cheng } 1276da14cebeSEric Cheng } 1277da14cebeSEric Cheng } 1278da14cebeSEric Cheng done: 1279da14cebeSEric Cheng srs_tx->st_func = mac_tx_get_func(srs_tx->st_mode); 1280da14cebeSEric Cheng mutex_exit(&srs->srs_bw->mac_bw_lock); 1281da14cebeSEric Cheng mutex_exit(&srs->srs_lock); 1282da14cebeSEric Cheng } 1283da14cebeSEric Cheng 1284da14cebeSEric Cheng /* 1285da14cebeSEric Cheng * The uber function that deals with any update to bandwidth limits. 1286da14cebeSEric Cheng */ 1287da14cebeSEric Cheng void 1288da14cebeSEric Cheng mac_srs_update_bwlimit(flow_entry_t *flent, mac_resource_props_t *mrp) 1289da14cebeSEric Cheng { 1290da14cebeSEric Cheng int count; 1291da14cebeSEric Cheng 1292da14cebeSEric Cheng for (count = 0; count < flent->fe_rx_srs_cnt; count++) 1293da14cebeSEric Cheng mac_rx_srs_update_bwlimit(flent->fe_rx_srs[count], mrp); 1294da14cebeSEric Cheng mac_tx_srs_update_bwlimit(flent->fe_tx_srs, mrp); 1295da14cebeSEric Cheng } 1296da14cebeSEric Cheng 1297da14cebeSEric Cheng void 1298da14cebeSEric Cheng mac_srs_change_upcall(void *arg, mac_direct_rx_t rx_func, void *rx_arg1) 1299da14cebeSEric Cheng { 1300da14cebeSEric Cheng mac_soft_ring_set_t *mac_srs = arg; 1301da14cebeSEric Cheng mac_srs_rx_t *srs_rx = &mac_srs->srs_rx; 1302da14cebeSEric Cheng mac_soft_ring_t *softring; 1303da14cebeSEric Cheng 1304da14cebeSEric Cheng mutex_enter(&mac_srs->srs_lock); 1305da14cebeSEric Cheng ASSERT((mac_srs->srs_type & SRST_TX) == 0); 1306da14cebeSEric Cheng srs_rx->sr_func = rx_func; 1307da14cebeSEric Cheng srs_rx->sr_arg1 = rx_arg1; 1308da14cebeSEric Cheng 1309da14cebeSEric Cheng softring = mac_srs->srs_soft_ring_head; 1310da14cebeSEric Cheng while (softring != NULL) { 1311da14cebeSEric Cheng mutex_enter(&softring->s_ring_lock); 1312da14cebeSEric Cheng softring->s_ring_rx_func = rx_func; 1313da14cebeSEric Cheng softring->s_ring_rx_arg1 = rx_arg1; 1314da14cebeSEric Cheng mutex_exit(&softring->s_ring_lock); 1315da14cebeSEric Cheng softring = softring->s_ring_next; 1316da14cebeSEric Cheng } 1317da14cebeSEric Cheng 1318da14cebeSEric Cheng mutex_exit(&mac_srs->srs_lock); 1319da14cebeSEric Cheng } 1320da14cebeSEric Cheng 1321da14cebeSEric Cheng /* 1322da14cebeSEric Cheng * When the first sub-flow is added to a link, we disable polling on the 1323da14cebeSEric Cheng * link and also modify the entry point to mac_rx_srs_subflow_process. 1324da14cebeSEric Cheng * (polling is disabled because with the subflow added, accounting 1325da14cebeSEric Cheng * for polling needs additional logic, it is assumed that when a subflow is 1326da14cebeSEric Cheng * added, we can take some hit as a result of disabling polling rather than 1327da14cebeSEric Cheng * adding more complexity - if this becomes a perf. issue we need to 1328da14cebeSEric Cheng * re-rvaluate this logic). When the last subflow is removed, we turn back 1329da14cebeSEric Cheng * polling and also reset the entry point to mac_rx_srs_process. 1330da14cebeSEric Cheng * 1331da14cebeSEric Cheng * In the future if there are multiple SRS, we can simply 1332da14cebeSEric Cheng * take one and give it to the flow rather than disabling polling and 1333da14cebeSEric Cheng * resetting the entry point. 1334da14cebeSEric Cheng */ 1335da14cebeSEric Cheng void 1336da14cebeSEric Cheng mac_client_update_classifier(mac_client_impl_t *mcip, boolean_t enable) 1337da14cebeSEric Cheng { 1338da14cebeSEric Cheng flow_entry_t *flent = mcip->mci_flent; 1339da14cebeSEric Cheng int i; 1340da14cebeSEric Cheng mac_impl_t *mip = mcip->mci_mip; 1341da14cebeSEric Cheng mac_rx_func_t rx_func; 1342da14cebeSEric Cheng uint_t rx_srs_cnt; 1343da14cebeSEric Cheng boolean_t enable_classifier; 1344da14cebeSEric Cheng 1345da14cebeSEric Cheng ASSERT(MAC_PERIM_HELD((mac_handle_t)mip)); 1346da14cebeSEric Cheng 1347da14cebeSEric Cheng enable_classifier = !FLOW_TAB_EMPTY(mcip->mci_subflow_tab) && enable; 1348da14cebeSEric Cheng 1349da14cebeSEric Cheng rx_func = enable_classifier ? mac_rx_srs_subflow_process : 1350da14cebeSEric Cheng mac_rx_srs_process; 1351da14cebeSEric Cheng 1352da14cebeSEric Cheng /* 1353da14cebeSEric Cheng * If receive function has already been configured correctly for 1354da14cebeSEric Cheng * current subflow configuration, do nothing. 1355da14cebeSEric Cheng */ 1356da14cebeSEric Cheng if (flent->fe_cb_fn == (flow_fn_t)rx_func) 1357da14cebeSEric Cheng return; 1358da14cebeSEric Cheng 1359da14cebeSEric Cheng rx_srs_cnt = flent->fe_rx_srs_cnt; 1360da14cebeSEric Cheng for (i = 0; i < rx_srs_cnt; i++) { 1361da14cebeSEric Cheng ASSERT(flent->fe_rx_srs[i] != NULL); 1362da14cebeSEric Cheng mac_srs_poll_state_change(flent->fe_rx_srs[i], 1363da14cebeSEric Cheng enable_classifier, rx_func); 1364da14cebeSEric Cheng } 1365da14cebeSEric Cheng 1366da14cebeSEric Cheng /* 1367da14cebeSEric Cheng * Change the S/W classifier so that we can land in the 1368da14cebeSEric Cheng * correct processing function with correct argument. 1369da14cebeSEric Cheng * If all subflows have been removed we can revert to 1370da14cebeSEric Cheng * mac_rx_srsprocess, else we need mac_rx_srs_subflow_process. 1371da14cebeSEric Cheng */ 1372da14cebeSEric Cheng mutex_enter(&flent->fe_lock); 1373da14cebeSEric Cheng flent->fe_cb_fn = (flow_fn_t)rx_func; 1374da14cebeSEric Cheng flent->fe_cb_arg1 = (void *)mip; 1375da14cebeSEric Cheng flent->fe_cb_arg2 = flent->fe_rx_srs[0]; 1376da14cebeSEric Cheng mutex_exit(&flent->fe_lock); 1377da14cebeSEric Cheng } 1378da14cebeSEric Cheng 1379da14cebeSEric Cheng static void 1380da14cebeSEric Cheng mac_srs_update_fanout_list(mac_soft_ring_set_t *mac_srs) 1381da14cebeSEric Cheng { 1382da14cebeSEric Cheng int tcp_count = 0; 1383da14cebeSEric Cheng int udp_count = 0; 1384da14cebeSEric Cheng int oth_count = 0; 1385da14cebeSEric Cheng mac_soft_ring_t *softring; 1386da14cebeSEric Cheng 1387da14cebeSEric Cheng softring = mac_srs->srs_soft_ring_head; 1388da14cebeSEric Cheng if (softring == NULL) { 1389da14cebeSEric Cheng ASSERT(mac_srs->srs_soft_ring_count == 0); 1390da14cebeSEric Cheng mac_srs->srs_tcp_ring_count = 0; 1391da14cebeSEric Cheng mac_srs->srs_udp_ring_count = 0; 1392da14cebeSEric Cheng mac_srs->srs_oth_ring_count = 0; 1393da14cebeSEric Cheng return; 1394da14cebeSEric Cheng } 1395da14cebeSEric Cheng 1396da14cebeSEric Cheng softring = mac_srs->srs_soft_ring_head; 1397da14cebeSEric Cheng tcp_count = udp_count = oth_count = 0; 1398da14cebeSEric Cheng 1399da14cebeSEric Cheng while (softring != NULL) { 1400da14cebeSEric Cheng if (softring->s_ring_type & ST_RING_TCP) 1401da14cebeSEric Cheng mac_srs->srs_tcp_soft_rings[tcp_count++] = softring; 1402da14cebeSEric Cheng else if (softring->s_ring_type & ST_RING_UDP) 1403da14cebeSEric Cheng mac_srs->srs_udp_soft_rings[udp_count++] = softring; 1404da14cebeSEric Cheng else 1405da14cebeSEric Cheng mac_srs->srs_oth_soft_rings[oth_count++] = softring; 1406da14cebeSEric Cheng softring = softring->s_ring_next; 1407da14cebeSEric Cheng } 1408da14cebeSEric Cheng 1409da14cebeSEric Cheng ASSERT(mac_srs->srs_soft_ring_count == 1410da14cebeSEric Cheng (tcp_count + udp_count + oth_count)); 1411da14cebeSEric Cheng 1412da14cebeSEric Cheng mac_srs->srs_tcp_ring_count = tcp_count; 1413da14cebeSEric Cheng mac_srs->srs_udp_ring_count = udp_count; 1414da14cebeSEric Cheng mac_srs->srs_oth_ring_count = oth_count; 1415da14cebeSEric Cheng } 1416da14cebeSEric Cheng 1417da14cebeSEric Cheng void 1418da14cebeSEric Cheng mac_srs_create_proto_softrings(int id, void *flent, uint16_t type, 1419da14cebeSEric Cheng pri_t pri, mac_client_impl_t *mcip, mac_soft_ring_set_t *mac_srs, 1420da14cebeSEric Cheng processorid_t cpuid, mac_direct_rx_t rx_func, void *x_arg1, 1421da14cebeSEric Cheng mac_resource_handle_t x_arg2, boolean_t set_bypass) 1422da14cebeSEric Cheng { 1423da14cebeSEric Cheng mac_soft_ring_t *softring; 1424da14cebeSEric Cheng mac_rx_fifo_t mrf; 1425da14cebeSEric Cheng 1426da14cebeSEric Cheng bzero(&mrf, sizeof (mac_rx_fifo_t)); 1427da14cebeSEric Cheng mrf.mrf_type = MAC_RX_FIFO; 1428da14cebeSEric Cheng mrf.mrf_receive = (mac_receive_t)mac_soft_ring_poll; 1429da14cebeSEric Cheng mrf.mrf_intr_enable = 1430da14cebeSEric Cheng (mac_intr_enable_t)mac_soft_ring_intr_enable; 1431da14cebeSEric Cheng mrf.mrf_intr_disable = 1432da14cebeSEric Cheng (mac_intr_disable_t)mac_soft_ring_intr_disable; 1433da14cebeSEric Cheng mrf.mrf_flow_priority = pri; 1434da14cebeSEric Cheng 1435da14cebeSEric Cheng softring = mac_soft_ring_create(id, mac_soft_ring_worker_wait, 1436da14cebeSEric Cheng (void *)flent, (type|ST_RING_TCP), pri, mcip, mac_srs, 1437da14cebeSEric Cheng cpuid, rx_func, x_arg1, x_arg2); 1438da14cebeSEric Cheng softring->s_ring_rx_arg2 = NULL; 1439da14cebeSEric Cheng 1440da14cebeSEric Cheng /* 1441da14cebeSEric Cheng * TCP and UDP support DLS bypass. In addition TCP 1442da14cebeSEric Cheng * squeue can also poll their corresponding soft rings. 1443da14cebeSEric Cheng */ 1444da14cebeSEric Cheng if (set_bypass && (mcip->mci_resource_arg != NULL)) { 1445da14cebeSEric Cheng mac_soft_ring_dls_bypass(softring, 1446da14cebeSEric Cheng mcip->mci_direct_rx_fn, 1447da14cebeSEric Cheng mcip->mci_direct_rx_arg); 1448da14cebeSEric Cheng 1449da14cebeSEric Cheng mrf.mrf_rx_arg = softring; 1450da14cebeSEric Cheng mrf.mrf_intr_handle = (mac_intr_handle_t)softring; 1451da14cebeSEric Cheng 1452da14cebeSEric Cheng /* 1453da14cebeSEric Cheng * Make a call in IP to get a TCP squeue assigned to 1454da14cebeSEric Cheng * this softring to maintain full CPU locality through 1455da14cebeSEric Cheng * the stack and allow the squeue to be able to poll 1456da14cebeSEric Cheng * the softring so the flow control can be pushed 1457da14cebeSEric Cheng * all the way to H/W. 1458da14cebeSEric Cheng */ 1459da14cebeSEric Cheng softring->s_ring_rx_arg2 = 1460da14cebeSEric Cheng mcip->mci_resource_add((void *)mcip->mci_resource_arg, 1461da14cebeSEric Cheng (mac_resource_t *)&mrf); 1462da14cebeSEric Cheng } 1463da14cebeSEric Cheng 1464da14cebeSEric Cheng /* 1465da14cebeSEric Cheng * Non-TCP protocols don't support squeues. Hence we 1466da14cebeSEric Cheng * don't make any ring addition callbacks for non-TCP 1467da14cebeSEric Cheng * rings. Now create the UDP softring and allow it to 1468da14cebeSEric Cheng * bypass the DLS layer. 1469da14cebeSEric Cheng */ 1470da14cebeSEric Cheng softring = mac_soft_ring_create(id, mac_soft_ring_worker_wait, 1471da14cebeSEric Cheng (void *)flent, (type|ST_RING_UDP), pri, mcip, mac_srs, 1472da14cebeSEric Cheng cpuid, rx_func, x_arg1, x_arg2); 1473da14cebeSEric Cheng softring->s_ring_rx_arg2 = NULL; 1474da14cebeSEric Cheng 1475da14cebeSEric Cheng if (set_bypass && (mcip->mci_resource_arg != NULL)) { 1476da14cebeSEric Cheng mac_soft_ring_dls_bypass(softring, 1477da14cebeSEric Cheng mcip->mci_direct_rx_fn, 1478da14cebeSEric Cheng mcip->mci_direct_rx_arg); 1479da14cebeSEric Cheng } 1480da14cebeSEric Cheng 1481da14cebeSEric Cheng /* Create the Oth softrings which has to go through the DLS */ 1482da14cebeSEric Cheng softring = mac_soft_ring_create(id, mac_soft_ring_worker_wait, 1483da14cebeSEric Cheng (void *)flent, (type|ST_RING_OTH), pri, mcip, mac_srs, 1484da14cebeSEric Cheng cpuid, rx_func, x_arg1, x_arg2); 1485da14cebeSEric Cheng softring->s_ring_rx_arg2 = NULL; 1486da14cebeSEric Cheng } 1487da14cebeSEric Cheng 1488da14cebeSEric Cheng /* 1489da14cebeSEric Cheng * This routine associates a CPU or a set of CPU to process incoming 1490da14cebeSEric Cheng * traffic from a mac client. If multiple CPUs are specified, then 1491da14cebeSEric Cheng * so many soft rings are created with each soft ring worker thread 1492da14cebeSEric Cheng * bound to a CPU in the set. Each soft ring in turn will be 1493da14cebeSEric Cheng * associated with an squeue and the squeue will be moved to the 1494da14cebeSEric Cheng * same CPU as that of the soft ring's. 1495da14cebeSEric Cheng */ 1496da14cebeSEric Cheng static void 1497da14cebeSEric Cheng mac_srs_fanout_modify(mac_client_impl_t *mcip, flow_entry_t *flent, 1498da14cebeSEric Cheng mac_resource_props_t *mrp, mac_direct_rx_t rx_func, void *x_arg1, 1499da14cebeSEric Cheng mac_resource_handle_t x_arg2, mac_soft_ring_set_t *mac_rx_srs, 1500da14cebeSEric Cheng mac_soft_ring_set_t *mac_tx_srs) 1501da14cebeSEric Cheng { 1502da14cebeSEric Cheng mac_soft_ring_t *softring; 1503*ae6aa22aSVenugopal Iyer uint32_t soft_ring_flag = 0; 1504da14cebeSEric Cheng processorid_t cpuid = -1; 1505da14cebeSEric Cheng boolean_t user_specified; 1506da14cebeSEric Cheng int i, srings_present, new_fanout_cnt; 1507da14cebeSEric Cheng mac_cpus_t *srs_cpu; 1508da14cebeSEric Cheng 1509da14cebeSEric Cheng user_specified = mrp->mrp_mask & MRP_CPUS_USERSPEC; 1510da14cebeSEric Cheng /* fanout state is REINIT. Set it back to INIT */ 1511da14cebeSEric Cheng ASSERT(mac_rx_srs->srs_fanout_state == SRS_FANOUT_REINIT); 1512da14cebeSEric Cheng mac_rx_srs->srs_fanout_state = SRS_FANOUT_INIT; 1513da14cebeSEric Cheng 1514da14cebeSEric Cheng /* how many are present right now */ 1515da14cebeSEric Cheng srings_present = mac_rx_srs->srs_tcp_ring_count; 1516da14cebeSEric Cheng /* new request */ 1517da14cebeSEric Cheng srs_cpu = &mac_rx_srs->srs_cpu; 1518da14cebeSEric Cheng new_fanout_cnt = srs_cpu->mc_fanout_cnt; 1519da14cebeSEric Cheng 1520da14cebeSEric Cheng mutex_enter(&mac_rx_srs->srs_lock); 1521da14cebeSEric Cheng if (mac_rx_srs->srs_type & SRST_BW_CONTROL) 1522da14cebeSEric Cheng soft_ring_flag |= ST_RING_BW_CTL; 1523da14cebeSEric Cheng mutex_exit(&mac_rx_srs->srs_lock); 1524da14cebeSEric Cheng 1525da14cebeSEric Cheng if (new_fanout_cnt > srings_present) { 1526da14cebeSEric Cheng /* soft rings increased */ 1527da14cebeSEric Cheng mutex_enter(&mac_rx_srs->srs_lock); 1528da14cebeSEric Cheng mac_rx_srs->srs_type |= SRST_FANOUT_SRC_IP; 1529da14cebeSEric Cheng mutex_exit(&mac_rx_srs->srs_lock); 1530da14cebeSEric Cheng 1531da14cebeSEric Cheng for (i = mac_rx_srs->srs_tcp_ring_count; 1532da14cebeSEric Cheng i < new_fanout_cnt; i++) { 1533da14cebeSEric Cheng /* 1534da14cebeSEric Cheng * Create the protocol softrings and set the 1535da14cebeSEric Cheng * DLS bypass where possible. 1536da14cebeSEric Cheng */ 1537da14cebeSEric Cheng mac_srs_create_proto_softrings(i, 1538da14cebeSEric Cheng (void *)flent, soft_ring_flag, 1539da14cebeSEric Cheng mac_rx_srs->srs_pri, mcip, mac_rx_srs, cpuid, 1540da14cebeSEric Cheng rx_func, x_arg1, x_arg2, B_TRUE); 1541da14cebeSEric Cheng } 1542da14cebeSEric Cheng mac_srs_update_fanout_list(mac_rx_srs); 1543da14cebeSEric Cheng } else if (new_fanout_cnt < srings_present) { 1544da14cebeSEric Cheng /* soft rings decreased */ 1545da14cebeSEric Cheng if (new_fanout_cnt == 1) { 1546da14cebeSEric Cheng mutex_enter(&mac_rx_srs->srs_lock); 1547da14cebeSEric Cheng mac_rx_srs->srs_type &= ~SRST_FANOUT_SRC_IP; 1548da14cebeSEric Cheng ASSERT(mac_rx_srs->srs_type & SRST_FANOUT_PROTO); 1549da14cebeSEric Cheng mutex_exit(&mac_rx_srs->srs_lock); 1550da14cebeSEric Cheng } 1551da14cebeSEric Cheng /* Get rid of extra soft rings */ 1552da14cebeSEric Cheng for (i = new_fanout_cnt; 1553da14cebeSEric Cheng i < mac_rx_srs->srs_tcp_ring_count; i++) { 1554da14cebeSEric Cheng softring = mac_rx_srs->srs_tcp_soft_rings[i]; 1555da14cebeSEric Cheng if (softring->s_ring_rx_arg2 != NULL) { 1556da14cebeSEric Cheng mcip->mci_resource_remove( 1557da14cebeSEric Cheng (void *)mcip->mci_resource_arg, 1558da14cebeSEric Cheng softring->s_ring_rx_arg2); 1559da14cebeSEric Cheng } 1560da14cebeSEric Cheng mac_soft_ring_remove(mac_rx_srs, 1561da14cebeSEric Cheng mac_rx_srs->srs_tcp_soft_rings[i]); 1562da14cebeSEric Cheng mac_soft_ring_remove(mac_rx_srs, 1563da14cebeSEric Cheng mac_rx_srs->srs_udp_soft_rings[i]); 1564da14cebeSEric Cheng mac_soft_ring_remove(mac_rx_srs, 1565da14cebeSEric Cheng mac_rx_srs->srs_oth_soft_rings[i]); 1566da14cebeSEric Cheng } 1567da14cebeSEric Cheng mac_srs_update_fanout_list(mac_rx_srs); 1568da14cebeSEric Cheng } 1569da14cebeSEric Cheng 1570da14cebeSEric Cheng ASSERT(new_fanout_cnt == mac_rx_srs->srs_tcp_ring_count); 1571da14cebeSEric Cheng mutex_enter(&cpu_lock); 1572da14cebeSEric Cheng for (i = 0; i < mac_rx_srs->srs_tcp_ring_count; i++) { 1573da14cebeSEric Cheng cpuid = srs_cpu->mc_fanout_cpus[i]; 1574da14cebeSEric Cheng (void) mac_soft_ring_bind(mac_rx_srs->srs_udp_soft_rings[i], 1575da14cebeSEric Cheng cpuid); 1576da14cebeSEric Cheng (void) mac_soft_ring_bind(mac_rx_srs->srs_oth_soft_rings[i], 1577da14cebeSEric Cheng cpuid); 1578da14cebeSEric Cheng (void) mac_soft_ring_bind(mac_rx_srs->srs_tcp_soft_rings[i], 1579da14cebeSEric Cheng cpuid); 1580da14cebeSEric Cheng softring = mac_rx_srs->srs_tcp_soft_rings[i]; 1581da14cebeSEric Cheng if (softring->s_ring_rx_arg2 != NULL) { 1582da14cebeSEric Cheng mcip->mci_resource_bind((void *)mcip->mci_resource_arg, 1583da14cebeSEric Cheng softring->s_ring_rx_arg2, cpuid); 1584da14cebeSEric Cheng } 1585da14cebeSEric Cheng } 1586da14cebeSEric Cheng 1587da14cebeSEric Cheng mac_srs_worker_bind(mac_rx_srs, srs_cpu->mc_pollid); 1588da14cebeSEric Cheng mac_srs_poll_bind(mac_rx_srs, srs_cpu->mc_workerid); 1589da14cebeSEric Cheng 1590da14cebeSEric Cheng /* 1591da14cebeSEric Cheng * Bind Tx srs and soft ring threads too. Let's bind tx 1592da14cebeSEric Cheng * srs to the last cpu in mrp list. 1593da14cebeSEric Cheng */ 1594da14cebeSEric Cheng if (mac_tx_srs != NULL && user_specified) { 1595da14cebeSEric Cheng BIND_TX_SRS_AND_SOFT_RINGS(mac_tx_srs, mrp); 1596da14cebeSEric Cheng } 1597da14cebeSEric Cheng mutex_exit(&cpu_lock); 1598da14cebeSEric Cheng } 1599da14cebeSEric Cheng 1600da14cebeSEric Cheng /* 1601da14cebeSEric Cheng * Bind SRS threads and soft rings to CPUs/create fanout list. 1602da14cebeSEric Cheng */ 1603da14cebeSEric Cheng void 1604da14cebeSEric Cheng mac_srs_fanout_init(mac_client_impl_t *mcip, flow_entry_t *flent, 1605da14cebeSEric Cheng mac_resource_props_t *mrp, mac_direct_rx_t rx_func, void *x_arg1, 1606da14cebeSEric Cheng mac_resource_handle_t x_arg2, mac_soft_ring_set_t *mac_rx_srs, 1607da14cebeSEric Cheng mac_soft_ring_set_t *mac_tx_srs) 1608da14cebeSEric Cheng { 1609da14cebeSEric Cheng int i; 1610da14cebeSEric Cheng processorid_t cpuid, worker_cpuid, poll_cpuid; 1611*ae6aa22aSVenugopal Iyer uint32_t soft_ring_flag = 0; 1612da14cebeSEric Cheng int soft_ring_cnt; 1613da14cebeSEric Cheng boolean_t user_specified = B_FALSE; 1614da14cebeSEric Cheng mac_cpus_t *srs_cpu = &mac_rx_srs->srs_cpu; 1615da14cebeSEric Cheng 1616da14cebeSEric Cheng /* 1617da14cebeSEric Cheng * Remove the no soft ring flag and we will adjust it 1618da14cebeSEric Cheng * appropriately further down. 1619da14cebeSEric Cheng */ 1620da14cebeSEric Cheng mutex_enter(&mac_rx_srs->srs_lock); 1621da14cebeSEric Cheng mac_rx_srs->srs_type &= ~SRST_NO_SOFT_RINGS; 1622da14cebeSEric Cheng mutex_exit(&mac_rx_srs->srs_lock); 1623da14cebeSEric Cheng 1624da14cebeSEric Cheng ASSERT(mac_rx_srs->srs_soft_ring_head == NULL); 1625da14cebeSEric Cheng 1626da14cebeSEric Cheng if (mac_rx_srs->srs_type & SRST_BW_CONTROL) 1627da14cebeSEric Cheng soft_ring_flag |= ST_RING_BW_CTL; 1628da14cebeSEric Cheng 1629da14cebeSEric Cheng ASSERT(mac_rx_srs->srs_fanout_state == SRS_FANOUT_UNINIT); 1630da14cebeSEric Cheng mac_rx_srs->srs_fanout_state = SRS_FANOUT_INIT; 1631da14cebeSEric Cheng user_specified = mrp->mrp_mask & MRP_CPUS_USERSPEC; 1632da14cebeSEric Cheng /* 1633da14cebeSEric Cheng * Ring count can be 0 if no fanout is required and no cpu 1634da14cebeSEric Cheng * were specified. Leave the SRS worker and poll thread 1635da14cebeSEric Cheng * unbound 1636da14cebeSEric Cheng */ 1637da14cebeSEric Cheng ASSERT(mrp != NULL); 1638da14cebeSEric Cheng soft_ring_cnt = srs_cpu->mc_fanout_cnt; 1639da14cebeSEric Cheng 1640da14cebeSEric Cheng /* Step 1: bind cpu contains cpu list where threads need to bind */ 1641da14cebeSEric Cheng if (soft_ring_cnt > 0) { 1642da14cebeSEric Cheng mutex_enter(&cpu_lock); 1643da14cebeSEric Cheng for (i = 0; i < soft_ring_cnt; i++) { 1644da14cebeSEric Cheng cpuid = srs_cpu->mc_fanout_cpus[i]; 1645da14cebeSEric Cheng /* Create the protocol softrings */ 1646da14cebeSEric Cheng mac_srs_create_proto_softrings(i, (void *)flent, 1647da14cebeSEric Cheng soft_ring_flag, mac_rx_srs->srs_pri, 1648da14cebeSEric Cheng mcip, mac_rx_srs, cpuid, rx_func, 1649da14cebeSEric Cheng x_arg1, x_arg2, B_FALSE); 1650da14cebeSEric Cheng } 1651da14cebeSEric Cheng worker_cpuid = srs_cpu->mc_workerid; 1652da14cebeSEric Cheng poll_cpuid = srs_cpu->mc_pollid; 1653da14cebeSEric Cheng mac_srs_worker_bind(mac_rx_srs, worker_cpuid); 1654da14cebeSEric Cheng mac_srs_poll_bind(mac_rx_srs, poll_cpuid); 1655da14cebeSEric Cheng 1656da14cebeSEric Cheng /* 1657da14cebeSEric Cheng * Bind Tx srs and soft ring threads too. 1658da14cebeSEric Cheng * Let's bind tx srs to the last cpu in 1659da14cebeSEric Cheng * mrp list. 1660da14cebeSEric Cheng */ 1661da14cebeSEric Cheng if (mac_tx_srs == NULL) { 1662da14cebeSEric Cheng mutex_exit(&cpu_lock); 1663da14cebeSEric Cheng goto alldone; 1664da14cebeSEric Cheng } 1665da14cebeSEric Cheng 1666da14cebeSEric Cheng if (user_specified) { 1667da14cebeSEric Cheng BIND_TX_SRS_AND_SOFT_RINGS(mac_tx_srs, mrp); 1668da14cebeSEric Cheng } 1669da14cebeSEric Cheng mutex_exit(&cpu_lock); 1670da14cebeSEric Cheng } else { 1671da14cebeSEric Cheng mutex_enter(&cpu_lock); 1672da14cebeSEric Cheng /* 1673da14cebeSEric Cheng * For a subflow, mrp_workerid and mrp_pollid 1674da14cebeSEric Cheng * is not set. 1675da14cebeSEric Cheng */ 1676da14cebeSEric Cheng mac_srs_worker_bind(mac_rx_srs, mrp->mrp_workerid); 1677da14cebeSEric Cheng mac_srs_poll_bind(mac_rx_srs, mrp->mrp_pollid); 1678da14cebeSEric Cheng mutex_exit(&cpu_lock); 1679da14cebeSEric Cheng goto no_softrings; 1680da14cebeSEric Cheng } 1681da14cebeSEric Cheng 1682da14cebeSEric Cheng alldone: 1683da14cebeSEric Cheng if (soft_ring_cnt > 1) 1684da14cebeSEric Cheng mac_rx_srs->srs_type |= SRST_FANOUT_SRC_IP; 1685da14cebeSEric Cheng mac_srs_update_fanout_list(mac_rx_srs); 1686da14cebeSEric Cheng mac_srs_client_poll_enable(mcip, mac_rx_srs); 1687da14cebeSEric Cheng return; 1688da14cebeSEric Cheng 1689da14cebeSEric Cheng no_softrings: 1690da14cebeSEric Cheng if (mac_rx_srs->srs_type & SRST_FANOUT_PROTO) { 1691da14cebeSEric Cheng mutex_enter(&cpu_lock); 1692da14cebeSEric Cheng cpuid = mac_next_bind_cpu(); 1693da14cebeSEric Cheng /* Create the protocol softrings */ 1694da14cebeSEric Cheng mac_srs_create_proto_softrings(0, (void *)flent, 1695da14cebeSEric Cheng soft_ring_flag, mac_rx_srs->srs_pri, 1696da14cebeSEric Cheng mcip, mac_rx_srs, cpuid, rx_func, 1697da14cebeSEric Cheng x_arg1, x_arg2, B_FALSE); 1698da14cebeSEric Cheng mutex_exit(&cpu_lock); 1699da14cebeSEric Cheng } else { 1700da14cebeSEric Cheng /* 1701da14cebeSEric Cheng * This is the case when there is no fanout which is 1702da14cebeSEric Cheng * true for subflows. 1703da14cebeSEric Cheng */ 1704da14cebeSEric Cheng mac_rx_srs->srs_type |= SRST_NO_SOFT_RINGS; 1705da14cebeSEric Cheng } 1706da14cebeSEric Cheng mac_srs_update_fanout_list(mac_rx_srs); 1707da14cebeSEric Cheng mac_srs_client_poll_enable(mcip, mac_rx_srs); 1708da14cebeSEric Cheng } 1709da14cebeSEric Cheng 1710da14cebeSEric Cheng /* 1711da14cebeSEric Cheng * mac_fanout_setup: 1712da14cebeSEric Cheng * 1713da14cebeSEric Cheng * Calls mac_srs_fanout_init() or modify() depending upon whether 1714da14cebeSEric Cheng * the SRS is getting initialized or re-initialized. 1715da14cebeSEric Cheng */ 1716da14cebeSEric Cheng void 1717da14cebeSEric Cheng mac_fanout_setup(mac_client_impl_t *mcip, flow_entry_t *flent, 1718da14cebeSEric Cheng mac_resource_props_t *mrp, mac_direct_rx_t rx_func, void *x_arg1, 1719da14cebeSEric Cheng mac_resource_handle_t x_arg2) 1720da14cebeSEric Cheng { 1721da14cebeSEric Cheng mac_soft_ring_set_t *mac_rx_srs, *mac_tx_srs; 1722da14cebeSEric Cheng int i, rx_srs_cnt; 1723da14cebeSEric Cheng 1724da14cebeSEric Cheng ASSERT(MAC_PERIM_HELD((mac_handle_t)mcip->mci_mip)); 1725da14cebeSEric Cheng /* 1726da14cebeSEric Cheng * This is an aggregation port. Fanout will be setup 1727da14cebeSEric Cheng * over the aggregation itself. 1728da14cebeSEric Cheng */ 1729da14cebeSEric Cheng if (mcip->mci_state_flags & MCIS_IS_AGGR_PORT) 1730da14cebeSEric Cheng return; 1731da14cebeSEric Cheng 1732da14cebeSEric Cheng mac_rx_srs = flent->fe_rx_srs[0]; 1733da14cebeSEric Cheng /* 1734da14cebeSEric Cheng * Set up the fanout on the tx side only once, with the 1735da14cebeSEric Cheng * first rx SRS. The CPU binding, fanout, and bandwidth 1736da14cebeSEric Cheng * criteria are common to both RX and TX, so 1737da14cebeSEric Cheng * initializing them along side avoids redundant code. 1738da14cebeSEric Cheng */ 1739da14cebeSEric Cheng mac_tx_srs = flent->fe_tx_srs; 1740da14cebeSEric Cheng rx_srs_cnt = flent->fe_rx_srs_cnt; 1741da14cebeSEric Cheng 1742da14cebeSEric Cheng /* No fanout for subflows */ 1743da14cebeSEric Cheng if (flent->fe_type & FLOW_USER) { 1744da14cebeSEric Cheng mac_srs_fanout_init(mcip, flent, mrp, rx_func, 1745da14cebeSEric Cheng x_arg1, x_arg2, mac_rx_srs, mac_tx_srs); 1746da14cebeSEric Cheng return; 1747da14cebeSEric Cheng } 1748da14cebeSEric Cheng 1749da14cebeSEric Cheng mac_flow_cpu_init(flent, mrp); 1750da14cebeSEric Cheng 1751da14cebeSEric Cheng /* 1752da14cebeSEric Cheng * Set up fanout for both SW (0th SRS) and HW classified 1753da14cebeSEric Cheng * SRS (the rest of Rx SRSs in flent). 1754da14cebeSEric Cheng */ 1755da14cebeSEric Cheng for (i = 0; i < rx_srs_cnt; i++) { 1756da14cebeSEric Cheng mac_rx_srs = flent->fe_rx_srs[i]; 1757da14cebeSEric Cheng if (i != 0) 1758da14cebeSEric Cheng mac_tx_srs = NULL; 1759da14cebeSEric Cheng switch (mac_rx_srs->srs_fanout_state) { 1760da14cebeSEric Cheng case SRS_FANOUT_UNINIT: 1761da14cebeSEric Cheng mac_srs_fanout_init(mcip, flent, mrp, rx_func, 1762da14cebeSEric Cheng x_arg1, x_arg2, mac_rx_srs, mac_tx_srs); 1763da14cebeSEric Cheng break; 1764da14cebeSEric Cheng case SRS_FANOUT_INIT: 1765da14cebeSEric Cheng break; 1766da14cebeSEric Cheng case SRS_FANOUT_REINIT: 1767da14cebeSEric Cheng mac_rx_srs_quiesce(mac_rx_srs, SRS_QUIESCE); 1768da14cebeSEric Cheng mac_srs_fanout_modify(mcip, flent, mrp, rx_func, 1769da14cebeSEric Cheng x_arg1, x_arg2, mac_rx_srs, mac_tx_srs); 1770da14cebeSEric Cheng mac_rx_srs_restart(mac_rx_srs); 1771da14cebeSEric Cheng break; 1772da14cebeSEric Cheng default: 1773da14cebeSEric Cheng VERIFY(mac_rx_srs->srs_fanout_state <= 1774da14cebeSEric Cheng SRS_FANOUT_REINIT); 1775da14cebeSEric Cheng break; 1776da14cebeSEric Cheng } 1777da14cebeSEric Cheng } 1778da14cebeSEric Cheng } 1779da14cebeSEric Cheng 1780da14cebeSEric Cheng /* 1781da14cebeSEric Cheng * mac_create_soft_ring_set: 1782da14cebeSEric Cheng * 1783da14cebeSEric Cheng * Create a mac_soft_ring_set_t (SRS). If soft_ring_fanout_type is 1784da14cebeSEric Cheng * SRST_TX, an SRS for Tx side is created. Otherwise an SRS for Rx side 1785da14cebeSEric Cheng * processing is created. 1786da14cebeSEric Cheng * 1787da14cebeSEric Cheng * Details on Rx SRS: 1788da14cebeSEric Cheng * Create a SRS and also add the necessary soft rings for TCP and 1789da14cebeSEric Cheng * non-TCP based on fanout type and count specified. 1790da14cebeSEric Cheng * 1791da14cebeSEric Cheng * mac_soft_ring_fanout, mac_srs_fanout_modify (?), 1792da14cebeSEric Cheng * mac_soft_ring_stop_workers, mac_soft_ring_set_destroy, etc need 1793da14cebeSEric Cheng * to be heavily modified. 1794da14cebeSEric Cheng * 1795da14cebeSEric Cheng * mi_soft_ring_list_size, mi_soft_ring_size, etc need to disappear. 1796da14cebeSEric Cheng */ 1797da14cebeSEric Cheng mac_soft_ring_set_t * 1798da14cebeSEric Cheng mac_srs_create(mac_client_impl_t *mcip, flow_entry_t *flent, uint32_t srs_type, 1799da14cebeSEric Cheng mac_direct_rx_t rx_func, void *x_arg1, mac_resource_handle_t x_arg2, 1800da14cebeSEric Cheng mac_ring_t *ring) 1801da14cebeSEric Cheng { 1802da14cebeSEric Cheng mac_soft_ring_set_t *mac_srs; 1803da14cebeSEric Cheng mac_srs_rx_t *srs_rx; 1804da14cebeSEric Cheng mac_srs_tx_t *srs_tx; 1805da14cebeSEric Cheng mac_bw_ctl_t *mac_bw; 1806da14cebeSEric Cheng mac_resource_props_t *mrp; 1807da14cebeSEric Cheng boolean_t is_tx_srs = ((srs_type & SRST_TX) != 0); 1808da14cebeSEric Cheng 1809da14cebeSEric Cheng mac_srs = kmem_cache_alloc(mac_srs_cache, KM_SLEEP); 1810da14cebeSEric Cheng bzero(mac_srs, sizeof (mac_soft_ring_set_t)); 1811da14cebeSEric Cheng srs_rx = &mac_srs->srs_rx; 1812da14cebeSEric Cheng srs_tx = &mac_srs->srs_tx; 1813da14cebeSEric Cheng 1814da14cebeSEric Cheng mutex_enter(&flent->fe_lock); 1815da14cebeSEric Cheng 1816da14cebeSEric Cheng /* 1817da14cebeSEric Cheng * Get the bandwidth control structure from the flent. Get 1818da14cebeSEric Cheng * rid of any residual values in the control structure for 1819da14cebeSEric Cheng * the tx bw struct and also for the rx, if the rx srs is 1820da14cebeSEric Cheng * the 1st one being brought up (the rx bw ctl struct may 1821da14cebeSEric Cheng * be shared by multiple SRSs) 1822da14cebeSEric Cheng */ 1823da14cebeSEric Cheng if (is_tx_srs) { 1824da14cebeSEric Cheng mac_srs->srs_bw = &flent->fe_tx_bw; 1825da14cebeSEric Cheng bzero(mac_srs->srs_bw, sizeof (mac_bw_ctl_t)); 1826da14cebeSEric Cheng flent->fe_tx_srs = mac_srs; 1827da14cebeSEric Cheng } else { 1828da14cebeSEric Cheng /* 1829da14cebeSEric Cheng * The bw counter (stored in the flent) is shared 1830da14cebeSEric Cheng * by SRS's within an rx group. 1831da14cebeSEric Cheng */ 1832da14cebeSEric Cheng mac_srs->srs_bw = &flent->fe_rx_bw; 1833da14cebeSEric Cheng /* First rx SRS, clear the bw structure */ 1834da14cebeSEric Cheng if (flent->fe_rx_srs_cnt == 0) 1835da14cebeSEric Cheng bzero(mac_srs->srs_bw, sizeof (mac_bw_ctl_t)); 1836da14cebeSEric Cheng ASSERT(flent->fe_rx_srs_cnt < MAX_RINGS_PER_GROUP); 1837da14cebeSEric Cheng flent->fe_rx_srs[flent->fe_rx_srs_cnt] = mac_srs; 1838da14cebeSEric Cheng flent->fe_rx_srs_cnt++; 1839da14cebeSEric Cheng } 1840da14cebeSEric Cheng mac_srs->srs_flent = flent; 1841da14cebeSEric Cheng mutex_exit(&flent->fe_lock); 1842da14cebeSEric Cheng 1843da14cebeSEric Cheng mac_srs->srs_state = 0; 1844da14cebeSEric Cheng mac_srs->srs_type = (srs_type | SRST_NO_SOFT_RINGS); 1845da14cebeSEric Cheng mac_srs->srs_worker_cpuid = mac_srs->srs_worker_cpuid_save = -1; 1846da14cebeSEric Cheng mac_srs->srs_poll_cpuid = mac_srs->srs_poll_cpuid_save = -1; 1847da14cebeSEric Cheng mac_srs_fanout_list_alloc(mac_srs); 1848da14cebeSEric Cheng 1849da14cebeSEric Cheng /* 1850da14cebeSEric Cheng * For a flow we use the underlying MAC client's priority range with 1851da14cebeSEric Cheng * the priority value to find an absolute priority value. For a MAC 1852da14cebeSEric Cheng * client we use the MAC client's maximum priority as the value. 1853da14cebeSEric Cheng */ 1854da14cebeSEric Cheng mrp = &flent->fe_effective_props; 1855da14cebeSEric Cheng if ((mac_srs->srs_type & SRST_FLOW) != 0) { 1856da14cebeSEric Cheng mac_srs->srs_pri = FLOW_PRIORITY(mcip->mci_min_pri, 1857da14cebeSEric Cheng mcip->mci_max_pri, mrp->mrp_priority); 1858da14cebeSEric Cheng } else { 1859da14cebeSEric Cheng mac_srs->srs_pri = mcip->mci_max_pri; 1860da14cebeSEric Cheng } 1861da14cebeSEric Cheng mac_srs->srs_mcip = mcip; 1862da14cebeSEric Cheng /* 1863da14cebeSEric Cheng * We need to insert the SRS in the global list before 1864da14cebeSEric Cheng * binding the SRS and SR threads. Otherwise there is a 1865da14cebeSEric Cheng * is a small window where the cpu reconfig callbacks 1866da14cebeSEric Cheng * may miss the SRS in the list walk and DR could fail 1867da14cebeSEric Cheng * as there are bound threads. 1868da14cebeSEric Cheng */ 1869da14cebeSEric Cheng mac_srs_add_glist(mac_srs); 1870da14cebeSEric Cheng 1871da14cebeSEric Cheng /* Initialize bw limit */ 1872da14cebeSEric Cheng if ((mrp->mrp_mask & MRP_MAXBW) != 0) { 1873da14cebeSEric Cheng mac_srs->srs_drain_func = mac_rx_srs_drain_bw; 1874da14cebeSEric Cheng 1875da14cebeSEric Cheng mac_bw = mac_srs->srs_bw; 1876da14cebeSEric Cheng mutex_enter(&mac_bw->mac_bw_lock); 1877da14cebeSEric Cheng mac_bw->mac_bw_limit = FLOW_BYTES_PER_TICK(mrp->mrp_maxbw); 1878da14cebeSEric Cheng 1879da14cebeSEric Cheng /* 1880da14cebeSEric Cheng * Give twice the queuing capability before 1881da14cebeSEric Cheng * dropping packets. The unit is bytes/tick. 1882da14cebeSEric Cheng */ 1883da14cebeSEric Cheng mac_bw->mac_bw_drop_threshold = mac_bw->mac_bw_limit << 1; 1884da14cebeSEric Cheng mutex_exit(&mac_bw->mac_bw_lock); 1885da14cebeSEric Cheng mac_srs->srs_type |= SRST_BW_CONTROL; 1886da14cebeSEric Cheng } else { 1887da14cebeSEric Cheng mac_srs->srs_drain_func = mac_rx_srs_drain; 1888da14cebeSEric Cheng } 1889da14cebeSEric Cheng 1890da14cebeSEric Cheng /* 1891da14cebeSEric Cheng * We use the following policy to control Receive 1892da14cebeSEric Cheng * Side Dynamic Polling: 1893da14cebeSEric Cheng * 1) We switch to poll mode anytime the processing thread causes 1894da14cebeSEric Cheng * a backlog to build up in SRS and its associated Soft Rings 1895da14cebeSEric Cheng * (sr_poll_pkt_cnt > 0). 1896da14cebeSEric Cheng * 2) As long as the backlog stays under the low water mark 1897da14cebeSEric Cheng * (sr_lowat), we poll the H/W for more packets. 1898da14cebeSEric Cheng * 3) If the backlog (sr_poll_pkt_cnt) exceeds low water mark, we 1899da14cebeSEric Cheng * stay in poll mode but don't poll the H/W for more packets. 1900da14cebeSEric Cheng * 4) Anytime in polling mode, if we poll the H/W for packets and 1901da14cebeSEric Cheng * find nothing plus we have an existing backlog 1902da14cebeSEric Cheng * (sr_poll_pkt_cnt > 0), we stay in polling mode but don't poll 1903da14cebeSEric Cheng * the H/W for packets anymore (let the polling thread go to sleep). 1904da14cebeSEric Cheng * 5) Once the backlog is relived (packets are processed) we reenable 1905da14cebeSEric Cheng * polling (by signalling the poll thread) only when the backlog 1906da14cebeSEric Cheng * dips below sr_poll_thres. 1907da14cebeSEric Cheng * 6) sr_hiwat is used exclusively when we are not polling capable 1908da14cebeSEric Cheng * and is used to decide when to drop packets so the SRS queue 1909da14cebeSEric Cheng * length doesn't grow infinitely. 1910da14cebeSEric Cheng */ 1911da14cebeSEric Cheng if (!is_tx_srs) { 1912da14cebeSEric Cheng srs_rx->sr_hiwat = mac_soft_ring_max_q_cnt; 1913da14cebeSEric Cheng /* Low water mark needs to be less than high water mark */ 1914da14cebeSEric Cheng srs_rx->sr_lowat = mac_soft_ring_min_q_cnt <= 1915da14cebeSEric Cheng mac_soft_ring_max_q_cnt ? mac_soft_ring_min_q_cnt : 1916da14cebeSEric Cheng (mac_soft_ring_max_q_cnt >> 2); 1917da14cebeSEric Cheng /* Poll threshold need to be half of low water mark or less */ 1918da14cebeSEric Cheng srs_rx->sr_poll_thres = mac_soft_ring_poll_thres <= 1919da14cebeSEric Cheng (srs_rx->sr_lowat >> 1) ? mac_soft_ring_poll_thres : 1920da14cebeSEric Cheng (srs_rx->sr_lowat >> 1); 1921da14cebeSEric Cheng if (mac_latency_optimize) 1922*ae6aa22aSVenugopal Iyer mac_srs->srs_state |= 1923*ae6aa22aSVenugopal Iyer (SRS_LATENCY_OPT|SRS_SOFTRING_QUEUE); 1924da14cebeSEric Cheng } 1925da14cebeSEric Cheng 1926da14cebeSEric Cheng mac_srs->srs_worker = thread_create(NULL, 0, 1927da14cebeSEric Cheng mac_srs_worker, mac_srs, 0, &p0, TS_RUN, mac_srs->srs_pri); 1928da14cebeSEric Cheng 1929da14cebeSEric Cheng if (is_tx_srs) { 1930da14cebeSEric Cheng /* Handle everything about Tx SRS and return */ 1931da14cebeSEric Cheng mac_srs->srs_drain_func = mac_tx_srs_drain; 1932da14cebeSEric Cheng srs_tx->st_max_q_cnt = mac_tx_srs_max_q_cnt; 1933da14cebeSEric Cheng srs_tx->st_hiwat = 1934da14cebeSEric Cheng (mac_tx_srs_hiwat > mac_tx_srs_max_q_cnt) ? 1935da14cebeSEric Cheng mac_tx_srs_max_q_cnt : mac_tx_srs_hiwat; 1936da14cebeSEric Cheng srs_tx->st_arg1 = x_arg1; 1937da14cebeSEric Cheng srs_tx->st_arg2 = x_arg2; 1938da14cebeSEric Cheng return (mac_srs); 1939da14cebeSEric Cheng } 1940da14cebeSEric Cheng 1941da14cebeSEric Cheng if ((srs_type & SRST_FLOW) != 0 || 1942da14cebeSEric Cheng FLOW_TAB_EMPTY(mcip->mci_subflow_tab)) 1943da14cebeSEric Cheng srs_rx->sr_lower_proc = mac_rx_srs_process; 1944da14cebeSEric Cheng else 1945da14cebeSEric Cheng srs_rx->sr_lower_proc = mac_rx_srs_subflow_process; 1946da14cebeSEric Cheng 1947da14cebeSEric Cheng srs_rx->sr_func = rx_func; 1948da14cebeSEric Cheng srs_rx->sr_arg1 = x_arg1; 1949da14cebeSEric Cheng srs_rx->sr_arg2 = x_arg2; 1950da14cebeSEric Cheng 1951da14cebeSEric Cheng if (ring != NULL) { 1952da14cebeSEric Cheng /* Is the mac_srs created over the RX default group? */ 1953da14cebeSEric Cheng if (ring->mr_gh == (mac_group_handle_t) 1954da14cebeSEric Cheng (&mcip->mci_mip->mi_rx_groups[0])) 1955da14cebeSEric Cheng mac_srs->srs_type |= SRST_DEFAULT_GRP; 1956da14cebeSEric Cheng 1957da14cebeSEric Cheng mac_srs->srs_ring = ring; 1958da14cebeSEric Cheng ring->mr_srs = mac_srs; 1959da14cebeSEric Cheng ring->mr_classify_type = MAC_HW_CLASSIFIER; 1960da14cebeSEric Cheng ring->mr_flag |= MR_INCIPIENT; 1961da14cebeSEric Cheng 1962*ae6aa22aSVenugopal Iyer if (FLOW_TAB_EMPTY(mcip->mci_subflow_tab) && mac_poll_enable) 1963da14cebeSEric Cheng mac_srs->srs_state |= SRS_POLLING_CAPAB; 1964da14cebeSEric Cheng 1965da14cebeSEric Cheng mac_srs->srs_poll_thr = thread_create(NULL, 0, 1966da14cebeSEric Cheng mac_rx_srs_poll_ring, mac_srs, 0, &p0, TS_RUN, 1967da14cebeSEric Cheng mac_srs->srs_pri); 1968*ae6aa22aSVenugopal Iyer /* 1969*ae6aa22aSVenugopal Iyer * Some drivers require serialization and don't send 1970*ae6aa22aSVenugopal Iyer * packet chains in interrupt context. For such 1971*ae6aa22aSVenugopal Iyer * drivers, we should always queue in soft ring 1972*ae6aa22aSVenugopal Iyer * so that we get a chance to switch into a polling 1973*ae6aa22aSVenugopal Iyer * mode under backlog. 1974*ae6aa22aSVenugopal Iyer */ 1975*ae6aa22aSVenugopal Iyer if (mcip->mci_mip->mi_v12n_level & MAC_VIRT_SERIALIZE) 1976*ae6aa22aSVenugopal Iyer mac_srs->srs_state |= SRS_SOFTRING_QUEUE; 1977da14cebeSEric Cheng } 1978da14cebeSEric Cheng return (mac_srs); 1979da14cebeSEric Cheng } 1980da14cebeSEric Cheng 1981da14cebeSEric Cheng /* 1982da14cebeSEric Cheng * Figure out the number of soft rings required. Its dependant on 1983da14cebeSEric Cheng * if protocol fanout is required (for LINKs), global settings 1984da14cebeSEric Cheng * require us to do fanout for performance (based on mac_soft_ring_enable), 1985da14cebeSEric Cheng * or user has specifically requested fanout. 1986da14cebeSEric Cheng */ 1987da14cebeSEric Cheng static uint32_t 1988da14cebeSEric Cheng mac_find_fanout(flow_entry_t *flent, uint32_t link_type) 1989da14cebeSEric Cheng { 1990da14cebeSEric Cheng uint32_t fanout_type; 1991da14cebeSEric Cheng mac_resource_props_t *mrp = &flent->fe_effective_props; 1992da14cebeSEric Cheng 1993da14cebeSEric Cheng /* no fanout for subflows */ 1994da14cebeSEric Cheng switch (link_type) { 1995da14cebeSEric Cheng case SRST_FLOW: 1996da14cebeSEric Cheng fanout_type = SRST_NO_SOFT_RINGS; 1997da14cebeSEric Cheng break; 1998da14cebeSEric Cheng case SRST_LINK: 1999da14cebeSEric Cheng fanout_type = SRST_FANOUT_PROTO; 2000da14cebeSEric Cheng break; 2001da14cebeSEric Cheng } 2002da14cebeSEric Cheng 2003da14cebeSEric Cheng /* A primary NIC/link is being plumbed */ 2004da14cebeSEric Cheng if (flent->fe_type & FLOW_PRIMARY_MAC) { 2005da14cebeSEric Cheng if (mac_soft_ring_enable && mac_rx_soft_ring_count > 1) { 2006da14cebeSEric Cheng fanout_type |= SRST_FANOUT_SRC_IP; 2007da14cebeSEric Cheng } 2008da14cebeSEric Cheng } else if (flent->fe_type & FLOW_VNIC) { 2009da14cebeSEric Cheng /* A VNIC is being created */ 2010da14cebeSEric Cheng if (mrp != NULL && mrp->mrp_ncpus > 0) { 2011da14cebeSEric Cheng fanout_type |= SRST_FANOUT_SRC_IP; 2012da14cebeSEric Cheng } 2013da14cebeSEric Cheng } 2014da14cebeSEric Cheng 2015da14cebeSEric Cheng return (fanout_type); 2016da14cebeSEric Cheng } 2017da14cebeSEric Cheng 2018da14cebeSEric Cheng /* 2019da14cebeSEric Cheng * Change a group from h/w to s/w classification. 2020da14cebeSEric Cheng */ 2021da14cebeSEric Cheng static void 2022da14cebeSEric Cheng mac_rx_switch_grp_to_sw(mac_group_t *group) 2023da14cebeSEric Cheng { 2024da14cebeSEric Cheng mac_ring_t *ring; 2025da14cebeSEric Cheng mac_soft_ring_set_t *mac_srs; 2026da14cebeSEric Cheng 2027da14cebeSEric Cheng for (ring = group->mrg_rings; ring != NULL; ring = ring->mr_next) { 2028da14cebeSEric Cheng if (ring->mr_classify_type == MAC_HW_CLASSIFIER) { 2029da14cebeSEric Cheng /* 2030da14cebeSEric Cheng * Remove the SRS associated with the HW ring. 2031da14cebeSEric Cheng * As a result, polling will be disabled. 2032da14cebeSEric Cheng */ 2033da14cebeSEric Cheng mac_srs = ring->mr_srs; 2034da14cebeSEric Cheng ASSERT(mac_srs != NULL); 2035da14cebeSEric Cheng mac_rx_srs_remove(mac_srs); 2036da14cebeSEric Cheng ring->mr_srs = NULL; 2037da14cebeSEric Cheng } 2038da14cebeSEric Cheng 2039da14cebeSEric Cheng if (ring->mr_state != MR_INUSE) 2040da14cebeSEric Cheng (void) mac_start_ring(ring); 2041da14cebeSEric Cheng /* 2042da14cebeSEric Cheng * We need to perform SW classification 2043da14cebeSEric Cheng * for packets landing in these rings 2044da14cebeSEric Cheng */ 2045da14cebeSEric Cheng ring->mr_state = MR_INUSE; 2046da14cebeSEric Cheng ring->mr_flag = 0; 2047da14cebeSEric Cheng ring->mr_classify_type = MAC_SW_CLASSIFIER; 2048da14cebeSEric Cheng } 2049da14cebeSEric Cheng } 2050da14cebeSEric Cheng 2051da14cebeSEric Cheng /* 2052da14cebeSEric Cheng * Create the Rx SRS for S/W classifier and for each ring in the 2053da14cebeSEric Cheng * group (if exclusive group). Also create the Tx SRS. 2054da14cebeSEric Cheng */ 2055da14cebeSEric Cheng void 2056da14cebeSEric Cheng mac_srs_group_setup(mac_client_impl_t *mcip, flow_entry_t *flent, 2057da14cebeSEric Cheng mac_group_t *group, uint32_t link_type) 2058da14cebeSEric Cheng { 2059da14cebeSEric Cheng mac_impl_t *mip = mcip->mci_mip; 2060da14cebeSEric Cheng mac_soft_ring_set_t *mac_srs; 2061da14cebeSEric Cheng mac_soft_ring_set_t *tx_srs = NULL; 2062da14cebeSEric Cheng mac_ring_t *ring; 2063da14cebeSEric Cheng uint32_t fanout_type; 2064da14cebeSEric Cheng boolean_t created_srs = B_FALSE; 2065da14cebeSEric Cheng 2066da14cebeSEric Cheng fanout_type = mac_find_fanout(flent, link_type); 2067da14cebeSEric Cheng 2068da14cebeSEric Cheng /* Create the SRS for S/W classification if none exists */ 2069da14cebeSEric Cheng if (flent->fe_rx_srs[0] == NULL) { 2070da14cebeSEric Cheng ASSERT(flent->fe_rx_srs_cnt == 0); 2071da14cebeSEric Cheng /* Setup the Rx SRS */ 2072da14cebeSEric Cheng mac_srs = mac_srs_create(mcip, flent, fanout_type | link_type, 2073da14cebeSEric Cheng mac_rx_deliver, mcip, NULL, NULL); 2074da14cebeSEric Cheng 2075da14cebeSEric Cheng mutex_enter(&flent->fe_lock); 2076da14cebeSEric Cheng flent->fe_cb_fn = (flow_fn_t)mac_srs->srs_rx.sr_lower_proc; 2077da14cebeSEric Cheng flent->fe_cb_arg1 = (void *)mip; 2078da14cebeSEric Cheng flent->fe_cb_arg2 = (void *)mac_srs; 2079da14cebeSEric Cheng mutex_exit(&flent->fe_lock); 2080da14cebeSEric Cheng 2081da14cebeSEric Cheng /* Setup the Tx SRS as well */ 2082da14cebeSEric Cheng ASSERT(flent->fe_tx_srs == NULL); 2083da14cebeSEric Cheng tx_srs = mac_srs_create(mcip, flent, SRST_TX | link_type, 2084da14cebeSEric Cheng NULL, mcip, NULL, NULL); 2085da14cebeSEric Cheng 2086da14cebeSEric Cheng if (mcip->mci_share != NULL) { 2087da14cebeSEric Cheng mac_srs_tx_t *tx = &tx_srs->srs_tx; 208808ac1c49SNicolas Droux ASSERT((mcip->mci_state_flags & MCIS_NO_HWRINGS) == 0); 2089da14cebeSEric Cheng /* 2090da14cebeSEric Cheng * A share requires a dedicated TX group. 2091da14cebeSEric Cheng * mac_reserve_tx_group() does the work needed to 2092da14cebeSEric Cheng * allocate a new group and populate that group 2093da14cebeSEric Cheng * with rings according to the driver requirements 2094da14cebeSEric Cheng * and limitations. 2095da14cebeSEric Cheng */ 2096da14cebeSEric Cheng tx->st_group = 2097da14cebeSEric Cheng mac_reserve_tx_group(mip, mcip->mci_share); 2098da14cebeSEric Cheng ASSERT(tx->st_group != NULL); 2099da14cebeSEric Cheng tx->st_group->mrg_tx_client = mcip; 2100da14cebeSEric Cheng } 2101da14cebeSEric Cheng mac_tx_srs_setup(mcip, flent, link_type); 2102da14cebeSEric Cheng created_srs = B_TRUE; 2103da14cebeSEric Cheng } 2104da14cebeSEric Cheng 2105da14cebeSEric Cheng if (group == NULL) { 2106da14cebeSEric Cheng if (created_srs) { 2107da14cebeSEric Cheng mac_fanout_setup(mcip, flent, 2108da14cebeSEric Cheng MCIP_RESOURCE_PROPS(mcip), mac_rx_deliver, 2109da14cebeSEric Cheng mcip, NULL); 2110da14cebeSEric Cheng } 2111da14cebeSEric Cheng return; 2112da14cebeSEric Cheng } 2113da14cebeSEric Cheng 2114da14cebeSEric Cheng /* 2115da14cebeSEric Cheng * fanout for default SRS is done when default SRS are created 2116da14cebeSEric Cheng * above. As each ring is added to the group, we setup the 2117da14cebeSEric Cheng * SRS and fanout to it. 2118da14cebeSEric Cheng */ 2119da14cebeSEric Cheng switch (group->mrg_state) { 2120da14cebeSEric Cheng case MAC_GROUP_STATE_RESERVED: 2121da14cebeSEric Cheng /* 2122da14cebeSEric Cheng * The group is exclusively ours. Create a SRS 2123da14cebeSEric Cheng * for each ring in the group and allow the 2124da14cebeSEric Cheng * individual SRS to dynamically poll their 2125da14cebeSEric Cheng * Rx ring. Do this only if the client is not 2126da14cebeSEric Cheng * a VLAN MAC client since for VLAN we do 2127da14cebeSEric Cheng * s/w classification for the VID check. 2128da14cebeSEric Cheng */ 2129da14cebeSEric Cheng if (i_mac_flow_vid(mcip->mci_flent) != VLAN_ID_NONE) 2130da14cebeSEric Cheng break; 2131da14cebeSEric Cheng for (ring = group->mrg_rings; ring != NULL; 2132da14cebeSEric Cheng ring = ring->mr_next) { 2133da14cebeSEric Cheng switch (ring->mr_state) { 2134da14cebeSEric Cheng case MR_INUSE: 2135da14cebeSEric Cheng case MR_FREE: 2136da14cebeSEric Cheng if (ring->mr_srs != NULL) 2137da14cebeSEric Cheng break; 2138da14cebeSEric Cheng if (ring->mr_state != MR_INUSE) 2139da14cebeSEric Cheng (void) mac_start_ring(ring); 2140da14cebeSEric Cheng 2141da14cebeSEric Cheng ring->mr_state = MR_INUSE; 2142da14cebeSEric Cheng 2143da14cebeSEric Cheng mac_srs = mac_srs_create(mcip, flent, 2144da14cebeSEric Cheng fanout_type | link_type, 2145da14cebeSEric Cheng mac_rx_deliver, mcip, NULL, ring); 2146da14cebeSEric Cheng break; 2147da14cebeSEric Cheng default: 2148da14cebeSEric Cheng cmn_err(CE_PANIC, "srs_setup: mcip = %p " 2149da14cebeSEric Cheng "trying to add UNKNOWN ring = %p\n", 2150da14cebeSEric Cheng (void *)mcip, (void *)ring); 2151da14cebeSEric Cheng break; 2152da14cebeSEric Cheng } 2153da14cebeSEric Cheng } 2154da14cebeSEric Cheng break; 2155da14cebeSEric Cheng case MAC_GROUP_STATE_SHARED: 2156da14cebeSEric Cheng /* 2157da14cebeSEric Cheng * Set all rings of this group to software classified. 2158da14cebeSEric Cheng * 2159da14cebeSEric Cheng * If the group is current RESERVED, the existing mac client 2160da14cebeSEric Cheng * (the only client on this group) is using this group 2161da14cebeSEric Cheng * exclusively. In that case we need to disable polling on 2162da14cebeSEric Cheng * the rings of the group (if it was enabled), and free the 2163da14cebeSEric Cheng * SRS associated with the rings. 2164da14cebeSEric Cheng */ 2165da14cebeSEric Cheng mac_rx_switch_grp_to_sw(group); 2166da14cebeSEric Cheng break; 2167da14cebeSEric Cheng default: 2168da14cebeSEric Cheng ASSERT(B_FALSE); 2169da14cebeSEric Cheng break; 2170da14cebeSEric Cheng } 2171da14cebeSEric Cheng mac_fanout_setup(mcip, flent, MCIP_RESOURCE_PROPS(mcip), 2172da14cebeSEric Cheng mac_rx_deliver, mcip, NULL); 2173da14cebeSEric Cheng } 2174da14cebeSEric Cheng 2175da14cebeSEric Cheng void 2176da14cebeSEric Cheng mac_srs_group_teardown(mac_client_impl_t *mcip, flow_entry_t *flent, 2177da14cebeSEric Cheng uint32_t link_type) 2178da14cebeSEric Cheng { 2179da14cebeSEric Cheng mac_soft_ring_set_t *mac_srs; 2180da14cebeSEric Cheng mac_soft_ring_set_t *tx_srs; 2181da14cebeSEric Cheng mac_srs_tx_t *tx; 2182da14cebeSEric Cheng int i; 2183da14cebeSEric Cheng 2184da14cebeSEric Cheng for (i = 0; i < flent->fe_rx_srs_cnt; i++) { 2185da14cebeSEric Cheng mac_srs = flent->fe_rx_srs[i]; 2186da14cebeSEric Cheng mac_rx_srs_quiesce(mac_srs, SRS_CONDEMNED); 2187da14cebeSEric Cheng /* 2188da14cebeSEric Cheng * Deal with all fanout tear down etc. 2189da14cebeSEric Cheng */ 2190da14cebeSEric Cheng mac_srs_free(mac_srs); 2191da14cebeSEric Cheng flent->fe_rx_srs[i] = NULL; 2192da14cebeSEric Cheng } 2193da14cebeSEric Cheng flent->fe_rx_srs_cnt = 0; 2194da14cebeSEric Cheng 2195da14cebeSEric Cheng tx_srs = flent->fe_tx_srs; 2196da14cebeSEric Cheng tx = &tx_srs->srs_tx; 2197da14cebeSEric Cheng switch (link_type) { 2198da14cebeSEric Cheng case SRST_FLOW: 2199da14cebeSEric Cheng /* 2200da14cebeSEric Cheng * For flows, we need to work with passed 2201da14cebeSEric Cheng * flent to find the Rx/Tx SRS. 2202da14cebeSEric Cheng */ 2203da14cebeSEric Cheng mac_tx_srs_quiesce(tx_srs, SRS_CONDEMNED); 2204da14cebeSEric Cheng break; 2205da14cebeSEric Cheng case SRST_LINK: 2206da14cebeSEric Cheng mac_tx_client_quiesce(mcip, SRS_CONDEMNED); 2207da14cebeSEric Cheng /* 2208da14cebeSEric Cheng * Release the TX resources. First the TX group, if any 2209da14cebeSEric Cheng * was assigned to the MAC client, which will cause the 2210da14cebeSEric Cheng * TX rings to be moved back to the pool. Then free the 2211da14cebeSEric Cheng * rings themselves. 2212da14cebeSEric Cheng */ 2213da14cebeSEric Cheng if (tx->st_group != NULL) { 2214da14cebeSEric Cheng mac_release_tx_group(tx_srs->srs_mcip->mci_mip, 2215da14cebeSEric Cheng tx->st_group); 2216da14cebeSEric Cheng tx->st_group = NULL; 2217da14cebeSEric Cheng } 2218da14cebeSEric Cheng if (tx->st_arg2 != NULL) { 2219da14cebeSEric Cheng ASSERT(tx_srs->srs_type & SRST_TX); 2220da14cebeSEric Cheng mac_release_tx_ring(tx->st_arg2); 2221da14cebeSEric Cheng } 2222da14cebeSEric Cheng break; 2223da14cebeSEric Cheng default: 2224da14cebeSEric Cheng ASSERT(B_FALSE); 2225da14cebeSEric Cheng break; 2226da14cebeSEric Cheng } 2227da14cebeSEric Cheng mac_srs_free(tx_srs); 2228da14cebeSEric Cheng flent->fe_tx_srs = NULL; 2229da14cebeSEric Cheng } 2230da14cebeSEric Cheng 2231da14cebeSEric Cheng /* 2232da14cebeSEric Cheng * This is the group state machine. The state of an Rx group is given by 2233da14cebeSEric Cheng * the following table. The default group and its rings are started in 2234da14cebeSEric Cheng * mac_start itself and the default group stays in SHARED state until 2235da14cebeSEric Cheng * mac_stop at which time the group and rings are stopped and and it 2236da14cebeSEric Cheng * reverts to the Registered state. 2237da14cebeSEric Cheng * 2238da14cebeSEric Cheng * Typically this function is called on a group after adding or removing a 2239da14cebeSEric Cheng * client from it, to find out what should be the new state of the group. 2240da14cebeSEric Cheng * If the new state is RESERVED, then the client that owns this group 2241da14cebeSEric Cheng * exclusively is also returned. Note that adding or removing a client from 2242da14cebeSEric Cheng * a group could also impact the default group and the caller needs to 2243da14cebeSEric Cheng * evaluate the effect on the default group. 2244da14cebeSEric Cheng * 2245da14cebeSEric Cheng * Group type # of clients mi_nactiveclients Group State 2246da14cebeSEric Cheng * in the group 2247da14cebeSEric Cheng * 2248da14cebeSEric Cheng * Non-default 0 N.A. REGISTERED 2249da14cebeSEric Cheng * Non-default 1 N.A. RESERVED 2250da14cebeSEric Cheng * Non-default > 1 N.A. SHARED 2251da14cebeSEric Cheng * 2252da14cebeSEric Cheng * Default 0 N.A. SHARED 2253da14cebeSEric Cheng * Default 1 1 RESERVED 2254da14cebeSEric Cheng * Default 1 > 1 SHARED 2255da14cebeSEric Cheng * Default > 1 N.A. SHARED 2256da14cebeSEric Cheng */ 2257da14cebeSEric Cheng mac_group_state_t 2258da14cebeSEric Cheng mac_rx_group_next_state(mac_group_t *grp, mac_client_impl_t **group_only_mcip) 2259da14cebeSEric Cheng { 2260da14cebeSEric Cheng mac_impl_t *mip = (mac_impl_t *)grp->mrg_mh; 2261da14cebeSEric Cheng 2262da14cebeSEric Cheng *group_only_mcip = NULL; 2263da14cebeSEric Cheng 2264da14cebeSEric Cheng /* Non-default group */ 2265da14cebeSEric Cheng 2266da14cebeSEric Cheng if (grp != mip->mi_rx_groups) { 2267da14cebeSEric Cheng if (MAC_RX_GROUP_NO_CLIENT(grp)) 2268da14cebeSEric Cheng return (MAC_GROUP_STATE_REGISTERED); 2269da14cebeSEric Cheng 2270da14cebeSEric Cheng *group_only_mcip = MAC_RX_GROUP_ONLY_CLIENT(grp); 2271da14cebeSEric Cheng if (*group_only_mcip != NULL) 2272da14cebeSEric Cheng return (MAC_GROUP_STATE_RESERVED); 2273da14cebeSEric Cheng 2274da14cebeSEric Cheng return (MAC_GROUP_STATE_SHARED); 2275da14cebeSEric Cheng } 2276da14cebeSEric Cheng 2277da14cebeSEric Cheng /* Default group */ 2278da14cebeSEric Cheng 2279da14cebeSEric Cheng if (MAC_RX_GROUP_NO_CLIENT(grp) || mip->mi_nactiveclients != 1) 2280da14cebeSEric Cheng return (MAC_GROUP_STATE_SHARED); 2281da14cebeSEric Cheng 2282da14cebeSEric Cheng *group_only_mcip = MAC_RX_GROUP_ONLY_CLIENT(grp); 2283da14cebeSEric Cheng ASSERT(*group_only_mcip != NULL); 2284da14cebeSEric Cheng return (MAC_GROUP_STATE_RESERVED); 2285da14cebeSEric Cheng } 2286da14cebeSEric Cheng 2287da14cebeSEric Cheng /* 2288da14cebeSEric Cheng * OVERVIEW NOTES FOR DATAPATH 2289da14cebeSEric Cheng * =========================== 2290da14cebeSEric Cheng * 2291da14cebeSEric Cheng * Create an SRS and setup the corresponding flow function and args. 2292da14cebeSEric Cheng * Add a classification rule for the flow specified by 'flent' and program 2293da14cebeSEric Cheng * the hardware classifier when applicable. 2294da14cebeSEric Cheng * 2295da14cebeSEric Cheng * Rx ring assignment, SRS, polling and B/W enforcement 2296da14cebeSEric Cheng * ---------------------------------------------------- 2297da14cebeSEric Cheng * 2298da14cebeSEric Cheng * We try to use H/W classification on NIC and assign traffic to a 2299da14cebeSEric Cheng * MAC address to a particular Rx ring. There is a 1-1 mapping 2300da14cebeSEric Cheng * between a SRS and a Rx ring. The SRS (short for soft ring set) 2301da14cebeSEric Cheng * dynamically switches the underlying Rx ring between interrupt 2302da14cebeSEric Cheng * and polling mode and enforces any specified B/W control. 2303da14cebeSEric Cheng * 2304da14cebeSEric Cheng * There is always a SRS created and tied to each H/W and S/W rule. 2305da14cebeSEric Cheng * Whenever we create a H/W rule, we always add the the same rule to 2306da14cebeSEric Cheng * S/W classifier and tie a SRS to it. 2307da14cebeSEric Cheng * 2308da14cebeSEric Cheng * In case a B/W control is specified, its broken into bytes 2309da14cebeSEric Cheng * per ticks and as soon as the quota for a tick is exhausted, 2310da14cebeSEric Cheng * the underlying Rx ring is forced into poll mode for remianing 2311da14cebeSEric Cheng * tick. The SRS poll thread only polls for bytes that are 2312da14cebeSEric Cheng * allowed to come in the SRS. We typically let 4x the configured 2313da14cebeSEric Cheng * B/W worth of packets to come in the SRS (to prevent unnecessary 2314da14cebeSEric Cheng * drops due to bursts) but only process the specified amount. 2315da14cebeSEric Cheng * 2316da14cebeSEric Cheng * A Link (primary NIC, VNIC, VLAN or aggr) can have 1 or more 2317da14cebeSEric Cheng * Rx rings (and corresponding SRSs) assigned to it. The SRS 2318da14cebeSEric Cheng * in turn can have softrings to do protocol level fanout or 2319da14cebeSEric Cheng * softrings to do S/W based fanout or both. In case the NIC 2320da14cebeSEric Cheng * has no Rx rings, we do S/W classification to respective SRS. 2321da14cebeSEric Cheng * The S/W classification rule is always setup and ready. This 2322da14cebeSEric Cheng * allows the MAC layer to reassign Rx rings whenever needed 2323da14cebeSEric Cheng * but packets still continue to flow via the default path and 2324da14cebeSEric Cheng * getting S/W classified to correct SRS. 2325da14cebeSEric Cheng * 2326da14cebeSEric Cheng * In other cases where a NIC or VNIC is plumbed, our goal is use 2327da14cebeSEric Cheng * H/W classifier and get two Rx ring assigned for the Link. One 2328da14cebeSEric Cheng * for TCP and one for UDP|SCTP. The respective SRS still do the 2329da14cebeSEric Cheng * polling on the Rx ring. For Link that is plumbed for IP, there 2330da14cebeSEric Cheng * is a TCP squeue which also does polling and can control the 2331da14cebeSEric Cheng * the Rx ring directly (where SRS is just pass through). For 2332da14cebeSEric Cheng * the following cases, the SRS does the polling underneath. 2333da14cebeSEric Cheng * 1) non IP based Links (Links which are not plumbed via ifconfig) 2334da14cebeSEric Cheng * and paths which have no IP squeues (UDP & SCTP) 2335da14cebeSEric Cheng * 2) If B/W control is specified on the Link 2336da14cebeSEric Cheng * 3) If S/W fanout is secified 2337da14cebeSEric Cheng * 2338da14cebeSEric Cheng * Note1: As of current implementation, we try to assign only 1 Rx 2339da14cebeSEric Cheng * ring per Link and more than 1 Rx ring for primary Link for 2340da14cebeSEric Cheng * H/W based fanout. We always create following softrings per SRS: 2341da14cebeSEric Cheng * 1) TCP softring which is polled by TCP squeue where possible 2342da14cebeSEric Cheng * (and also bypasses DLS) 2343da14cebeSEric Cheng * 2) UDP/SCTP based which bypasses DLS 2344da14cebeSEric Cheng * 3) OTH softring which goes via DLS (currently deal with IPv6 2345da14cebeSEric Cheng * and non TCP/UDP/SCTP for IPv4 packets). 2346da14cebeSEric Cheng * 2347da14cebeSEric Cheng * It is necessary to create 3 softrings since SRS has to poll 2348da14cebeSEric Cheng * the single Rx ring underneath and enforce any link level B/W 2349da14cebeSEric Cheng * control (we can't switch the Rx ring in poll mode just based 2350da14cebeSEric Cheng * on TCP squeue if the same Rx ring is sharing UDP and other 2351da14cebeSEric Cheng * traffic as well). Once polling is done and any Link level B/W 2352da14cebeSEric Cheng * control is specified, the packets are assigned to respective 2353da14cebeSEric Cheng * softring based on protocol. Since TCP has IP based squeue 2354da14cebeSEric Cheng * which benefits by polling, we separate TCP packets into 2355da14cebeSEric Cheng * its own softring which can be polled by IP squeue. We need 2356da14cebeSEric Cheng * to separate out UDP/SCTP to UDP softring since it can bypass 2357da14cebeSEric Cheng * the DLS layer which has heavy performance advanatges and we 2358da14cebeSEric Cheng * need a softring (OTH) for rest. 2359da14cebeSEric Cheng * 2360da14cebeSEric Cheng * ToDo: The 3 softrings for protocol are needed only till we can 2361da14cebeSEric Cheng * get rid of DLS from datapath, make IPv4 and IPv6 paths 2362da14cebeSEric Cheng * symmetric (deal with mac_header_info for v6 and polling for 2363da14cebeSEric Cheng * IPv4 TCP - ip_accept_tcp is IPv4 specific although squeues 2364da14cebeSEric Cheng * are generic), and bring SAP based classification to MAC layer 2365da14cebeSEric Cheng * 2366da14cebeSEric Cheng * H/W and S/W based fanout and multiple Rx rings per Link 2367da14cebeSEric Cheng * ------------------------------------------------------- 2368da14cebeSEric Cheng * 2369da14cebeSEric Cheng * In case, fanout is requested (or determined automatically based 2370da14cebeSEric Cheng * on Link speed and processor speed), we try to assign multiple 2371da14cebeSEric Cheng * Rx rings per Link with their respective SRS. In this case 2372da14cebeSEric Cheng * the NIC should be capable of fanning out incoming packets between 2373da14cebeSEric Cheng * the assigned Rx rings (H/W based fanout). All the SRS 2374da14cebeSEric Cheng * individually switch their Rx ring between interrupt and polling 2375da14cebeSEric Cheng * mode but share a common B/W control counter in case of Link 2376da14cebeSEric Cheng * level B/W is specified. 2377da14cebeSEric Cheng * 2378da14cebeSEric Cheng * If S/W based fanout is specified in lieu of H/W based fanout, 2379da14cebeSEric Cheng * the Link SRS creates the specified number of softrings for 2380da14cebeSEric Cheng * each protocol (TCP, UDP, OTH). Incoming packets are fanned 2381da14cebeSEric Cheng * out to the correct softring based on their protocol and 2382da14cebeSEric Cheng * protocol specific hash function. 2383da14cebeSEric Cheng * 2384da14cebeSEric Cheng * Primary and non primary MAC clients 2385da14cebeSEric Cheng * ----------------------------------- 2386da14cebeSEric Cheng * 2387da14cebeSEric Cheng * The NICs, VNICs, Vlans, and Aggrs are typically termed as Links 2388da14cebeSEric Cheng * and are a Layer 2 construct. 2389da14cebeSEric Cheng * 2390da14cebeSEric Cheng * Primary NIC: 2391da14cebeSEric Cheng * The Link that owns the primary MAC address and typically 2392da14cebeSEric Cheng * is used as the data NIC in non virtualized cases. As such 2393da14cebeSEric Cheng * H/W resources are preferntially given to primary NIC. As 2394da14cebeSEric Cheng * far as code is concerned, there is no difference in the 2395da14cebeSEric Cheng * primary NIC vs VNICs. They are all treated as Links. 2396da14cebeSEric Cheng * At the very first call to mac_unicast_add() we program the S/W 2397da14cebeSEric Cheng * classifier for the primary MAC address, get a soft ring set 2398da14cebeSEric Cheng * (and soft rings based on 'ip_soft_ring_cnt') 2399da14cebeSEric Cheng * and a Rx ring assigned for polling to get enabled. 2400da14cebeSEric Cheng * When IP get plumbed and negotiates polling, we can 2401da14cebeSEric Cheng * let squeue do the polling on TCP softring. 2402da14cebeSEric Cheng * 2403da14cebeSEric Cheng * VNICs: 2404da14cebeSEric Cheng * Same as any other Link. As long as the H/W resource assignments 2405da14cebeSEric Cheng * are equal, the data path and setup for all Links is same. 2406da14cebeSEric Cheng * 2407da14cebeSEric Cheng * Flows: 2408da14cebeSEric Cheng * Can be configured on Links. They have their own SRS and the 2409da14cebeSEric Cheng * S/W classifier is programmed appropriately based on the flow. 2410da14cebeSEric Cheng * The flows typically deal with layer 3 and above and 2411da14cebeSEric Cheng * creates a soft ring set specific to the flow. The receive 2412da14cebeSEric Cheng * side function is switched from mac_rx_srs_process to 2413da14cebeSEric Cheng * mac_rx_srs_subflow_process which first tries to assign the 2414da14cebeSEric Cheng * packet to appropriate flow SRS and failing which assigns it 2415da14cebeSEric Cheng * to link SRS. This allows us to avoid the layered approach 2416da14cebeSEric Cheng * which gets complex. 2417da14cebeSEric Cheng * 2418da14cebeSEric Cheng * By the time mac_datapath_setup() completes, we already have the 2419da14cebeSEric Cheng * soft rings set, Rx rings, soft rings, etc figured out and both H/W 2420da14cebeSEric Cheng * and S/W classifiers programmed. IP is not plumbed yet (and might 2421da14cebeSEric Cheng * never be for Virtual Machines guest OS path). When IP is plumbed 2422da14cebeSEric Cheng * (for both NIC and VNIC), we do a capability negotiation for polling 2423da14cebeSEric Cheng * and upcall functions etc. 2424da14cebeSEric Cheng * 2425da14cebeSEric Cheng * Rx ring Assignement NOTES 2426da14cebeSEric Cheng * ------------------------- 2427da14cebeSEric Cheng * 2428da14cebeSEric Cheng * For NICs which have only 1 Rx ring (we treat NICs with no Rx rings 2429da14cebeSEric Cheng * as NIC with a single default ring), we assign the only ring to 2430da14cebeSEric Cheng * primary Link as MAC_RX_HW_DEFAULT_RING. The primary Link SRS can do 2431da14cebeSEric Cheng * polling on it as long as it is the only link in use and we compare 2432da14cebeSEric Cheng * the MAC address for unicast packets before accepting an incoming 2433da14cebeSEric Cheng * packet (there is no need for S/W classification in this case). We 2434da14cebeSEric Cheng * disable polling on the only ring the moment 2nd link gets created 2435da14cebeSEric Cheng * (the polling remains enabled even though there are broadcast and 2436da14cebeSEric Cheng * multicast flows created). 2437da14cebeSEric Cheng * 2438da14cebeSEric Cheng * If the NIC has more than 1 Rx ring, we assign the default ring (the 2439da14cebeSEric Cheng * 1st ring) to deal with broadcast, multicast and traffic for other 2440da14cebeSEric Cheng * NICs which needs S/W classification. We assign the primary mac 2441da14cebeSEric Cheng * addresses to another ring by specifiying a classification rule for 2442da14cebeSEric Cheng * primary unicast MAC address to the selected ring. The primary Link 2443da14cebeSEric Cheng * (and its SRS) can continue to poll the assigned Rx ring at all times 2444da14cebeSEric Cheng * independantly. 2445da14cebeSEric Cheng * 2446da14cebeSEric Cheng * Right now we just assign MAC_RX_HW_DEFAULT_RING to note that it is 2447da14cebeSEric Cheng * primary NIC and later we will check to see how many Rx rings we 2448da14cebeSEric Cheng * have and can we get a non default Rx ring for the primary MAC. 2449da14cebeSEric Cheng * 2450da14cebeSEric Cheng * Note: In future, if no fanout is specified, we try to assign 2 Rx 2451da14cebeSEric Cheng * rings for the primary Link with the primary MAC address + TCP going 2452da14cebeSEric Cheng * to one ring and primary MAC address + UDP|SCTP going to other ring. 2453da14cebeSEric Cheng * Any remaining traffic for primary MAC address can go to the default 2454da14cebeSEric Cheng * Rx ring and get S/W classified. This way the respective SRSs don't 2455da14cebeSEric Cheng * need to do proto fanout and don't need to have softrings at all and 2456da14cebeSEric Cheng * can poll their respective Rx rings. 2457da14cebeSEric Cheng * 2458da14cebeSEric Cheng * As an optimization, when a new NIC or VNIC is created, we can get 2459da14cebeSEric Cheng * only one Rx ring and make it a TCP specific Rx ring and use the 2460da14cebeSEric Cheng * H/W default Rx ring for the rest (this Rx ring is never polled). 2461da14cebeSEric Cheng */ 2462da14cebeSEric Cheng int 2463da14cebeSEric Cheng mac_datapath_setup(mac_client_impl_t *mcip, flow_entry_t *flent, 2464da14cebeSEric Cheng uint32_t link_type) 2465da14cebeSEric Cheng { 2466da14cebeSEric Cheng mac_impl_t *mip = mcip->mci_mip; 2467da14cebeSEric Cheng mac_group_t *group = NULL; 2468da14cebeSEric Cheng mac_group_t *default_group; 2469da14cebeSEric Cheng int err; 2470da14cebeSEric Cheng uint8_t *mac_addr; 2471da14cebeSEric Cheng mac_rx_group_reserve_type_t rtype = MAC_RX_RESERVE_NONDEFAULT; 2472da14cebeSEric Cheng mac_group_state_t next_state; 2473da14cebeSEric Cheng mac_client_impl_t *group_only_mcip; 2474da14cebeSEric Cheng 2475da14cebeSEric Cheng ASSERT(MAC_PERIM_HELD((mac_handle_t)mip)); 2476da14cebeSEric Cheng 2477da14cebeSEric Cheng switch (link_type) { 2478da14cebeSEric Cheng case SRST_FLOW: 2479da14cebeSEric Cheng mac_srs_group_setup(mcip, flent, NULL, link_type); 2480da14cebeSEric Cheng return (0); 2481da14cebeSEric Cheng 2482da14cebeSEric Cheng case SRST_LINK: 2483da14cebeSEric Cheng mac_addr = flent->fe_flow_desc.fd_dst_mac; 2484da14cebeSEric Cheng 2485da14cebeSEric Cheng /* Check if we need to reserve the default group */ 2486da14cebeSEric Cheng if (flent->fe_type & FLOW_PRIMARY_MAC) 2487da14cebeSEric Cheng rtype = MAC_RX_RESERVE_DEFAULT; 2488da14cebeSEric Cheng 248908ac1c49SNicolas Droux if ((mcip->mci_state_flags & MCIS_NO_HWRINGS) == 0) { 2490da14cebeSEric Cheng /* 2491da14cebeSEric Cheng * Check to see if we can get an exclusive group for 2492da14cebeSEric Cheng * this mac address or if there already exists a 2493da14cebeSEric Cheng * group that has this mac address (case of VLANs). 2494da14cebeSEric Cheng * If no groups are available, use the default group. 2495da14cebeSEric Cheng */ 2496da14cebeSEric Cheng group = mac_reserve_rx_group(mcip, mac_addr, rtype); 2497da14cebeSEric Cheng } 2498da14cebeSEric Cheng 2499da14cebeSEric Cheng if (group == NULL) { 250008ac1c49SNicolas Droux if ((mcip->mci_state_flags & MCIS_REQ_HWRINGS) != 0) 2501da14cebeSEric Cheng return (ENOSPC); 2502da14cebeSEric Cheng group = &mip->mi_rx_groups[0]; 2503da14cebeSEric Cheng } 2504da14cebeSEric Cheng 2505da14cebeSEric Cheng /* 2506da14cebeSEric Cheng * Some NICs don't support any Rx rings, so there may not 2507da14cebeSEric Cheng * even be a default group. 2508da14cebeSEric Cheng */ 2509da14cebeSEric Cheng if (group != NULL) { 2510da14cebeSEric Cheng flent->fe_rx_ring_group = group; 2511da14cebeSEric Cheng /* 2512da14cebeSEric Cheng * Add the client to the group. This could cause 2513da14cebeSEric Cheng * either this group to move to the shared state or 2514da14cebeSEric Cheng * cause the default group to move to the shared state. 2515da14cebeSEric Cheng * The actions on this group are done here, while the 2516da14cebeSEric Cheng * actions on the default group are postponed to 2517da14cebeSEric Cheng * the end of this function. 2518da14cebeSEric Cheng */ 2519da14cebeSEric Cheng mac_rx_group_add_client(group, mcip); 2520da14cebeSEric Cheng next_state = mac_rx_group_next_state(group, 2521da14cebeSEric Cheng &group_only_mcip); 2522da14cebeSEric Cheng 2523da14cebeSEric Cheng ASSERT((next_state == MAC_GROUP_STATE_RESERVED && 2524da14cebeSEric Cheng mcip == group_only_mcip) || 2525da14cebeSEric Cheng (next_state == MAC_GROUP_STATE_SHARED && 2526da14cebeSEric Cheng group_only_mcip == NULL)); 2527da14cebeSEric Cheng 2528da14cebeSEric Cheng mac_set_rx_group_state(group, next_state); 2529da14cebeSEric Cheng } 2530da14cebeSEric Cheng 2531da14cebeSEric Cheng /* 2532da14cebeSEric Cheng * Setup the Rx and Tx SRSes. If we got a pristine group 2533da14cebeSEric Cheng * exclusively above, mac_srs_group_setup would simply create 2534da14cebeSEric Cheng * the required SRSes. If we ended up sharing a previously 2535da14cebeSEric Cheng * reserved group, mac_srs_group_setup would also dismantle the 2536da14cebeSEric Cheng * SRSes of the previously exclusive group 2537da14cebeSEric Cheng */ 2538da14cebeSEric Cheng mac_srs_group_setup(mcip, flent, group, link_type); 2539da14cebeSEric Cheng 2540da14cebeSEric Cheng /* Program the S/W Classifer */ 2541da14cebeSEric Cheng if ((err = mac_flow_add(mip->mi_flow_tab, flent)) != 0) 2542da14cebeSEric Cheng goto setup_failed; 2543da14cebeSEric Cheng 2544da14cebeSEric Cheng /* Program the H/W Classifier */ 254508ac1c49SNicolas Droux if ((err = mac_add_macaddr(mip, group, mac_addr, 254608ac1c49SNicolas Droux (mcip->mci_state_flags & MCIS_UNICAST_HW) != 0)) != 0) 2547da14cebeSEric Cheng goto setup_failed; 2548da14cebeSEric Cheng mcip->mci_unicast = mac_find_macaddr(mip, mac_addr); 2549da14cebeSEric Cheng ASSERT(mcip->mci_unicast != NULL); 2550da14cebeSEric Cheng break; 2551da14cebeSEric Cheng 2552da14cebeSEric Cheng default: 2553da14cebeSEric Cheng ASSERT(B_FALSE); 2554da14cebeSEric Cheng break; 2555da14cebeSEric Cheng } 2556da14cebeSEric Cheng 2557da14cebeSEric Cheng /* 2558da14cebeSEric Cheng * All broadcast and multicast traffic is received only on the default 2559da14cebeSEric Cheng * group. If we have setup the datapath for a non-default group above 2560da14cebeSEric Cheng * then move the default group to shared state to allow distribution of 2561da14cebeSEric Cheng * incoming broadcast traffic to the other groups and dismantle the 2562da14cebeSEric Cheng * SRSes over the default group. 2563da14cebeSEric Cheng */ 2564da14cebeSEric Cheng if (group != NULL) { 2565da14cebeSEric Cheng if (group != mip->mi_rx_groups) { 2566da14cebeSEric Cheng default_group = mip->mi_rx_groups; 2567da14cebeSEric Cheng if (default_group->mrg_state == 2568da14cebeSEric Cheng MAC_GROUP_STATE_RESERVED) { 2569da14cebeSEric Cheng group_only_mcip = MAC_RX_GROUP_ONLY_CLIENT( 2570da14cebeSEric Cheng default_group); 2571da14cebeSEric Cheng ASSERT(group_only_mcip != NULL && 2572da14cebeSEric Cheng mip->mi_nactiveclients > 1); 2573da14cebeSEric Cheng 2574da14cebeSEric Cheng mac_set_rx_group_state(default_group, 2575da14cebeSEric Cheng MAC_GROUP_STATE_SHARED); 2576da14cebeSEric Cheng mac_srs_group_setup(group_only_mcip, 2577da14cebeSEric Cheng group_only_mcip->mci_flent, 2578da14cebeSEric Cheng default_group, SRST_LINK); 2579da14cebeSEric Cheng } 2580da14cebeSEric Cheng ASSERT(default_group->mrg_state == 2581da14cebeSEric Cheng MAC_GROUP_STATE_SHARED); 2582da14cebeSEric Cheng } 2583da14cebeSEric Cheng /* 2584da14cebeSEric Cheng * If we get an exclusive group for a VLAN MAC client we 2585da14cebeSEric Cheng * need to take the s/w path to make the additional check for 2586da14cebeSEric Cheng * the vid. Disable polling and set it to s/w classification. 2587da14cebeSEric Cheng */ 2588da14cebeSEric Cheng if (group->mrg_state == MAC_GROUP_STATE_RESERVED && 2589da14cebeSEric Cheng i_mac_flow_vid(mcip->mci_flent) != VLAN_ID_NONE) { 2590da14cebeSEric Cheng mac_rx_switch_grp_to_sw(group); 2591da14cebeSEric Cheng } 2592da14cebeSEric Cheng } 2593da14cebeSEric Cheng return (0); 2594da14cebeSEric Cheng 2595da14cebeSEric Cheng setup_failed: 2596da14cebeSEric Cheng mac_datapath_teardown(mcip, flent, link_type); 2597da14cebeSEric Cheng return (err); 2598da14cebeSEric Cheng } 2599da14cebeSEric Cheng 2600da14cebeSEric Cheng void 2601da14cebeSEric Cheng mac_datapath_teardown(mac_client_impl_t *mcip, flow_entry_t *flent, 2602da14cebeSEric Cheng uint32_t link_type) 2603da14cebeSEric Cheng { 2604da14cebeSEric Cheng mac_impl_t *mip = mcip->mci_mip; 2605da14cebeSEric Cheng mac_group_t *group = NULL; 2606da14cebeSEric Cheng mac_client_impl_t *grp_only_mcip; 2607da14cebeSEric Cheng flow_entry_t *group_only_flent; 2608da14cebeSEric Cheng mac_group_t *default_group; 2609da14cebeSEric Cheng boolean_t check_default_group = B_FALSE; 2610da14cebeSEric Cheng mac_group_state_t next_state; 2611da14cebeSEric Cheng 2612da14cebeSEric Cheng ASSERT(MAC_PERIM_HELD((mac_handle_t)mip)); 2613da14cebeSEric Cheng 2614da14cebeSEric Cheng switch (link_type) { 2615da14cebeSEric Cheng case SRST_FLOW: 2616da14cebeSEric Cheng mac_srs_group_teardown(mcip, flent, SRST_FLOW); 2617da14cebeSEric Cheng return; 2618da14cebeSEric Cheng 2619da14cebeSEric Cheng case SRST_LINK: 2620da14cebeSEric Cheng /* Stop sending packets */ 2621da14cebeSEric Cheng mac_tx_client_block(mcip); 2622da14cebeSEric Cheng 2623da14cebeSEric Cheng /* Stop the packets coming from the H/W */ 2624da14cebeSEric Cheng if (mcip->mci_unicast != NULL) { 2625da14cebeSEric Cheng int err; 2626da14cebeSEric Cheng err = mac_remove_macaddr(mcip->mci_unicast); 2627da14cebeSEric Cheng if (err != 0) { 2628da14cebeSEric Cheng cmn_err(CE_WARN, "%s: failed to remove a MAC" 2629da14cebeSEric Cheng " address because of error 0x%x", 2630da14cebeSEric Cheng mip->mi_name, err); 2631da14cebeSEric Cheng } 2632da14cebeSEric Cheng mcip->mci_unicast = NULL; 2633da14cebeSEric Cheng } 2634da14cebeSEric Cheng 2635da14cebeSEric Cheng /* Stop the packets coming from the S/W classifier */ 2636da14cebeSEric Cheng mac_flow_remove(mip->mi_flow_tab, flent, B_FALSE); 2637da14cebeSEric Cheng mac_flow_wait(flent, FLOW_DRIVER_UPCALL); 2638da14cebeSEric Cheng 2639da14cebeSEric Cheng /* Now quiesce and destroy all SRS and soft rings */ 2640da14cebeSEric Cheng mac_srs_group_teardown(mcip, flent, SRST_LINK); 2641da14cebeSEric Cheng ASSERT((mcip->mci_flent == flent) && 2642da14cebeSEric Cheng (flent->fe_next == NULL)); 2643da14cebeSEric Cheng 2644da14cebeSEric Cheng /* 2645da14cebeSEric Cheng * Release our hold on the group as well. We need 2646da14cebeSEric Cheng * to check if the shared group has only one client 2647da14cebeSEric Cheng * left who can use it exclusively. Also, if we 2648da14cebeSEric Cheng * were the last client, release the group. 2649da14cebeSEric Cheng */ 2650da14cebeSEric Cheng group = flent->fe_rx_ring_group; 2651da14cebeSEric Cheng if (group != NULL) { 2652da14cebeSEric Cheng mac_rx_group_remove_client(group, mcip); 2653da14cebeSEric Cheng next_state = mac_rx_group_next_state(group, 2654da14cebeSEric Cheng &grp_only_mcip); 2655da14cebeSEric Cheng if (next_state == MAC_GROUP_STATE_RESERVED) { 2656da14cebeSEric Cheng /* 2657da14cebeSEric Cheng * Only one client left on this RX group. 2658da14cebeSEric Cheng */ 2659da14cebeSEric Cheng ASSERT(grp_only_mcip != NULL); 2660da14cebeSEric Cheng mac_set_rx_group_state(group, 2661da14cebeSEric Cheng MAC_GROUP_STATE_RESERVED); 2662da14cebeSEric Cheng group_only_flent = grp_only_mcip->mci_flent; 2663da14cebeSEric Cheng 2664da14cebeSEric Cheng /* 2665da14cebeSEric Cheng * The only remaining client has exclusive 2666da14cebeSEric Cheng * access on the group. Allow it to 2667da14cebeSEric Cheng * dynamically poll the H/W rings etc. 2668da14cebeSEric Cheng */ 2669da14cebeSEric Cheng mac_srs_group_setup(grp_only_mcip, 2670da14cebeSEric Cheng group_only_flent, group, SRST_LINK); 2671da14cebeSEric Cheng mac_rx_group_unmark(group, MR_INCIPIENT); 2672da14cebeSEric Cheng } else if (next_state == MAC_GROUP_STATE_REGISTERED) { 2673da14cebeSEric Cheng /* 2674da14cebeSEric Cheng * This is a non-default group being freed up. 2675da14cebeSEric Cheng * We need to reevaluate the default group 2676da14cebeSEric Cheng * to see if the primary client can get 2677da14cebeSEric Cheng * exclusive access to the default group. 2678da14cebeSEric Cheng */ 2679da14cebeSEric Cheng ASSERT(group != mip->mi_rx_groups); 2680da14cebeSEric Cheng mac_release_rx_group(mcip, group); 2681da14cebeSEric Cheng mac_set_rx_group_state(group, 2682da14cebeSEric Cheng MAC_GROUP_STATE_REGISTERED); 2683da14cebeSEric Cheng check_default_group = B_TRUE; 2684da14cebeSEric Cheng } else { 2685da14cebeSEric Cheng ASSERT(next_state == MAC_GROUP_STATE_SHARED); 2686da14cebeSEric Cheng mac_set_rx_group_state(group, 2687da14cebeSEric Cheng MAC_GROUP_STATE_SHARED); 2688da14cebeSEric Cheng mac_rx_group_unmark(group, MR_CONDEMNED); 2689da14cebeSEric Cheng } 2690da14cebeSEric Cheng flent->fe_rx_ring_group = NULL; 2691da14cebeSEric Cheng } 2692da14cebeSEric Cheng break; 2693da14cebeSEric Cheng default: 2694da14cebeSEric Cheng ASSERT(B_FALSE); 2695da14cebeSEric Cheng break; 2696da14cebeSEric Cheng } 2697da14cebeSEric Cheng 2698da14cebeSEric Cheng /* 2699da14cebeSEric Cheng * The mac client using the default group gets exclusive access to the 2700da14cebeSEric Cheng * default group if and only if it is the sole client on the entire 2701da14cebeSEric Cheng * mip. If so set the group state to reserved, and set up the SRSes 2702da14cebeSEric Cheng * over the default group. 2703da14cebeSEric Cheng */ 2704da14cebeSEric Cheng if (check_default_group) { 2705da14cebeSEric Cheng default_group = mip->mi_rx_groups; 2706da14cebeSEric Cheng ASSERT(default_group->mrg_state == MAC_GROUP_STATE_SHARED); 2707da14cebeSEric Cheng next_state = mac_rx_group_next_state(default_group, 2708da14cebeSEric Cheng &grp_only_mcip); 2709da14cebeSEric Cheng if (next_state == MAC_GROUP_STATE_RESERVED) { 2710da14cebeSEric Cheng ASSERT(grp_only_mcip != NULL && 2711da14cebeSEric Cheng mip->mi_nactiveclients == 1); 2712da14cebeSEric Cheng mac_set_rx_group_state(default_group, 2713da14cebeSEric Cheng MAC_GROUP_STATE_RESERVED); 2714da14cebeSEric Cheng mac_srs_group_setup(grp_only_mcip, 2715da14cebeSEric Cheng grp_only_mcip->mci_flent, 2716da14cebeSEric Cheng default_group, SRST_LINK); 2717*ae6aa22aSVenugopal Iyer mac_rx_group_unmark(default_group, MR_INCIPIENT); 2718da14cebeSEric Cheng } 2719da14cebeSEric Cheng } 2720da14cebeSEric Cheng } 2721da14cebeSEric Cheng 2722da14cebeSEric Cheng /* DATAPATH TEAR DOWN ROUTINES (SRS and FANOUT teardown) */ 2723da14cebeSEric Cheng 2724da14cebeSEric Cheng static void 2725da14cebeSEric Cheng mac_srs_fanout_list_free(mac_soft_ring_set_t *mac_srs) 2726da14cebeSEric Cheng { 2727da14cebeSEric Cheng ASSERT(mac_srs->srs_tcp_soft_rings != NULL); 2728da14cebeSEric Cheng kmem_free(mac_srs->srs_tcp_soft_rings, 2729da14cebeSEric Cheng sizeof (mac_soft_ring_t *) * MAX_SR_FANOUT); 2730da14cebeSEric Cheng mac_srs->srs_tcp_soft_rings = NULL; 2731da14cebeSEric Cheng ASSERT(mac_srs->srs_udp_soft_rings != NULL); 2732da14cebeSEric Cheng kmem_free(mac_srs->srs_udp_soft_rings, 2733da14cebeSEric Cheng sizeof (mac_soft_ring_t *) * MAX_SR_FANOUT); 2734da14cebeSEric Cheng mac_srs->srs_udp_soft_rings = NULL; 2735da14cebeSEric Cheng ASSERT(mac_srs->srs_oth_soft_rings != NULL); 2736da14cebeSEric Cheng kmem_free(mac_srs->srs_oth_soft_rings, 2737da14cebeSEric Cheng sizeof (mac_soft_ring_t *) * MAX_SR_FANOUT); 2738da14cebeSEric Cheng mac_srs->srs_oth_soft_rings = NULL; 2739da14cebeSEric Cheng } 2740da14cebeSEric Cheng 2741da14cebeSEric Cheng /* 2742da14cebeSEric Cheng * An RX SRS is attached to at most one mac_ring. 2743da14cebeSEric Cheng * A TX SRS has no rings. 2744da14cebeSEric Cheng */ 2745da14cebeSEric Cheng static void 2746da14cebeSEric Cheng mac_srs_ring_free(mac_soft_ring_set_t *mac_srs) 2747da14cebeSEric Cheng { 2748da14cebeSEric Cheng mac_client_impl_t *mcip; 2749da14cebeSEric Cheng mac_ring_t *ring; 2750da14cebeSEric Cheng flow_entry_t *flent; 2751da14cebeSEric Cheng 2752da14cebeSEric Cheng ring = mac_srs->srs_ring; 2753da14cebeSEric Cheng if (mac_srs->srs_type & SRST_TX) { 2754da14cebeSEric Cheng ASSERT(ring == NULL); 2755da14cebeSEric Cheng return; 2756da14cebeSEric Cheng } 2757da14cebeSEric Cheng 2758da14cebeSEric Cheng if (ring == NULL) 2759da14cebeSEric Cheng return; 2760da14cebeSEric Cheng 2761da14cebeSEric Cheng /* 2762da14cebeSEric Cheng * Broadcast flows don't have a client impl association, but they 2763da14cebeSEric Cheng * use only soft rings. 2764da14cebeSEric Cheng */ 2765da14cebeSEric Cheng flent = mac_srs->srs_flent; 2766da14cebeSEric Cheng mcip = flent->fe_mcip; 2767da14cebeSEric Cheng ASSERT(mcip != NULL); 2768da14cebeSEric Cheng 2769da14cebeSEric Cheng ring->mr_classify_type = MAC_NO_CLASSIFIER; 2770da14cebeSEric Cheng ring->mr_srs = NULL; 2771da14cebeSEric Cheng } 2772da14cebeSEric Cheng 2773da14cebeSEric Cheng /* 2774da14cebeSEric Cheng * Physical unlink and free of the data structures happen below. This is 2775da14cebeSEric Cheng * driven from mac_flow_destroy(), on the last refrele of a flow. 2776da14cebeSEric Cheng * 2777da14cebeSEric Cheng * Assumes Rx srs is 1-1 mapped with an ring. 2778da14cebeSEric Cheng */ 2779da14cebeSEric Cheng void 2780da14cebeSEric Cheng mac_srs_free(mac_soft_ring_set_t *mac_srs) 2781da14cebeSEric Cheng { 2782da14cebeSEric Cheng ASSERT(mac_srs->srs_mcip == NULL || 2783da14cebeSEric Cheng MAC_PERIM_HELD((mac_handle_t)mac_srs->srs_mcip->mci_mip)); 2784da14cebeSEric Cheng ASSERT((mac_srs->srs_state & (SRS_CONDEMNED | SRS_CONDEMNED_DONE | 2785da14cebeSEric Cheng SRS_PROC | SRS_PROC_FAST)) == (SRS_CONDEMNED | SRS_CONDEMNED_DONE)); 2786da14cebeSEric Cheng 2787da14cebeSEric Cheng mac_pkt_drop(NULL, NULL, mac_srs->srs_first, B_FALSE); 2788da14cebeSEric Cheng mac_srs_ring_free(mac_srs); 2789da14cebeSEric Cheng mac_srs_soft_rings_free(mac_srs, B_TRUE); 2790da14cebeSEric Cheng mac_srs_fanout_list_free(mac_srs); 2791da14cebeSEric Cheng 2792da14cebeSEric Cheng mac_srs->srs_bw = NULL; 2793da14cebeSEric Cheng kmem_cache_free(mac_srs_cache, mac_srs); 2794da14cebeSEric Cheng } 2795da14cebeSEric Cheng 2796da14cebeSEric Cheng static void 2797da14cebeSEric Cheng mac_srs_soft_rings_quiesce(mac_soft_ring_set_t *mac_srs, uint_t s_ring_flag) 2798da14cebeSEric Cheng { 2799da14cebeSEric Cheng mac_soft_ring_t *softring; 2800da14cebeSEric Cheng 2801da14cebeSEric Cheng ASSERT(MUTEX_HELD(&mac_srs->srs_lock)); 2802da14cebeSEric Cheng 2803da14cebeSEric Cheng mac_srs_soft_rings_signal(mac_srs, s_ring_flag); 2804da14cebeSEric Cheng if (s_ring_flag == S_RING_CONDEMNED) { 2805da14cebeSEric Cheng while (mac_srs->srs_soft_ring_condemned_count != 2806da14cebeSEric Cheng mac_srs->srs_soft_ring_count) 2807da14cebeSEric Cheng cv_wait(&mac_srs->srs_async, &mac_srs->srs_lock); 2808da14cebeSEric Cheng } else { 2809da14cebeSEric Cheng while (mac_srs->srs_soft_ring_quiesced_count != 2810da14cebeSEric Cheng mac_srs->srs_soft_ring_count) 2811da14cebeSEric Cheng cv_wait(&mac_srs->srs_async, &mac_srs->srs_lock); 2812da14cebeSEric Cheng } 2813da14cebeSEric Cheng mutex_exit(&mac_srs->srs_lock); 2814da14cebeSEric Cheng 2815da14cebeSEric Cheng for (softring = mac_srs->srs_soft_ring_head; softring != NULL; 2816da14cebeSEric Cheng softring = softring->s_ring_next) 2817da14cebeSEric Cheng (void) untimeout(softring->s_ring_tid); 2818da14cebeSEric Cheng 2819da14cebeSEric Cheng (void) untimeout(mac_srs->srs_tid); 2820da14cebeSEric Cheng 2821da14cebeSEric Cheng mutex_enter(&mac_srs->srs_lock); 2822da14cebeSEric Cheng } 2823da14cebeSEric Cheng 2824da14cebeSEric Cheng /* 2825da14cebeSEric Cheng * The block comment above mac_rx_classify_flow_state_change explains the 2826da14cebeSEric Cheng * background. At this point upcalls from the driver (both hardware classified 2827da14cebeSEric Cheng * and software classified) have been cut off. We now need to quiesce the 2828da14cebeSEric Cheng * SRS worker, poll, and softring threads. The SRS worker thread serves as 2829da14cebeSEric Cheng * the master controller. The steps involved are described below in the function 2830da14cebeSEric Cheng */ 2831da14cebeSEric Cheng void 2832da14cebeSEric Cheng mac_srs_worker_quiesce(mac_soft_ring_set_t *mac_srs) 2833da14cebeSEric Cheng { 2834da14cebeSEric Cheng uint_t s_ring_flag; 2835da14cebeSEric Cheng uint_t srs_poll_wait_flag; 2836da14cebeSEric Cheng 2837da14cebeSEric Cheng ASSERT(MUTEX_HELD(&mac_srs->srs_lock)); 2838da14cebeSEric Cheng ASSERT(mac_srs->srs_state & (SRS_CONDEMNED | SRS_QUIESCE)); 2839da14cebeSEric Cheng 2840da14cebeSEric Cheng if (mac_srs->srs_state & SRS_CONDEMNED) { 2841da14cebeSEric Cheng s_ring_flag = S_RING_CONDEMNED; 2842da14cebeSEric Cheng srs_poll_wait_flag = SRS_POLL_THR_EXITED; 2843da14cebeSEric Cheng } else { 2844da14cebeSEric Cheng s_ring_flag = S_RING_QUIESCE; 2845da14cebeSEric Cheng srs_poll_wait_flag = SRS_POLL_THR_QUIESCED; 2846da14cebeSEric Cheng } 2847da14cebeSEric Cheng 2848da14cebeSEric Cheng /* 2849da14cebeSEric Cheng * In the case of Rx SRS wait till the poll thread is done. 2850da14cebeSEric Cheng */ 2851da14cebeSEric Cheng if ((mac_srs->srs_type & SRST_TX) == 0 && 2852da14cebeSEric Cheng mac_srs->srs_poll_thr != NULL) { 2853da14cebeSEric Cheng while (!(mac_srs->srs_state & srs_poll_wait_flag)) 2854da14cebeSEric Cheng cv_wait(&mac_srs->srs_async, &mac_srs->srs_lock); 2855da14cebeSEric Cheng 2856da14cebeSEric Cheng /* 2857da14cebeSEric Cheng * Turn off polling as part of the quiesce operation. 2858da14cebeSEric Cheng */ 2859da14cebeSEric Cheng MAC_SRS_POLLING_OFF(mac_srs); 2860da14cebeSEric Cheng mac_srs->srs_state &= ~(SRS_POLLING | SRS_GET_PKTS); 2861da14cebeSEric Cheng } 2862da14cebeSEric Cheng 2863da14cebeSEric Cheng /* 2864da14cebeSEric Cheng * Then signal the soft ring worker threads to quiesce or quit 2865da14cebeSEric Cheng * as needed and then wait till that happens. 2866da14cebeSEric Cheng */ 2867da14cebeSEric Cheng mac_srs_soft_rings_quiesce(mac_srs, s_ring_flag); 2868da14cebeSEric Cheng 2869da14cebeSEric Cheng if (mac_srs->srs_state & SRS_CONDEMNED) 2870da14cebeSEric Cheng mac_srs->srs_state |= (SRS_QUIESCE_DONE | SRS_CONDEMNED_DONE); 2871da14cebeSEric Cheng else 2872da14cebeSEric Cheng mac_srs->srs_state |= SRS_QUIESCE_DONE; 2873da14cebeSEric Cheng cv_signal(&mac_srs->srs_quiesce_done_cv); 2874da14cebeSEric Cheng } 2875da14cebeSEric Cheng 2876da14cebeSEric Cheng /* 2877da14cebeSEric Cheng * Signal an SRS to start a temporary quiesce, or permanent removal, or restart 2878da14cebeSEric Cheng * a quiesced SRS by setting the appropriate flags and signaling the SRS worker 2879da14cebeSEric Cheng * or poll thread. This function is internal to the quiescing logic and is 2880da14cebeSEric Cheng * called internally from the SRS quiesce or flow quiesce or client quiesce 2881da14cebeSEric Cheng * higher level functions. 2882da14cebeSEric Cheng */ 2883da14cebeSEric Cheng void 2884da14cebeSEric Cheng mac_srs_signal(mac_soft_ring_set_t *mac_srs, uint_t srs_flag) 2885da14cebeSEric Cheng { 2886da14cebeSEric Cheng mac_ring_t *ring; 2887da14cebeSEric Cheng 2888da14cebeSEric Cheng ring = mac_srs->srs_ring; 2889da14cebeSEric Cheng ASSERT(ring == NULL || ring->mr_refcnt == 0); 2890da14cebeSEric Cheng 2891da14cebeSEric Cheng if (srs_flag == SRS_CONDEMNED) { 2892da14cebeSEric Cheng /* 2893da14cebeSEric Cheng * The SRS is going away. We need to unbind the SRS and SR 2894da14cebeSEric Cheng * threads before removing from the global SRS list. Otherwise 2895da14cebeSEric Cheng * there is a small window where the cpu reconfig callbacks 2896da14cebeSEric Cheng * may miss the SRS in the list walk and DR could fail since 2897da14cebeSEric Cheng * there are still bound threads. 2898da14cebeSEric Cheng */ 2899da14cebeSEric Cheng mac_srs_threads_unbind(mac_srs); 2900da14cebeSEric Cheng mac_srs_remove_glist(mac_srs); 2901da14cebeSEric Cheng } 2902da14cebeSEric Cheng /* 2903da14cebeSEric Cheng * Wakeup the SRS worker and poll threads. 2904da14cebeSEric Cheng */ 2905da14cebeSEric Cheng mutex_enter(&mac_srs->srs_lock); 2906da14cebeSEric Cheng mac_srs->srs_state |= srs_flag; 2907da14cebeSEric Cheng cv_signal(&mac_srs->srs_async); 2908da14cebeSEric Cheng cv_signal(&mac_srs->srs_cv); 2909da14cebeSEric Cheng mutex_exit(&mac_srs->srs_lock); 2910da14cebeSEric Cheng } 2911da14cebeSEric Cheng 2912da14cebeSEric Cheng /* 2913da14cebeSEric Cheng * In the Rx side, the quiescing is done bottom up. After the Rx upcalls 2914da14cebeSEric Cheng * from the driver are done, then the Rx SRS is quiesced and only then can 2915da14cebeSEric Cheng * we signal the soft rings. Thus this function can't be called arbitrarily 2916da14cebeSEric Cheng * without satisfying the prerequisites. On the Tx side, the threads from 2917da14cebeSEric Cheng * top need to quiesced, then the Tx SRS and only then can we signal the 2918da14cebeSEric Cheng * Tx soft rings. 2919da14cebeSEric Cheng */ 2920da14cebeSEric Cheng static void 2921da14cebeSEric Cheng mac_srs_soft_rings_signal(mac_soft_ring_set_t *mac_srs, uint_t sr_flag) 2922da14cebeSEric Cheng { 2923da14cebeSEric Cheng mac_soft_ring_t *softring; 2924da14cebeSEric Cheng 2925da14cebeSEric Cheng for (softring = mac_srs->srs_soft_ring_head; softring != NULL; 2926da14cebeSEric Cheng softring = softring->s_ring_next) 2927da14cebeSEric Cheng mac_soft_ring_signal(softring, sr_flag); 2928da14cebeSEric Cheng } 2929da14cebeSEric Cheng 2930da14cebeSEric Cheng /* 2931da14cebeSEric Cheng * The block comment above mac_rx_classify_flow_state_change explains the 2932da14cebeSEric Cheng * background. At this point the SRS is quiesced and we need to restart the 2933da14cebeSEric Cheng * SRS worker, poll, and softring threads. The SRS worker thread serves as 2934da14cebeSEric Cheng * the master controller. The steps involved are described below in the function 2935da14cebeSEric Cheng */ 2936da14cebeSEric Cheng void 2937da14cebeSEric Cheng mac_srs_worker_restart(mac_soft_ring_set_t *mac_srs) 2938da14cebeSEric Cheng { 2939da14cebeSEric Cheng boolean_t iam_rx_srs; 2940da14cebeSEric Cheng mac_soft_ring_t *softring; 2941da14cebeSEric Cheng 2942da14cebeSEric Cheng ASSERT(MUTEX_HELD(&mac_srs->srs_lock)); 2943da14cebeSEric Cheng if ((mac_srs->srs_type & SRST_TX) != 0) { 2944da14cebeSEric Cheng iam_rx_srs = B_FALSE; 2945da14cebeSEric Cheng ASSERT((mac_srs->srs_state & 2946da14cebeSEric Cheng (SRS_POLL_THR_QUIESCED | SRS_QUIESCE_DONE | SRS_QUIESCE)) == 2947da14cebeSEric Cheng (SRS_QUIESCE_DONE | SRS_QUIESCE)); 2948da14cebeSEric Cheng } else { 2949da14cebeSEric Cheng iam_rx_srs = B_TRUE; 2950da14cebeSEric Cheng ASSERT((mac_srs->srs_state & 2951da14cebeSEric Cheng (SRS_QUIESCE_DONE | SRS_QUIESCE)) == 2952da14cebeSEric Cheng (SRS_QUIESCE_DONE | SRS_QUIESCE)); 2953da14cebeSEric Cheng if (mac_srs->srs_poll_thr != NULL) { 2954da14cebeSEric Cheng ASSERT((mac_srs->srs_state & SRS_POLL_THR_QUIESCED) == 2955da14cebeSEric Cheng SRS_POLL_THR_QUIESCED); 2956da14cebeSEric Cheng } 2957da14cebeSEric Cheng } 2958da14cebeSEric Cheng 2959da14cebeSEric Cheng /* 2960da14cebeSEric Cheng * Signal any quiesced soft ring workers to restart and wait for the 2961da14cebeSEric Cheng * soft ring down count to come down to zero. 2962da14cebeSEric Cheng */ 2963da14cebeSEric Cheng if (mac_srs->srs_soft_ring_quiesced_count != 0) { 2964da14cebeSEric Cheng for (softring = mac_srs->srs_soft_ring_head; softring != NULL; 2965da14cebeSEric Cheng softring = softring->s_ring_next) { 2966da14cebeSEric Cheng if (!(softring->s_ring_state & S_RING_QUIESCE)) 2967da14cebeSEric Cheng continue; 2968da14cebeSEric Cheng mac_soft_ring_signal(softring, S_RING_RESTART); 2969da14cebeSEric Cheng } 2970da14cebeSEric Cheng while (mac_srs->srs_soft_ring_quiesced_count != 0) 2971da14cebeSEric Cheng cv_wait(&mac_srs->srs_async, &mac_srs->srs_lock); 2972da14cebeSEric Cheng } 2973da14cebeSEric Cheng 2974da14cebeSEric Cheng mac_srs->srs_state &= ~(SRS_QUIESCE_DONE | SRS_QUIESCE | SRS_RESTART); 2975da14cebeSEric Cheng if (iam_rx_srs && mac_srs->srs_poll_thr != NULL) { 2976da14cebeSEric Cheng /* 2977da14cebeSEric Cheng * Signal the poll thread and ask it to restart. Wait till it 2978da14cebeSEric Cheng * actually restarts and the SRS_POLL_THR_QUIESCED flag gets 2979da14cebeSEric Cheng * cleared. 2980da14cebeSEric Cheng */ 2981da14cebeSEric Cheng mac_srs->srs_state |= SRS_POLL_THR_RESTART; 2982da14cebeSEric Cheng cv_signal(&mac_srs->srs_cv); 2983da14cebeSEric Cheng while (mac_srs->srs_state & SRS_POLL_THR_QUIESCED) 2984da14cebeSEric Cheng cv_wait(&mac_srs->srs_async, &mac_srs->srs_lock); 2985da14cebeSEric Cheng ASSERT(!(mac_srs->srs_state & SRS_POLL_THR_RESTART)); 2986da14cebeSEric Cheng } 2987da14cebeSEric Cheng /* Wake up any waiter waiting for the restart to complete */ 2988da14cebeSEric Cheng mac_srs->srs_state |= SRS_RESTART_DONE; 2989da14cebeSEric Cheng cv_signal(&mac_srs->srs_quiesce_done_cv); 2990da14cebeSEric Cheng } 2991da14cebeSEric Cheng 2992da14cebeSEric Cheng static void 2993da14cebeSEric Cheng mac_srs_worker_unbind(mac_soft_ring_set_t *mac_srs) 2994da14cebeSEric Cheng { 2995da14cebeSEric Cheng mutex_enter(&mac_srs->srs_lock); 2996da14cebeSEric Cheng if (!(mac_srs->srs_state & SRS_WORKER_BOUND)) { 2997da14cebeSEric Cheng ASSERT(mac_srs->srs_worker_cpuid == -1); 2998da14cebeSEric Cheng mutex_exit(&mac_srs->srs_lock); 2999da14cebeSEric Cheng return; 3000da14cebeSEric Cheng } 3001da14cebeSEric Cheng 3002da14cebeSEric Cheng mac_srs->srs_worker_cpuid = -1; 3003da14cebeSEric Cheng mac_srs->srs_state &= ~SRS_WORKER_BOUND; 3004da14cebeSEric Cheng thread_affinity_clear(mac_srs->srs_worker); 3005da14cebeSEric Cheng mutex_exit(&mac_srs->srs_lock); 3006da14cebeSEric Cheng } 3007da14cebeSEric Cheng 3008da14cebeSEric Cheng static void 3009da14cebeSEric Cheng mac_srs_poll_unbind(mac_soft_ring_set_t *mac_srs) 3010da14cebeSEric Cheng { 3011da14cebeSEric Cheng mutex_enter(&mac_srs->srs_lock); 3012da14cebeSEric Cheng if (mac_srs->srs_poll_thr == NULL || 3013da14cebeSEric Cheng (mac_srs->srs_state & SRS_POLL_BOUND) == 0) { 3014da14cebeSEric Cheng ASSERT(mac_srs->srs_poll_cpuid == -1); 3015da14cebeSEric Cheng mutex_exit(&mac_srs->srs_lock); 3016da14cebeSEric Cheng return; 3017da14cebeSEric Cheng } 3018da14cebeSEric Cheng 3019da14cebeSEric Cheng mac_srs->srs_poll_cpuid = -1; 3020da14cebeSEric Cheng mac_srs->srs_state &= ~SRS_POLL_BOUND; 3021da14cebeSEric Cheng thread_affinity_clear(mac_srs->srs_poll_thr); 3022da14cebeSEric Cheng mutex_exit(&mac_srs->srs_lock); 3023da14cebeSEric Cheng } 3024da14cebeSEric Cheng 3025da14cebeSEric Cheng static void 3026da14cebeSEric Cheng mac_srs_threads_unbind(mac_soft_ring_set_t *mac_srs) 3027da14cebeSEric Cheng { 3028da14cebeSEric Cheng mac_soft_ring_t *soft_ring; 3029da14cebeSEric Cheng 3030da14cebeSEric Cheng ASSERT(MAC_PERIM_HELD((mac_handle_t)mac_srs->srs_mcip->mci_mip)); 3031da14cebeSEric Cheng 3032da14cebeSEric Cheng mutex_enter(&cpu_lock); 3033da14cebeSEric Cheng mac_srs_worker_unbind(mac_srs); 3034da14cebeSEric Cheng if (!(mac_srs->srs_type & SRST_TX)) 3035da14cebeSEric Cheng mac_srs_poll_unbind(mac_srs); 3036da14cebeSEric Cheng 3037da14cebeSEric Cheng for (soft_ring = mac_srs->srs_soft_ring_head; soft_ring != NULL; 3038da14cebeSEric Cheng soft_ring = soft_ring->s_ring_next) { 3039da14cebeSEric Cheng mac_soft_ring_unbind(soft_ring); 3040da14cebeSEric Cheng } 3041da14cebeSEric Cheng mutex_exit(&cpu_lock); 3042da14cebeSEric Cheng } 3043da14cebeSEric Cheng 3044da14cebeSEric Cheng /* 3045da14cebeSEric Cheng * When a CPU is going away, unbind all MAC threads which are bound 3046da14cebeSEric Cheng * to that CPU. The affinity of the thread to the CPU is saved to allow 3047da14cebeSEric Cheng * the thread to be rebound to the CPU if it comes back online. 3048da14cebeSEric Cheng */ 3049da14cebeSEric Cheng static void 3050da14cebeSEric Cheng mac_walk_srs_and_unbind(int cpuid) 3051da14cebeSEric Cheng { 3052da14cebeSEric Cheng mac_soft_ring_set_t *mac_srs; 3053da14cebeSEric Cheng mac_soft_ring_t *soft_ring; 3054da14cebeSEric Cheng 3055da14cebeSEric Cheng rw_enter(&mac_srs_g_lock, RW_READER); 3056da14cebeSEric Cheng 3057da14cebeSEric Cheng if ((mac_srs = mac_srs_g_list) == NULL) 3058da14cebeSEric Cheng goto done; 3059da14cebeSEric Cheng 3060da14cebeSEric Cheng for (; mac_srs != NULL; mac_srs = mac_srs->srs_next) { 3061da14cebeSEric Cheng if (mac_srs->srs_worker_cpuid == cpuid) { 3062da14cebeSEric Cheng mac_srs->srs_worker_cpuid_save = cpuid; 3063da14cebeSEric Cheng mac_srs_worker_unbind(mac_srs); 3064da14cebeSEric Cheng } 3065da14cebeSEric Cheng 3066da14cebeSEric Cheng if (!(mac_srs->srs_type & SRST_TX)) { 3067da14cebeSEric Cheng if (mac_srs->srs_poll_cpuid == cpuid) { 3068da14cebeSEric Cheng mac_srs->srs_poll_cpuid_save = cpuid; 3069da14cebeSEric Cheng mac_srs_poll_unbind(mac_srs); 3070da14cebeSEric Cheng } 3071da14cebeSEric Cheng } 3072da14cebeSEric Cheng 3073da14cebeSEric Cheng /* Next tackle the soft rings associated with the srs */ 3074da14cebeSEric Cheng mutex_enter(&mac_srs->srs_lock); 3075da14cebeSEric Cheng for (soft_ring = mac_srs->srs_soft_ring_head; soft_ring != NULL; 3076da14cebeSEric Cheng soft_ring = soft_ring->s_ring_next) { 3077da14cebeSEric Cheng if (soft_ring->s_ring_cpuid == cpuid) { 3078da14cebeSEric Cheng soft_ring->s_ring_cpuid_save = cpuid; 3079da14cebeSEric Cheng mac_soft_ring_unbind(soft_ring); 3080da14cebeSEric Cheng } 3081da14cebeSEric Cheng } 3082da14cebeSEric Cheng mutex_exit(&mac_srs->srs_lock); 3083da14cebeSEric Cheng } 3084da14cebeSEric Cheng done: 3085da14cebeSEric Cheng rw_exit(&mac_srs_g_lock); 3086da14cebeSEric Cheng } 3087da14cebeSEric Cheng 3088da14cebeSEric Cheng /* TX SETUP and TEARDOWN ROUTINES */ 3089da14cebeSEric Cheng 3090da14cebeSEric Cheng /* 3091da14cebeSEric Cheng * XXXHIO need to make sure the two mac_tx_srs_{add,del}_ring() 3092da14cebeSEric Cheng * handle the case where the number of rings is one. I.e. there is 3093da14cebeSEric Cheng * a ring pointed to by mac_srs->srs_tx_arg2. 3094da14cebeSEric Cheng */ 3095da14cebeSEric Cheng void 3096da14cebeSEric Cheng mac_tx_srs_add_ring(mac_soft_ring_set_t *mac_srs, mac_ring_t *tx_ring) 3097da14cebeSEric Cheng { 3098da14cebeSEric Cheng mac_client_impl_t *mcip = mac_srs->srs_mcip; 3099da14cebeSEric Cheng mac_soft_ring_t *soft_ring; 3100da14cebeSEric Cheng int count = mac_srs->srs_oth_ring_count; 3101da14cebeSEric Cheng 3102da14cebeSEric Cheng ASSERT(mac_srs->srs_state & SRS_QUIESCE); 3103da14cebeSEric Cheng soft_ring = mac_soft_ring_create(count, 0, NULL, 3104da14cebeSEric Cheng (ST_RING_OTH | ST_RING_TX), maxclsyspri, mcip, mac_srs, -1, 3105da14cebeSEric Cheng NULL, mcip, (mac_resource_handle_t)tx_ring); 3106da14cebeSEric Cheng mac_srs->srs_oth_ring_count++; 3107da14cebeSEric Cheng /* 3108da14cebeSEric Cheng * put this soft ring in quiesce mode too so when we restart 3109da14cebeSEric Cheng * all soft rings in the srs are in the same state. 3110da14cebeSEric Cheng */ 3111da14cebeSEric Cheng mac_soft_ring_signal(soft_ring, S_RING_QUIESCE); 3112da14cebeSEric Cheng } 3113da14cebeSEric Cheng 3114da14cebeSEric Cheng static void 3115da14cebeSEric Cheng mac_soft_ring_remove(mac_soft_ring_set_t *mac_srs, mac_soft_ring_t *softring) 3116da14cebeSEric Cheng { 3117da14cebeSEric Cheng int sringcnt; 3118da14cebeSEric Cheng 3119da14cebeSEric Cheng mutex_enter(&mac_srs->srs_lock); 3120da14cebeSEric Cheng sringcnt = mac_srs->srs_soft_ring_count; 3121da14cebeSEric Cheng ASSERT(sringcnt > 0); 3122da14cebeSEric Cheng mac_soft_ring_signal(softring, S_RING_CONDEMNED); 3123da14cebeSEric Cheng 3124da14cebeSEric Cheng ASSERT(mac_srs->srs_soft_ring_condemned_count == 0); 3125da14cebeSEric Cheng while (mac_srs->srs_soft_ring_condemned_count != 1) 3126da14cebeSEric Cheng cv_wait(&mac_srs->srs_async, &mac_srs->srs_lock); 3127da14cebeSEric Cheng 3128da14cebeSEric Cheng if (softring == mac_srs->srs_soft_ring_head) { 3129da14cebeSEric Cheng mac_srs->srs_soft_ring_head = softring->s_ring_next; 3130da14cebeSEric Cheng if (mac_srs->srs_soft_ring_head != NULL) { 3131da14cebeSEric Cheng mac_srs->srs_soft_ring_head->s_ring_prev = NULL; 3132da14cebeSEric Cheng } else { 3133da14cebeSEric Cheng mac_srs->srs_soft_ring_tail = NULL; 3134da14cebeSEric Cheng } 3135da14cebeSEric Cheng } else { 3136da14cebeSEric Cheng softring->s_ring_prev->s_ring_next = 3137da14cebeSEric Cheng softring->s_ring_next; 3138da14cebeSEric Cheng if (softring->s_ring_next != NULL) { 3139da14cebeSEric Cheng softring->s_ring_next->s_ring_prev = 3140da14cebeSEric Cheng softring->s_ring_prev; 3141da14cebeSEric Cheng } else { 3142da14cebeSEric Cheng mac_srs->srs_soft_ring_tail = 3143da14cebeSEric Cheng softring->s_ring_prev; 3144da14cebeSEric Cheng } 3145da14cebeSEric Cheng } 3146da14cebeSEric Cheng mac_srs->srs_soft_ring_count--; 3147da14cebeSEric Cheng 3148da14cebeSEric Cheng mac_srs->srs_soft_ring_condemned_count--; 3149da14cebeSEric Cheng mutex_exit(&mac_srs->srs_lock); 3150da14cebeSEric Cheng 3151da14cebeSEric Cheng mac_soft_ring_free(softring, B_FALSE); 3152da14cebeSEric Cheng } 3153da14cebeSEric Cheng 3154da14cebeSEric Cheng void 3155da14cebeSEric Cheng mac_tx_srs_del_ring(mac_soft_ring_set_t *mac_srs, mac_ring_t *tx_ring) 3156da14cebeSEric Cheng { 3157da14cebeSEric Cheng int i; 3158da14cebeSEric Cheng mac_soft_ring_t *soft_ring, *remove_sring; 3159da14cebeSEric Cheng 3160da14cebeSEric Cheng mutex_enter(&mac_srs->srs_lock); 3161da14cebeSEric Cheng for (i = 0; i < mac_srs->srs_oth_ring_count; i++) { 3162da14cebeSEric Cheng soft_ring = mac_srs->srs_oth_soft_rings[i]; 3163da14cebeSEric Cheng if (soft_ring->s_ring_tx_arg2 == tx_ring) 3164da14cebeSEric Cheng break; 3165da14cebeSEric Cheng } 3166da14cebeSEric Cheng mutex_exit(&mac_srs->srs_lock); 3167da14cebeSEric Cheng ASSERT(i < mac_srs->srs_oth_ring_count); 3168da14cebeSEric Cheng remove_sring = soft_ring; 3169da14cebeSEric Cheng mac_soft_ring_remove(mac_srs, remove_sring); 3170da14cebeSEric Cheng mac_srs_update_fanout_list(mac_srs); 3171da14cebeSEric Cheng } 3172da14cebeSEric Cheng 3173da14cebeSEric Cheng /* 3174da14cebeSEric Cheng * mac_tx_srs_setup(): 3175da14cebeSEric Cheng * 3176da14cebeSEric Cheng * Used to setup Tx rings. If no free Tx ring is available, then default 3177da14cebeSEric Cheng * Tx ring is used. 3178da14cebeSEric Cheng */ 3179da14cebeSEric Cheng void 3180da14cebeSEric Cheng mac_tx_srs_setup(mac_client_impl_t *mcip, flow_entry_t *flent, 3181da14cebeSEric Cheng uint32_t srs_type) 3182da14cebeSEric Cheng { 3183da14cebeSEric Cheng mac_impl_t *mip = mcip->mci_mip; 3184da14cebeSEric Cheng mac_soft_ring_set_t *tx_srs; 3185*ae6aa22aSVenugopal Iyer int i, tx_ring_count = 0, tx_rings_reserved = 0; 3186da14cebeSEric Cheng mac_ring_handle_t *tx_ring = NULL; 3187da14cebeSEric Cheng uint32_t soft_ring_type; 3188da14cebeSEric Cheng mac_group_t *grp = NULL; 3189da14cebeSEric Cheng mac_ring_t *ring; 3190da14cebeSEric Cheng mac_srs_tx_t *tx; 3191da14cebeSEric Cheng boolean_t serialize = B_FALSE; 3192da14cebeSEric Cheng 3193da14cebeSEric Cheng tx_srs = flent->fe_tx_srs; 3194da14cebeSEric Cheng tx = &tx_srs->srs_tx; 3195da14cebeSEric Cheng 3196da14cebeSEric Cheng if (tx->st_group != NULL) { 3197da14cebeSEric Cheng grp = tx->st_group; 3198da14cebeSEric Cheng tx_ring_count = grp->mrg_cur_count; 3199da14cebeSEric Cheng } else { 3200da14cebeSEric Cheng tx_ring_count = mac_tx_ring_count; 3201da14cebeSEric Cheng } 3202da14cebeSEric Cheng 3203da14cebeSEric Cheng if (tx_ring_count != 0) { 3204da14cebeSEric Cheng tx_ring = kmem_zalloc(sizeof (mac_ring_handle_t) * 3205da14cebeSEric Cheng tx_ring_count, KM_SLEEP); 3206da14cebeSEric Cheng } 3207da14cebeSEric Cheng 3208da14cebeSEric Cheng /* 3209da14cebeSEric Cheng * Just use the default ring for now. We need to use 3210da14cebeSEric Cheng * the underlying link's ring set instead of the underlying 3211da14cebeSEric Cheng * NIC's. 3212da14cebeSEric Cheng */ 321308ac1c49SNicolas Droux if (srs_type == SRST_FLOW || 321408ac1c49SNicolas Droux (mcip->mci_state_flags & MCIS_NO_HWRINGS) != 0) 3215da14cebeSEric Cheng goto use_default_ring; 3216da14cebeSEric Cheng 3217da14cebeSEric Cheng if (mcip->mci_share != NULL) 3218da14cebeSEric Cheng ring = grp->mrg_rings; 3219da14cebeSEric Cheng /* 3220da14cebeSEric Cheng * An attempt is made to reserve 'tx_ring_count' number 3221da14cebeSEric Cheng * of Tx rings. If tx_ring_count is 0, default Tx ring 3222da14cebeSEric Cheng * is used. If it is 1, an attempt is made to reserve one 3223da14cebeSEric Cheng * Tx ring. In both the cases, the ring information is 3224da14cebeSEric Cheng * stored in Tx SRS. If multiple Tx rings are specified, 3225da14cebeSEric Cheng * then each Tx ring will have a Tx-side soft ring. All 3226da14cebeSEric Cheng * these soft rings will be hang off Tx SRS. 3227da14cebeSEric Cheng */ 3228da14cebeSEric Cheng for (i = 0, tx_rings_reserved = 0; 3229da14cebeSEric Cheng i < tx_ring_count; i++, tx_rings_reserved++) { 3230da14cebeSEric Cheng if (mcip->mci_share != NULL) { 3231da14cebeSEric Cheng /* 3232da14cebeSEric Cheng * The ring was already chosen and associated 3233da14cebeSEric Cheng * with the TX group. Save it in the new 3234da14cebeSEric Cheng * array to keep as much of the code below common 3235da14cebeSEric Cheng * between the share and non-share cases. 3236da14cebeSEric Cheng */ 3237da14cebeSEric Cheng ASSERT(ring != NULL); 3238da14cebeSEric Cheng tx_ring[i] = (mac_ring_handle_t)ring; 3239da14cebeSEric Cheng ring = ring->mr_next; 3240da14cebeSEric Cheng } else { 3241da14cebeSEric Cheng tx_ring[i] = 3242da14cebeSEric Cheng (mac_ring_handle_t)mac_reserve_tx_ring(mip, NULL); 3243da14cebeSEric Cheng if (tx_ring[i] == NULL) 3244da14cebeSEric Cheng break; 3245da14cebeSEric Cheng } 3246da14cebeSEric Cheng } 3247da14cebeSEric Cheng if (mac_tx_serialize || (mip->mi_v12n_level & MAC_VIRT_SERIALIZE)) 3248da14cebeSEric Cheng serialize = B_TRUE; 3249da14cebeSEric Cheng /* 3250da14cebeSEric Cheng * Did we get the requested number of tx rings? 3251da14cebeSEric Cheng * There are 3 actions we can take depending upon the number 3252da14cebeSEric Cheng * of tx_rings we got. 3253da14cebeSEric Cheng * 1) If we got none, then hook up the tx_srs with the 3254da14cebeSEric Cheng * default ring. 3255da14cebeSEric Cheng * 2) If we got one, then get the tx_ring from the soft ring, 3256da14cebeSEric Cheng * save it in SRS and free up the soft ring. 3257da14cebeSEric Cheng * 3) If we got more than 1, then do the tx fanout among the 3258da14cebeSEric Cheng * rings we obtained. 3259da14cebeSEric Cheng */ 3260da14cebeSEric Cheng switch (tx_rings_reserved) { 3261da14cebeSEric Cheng case 1: 3262da14cebeSEric Cheng /* 3263da14cebeSEric Cheng * No need to allocate Tx soft rings. Tx-side soft 3264da14cebeSEric Cheng * rings are for Tx fanout case. Just use Tx SRS. 3265da14cebeSEric Cheng */ 3266da14cebeSEric Cheng /* FALLTHRU */ 3267da14cebeSEric Cheng 3268da14cebeSEric Cheng case 0: 3269da14cebeSEric Cheng use_default_ring: 3270da14cebeSEric Cheng if (tx_rings_reserved == 0) 3271da14cebeSEric Cheng tx->st_arg2 = (void *)mip->mi_default_tx_ring; 3272da14cebeSEric Cheng else 3273da14cebeSEric Cheng tx->st_arg2 = (void *)tx_ring[0]; 3274da14cebeSEric Cheng /* For ring_count of 0 or 1, set the tx_mode and return */ 3275da14cebeSEric Cheng if (tx_srs->srs_type & SRST_BW_CONTROL) 3276da14cebeSEric Cheng tx->st_mode = SRS_TX_BW; 3277da14cebeSEric Cheng else if (serialize) 3278da14cebeSEric Cheng tx->st_mode = SRS_TX_SERIALIZE; 3279da14cebeSEric Cheng else 3280da14cebeSEric Cheng tx->st_mode = SRS_TX_DEFAULT; 3281da14cebeSEric Cheng break; 3282da14cebeSEric Cheng 3283da14cebeSEric Cheng default: 3284da14cebeSEric Cheng /* 3285da14cebeSEric Cheng * We got multiple Tx rings for Tx fanout. 3286da14cebeSEric Cheng * 3287da14cebeSEric Cheng * cpuid of -1 is passed. This creates an unbound 3288da14cebeSEric Cheng * worker thread. Instead the code should get CPU 3289da14cebeSEric Cheng * binding information and pass that to 3290da14cebeSEric Cheng * mac_soft_ring_create(). This needs to be done 3291da14cebeSEric Cheng * in conjunction with Rx-side soft ring 3292da14cebeSEric Cheng * bindings. 3293da14cebeSEric Cheng */ 3294da14cebeSEric Cheng soft_ring_type = ST_RING_OTH | ST_RING_TX; 3295da14cebeSEric Cheng if (tx_srs->srs_type & SRST_BW_CONTROL) { 3296da14cebeSEric Cheng tx->st_mode = SRS_TX_BW_FANOUT; 3297da14cebeSEric Cheng } else { 3298da14cebeSEric Cheng tx->st_mode = SRS_TX_FANOUT; 3299da14cebeSEric Cheng if (serialize) 3300da14cebeSEric Cheng soft_ring_type |= ST_RING_WORKER_ONLY; 3301da14cebeSEric Cheng } 3302da14cebeSEric Cheng for (i = 0; i < tx_rings_reserved; i++) { 3303da14cebeSEric Cheng (void) mac_soft_ring_create(i, 0, NULL, soft_ring_type, 3304da14cebeSEric Cheng maxclsyspri, mcip, tx_srs, -1, NULL, mcip, 3305da14cebeSEric Cheng (mac_resource_handle_t)tx_ring[i]); 3306da14cebeSEric Cheng } 3307da14cebeSEric Cheng mac_srs_update_fanout_list(tx_srs); 3308da14cebeSEric Cheng } 3309da14cebeSEric Cheng tx->st_func = mac_tx_get_func(tx->st_mode); 3310da14cebeSEric Cheng 3311da14cebeSEric Cheng DTRACE_PROBE3(tx__srs___setup__return, mac_soft_ring_set_t *, tx_srs, 3312da14cebeSEric Cheng int, tx->st_mode, int, tx_srs->srs_oth_ring_count); 3313da14cebeSEric Cheng 3314da14cebeSEric Cheng if (tx_ring_count != 0) { 3315da14cebeSEric Cheng kmem_free(tx_ring, 3316da14cebeSEric Cheng sizeof (mac_ring_handle_t) * tx_ring_count); 3317da14cebeSEric Cheng } 3318da14cebeSEric Cheng } 3319da14cebeSEric Cheng 3320da14cebeSEric Cheng /* 332108ac1c49SNicolas Droux * Update the fanout of a client if its recorded link speed doesn't match 332208ac1c49SNicolas Droux * its current link speed. 332308ac1c49SNicolas Droux */ 332408ac1c49SNicolas Droux void 332508ac1c49SNicolas Droux mac_fanout_recompute_client(mac_client_impl_t *mcip) 332608ac1c49SNicolas Droux { 332708ac1c49SNicolas Droux uint64_t link_speed; 332808ac1c49SNicolas Droux mac_resource_props_t *mcip_mrp; 332908ac1c49SNicolas Droux 333008ac1c49SNicolas Droux ASSERT(MAC_PERIM_HELD((mac_handle_t)mcip->mci_mip)); 333108ac1c49SNicolas Droux 333208ac1c49SNicolas Droux link_speed = mac_client_stat_get(mcip->mci_flent->fe_mcip, 333308ac1c49SNicolas Droux MAC_STAT_IFSPEED); 333408ac1c49SNicolas Droux 333508ac1c49SNicolas Droux if ((link_speed != 0) && 333608ac1c49SNicolas Droux (link_speed != mcip->mci_flent->fe_nic_speed)) { 333708ac1c49SNicolas Droux mcip_mrp = MCIP_RESOURCE_PROPS(mcip); 333808ac1c49SNicolas Droux mac_fanout_setup(mcip, mcip->mci_flent, 333908ac1c49SNicolas Droux mcip_mrp, mac_rx_deliver, mcip, NULL); 334008ac1c49SNicolas Droux } 334108ac1c49SNicolas Droux } 334208ac1c49SNicolas Droux 334308ac1c49SNicolas Droux /* 3344da14cebeSEric Cheng * Walk through the list of mac clients for the MAC. 3345da14cebeSEric Cheng * For each active mac client, recompute the number of soft rings 3346da14cebeSEric Cheng * associated with every client, only if current speed is different 3347da14cebeSEric Cheng * from the speed that was previously used for soft ring computation. 3348da14cebeSEric Cheng * If the cable is disconnected whlie the NIC is started, we would get 3349da14cebeSEric Cheng * notification with speed set to 0. We do not recompute in that case. 3350da14cebeSEric Cheng */ 3351da14cebeSEric Cheng void 3352da14cebeSEric Cheng mac_fanout_recompute(mac_impl_t *mip) 3353da14cebeSEric Cheng { 3354da14cebeSEric Cheng mac_client_impl_t *mcip; 335508ac1c49SNicolas Droux 3356da14cebeSEric Cheng 3357da14cebeSEric Cheng i_mac_perim_enter(mip); 3358da14cebeSEric Cheng ASSERT(!(mip->mi_state_flags & MIS_IS_VNIC)); 3359da14cebeSEric Cheng 3360da14cebeSEric Cheng if (mip->mi_linkstate != LINK_STATE_UP) { 3361da14cebeSEric Cheng i_mac_perim_exit(mip); 3362da14cebeSEric Cheng return; 3363da14cebeSEric Cheng } 3364da14cebeSEric Cheng 3365da14cebeSEric Cheng for (mcip = mip->mi_clients_list; mcip != NULL; 3366da14cebeSEric Cheng mcip = mcip->mci_client_next) { 336708ac1c49SNicolas Droux if ((mcip->mci_state_flags & MCIS_SHARE_BOUND) != 0 || 336808ac1c49SNicolas Droux !MCIP_DATAPATH_SETUP(mcip)) 3369da14cebeSEric Cheng continue; 337008ac1c49SNicolas Droux mac_fanout_recompute_client(mcip); 3371da14cebeSEric Cheng } 3372da14cebeSEric Cheng i_mac_perim_exit(mip); 3373da14cebeSEric Cheng } 3374