1da14cebeSEric Cheng /* 2da14cebeSEric Cheng * CDDL HEADER START 3da14cebeSEric Cheng * 4da14cebeSEric Cheng * The contents of this file are subject to the terms of the 5da14cebeSEric Cheng * Common Development and Distribution License (the "License"). 6da14cebeSEric Cheng * You may not use this file except in compliance with the License. 7da14cebeSEric Cheng * 8da14cebeSEric Cheng * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE 9da14cebeSEric Cheng * or http://www.opensolaris.org/os/licensing. 10da14cebeSEric Cheng * See the License for the specific language governing permissions 11da14cebeSEric Cheng * and limitations under the License. 12da14cebeSEric Cheng * 13da14cebeSEric Cheng * When distributing Covered Code, include this CDDL HEADER in each 14da14cebeSEric Cheng * file and include the License file at usr/src/OPENSOLARIS.LICENSE. 15da14cebeSEric Cheng * If applicable, add the following below this CDDL HEADER, with the 16da14cebeSEric Cheng * fields enclosed by brackets "[]" replaced with your own identifying 17da14cebeSEric Cheng * information: Portions Copyright [yyyy] [name of copyright owner] 18da14cebeSEric Cheng * 19da14cebeSEric Cheng * CDDL HEADER END 20da14cebeSEric Cheng */ 21da14cebeSEric Cheng /* 225adf34bdSRajagopal Kunhappan * Copyright (c) 2008, 2010, Oracle and/or its affiliates. All rights reserved. 2384de666eSRyan Zezeski * Copyright 2018 Joyent, Inc. 24*3714f7beSPaul Winder * Copyright 2020 RackTop Systems. 25da14cebeSEric Cheng */ 26da14cebeSEric Cheng 27da14cebeSEric Cheng #include <sys/types.h> 28da14cebeSEric Cheng #include <sys/callb.h> 290dc2366fSVenugopal Iyer #include <sys/cpupart.h> 300dc2366fSVenugopal Iyer #include <sys/pool.h> 310dc2366fSVenugopal Iyer #include <sys/pool_pset.h> 32da14cebeSEric Cheng #include <sys/sdt.h> 33da14cebeSEric Cheng #include <sys/strsubr.h> 34da14cebeSEric Cheng #include <sys/strsun.h> 35da14cebeSEric Cheng #include <sys/vlan.h> 36da14cebeSEric Cheng #include <inet/ipsec_impl.h> 37da14cebeSEric Cheng #include <inet/ip_impl.h> 38da14cebeSEric Cheng #include <inet/sadb.h> 39da14cebeSEric Cheng #include <inet/ipsecesp.h> 40da14cebeSEric Cheng #include <inet/ipsecah.h> 41da14cebeSEric Cheng 42da14cebeSEric Cheng #include <sys/mac_impl.h> 43da14cebeSEric Cheng #include <sys/mac_client_impl.h> 44da14cebeSEric Cheng #include <sys/mac_client_priv.h> 45da14cebeSEric Cheng #include <sys/mac_soft_ring.h> 46da14cebeSEric Cheng #include <sys/mac_flow_impl.h> 470dc2366fSVenugopal Iyer #include <sys/mac_stat.h> 48da14cebeSEric Cheng 49da14cebeSEric Cheng static void mac_srs_soft_rings_signal(mac_soft_ring_set_t *, uint_t); 50da14cebeSEric Cheng static void mac_srs_update_fanout_list(mac_soft_ring_set_t *); 51da14cebeSEric Cheng static void mac_srs_poll_unbind(mac_soft_ring_set_t *); 52da14cebeSEric Cheng static void mac_srs_worker_unbind(mac_soft_ring_set_t *); 53da14cebeSEric Cheng static void mac_srs_soft_rings_quiesce(mac_soft_ring_set_t *, uint_t); 54da14cebeSEric Cheng 55da14cebeSEric Cheng static int mac_srs_cpu_setup(cpu_setup_t, int, void *); 56da14cebeSEric Cheng static void mac_srs_worker_bind(mac_soft_ring_set_t *, processorid_t); 57da14cebeSEric Cheng static void mac_srs_poll_bind(mac_soft_ring_set_t *, processorid_t); 58da14cebeSEric Cheng static void mac_srs_threads_unbind(mac_soft_ring_set_t *); 59da14cebeSEric Cheng static void mac_srs_add_glist(mac_soft_ring_set_t *); 60da14cebeSEric Cheng static void mac_srs_remove_glist(mac_soft_ring_set_t *); 61da14cebeSEric Cheng static void mac_srs_fanout_list_free(mac_soft_ring_set_t *); 62da14cebeSEric Cheng static void mac_soft_ring_remove(mac_soft_ring_set_t *, mac_soft_ring_t *); 63da14cebeSEric Cheng 640dc2366fSVenugopal Iyer static int mac_compute_soft_ring_count(flow_entry_t *, int, int); 65da14cebeSEric Cheng static void mac_walk_srs_and_bind(int); 66da14cebeSEric Cheng static void mac_walk_srs_and_unbind(int); 67da14cebeSEric Cheng 68da14cebeSEric Cheng extern boolean_t mac_latency_optimize; 69da14cebeSEric Cheng 70da14cebeSEric Cheng static kmem_cache_t *mac_srs_cache; 71da14cebeSEric Cheng kmem_cache_t *mac_soft_ring_cache; 72da14cebeSEric Cheng 73da14cebeSEric Cheng /* 74da14cebeSEric Cheng * The duration in msec we wait before signalling the soft ring 75da14cebeSEric Cheng * worker thread in case packets get queued. 76da14cebeSEric Cheng */ 77ae6aa22aSVenugopal Iyer uint32_t mac_soft_ring_worker_wait = 0; 78ae6aa22aSVenugopal Iyer 79ae6aa22aSVenugopal Iyer /* 80ae6aa22aSVenugopal Iyer * A global tunable for turning polling on/off. By default, dynamic 81ae6aa22aSVenugopal Iyer * polling is always on and is always very beneficial. It should be 82ae6aa22aSVenugopal Iyer * turned off with absolute care and for the rare workload (very 83ae6aa22aSVenugopal Iyer * low latency sensitive traffic). 84ae6aa22aSVenugopal Iyer */ 85ae6aa22aSVenugopal Iyer int mac_poll_enable = B_TRUE; 86da14cebeSEric Cheng 87da14cebeSEric Cheng /* 88da14cebeSEric Cheng * Need to set mac_soft_ring_max_q_cnt based on bandwidth and perhaps latency. 89da14cebeSEric Cheng * Large values could end up in consuming lot of system memory and cause 90da14cebeSEric Cheng * system hang. 91da14cebeSEric Cheng */ 92ae6aa22aSVenugopal Iyer int mac_soft_ring_max_q_cnt = 1024; 93ae6aa22aSVenugopal Iyer int mac_soft_ring_min_q_cnt = 256; 94ae6aa22aSVenugopal Iyer int mac_soft_ring_poll_thres = 16; 95da14cebeSEric Cheng 96ae6aa22aSVenugopal Iyer boolean_t mac_tx_serialize = B_FALSE; 97da14cebeSEric Cheng 98da14cebeSEric Cheng /* 99da14cebeSEric Cheng * mac_tx_srs_hiwat is the queue depth threshold at which callers of 100da14cebeSEric Cheng * mac_tx() will be notified of flow control condition. 101da14cebeSEric Cheng * 102da14cebeSEric Cheng * TCP does not honour flow control condition sent up by mac_tx(). 103da14cebeSEric Cheng * Thus provision is made for TCP to allow more packets to be queued 104da14cebeSEric Cheng * in SRS upto a maximum of mac_tx_srs_max_q_cnt. 105da14cebeSEric Cheng * 106da14cebeSEric Cheng * Note that mac_tx_srs_hiwat is always be lesser than 107da14cebeSEric Cheng * mac_tx_srs_max_q_cnt. 108da14cebeSEric Cheng */ 109ae6aa22aSVenugopal Iyer uint32_t mac_tx_srs_max_q_cnt = 100000; 110ae6aa22aSVenugopal Iyer uint32_t mac_tx_srs_hiwat = 1000; 111da14cebeSEric Cheng 112da14cebeSEric Cheng /* 113da14cebeSEric Cheng * mac_rx_soft_ring_count, mac_soft_ring_10gig_count: 114da14cebeSEric Cheng * 115da14cebeSEric Cheng * Global tunables that determines the number of soft rings to be used for 116da14cebeSEric Cheng * fanning out incoming traffic on a link. These count will be used only 117da14cebeSEric Cheng * when no explicit set of CPUs was assigned to the data-links. 118da14cebeSEric Cheng * 119da14cebeSEric Cheng * mac_rx_soft_ring_count tunable will come into effect only if 120da14cebeSEric Cheng * mac_soft_ring_enable is set. mac_soft_ring_enable is turned on by 121da14cebeSEric Cheng * default only for sun4v platforms. 122da14cebeSEric Cheng * 123da14cebeSEric Cheng * mac_rx_soft_ring_10gig_count will come into effect if you are running on a 124da14cebeSEric Cheng * 10Gbps link and is not dependent upon mac_soft_ring_enable. 125da14cebeSEric Cheng * 126da14cebeSEric Cheng * The number of soft rings for fanout for a link or a flow is determined 127da14cebeSEric Cheng * by mac_compute_soft_ring_count() routine. This routine will take into 128da14cebeSEric Cheng * account mac_soft_ring_enable, mac_rx_soft_ring_count and 129da14cebeSEric Cheng * mac_rx_soft_ring_10gig_count to determine the soft ring count for a link. 130da14cebeSEric Cheng * 131da14cebeSEric Cheng * If a bandwidth is specified, the determination of the number of soft 132da14cebeSEric Cheng * rings is based on specified bandwidth, CPU speed and number of CPUs in 133da14cebeSEric Cheng * the system. 134da14cebeSEric Cheng */ 135ae6aa22aSVenugopal Iyer uint_t mac_rx_soft_ring_count = 8; 136ae6aa22aSVenugopal Iyer uint_t mac_rx_soft_ring_10gig_count = 8; 137da14cebeSEric Cheng 138da14cebeSEric Cheng /* 139da14cebeSEric Cheng * Every Tx and Rx mac_soft_ring_set_t (mac_srs) created gets added 140da14cebeSEric Cheng * to mac_srs_g_list and mac_srs_g_lock protects mac_srs_g_list. The 141da14cebeSEric Cheng * list is used to walk the list of all MAC threads when a CPU is 142da14cebeSEric Cheng * coming online or going offline. 143da14cebeSEric Cheng */ 144da14cebeSEric Cheng static mac_soft_ring_set_t *mac_srs_g_list = NULL; 145da14cebeSEric Cheng static krwlock_t mac_srs_g_lock; 146da14cebeSEric Cheng 147da14cebeSEric Cheng /* 148da14cebeSEric Cheng * Whether the SRS threads should be bound, or not. 149da14cebeSEric Cheng */ 150ae6aa22aSVenugopal Iyer boolean_t mac_srs_thread_bind = B_TRUE; 151da14cebeSEric Cheng 152da14cebeSEric Cheng /* 1530dc2366fSVenugopal Iyer * Whether Rx/Tx interrupts should be re-targeted. Disabled by default. 1540dc2366fSVenugopal Iyer * dladm command would override this. 155da14cebeSEric Cheng */ 1560dc2366fSVenugopal Iyer boolean_t mac_tx_intr_retarget = B_FALSE; 1570dc2366fSVenugopal Iyer boolean_t mac_rx_intr_retarget = B_FALSE; 158da14cebeSEric Cheng 159da14cebeSEric Cheng /* 160da14cebeSEric Cheng * If cpu bindings are specified by user, then Tx SRS and its soft 161da14cebeSEric Cheng * rings should also be bound to the CPUs specified by user. The 162da14cebeSEric Cheng * CPUs for Tx bindings are at the end of the cpu list provided by 163da14cebeSEric Cheng * the user. If enough CPUs are not available (for Tx and Rx 164da14cebeSEric Cheng * SRSes), then the CPUs are shared by both Tx and Rx SRSes. 165da14cebeSEric Cheng */ 166da14cebeSEric Cheng #define BIND_TX_SRS_AND_SOFT_RINGS(mac_tx_srs, mrp) { \ 167da14cebeSEric Cheng processorid_t cpuid; \ 1680dc2366fSVenugopal Iyer int i; \ 169da14cebeSEric Cheng mac_soft_ring_t *softring; \ 1700dc2366fSVenugopal Iyer mac_cpus_t *srs_cpu; \ 171da14cebeSEric Cheng \ 1720dc2366fSVenugopal Iyer srs_cpu = &mac_tx_srs->srs_cpu; \ 1730dc2366fSVenugopal Iyer cpuid = srs_cpu->mc_tx_fanout_cpus[0]; \ 174da14cebeSEric Cheng mac_srs_worker_bind(mac_tx_srs, cpuid); \ 1750dc2366fSVenugopal Iyer if (MAC_TX_SOFT_RINGS(mac_tx_srs)) { \ 1760dc2366fSVenugopal Iyer for (i = 0; i < mac_tx_srs->srs_tx_ring_count; i++) { \ 1770dc2366fSVenugopal Iyer cpuid = srs_cpu->mc_tx_fanout_cpus[i]; \ 1780dc2366fSVenugopal Iyer softring = mac_tx_srs->srs_tx_soft_rings[i]; \ 1790dc2366fSVenugopal Iyer if (cpuid != -1) { \ 1800dc2366fSVenugopal Iyer (void) mac_soft_ring_bind(softring, \ 1810dc2366fSVenugopal Iyer cpuid); \ 1820dc2366fSVenugopal Iyer } \ 183da14cebeSEric Cheng } \ 184da14cebeSEric Cheng } \ 185da14cebeSEric Cheng } 186da14cebeSEric Cheng 1870dc2366fSVenugopal Iyer /* 1880dc2366fSVenugopal Iyer * Re-targeting is allowed only for exclusive group or for primary. 1890dc2366fSVenugopal Iyer */ 1900dc2366fSVenugopal Iyer #define RETARGETABLE_CLIENT(group, mcip) \ 1910dc2366fSVenugopal Iyer ((((group) != NULL) && \ 1920dc2366fSVenugopal Iyer ((group)->mrg_state == MAC_GROUP_STATE_RESERVED)) || \ 1930dc2366fSVenugopal Iyer mac_is_primary_client(mcip)) 1940dc2366fSVenugopal Iyer 1950dc2366fSVenugopal Iyer #define MAC_RING_RETARGETABLE(ring) \ 1960dc2366fSVenugopal Iyer (((ring) != NULL) && \ 1970dc2366fSVenugopal Iyer ((ring)->mr_info.mri_intr.mi_ddi_handle != NULL) && \ 1980dc2366fSVenugopal Iyer !((ring)->mr_info.mri_intr.mi_ddi_shared)) 1990dc2366fSVenugopal Iyer 2000dc2366fSVenugopal Iyer 201da14cebeSEric Cheng /* INIT and FINI ROUTINES */ 202da14cebeSEric Cheng 203da14cebeSEric Cheng void 204da14cebeSEric Cheng mac_soft_ring_init(void) 205da14cebeSEric Cheng { 206da14cebeSEric Cheng mac_soft_ring_cache = kmem_cache_create("mac_soft_ring_cache", 207da14cebeSEric Cheng sizeof (mac_soft_ring_t), 64, NULL, NULL, NULL, NULL, NULL, 0); 208da14cebeSEric Cheng 209da14cebeSEric Cheng mac_srs_cache = kmem_cache_create("mac_srs_cache", 210da14cebeSEric Cheng sizeof (mac_soft_ring_set_t), 211da14cebeSEric Cheng 64, NULL, NULL, NULL, NULL, NULL, 0); 212da14cebeSEric Cheng 213da14cebeSEric Cheng rw_init(&mac_srs_g_lock, NULL, RW_DEFAULT, NULL); 214da14cebeSEric Cheng mutex_enter(&cpu_lock); 215da14cebeSEric Cheng register_cpu_setup_func(mac_srs_cpu_setup, NULL); 216da14cebeSEric Cheng mutex_exit(&cpu_lock); 217da14cebeSEric Cheng } 218da14cebeSEric Cheng 219da14cebeSEric Cheng void 220da14cebeSEric Cheng mac_soft_ring_finish(void) 221da14cebeSEric Cheng { 222da14cebeSEric Cheng mutex_enter(&cpu_lock); 223da14cebeSEric Cheng unregister_cpu_setup_func(mac_srs_cpu_setup, NULL); 224da14cebeSEric Cheng mutex_exit(&cpu_lock); 225da14cebeSEric Cheng rw_destroy(&mac_srs_g_lock); 226da14cebeSEric Cheng kmem_cache_destroy(mac_soft_ring_cache); 227da14cebeSEric Cheng kmem_cache_destroy(mac_srs_cache); 228da14cebeSEric Cheng } 229da14cebeSEric Cheng 230da14cebeSEric Cheng static void 2310dc2366fSVenugopal Iyer mac_srs_soft_rings_free(mac_soft_ring_set_t *mac_srs) 232da14cebeSEric Cheng { 233da14cebeSEric Cheng mac_soft_ring_t *softring, *next, *head; 234da14cebeSEric Cheng 235da14cebeSEric Cheng /* 236da14cebeSEric Cheng * Synchronize with mac_walk_srs_bind/unbind which are callbacks from 237da14cebeSEric Cheng * DR. The callbacks from DR are called with cpu_lock held, and hence 238da14cebeSEric Cheng * can't wait to grab the mac perimeter. The soft ring list is hence 239da14cebeSEric Cheng * protected for read access by srs_lock. Changing the soft ring list 240da14cebeSEric Cheng * needs the mac perimeter and the srs_lock. 241da14cebeSEric Cheng */ 242da14cebeSEric Cheng mutex_enter(&mac_srs->srs_lock); 243da14cebeSEric Cheng 244da14cebeSEric Cheng head = mac_srs->srs_soft_ring_head; 245da14cebeSEric Cheng mac_srs->srs_soft_ring_head = NULL; 246da14cebeSEric Cheng mac_srs->srs_soft_ring_tail = NULL; 247da14cebeSEric Cheng mac_srs->srs_soft_ring_count = 0; 248da14cebeSEric Cheng 249da14cebeSEric Cheng mutex_exit(&mac_srs->srs_lock); 250da14cebeSEric Cheng 251da14cebeSEric Cheng for (softring = head; softring != NULL; softring = next) { 252da14cebeSEric Cheng next = softring->s_ring_next; 2530dc2366fSVenugopal Iyer mac_soft_ring_free(softring); 254da14cebeSEric Cheng } 255da14cebeSEric Cheng } 256da14cebeSEric Cheng 257da14cebeSEric Cheng static void 258da14cebeSEric Cheng mac_srs_add_glist(mac_soft_ring_set_t *mac_srs) 259da14cebeSEric Cheng { 260da14cebeSEric Cheng ASSERT(mac_srs->srs_next == NULL && mac_srs->srs_prev == NULL); 261da14cebeSEric Cheng ASSERT(MAC_PERIM_HELD((mac_handle_t)mac_srs->srs_mcip->mci_mip)); 262da14cebeSEric Cheng 263da14cebeSEric Cheng rw_enter(&mac_srs_g_lock, RW_WRITER); 264da14cebeSEric Cheng mutex_enter(&mac_srs->srs_lock); 265da14cebeSEric Cheng 266da14cebeSEric Cheng ASSERT((mac_srs->srs_state & SRS_IN_GLIST) == 0); 267da14cebeSEric Cheng 268da14cebeSEric Cheng if (mac_srs_g_list == NULL) { 269da14cebeSEric Cheng mac_srs_g_list = mac_srs; 270da14cebeSEric Cheng } else { 271da14cebeSEric Cheng mac_srs->srs_next = mac_srs_g_list; 272da14cebeSEric Cheng mac_srs_g_list->srs_prev = mac_srs; 273da14cebeSEric Cheng mac_srs->srs_prev = NULL; 274da14cebeSEric Cheng mac_srs_g_list = mac_srs; 275da14cebeSEric Cheng } 276da14cebeSEric Cheng mac_srs->srs_state |= SRS_IN_GLIST; 277da14cebeSEric Cheng 278da14cebeSEric Cheng mutex_exit(&mac_srs->srs_lock); 279da14cebeSEric Cheng rw_exit(&mac_srs_g_lock); 280da14cebeSEric Cheng } 281da14cebeSEric Cheng 282da14cebeSEric Cheng static void 283da14cebeSEric Cheng mac_srs_remove_glist(mac_soft_ring_set_t *mac_srs) 284da14cebeSEric Cheng { 285da14cebeSEric Cheng ASSERT(MAC_PERIM_HELD((mac_handle_t)mac_srs->srs_mcip->mci_mip)); 286da14cebeSEric Cheng 287da14cebeSEric Cheng rw_enter(&mac_srs_g_lock, RW_WRITER); 288da14cebeSEric Cheng mutex_enter(&mac_srs->srs_lock); 289da14cebeSEric Cheng 290da14cebeSEric Cheng ASSERT((mac_srs->srs_state & SRS_IN_GLIST) != 0); 291da14cebeSEric Cheng 292da14cebeSEric Cheng if (mac_srs == mac_srs_g_list) { 293da14cebeSEric Cheng mac_srs_g_list = mac_srs->srs_next; 294da14cebeSEric Cheng if (mac_srs_g_list != NULL) 295da14cebeSEric Cheng mac_srs_g_list->srs_prev = NULL; 296da14cebeSEric Cheng } else { 297da14cebeSEric Cheng mac_srs->srs_prev->srs_next = mac_srs->srs_next; 298da14cebeSEric Cheng if (mac_srs->srs_next != NULL) 299da14cebeSEric Cheng mac_srs->srs_next->srs_prev = mac_srs->srs_prev; 300da14cebeSEric Cheng } 301da14cebeSEric Cheng mac_srs->srs_state &= ~SRS_IN_GLIST; 302da14cebeSEric Cheng 303da14cebeSEric Cheng mutex_exit(&mac_srs->srs_lock); 304da14cebeSEric Cheng rw_exit(&mac_srs_g_lock); 305da14cebeSEric Cheng } 306da14cebeSEric Cheng 307da14cebeSEric Cheng /* POLLING SETUP AND TEAR DOWN ROUTINES */ 308da14cebeSEric Cheng 309da14cebeSEric Cheng /* 310da14cebeSEric Cheng * mac_srs_client_poll_quiesce and mac_srs_client_poll_restart 311da14cebeSEric Cheng * 312da14cebeSEric Cheng * These routines are used to call back into the upper layer 313da14cebeSEric Cheng * (primarily TCP squeue) to stop polling the soft rings or 314da14cebeSEric Cheng * restart polling. 315da14cebeSEric Cheng */ 316da14cebeSEric Cheng void 317da14cebeSEric Cheng mac_srs_client_poll_quiesce(mac_client_impl_t *mcip, 318da14cebeSEric Cheng mac_soft_ring_set_t *mac_srs) 319da14cebeSEric Cheng { 320da14cebeSEric Cheng mac_soft_ring_t *softring; 321da14cebeSEric Cheng 322da14cebeSEric Cheng ASSERT(MAC_PERIM_HELD((mac_handle_t)mcip->mci_mip)); 323da14cebeSEric Cheng 324da14cebeSEric Cheng if (!(mac_srs->srs_type & SRST_CLIENT_POLL_ENABLED)) { 325da14cebeSEric Cheng ASSERT(!(mac_srs->srs_type & SRST_DLS_BYPASS)); 326da14cebeSEric Cheng return; 327da14cebeSEric Cheng } 328da14cebeSEric Cheng 329da14cebeSEric Cheng for (softring = mac_srs->srs_soft_ring_head; 330da14cebeSEric Cheng softring != NULL; softring = softring->s_ring_next) { 331da14cebeSEric Cheng if ((softring->s_ring_type & ST_RING_TCP) && 332da14cebeSEric Cheng (softring->s_ring_rx_arg2 != NULL)) { 333da14cebeSEric Cheng mcip->mci_resource_quiesce(mcip->mci_resource_arg, 334da14cebeSEric Cheng softring->s_ring_rx_arg2); 335da14cebeSEric Cheng } 336da14cebeSEric Cheng } 337da14cebeSEric Cheng } 338da14cebeSEric Cheng 339da14cebeSEric Cheng void 340da14cebeSEric Cheng mac_srs_client_poll_restart(mac_client_impl_t *mcip, 341da14cebeSEric Cheng mac_soft_ring_set_t *mac_srs) 342da14cebeSEric Cheng { 343da14cebeSEric Cheng mac_soft_ring_t *softring; 344da14cebeSEric Cheng 345da14cebeSEric Cheng ASSERT(MAC_PERIM_HELD((mac_handle_t)mcip->mci_mip)); 346da14cebeSEric Cheng 347da14cebeSEric Cheng if (!(mac_srs->srs_type & SRST_CLIENT_POLL_ENABLED)) { 348da14cebeSEric Cheng ASSERT(!(mac_srs->srs_type & SRST_DLS_BYPASS)); 349da14cebeSEric Cheng return; 350da14cebeSEric Cheng } 351da14cebeSEric Cheng 352da14cebeSEric Cheng for (softring = mac_srs->srs_soft_ring_head; 353da14cebeSEric Cheng softring != NULL; softring = softring->s_ring_next) { 354da14cebeSEric Cheng if ((softring->s_ring_type & ST_RING_TCP) && 355da14cebeSEric Cheng (softring->s_ring_rx_arg2 != NULL)) { 356da14cebeSEric Cheng mcip->mci_resource_restart(mcip->mci_resource_arg, 357da14cebeSEric Cheng softring->s_ring_rx_arg2); 358da14cebeSEric Cheng } 359da14cebeSEric Cheng } 360da14cebeSEric Cheng } 361da14cebeSEric Cheng 362da14cebeSEric Cheng /* 363da14cebeSEric Cheng * Register the given SRS and associated soft rings with the consumer and 364da14cebeSEric Cheng * enable the polling interface used by the consumer.(i.e IP) over this 365da14cebeSEric Cheng * SRS and associated soft rings. 366da14cebeSEric Cheng */ 367da14cebeSEric Cheng void 368da14cebeSEric Cheng mac_srs_client_poll_enable(mac_client_impl_t *mcip, 369da14cebeSEric Cheng mac_soft_ring_set_t *mac_srs) 370da14cebeSEric Cheng { 371da14cebeSEric Cheng mac_rx_fifo_t mrf; 372da14cebeSEric Cheng mac_soft_ring_t *softring; 373da14cebeSEric Cheng 374da14cebeSEric Cheng ASSERT(mac_srs->srs_mcip == mcip); 375da14cebeSEric Cheng ASSERT(MAC_PERIM_HELD((mac_handle_t)mcip->mci_mip)); 376da14cebeSEric Cheng 377da14cebeSEric Cheng if (!(mcip->mci_state_flags & MCIS_CLIENT_POLL_CAPABLE)) 378da14cebeSEric Cheng return; 379da14cebeSEric Cheng 380da14cebeSEric Cheng bzero(&mrf, sizeof (mac_rx_fifo_t)); 381da14cebeSEric Cheng mrf.mrf_type = MAC_RX_FIFO; 382da14cebeSEric Cheng 383da14cebeSEric Cheng /* 384da14cebeSEric Cheng * A SRS is capable of acting as a soft ring for cases 385da14cebeSEric Cheng * where no fanout is needed. This is the case for userland 386da14cebeSEric Cheng * flows. 387da14cebeSEric Cheng */ 388da14cebeSEric Cheng if (mac_srs->srs_type & SRST_NO_SOFT_RINGS) 389da14cebeSEric Cheng return; 390da14cebeSEric Cheng 391da14cebeSEric Cheng mrf.mrf_receive = (mac_receive_t)mac_soft_ring_poll; 392da14cebeSEric Cheng mrf.mrf_intr_enable = (mac_intr_enable_t)mac_soft_ring_intr_enable; 393da14cebeSEric Cheng mrf.mrf_intr_disable = (mac_intr_disable_t)mac_soft_ring_intr_disable; 394da14cebeSEric Cheng mac_srs->srs_type |= SRST_CLIENT_POLL_ENABLED; 395da14cebeSEric Cheng 396da14cebeSEric Cheng softring = mac_srs->srs_soft_ring_head; 397da14cebeSEric Cheng while (softring != NULL) { 398da14cebeSEric Cheng if (softring->s_ring_type & (ST_RING_TCP | ST_RING_UDP)) { 399da14cebeSEric Cheng /* 400da14cebeSEric Cheng * TCP and UDP support DLS bypass. Squeue polling 401da14cebeSEric Cheng * support implies DLS bypass since the squeue poll 402da14cebeSEric Cheng * path does not have DLS processing. 403da14cebeSEric Cheng */ 404da14cebeSEric Cheng mac_soft_ring_dls_bypass(softring, 405da14cebeSEric Cheng mcip->mci_direct_rx_fn, mcip->mci_direct_rx_arg); 406da14cebeSEric Cheng } 407da14cebeSEric Cheng /* 408da14cebeSEric Cheng * Non-TCP protocols don't support squeues. Hence we don't 409da14cebeSEric Cheng * make any ring addition callbacks for non-TCP rings 410da14cebeSEric Cheng */ 411da14cebeSEric Cheng if (!(softring->s_ring_type & ST_RING_TCP)) { 412da14cebeSEric Cheng softring->s_ring_rx_arg2 = NULL; 413da14cebeSEric Cheng softring = softring->s_ring_next; 414da14cebeSEric Cheng continue; 415da14cebeSEric Cheng } 416da14cebeSEric Cheng mrf.mrf_rx_arg = softring; 417da14cebeSEric Cheng mrf.mrf_intr_handle = (mac_intr_handle_t)softring; 418da14cebeSEric Cheng mrf.mrf_cpu_id = softring->s_ring_cpuid; 419da14cebeSEric Cheng mrf.mrf_flow_priority = mac_srs->srs_pri; 420da14cebeSEric Cheng 421da14cebeSEric Cheng softring->s_ring_rx_arg2 = mcip->mci_resource_add( 422da14cebeSEric Cheng mcip->mci_resource_arg, (mac_resource_t *)&mrf); 423da14cebeSEric Cheng 424da14cebeSEric Cheng softring = softring->s_ring_next; 425da14cebeSEric Cheng } 426da14cebeSEric Cheng } 427da14cebeSEric Cheng 428da14cebeSEric Cheng /* 429da14cebeSEric Cheng * Unregister the given SRS and associated soft rings with the consumer and 430da14cebeSEric Cheng * disable the polling interface used by the consumer.(i.e IP) over this 431da14cebeSEric Cheng * SRS and associated soft rings. 432da14cebeSEric Cheng */ 433da14cebeSEric Cheng void 434da14cebeSEric Cheng mac_srs_client_poll_disable(mac_client_impl_t *mcip, 435da14cebeSEric Cheng mac_soft_ring_set_t *mac_srs) 436da14cebeSEric Cheng { 437da14cebeSEric Cheng mac_soft_ring_t *softring; 438da14cebeSEric Cheng 439da14cebeSEric Cheng ASSERT(MAC_PERIM_HELD((mac_handle_t)mcip->mci_mip)); 440da14cebeSEric Cheng 441da14cebeSEric Cheng /* 442da14cebeSEric Cheng * A SRS is capable of acting as a soft ring for cases 443da14cebeSEric Cheng * where no protocol fanout is needed. This is the case 444da14cebeSEric Cheng * for userland flows. Nothing to do here. 445da14cebeSEric Cheng */ 446da14cebeSEric Cheng if (mac_srs->srs_type & SRST_NO_SOFT_RINGS) 447da14cebeSEric Cheng return; 448da14cebeSEric Cheng 449da14cebeSEric Cheng mutex_enter(&mac_srs->srs_lock); 450da14cebeSEric Cheng if (!(mac_srs->srs_type & SRST_CLIENT_POLL_ENABLED)) { 451da14cebeSEric Cheng ASSERT(!(mac_srs->srs_type & SRST_DLS_BYPASS)); 452da14cebeSEric Cheng mutex_exit(&mac_srs->srs_lock); 453da14cebeSEric Cheng return; 454da14cebeSEric Cheng } 455da14cebeSEric Cheng mac_srs->srs_type &= ~(SRST_CLIENT_POLL_ENABLED | SRST_DLS_BYPASS); 456da14cebeSEric Cheng mutex_exit(&mac_srs->srs_lock); 457da14cebeSEric Cheng 458da14cebeSEric Cheng /* 459da14cebeSEric Cheng * DLS bypass is now disabled in the case of both TCP and UDP. 460da14cebeSEric Cheng * Reset the soft ring callbacks to the standard 'mac_rx_deliver' 461da14cebeSEric Cheng * callback. In addition, in the case of TCP, invoke IP's callback 462da14cebeSEric Cheng * for ring removal. 463da14cebeSEric Cheng */ 464da14cebeSEric Cheng for (softring = mac_srs->srs_soft_ring_head; 465da14cebeSEric Cheng softring != NULL; softring = softring->s_ring_next) { 466da14cebeSEric Cheng if (!(softring->s_ring_type & (ST_RING_UDP | ST_RING_TCP))) 467da14cebeSEric Cheng continue; 468da14cebeSEric Cheng 469da14cebeSEric Cheng if ((softring->s_ring_type & ST_RING_TCP) && 470da14cebeSEric Cheng softring->s_ring_rx_arg2 != NULL) { 471da14cebeSEric Cheng mcip->mci_resource_remove(mcip->mci_resource_arg, 472da14cebeSEric Cheng softring->s_ring_rx_arg2); 473da14cebeSEric Cheng } 474da14cebeSEric Cheng 475da14cebeSEric Cheng mutex_enter(&softring->s_ring_lock); 476da14cebeSEric Cheng while (softring->s_ring_state & S_RING_PROC) { 477da14cebeSEric Cheng softring->s_ring_state |= S_RING_CLIENT_WAIT; 478da14cebeSEric Cheng cv_wait(&softring->s_ring_client_cv, 479da14cebeSEric Cheng &softring->s_ring_lock); 480da14cebeSEric Cheng } 481da14cebeSEric Cheng softring->s_ring_state &= ~S_RING_CLIENT_WAIT; 482da14cebeSEric Cheng softring->s_ring_rx_arg2 = NULL; 483da14cebeSEric Cheng softring->s_ring_rx_func = mac_rx_deliver; 484da14cebeSEric Cheng softring->s_ring_rx_arg1 = mcip; 485da14cebeSEric Cheng mutex_exit(&softring->s_ring_lock); 486da14cebeSEric Cheng } 487da14cebeSEric Cheng } 488da14cebeSEric Cheng 489da14cebeSEric Cheng /* 490da14cebeSEric Cheng * Enable or disable poll capability of the SRS on the underlying Rx ring. 491da14cebeSEric Cheng * 492da14cebeSEric Cheng * There is a need to enable or disable the poll capability of an SRS over an 493da14cebeSEric Cheng * Rx ring depending on the number of mac clients sharing the ring and also 494da14cebeSEric Cheng * whether user flows are configured on it. However the poll state is actively 495da14cebeSEric Cheng * manipulated by the SRS worker and poll threads and uncoordinated changes by 496da14cebeSEric Cheng * yet another thread to the underlying capability can surprise them leading 497da14cebeSEric Cheng * to assert failures. Instead we quiesce the SRS, make the changes and then 498da14cebeSEric Cheng * restart the SRS. 499da14cebeSEric Cheng */ 500da14cebeSEric Cheng static void 501da14cebeSEric Cheng mac_srs_poll_state_change(mac_soft_ring_set_t *mac_srs, 502da14cebeSEric Cheng boolean_t turn_off_poll_capab, mac_rx_func_t rx_func) 503da14cebeSEric Cheng { 504da14cebeSEric Cheng boolean_t need_restart = B_FALSE; 505da14cebeSEric Cheng mac_srs_rx_t *srs_rx = &mac_srs->srs_rx; 506da14cebeSEric Cheng mac_ring_t *ring; 507da14cebeSEric Cheng 508da14cebeSEric Cheng if (!SRS_QUIESCED(mac_srs)) { 509da14cebeSEric Cheng mac_rx_srs_quiesce(mac_srs, SRS_QUIESCE); 510da14cebeSEric Cheng need_restart = B_TRUE; 511da14cebeSEric Cheng } 512da14cebeSEric Cheng 513da14cebeSEric Cheng ring = mac_srs->srs_ring; 514da14cebeSEric Cheng if ((ring != NULL) && 515da14cebeSEric Cheng (ring->mr_classify_type == MAC_HW_CLASSIFIER)) { 516da14cebeSEric Cheng if (turn_off_poll_capab) 517da14cebeSEric Cheng mac_srs->srs_state &= ~SRS_POLLING_CAPAB; 518ae6aa22aSVenugopal Iyer else if (mac_poll_enable) 519da14cebeSEric Cheng mac_srs->srs_state |= SRS_POLLING_CAPAB; 520da14cebeSEric Cheng } 521da14cebeSEric Cheng srs_rx->sr_lower_proc = rx_func; 522da14cebeSEric Cheng 523da14cebeSEric Cheng if (need_restart) 524da14cebeSEric Cheng mac_rx_srs_restart(mac_srs); 525da14cebeSEric Cheng } 526da14cebeSEric Cheng 527da14cebeSEric Cheng /* CPU RECONFIGURATION AND FANOUT COMPUTATION ROUTINES */ 528da14cebeSEric Cheng 529da14cebeSEric Cheng /* 530da14cebeSEric Cheng * Return the next CPU to be used to bind a MAC kernel thread. 5310dc2366fSVenugopal Iyer * If a cpupart is specified, the cpu chosen must be from that 5320dc2366fSVenugopal Iyer * cpu partition. 533da14cebeSEric Cheng */ 534da14cebeSEric Cheng static processorid_t 5350dc2366fSVenugopal Iyer mac_next_bind_cpu(cpupart_t *cpupart) 536da14cebeSEric Cheng { 5370dc2366fSVenugopal Iyer static cpu_t *cp = NULL; 5380dc2366fSVenugopal Iyer cpu_t *cp_start; 539da14cebeSEric Cheng 540da14cebeSEric Cheng ASSERT(MUTEX_HELD(&cpu_lock)); 541da14cebeSEric Cheng 5420dc2366fSVenugopal Iyer if (cp == NULL) 5430dc2366fSVenugopal Iyer cp = cpu_list; 544da14cebeSEric Cheng 5450dc2366fSVenugopal Iyer cp = cp->cpu_next_onln; 5460dc2366fSVenugopal Iyer cp_start = cp; 5470dc2366fSVenugopal Iyer 5480dc2366fSVenugopal Iyer do { 5490dc2366fSVenugopal Iyer if ((cpupart == NULL) || (cp->cpu_part == cpupart)) 5500dc2366fSVenugopal Iyer return (cp->cpu_id); 5510dc2366fSVenugopal Iyer 5520dc2366fSVenugopal Iyer } while ((cp = cp->cpu_next_onln) != cp_start); 5530dc2366fSVenugopal Iyer 55436f99a58SToomas Soome return (-1); /* No matching CPU found online */ 555da14cebeSEric Cheng } 556da14cebeSEric Cheng 557da14cebeSEric Cheng /* ARGSUSED */ 558da14cebeSEric Cheng static int 559da14cebeSEric Cheng mac_srs_cpu_setup(cpu_setup_t what, int id, void *arg) 560da14cebeSEric Cheng { 561da14cebeSEric Cheng ASSERT(MUTEX_HELD(&cpu_lock)); 562da14cebeSEric Cheng switch (what) { 563da14cebeSEric Cheng case CPU_CONFIG: 564da14cebeSEric Cheng case CPU_ON: 565da14cebeSEric Cheng case CPU_CPUPART_IN: 566da14cebeSEric Cheng mac_walk_srs_and_bind(id); 567da14cebeSEric Cheng break; 568da14cebeSEric Cheng 569da14cebeSEric Cheng case CPU_UNCONFIG: 570da14cebeSEric Cheng case CPU_OFF: 571da14cebeSEric Cheng case CPU_CPUPART_OUT: 572da14cebeSEric Cheng mac_walk_srs_and_unbind(id); 573da14cebeSEric Cheng break; 574da14cebeSEric Cheng 575da14cebeSEric Cheng default: 576da14cebeSEric Cheng break; 577da14cebeSEric Cheng } 578da14cebeSEric Cheng return (0); 579da14cebeSEric Cheng } 580da14cebeSEric Cheng 581da14cebeSEric Cheng /* 582da14cebeSEric Cheng * mac_compute_soft_ring_count(): 583da14cebeSEric Cheng * 584da14cebeSEric Cheng * This routine computes the number of soft rings needed to handle incoming 585da14cebeSEric Cheng * load given a flow_entry. 586da14cebeSEric Cheng * 587da14cebeSEric Cheng * The routine does the following: 588da14cebeSEric Cheng * 1) soft rings will be created if mac_soft_ring_enable is set. 589da14cebeSEric Cheng * 2) If the underlying link is a 10Gbps link, then soft rings will be 590da14cebeSEric Cheng * created even if mac_soft_ring_enable is not set. The number of soft 591da14cebeSEric Cheng * rings, so created, will equal mac_rx_soft_ring_10gig_count. 592da14cebeSEric Cheng * 3) On a sun4v platform (i.e., mac_soft_ring_enable is set), 2 times the 593da14cebeSEric Cheng * mac_rx_soft_ring_10gig_count number of soft rings will be created for a 594da14cebeSEric Cheng * 10Gbps link. 595da14cebeSEric Cheng * 596da14cebeSEric Cheng * If a bandwidth limit is specified, the number that gets computed is 597da14cebeSEric Cheng * dependent upon CPU speed, the number of Rx rings configured, and 598da14cebeSEric Cheng * the bandwidth limit. 599da14cebeSEric Cheng * If more Rx rings are available, less number of soft rings is needed. 600da14cebeSEric Cheng * 601da14cebeSEric Cheng * mac_use_bw_heuristic is another "hidden" variable that can be used to 602da14cebeSEric Cheng * override the default use of soft ring count computation. Depending upon 603da14cebeSEric Cheng * the usefulness of it, mac_use_bw_heuristic can later be made into a 604da14cebeSEric Cheng * data-link property or removed altogether. 605da14cebeSEric Cheng * 606da14cebeSEric Cheng * TODO: Cleanup and tighten some of the assumptions. 607da14cebeSEric Cheng */ 608da14cebeSEric Cheng boolean_t mac_use_bw_heuristic = B_TRUE; 609da14cebeSEric Cheng static int 6100dc2366fSVenugopal Iyer mac_compute_soft_ring_count(flow_entry_t *flent, int rx_srs_cnt, int maxcpus) 611da14cebeSEric Cheng { 612da14cebeSEric Cheng uint64_t cpu_speed, bw = 0; 613da14cebeSEric Cheng int srings = 0; 614da14cebeSEric Cheng boolean_t bw_enabled = B_FALSE; 615da14cebeSEric Cheng 616da14cebeSEric Cheng ASSERT(!(flent->fe_type & FLOW_USER)); 617da14cebeSEric Cheng if (flent->fe_resource_props.mrp_mask & MRP_MAXBW && 618da14cebeSEric Cheng mac_use_bw_heuristic) { 619da14cebeSEric Cheng /* bandwidth enabled */ 620da14cebeSEric Cheng bw_enabled = B_TRUE; 621da14cebeSEric Cheng bw = flent->fe_resource_props.mrp_maxbw; 622da14cebeSEric Cheng } 623da14cebeSEric Cheng if (!bw_enabled) { 624da14cebeSEric Cheng /* No bandwidth enabled */ 625da14cebeSEric Cheng if (mac_soft_ring_enable) 626da14cebeSEric Cheng srings = mac_rx_soft_ring_count; 627da14cebeSEric Cheng 628da14cebeSEric Cheng /* Is this a 10Gig link? */ 629da14cebeSEric Cheng flent->fe_nic_speed = mac_client_stat_get(flent->fe_mcip, 630da14cebeSEric Cheng MAC_STAT_IFSPEED); 631da14cebeSEric Cheng /* convert to Mbps */ 632da14cebeSEric Cheng if (((flent->fe_nic_speed)/1000000) > 1000 && 633da14cebeSEric Cheng mac_rx_soft_ring_10gig_count > 0) { 634da14cebeSEric Cheng /* This is a 10Gig link */ 635da14cebeSEric Cheng srings = mac_rx_soft_ring_10gig_count; 636da14cebeSEric Cheng /* 637da14cebeSEric Cheng * Use 2 times mac_rx_soft_ring_10gig_count for 638da14cebeSEric Cheng * sun4v systems. 639da14cebeSEric Cheng */ 640da14cebeSEric Cheng if (mac_soft_ring_enable) 641da14cebeSEric Cheng srings = srings * 2; 642da14cebeSEric Cheng } 643da14cebeSEric Cheng } else { 644da14cebeSEric Cheng /* 645da14cebeSEric Cheng * Soft ring computation using CPU speed and specified 646da14cebeSEric Cheng * bandwidth limit. 647da14cebeSEric Cheng */ 648da14cebeSEric Cheng /* Assumption: all CPUs have the same frequency */ 649da14cebeSEric Cheng cpu_speed = (uint64_t)CPU->cpu_type_info.pi_clock; 650da14cebeSEric Cheng 651da14cebeSEric Cheng /* cpu_speed is in MHz; make bw in units of Mbps. */ 652da14cebeSEric Cheng bw = bw/1000000; 653da14cebeSEric Cheng 654da14cebeSEric Cheng if (bw >= 1000) { 655da14cebeSEric Cheng /* 656da14cebeSEric Cheng * bw is greater than or equal to 1Gbps. 657da14cebeSEric Cheng * The number of soft rings required is a function 658da14cebeSEric Cheng * of bandwidth and CPU speed. To keep this simple, 659da14cebeSEric Cheng * let's use this rule: 1GHz CPU can handle 1Gbps. 660da14cebeSEric Cheng * If bw is less than 1 Gbps, then there is no need 661da14cebeSEric Cheng * for soft rings. Assumption is that CPU speeds 662da14cebeSEric Cheng * (on modern systems) are at least 1GHz. 663da14cebeSEric Cheng */ 664da14cebeSEric Cheng srings = bw/cpu_speed; 665da14cebeSEric Cheng if (srings <= 1 && mac_soft_ring_enable) { 666da14cebeSEric Cheng /* 667da14cebeSEric Cheng * Give at least 2 soft rings 668da14cebeSEric Cheng * for sun4v systems 669da14cebeSEric Cheng */ 670da14cebeSEric Cheng srings = 2; 671da14cebeSEric Cheng } 672da14cebeSEric Cheng } 673da14cebeSEric Cheng } 674da14cebeSEric Cheng /* 675da14cebeSEric Cheng * If the flent has multiple Rx SRSs, then each SRS need not 676da14cebeSEric Cheng * have that many soft rings on top of it. The number of 677da14cebeSEric Cheng * soft rings for each Rx SRS is found by dividing srings by 678da14cebeSEric Cheng * rx_srs_cnt. 679da14cebeSEric Cheng */ 680da14cebeSEric Cheng if (rx_srs_cnt > 1) { 681da14cebeSEric Cheng int remainder; 682da14cebeSEric Cheng 683da14cebeSEric Cheng remainder = srings%rx_srs_cnt; 684da14cebeSEric Cheng srings = srings/rx_srs_cnt; 685da14cebeSEric Cheng if (remainder != 0) 686da14cebeSEric Cheng srings++; 687da14cebeSEric Cheng /* 688da14cebeSEric Cheng * Fanning out to 1 soft ring is not very useful. 689da14cebeSEric Cheng * Set it as well to 0 and mac_srs_fanout_init() 690da14cebeSEric Cheng * will take care of creating a single soft ring 691da14cebeSEric Cheng * for proto fanout. 692da14cebeSEric Cheng */ 693da14cebeSEric Cheng if (srings == 1) 694da14cebeSEric Cheng srings = 0; 695da14cebeSEric Cheng } 696da14cebeSEric Cheng /* Do some more massaging */ 6970dc2366fSVenugopal Iyer srings = min(srings, maxcpus); 698da14cebeSEric Cheng srings = min(srings, MAX_SR_FANOUT); 699da14cebeSEric Cheng return (srings); 700da14cebeSEric Cheng } 701da14cebeSEric Cheng 702da14cebeSEric Cheng /* 7030dc2366fSVenugopal Iyer * mac_tx_cpu_init: 7040dc2366fSVenugopal Iyer * set up CPUs for Tx interrupt re-targeting and Tx worker 7050dc2366fSVenugopal Iyer * thread binding 7060dc2366fSVenugopal Iyer */ 7070dc2366fSVenugopal Iyer static void 7080dc2366fSVenugopal Iyer mac_tx_cpu_init(flow_entry_t *flent, mac_resource_props_t *mrp, 7090dc2366fSVenugopal Iyer cpupart_t *cpupart) 7100dc2366fSVenugopal Iyer { 7110dc2366fSVenugopal Iyer mac_soft_ring_set_t *tx_srs = flent->fe_tx_srs; 7120dc2366fSVenugopal Iyer mac_srs_tx_t *srs_tx = &tx_srs->srs_tx; 7130dc2366fSVenugopal Iyer mac_cpus_t *srs_cpu = &tx_srs->srs_cpu; 7140dc2366fSVenugopal Iyer mac_soft_ring_t *sringp; 7150dc2366fSVenugopal Iyer mac_ring_t *ring; 7160dc2366fSVenugopal Iyer processorid_t worker_cpuid; 7170dc2366fSVenugopal Iyer boolean_t retargetable_client = B_FALSE; 7180dc2366fSVenugopal Iyer int i, j; 7190dc2366fSVenugopal Iyer 7200dc2366fSVenugopal Iyer if (RETARGETABLE_CLIENT((mac_group_t *)flent->fe_tx_ring_group, 7210dc2366fSVenugopal Iyer flent->fe_mcip)) { 7220dc2366fSVenugopal Iyer retargetable_client = B_TRUE; 7230dc2366fSVenugopal Iyer } 7240dc2366fSVenugopal Iyer 7250dc2366fSVenugopal Iyer if (MAC_TX_SOFT_RINGS(tx_srs)) { 7260dc2366fSVenugopal Iyer if (mrp != NULL) 7270dc2366fSVenugopal Iyer j = mrp->mrp_ncpus - 1; 7280dc2366fSVenugopal Iyer for (i = 0; i < tx_srs->srs_tx_ring_count; i++) { 7290dc2366fSVenugopal Iyer if (mrp != NULL) { 7300dc2366fSVenugopal Iyer if (j < 0) 7310dc2366fSVenugopal Iyer j = mrp->mrp_ncpus - 1; 7320dc2366fSVenugopal Iyer worker_cpuid = mrp->mrp_cpu[j]; 7330dc2366fSVenugopal Iyer } else { 7340dc2366fSVenugopal Iyer /* 7350dc2366fSVenugopal Iyer * Bind interrupt to the next CPU available 7360dc2366fSVenugopal Iyer * and leave the worker unbound. 7370dc2366fSVenugopal Iyer */ 7380dc2366fSVenugopal Iyer worker_cpuid = -1; 7390dc2366fSVenugopal Iyer } 7400dc2366fSVenugopal Iyer sringp = tx_srs->srs_tx_soft_rings[i]; 7410dc2366fSVenugopal Iyer ring = (mac_ring_t *)sringp->s_ring_tx_arg2; 7420dc2366fSVenugopal Iyer srs_cpu->mc_tx_fanout_cpus[i] = worker_cpuid; 7430dc2366fSVenugopal Iyer if (MAC_RING_RETARGETABLE(ring) && 7440dc2366fSVenugopal Iyer retargetable_client) { 7450dc2366fSVenugopal Iyer mutex_enter(&cpu_lock); 7460dc2366fSVenugopal Iyer srs_cpu->mc_tx_intr_cpu[i] = 7470dc2366fSVenugopal Iyer (mrp != NULL) ? mrp->mrp_cpu[j] : 7480dc2366fSVenugopal Iyer (mac_tx_intr_retarget ? 7490dc2366fSVenugopal Iyer mac_next_bind_cpu(cpupart) : -1); 7500dc2366fSVenugopal Iyer mutex_exit(&cpu_lock); 7510dc2366fSVenugopal Iyer } else { 7520dc2366fSVenugopal Iyer srs_cpu->mc_tx_intr_cpu[i] = -1; 7530dc2366fSVenugopal Iyer } 7540dc2366fSVenugopal Iyer if (mrp != NULL) 7550dc2366fSVenugopal Iyer j--; 7560dc2366fSVenugopal Iyer } 7570dc2366fSVenugopal Iyer } else { 7580dc2366fSVenugopal Iyer /* Tx mac_ring_handle_t is stored in st_arg2 */ 7590dc2366fSVenugopal Iyer srs_cpu->mc_tx_fanout_cpus[0] = 7600dc2366fSVenugopal Iyer (mrp != NULL) ? mrp->mrp_cpu[mrp->mrp_ncpus - 1] : -1; 7610dc2366fSVenugopal Iyer ring = (mac_ring_t *)srs_tx->st_arg2; 7620dc2366fSVenugopal Iyer if (MAC_RING_RETARGETABLE(ring) && retargetable_client) { 7630dc2366fSVenugopal Iyer mutex_enter(&cpu_lock); 7640dc2366fSVenugopal Iyer srs_cpu->mc_tx_intr_cpu[0] = (mrp != NULL) ? 7650dc2366fSVenugopal Iyer mrp->mrp_cpu[mrp->mrp_ncpus - 1] : 7660dc2366fSVenugopal Iyer (mac_tx_intr_retarget ? 7670dc2366fSVenugopal Iyer mac_next_bind_cpu(cpupart) : -1); 7680dc2366fSVenugopal Iyer mutex_exit(&cpu_lock); 7690dc2366fSVenugopal Iyer } else { 7700dc2366fSVenugopal Iyer srs_cpu->mc_tx_intr_cpu[0] = -1; 7710dc2366fSVenugopal Iyer } 7720dc2366fSVenugopal Iyer } 7730dc2366fSVenugopal Iyer } 7740dc2366fSVenugopal Iyer 7750dc2366fSVenugopal Iyer /* 776da14cebeSEric Cheng * Assignment of user specified CPUs to a link. 777da14cebeSEric Cheng * 778da14cebeSEric Cheng * Minimum CPUs required to get an optimal assignmet: 779da14cebeSEric Cheng * For each Rx SRS, atleast two CPUs are needed if mac_latency_optimize 780da14cebeSEric Cheng * flag is set -- one for polling, one for fanout soft ring. 781da14cebeSEric Cheng * If mac_latency_optimize is not set, then 3 CPUs are needed -- one 782da14cebeSEric Cheng * for polling, one for SRS worker thread and one for fanout soft ring. 783da14cebeSEric Cheng * 784da14cebeSEric Cheng * The CPUs needed for Tx side is equal to the number of Tx rings 785da14cebeSEric Cheng * the link is using. 786da14cebeSEric Cheng * 787da14cebeSEric Cheng * mac_flow_user_cpu_init() categorizes the CPU assignment depending 788da14cebeSEric Cheng * upon the number of CPUs in 3 different buckets. 789da14cebeSEric Cheng * 790da14cebeSEric Cheng * In the first bucket, the most optimal case is handled. The user has 791da14cebeSEric Cheng * passed enough number of CPUs and every thread gets its own CPU. 792da14cebeSEric Cheng * 793da14cebeSEric Cheng * The second and third are the sub-optimal cases. Enough CPUs are not 794da14cebeSEric Cheng * available. 795da14cebeSEric Cheng * 796da14cebeSEric Cheng * The second bucket handles the case where atleast one distinct CPU is 797da14cebeSEric Cheng * is available for each of the Rx rings (Rx SRSes) and Tx rings (Tx 798da14cebeSEric Cheng * SRS or soft rings). 799da14cebeSEric Cheng * 800da14cebeSEric Cheng * In the third case (worst case scenario), specified CPU count is less 801da14cebeSEric Cheng * than the Rx rings configured for the link. In this case, we round 802da14cebeSEric Cheng * robin the CPUs among the Rx SRSes and Tx SRS/soft rings. 803da14cebeSEric Cheng */ 804da14cebeSEric Cheng static void 805da14cebeSEric Cheng mac_flow_user_cpu_init(flow_entry_t *flent, mac_resource_props_t *mrp) 806da14cebeSEric Cheng { 807da14cebeSEric Cheng mac_soft_ring_set_t *rx_srs, *tx_srs; 808da14cebeSEric Cheng int i, srs_cnt; 809da14cebeSEric Cheng mac_cpus_t *srs_cpu; 810da14cebeSEric Cheng int no_of_cpus, cpu_cnt; 811da14cebeSEric Cheng int rx_srs_cnt, reqd_rx_cpu_cnt; 812da14cebeSEric Cheng int fanout_cpu_cnt, reqd_tx_cpu_cnt; 813da14cebeSEric Cheng int reqd_poll_worker_cnt, fanout_cnt_per_srs; 8140dc2366fSVenugopal Iyer mac_resource_props_t *emrp = &flent->fe_effective_props; 815da14cebeSEric Cheng 816da14cebeSEric Cheng ASSERT(mrp->mrp_fanout_mode == MCM_CPUS); 817da14cebeSEric Cheng /* 818da14cebeSEric Cheng * The check for nbc_ncpus to be within limits for 819da14cebeSEric Cheng * the user specified case was done earlier and if 820da14cebeSEric Cheng * not within limits, an error would have been 821da14cebeSEric Cheng * returned to the user. 822da14cebeSEric Cheng */ 8235adf34bdSRajagopal Kunhappan ASSERT(mrp->mrp_ncpus > 0); 824da14cebeSEric Cheng 825da14cebeSEric Cheng no_of_cpus = mrp->mrp_ncpus; 826da14cebeSEric Cheng 8270dc2366fSVenugopal Iyer if (mrp->mrp_rx_intr_cpu != -1) { 828da14cebeSEric Cheng /* 829da14cebeSEric Cheng * interrupt has been re-targetted. Poll 830da14cebeSEric Cheng * thread needs to be bound to interrupt 8310dc2366fSVenugopal Iyer * CPU. 832da14cebeSEric Cheng * 833da14cebeSEric Cheng * Find where in the list is the intr 834da14cebeSEric Cheng * CPU and swap it with the first one. 835da14cebeSEric Cheng * We will be using the first CPU in the 836da14cebeSEric Cheng * list for poll. 837da14cebeSEric Cheng */ 838da14cebeSEric Cheng for (i = 0; i < no_of_cpus; i++) { 8390dc2366fSVenugopal Iyer if (mrp->mrp_cpu[i] == mrp->mrp_rx_intr_cpu) 840da14cebeSEric Cheng break; 841da14cebeSEric Cheng } 842da14cebeSEric Cheng mrp->mrp_cpu[i] = mrp->mrp_cpu[0]; 8430dc2366fSVenugopal Iyer mrp->mrp_cpu[0] = mrp->mrp_rx_intr_cpu; 844da14cebeSEric Cheng } 845da14cebeSEric Cheng 846da14cebeSEric Cheng /* 847da14cebeSEric Cheng * Requirements: 848da14cebeSEric Cheng * The number of CPUs that each Rx ring needs is dependent 849da14cebeSEric Cheng * upon mac_latency_optimize flag. 850da14cebeSEric Cheng * 1) If set, atleast 2 CPUs are needed -- one for 851da14cebeSEric Cheng * polling, one for fanout soft ring. 852da14cebeSEric Cheng * 2) If not set, then atleast 3 CPUs are needed -- one 853da14cebeSEric Cheng * for polling, one for srs worker thread, and one for 854da14cebeSEric Cheng * fanout soft ring. 855da14cebeSEric Cheng */ 856da14cebeSEric Cheng rx_srs_cnt = (flent->fe_rx_srs_cnt > 1) ? 857da14cebeSEric Cheng (flent->fe_rx_srs_cnt - 1) : flent->fe_rx_srs_cnt; 858da14cebeSEric Cheng reqd_rx_cpu_cnt = mac_latency_optimize ? 859da14cebeSEric Cheng (rx_srs_cnt * 2) : (rx_srs_cnt * 3); 860da14cebeSEric Cheng 861da14cebeSEric Cheng /* How many CPUs are needed for Tx side? */ 862da14cebeSEric Cheng tx_srs = flent->fe_tx_srs; 8630dc2366fSVenugopal Iyer reqd_tx_cpu_cnt = MAC_TX_SOFT_RINGS(tx_srs) ? 8640dc2366fSVenugopal Iyer tx_srs->srs_tx_ring_count : 1; 865da14cebeSEric Cheng 866da14cebeSEric Cheng /* CPUs needed for Rx SRSes poll and worker threads */ 867da14cebeSEric Cheng reqd_poll_worker_cnt = mac_latency_optimize ? 868da14cebeSEric Cheng rx_srs_cnt : rx_srs_cnt * 2; 869da14cebeSEric Cheng 870da14cebeSEric Cheng /* Has the user provided enough CPUs? */ 871da14cebeSEric Cheng if (no_of_cpus >= (reqd_rx_cpu_cnt + reqd_tx_cpu_cnt)) { 872da14cebeSEric Cheng /* 873da14cebeSEric Cheng * Best case scenario. There is enough CPUs. All 874da14cebeSEric Cheng * Rx rings will get their own set of CPUs plus 875da14cebeSEric Cheng * Tx soft rings will get their own. 876da14cebeSEric Cheng */ 877da14cebeSEric Cheng /* 878da14cebeSEric Cheng * fanout_cpu_cnt is the number of CPUs available 879da14cebeSEric Cheng * for Rx side fanout soft rings. 880da14cebeSEric Cheng */ 881da14cebeSEric Cheng fanout_cpu_cnt = no_of_cpus - 882da14cebeSEric Cheng reqd_poll_worker_cnt - reqd_tx_cpu_cnt; 883da14cebeSEric Cheng 884da14cebeSEric Cheng /* 885da14cebeSEric Cheng * Divide fanout_cpu_cnt by rx_srs_cnt to find 886da14cebeSEric Cheng * out how many fanout soft rings each Rx SRS 887da14cebeSEric Cheng * can have. 888da14cebeSEric Cheng */ 889da14cebeSEric Cheng fanout_cnt_per_srs = fanout_cpu_cnt/rx_srs_cnt; 890da14cebeSEric Cheng 8915adf34bdSRajagopal Kunhappan /* fanout_cnt_per_srs should not be > MAX_SR_FANOUT */ 8925adf34bdSRajagopal Kunhappan fanout_cnt_per_srs = min(fanout_cnt_per_srs, MAX_SR_FANOUT); 8935adf34bdSRajagopal Kunhappan 894da14cebeSEric Cheng /* Do the assignment for the default Rx ring */ 895da14cebeSEric Cheng cpu_cnt = 0; 896da14cebeSEric Cheng rx_srs = flent->fe_rx_srs[0]; 897da14cebeSEric Cheng ASSERT(rx_srs->srs_ring == NULL); 898da14cebeSEric Cheng if (rx_srs->srs_fanout_state == SRS_FANOUT_INIT) 899da14cebeSEric Cheng rx_srs->srs_fanout_state = SRS_FANOUT_REINIT; 900da14cebeSEric Cheng srs_cpu = &rx_srs->srs_cpu; 901da14cebeSEric Cheng srs_cpu->mc_ncpus = no_of_cpus; 902da14cebeSEric Cheng bcopy(mrp->mrp_cpu, 903da14cebeSEric Cheng srs_cpu->mc_cpus, sizeof (srs_cpu->mc_cpus)); 9040dc2366fSVenugopal Iyer srs_cpu->mc_rx_fanout_cnt = fanout_cnt_per_srs; 9050dc2366fSVenugopal Iyer srs_cpu->mc_rx_pollid = mrp->mrp_cpu[cpu_cnt++]; 9060dc2366fSVenugopal Iyer /* Retarget the interrupt to the same CPU as the poll */ 9070dc2366fSVenugopal Iyer srs_cpu->mc_rx_intr_cpu = srs_cpu->mc_rx_pollid; 9080dc2366fSVenugopal Iyer srs_cpu->mc_rx_workerid = (mac_latency_optimize ? 9090dc2366fSVenugopal Iyer srs_cpu->mc_rx_pollid : mrp->mrp_cpu[cpu_cnt++]); 910da14cebeSEric Cheng for (i = 0; i < fanout_cnt_per_srs; i++) 9110dc2366fSVenugopal Iyer srs_cpu->mc_rx_fanout_cpus[i] = mrp->mrp_cpu[cpu_cnt++]; 912da14cebeSEric Cheng 913da14cebeSEric Cheng /* Do the assignment for h/w Rx SRSes */ 914da14cebeSEric Cheng if (flent->fe_rx_srs_cnt > 1) { 915da14cebeSEric Cheng cpu_cnt = 0; 916da14cebeSEric Cheng for (srs_cnt = 1; 917da14cebeSEric Cheng srs_cnt < flent->fe_rx_srs_cnt; srs_cnt++) { 918da14cebeSEric Cheng rx_srs = flent->fe_rx_srs[srs_cnt]; 919da14cebeSEric Cheng ASSERT(rx_srs->srs_ring != NULL); 920da14cebeSEric Cheng if (rx_srs->srs_fanout_state == 921da14cebeSEric Cheng SRS_FANOUT_INIT) { 922da14cebeSEric Cheng rx_srs->srs_fanout_state = 923da14cebeSEric Cheng SRS_FANOUT_REINIT; 924da14cebeSEric Cheng } 925da14cebeSEric Cheng srs_cpu = &rx_srs->srs_cpu; 926da14cebeSEric Cheng srs_cpu->mc_ncpus = no_of_cpus; 927da14cebeSEric Cheng bcopy(mrp->mrp_cpu, srs_cpu->mc_cpus, 928da14cebeSEric Cheng sizeof (srs_cpu->mc_cpus)); 9290dc2366fSVenugopal Iyer srs_cpu->mc_rx_fanout_cnt = fanout_cnt_per_srs; 930da14cebeSEric Cheng /* The first CPU in the list is the intr CPU */ 9310dc2366fSVenugopal Iyer srs_cpu->mc_rx_pollid = mrp->mrp_cpu[cpu_cnt++]; 9320dc2366fSVenugopal Iyer srs_cpu->mc_rx_intr_cpu = srs_cpu->mc_rx_pollid; 9330dc2366fSVenugopal Iyer srs_cpu->mc_rx_workerid = 9340dc2366fSVenugopal Iyer (mac_latency_optimize ? 9350dc2366fSVenugopal Iyer srs_cpu->mc_rx_pollid : 9360dc2366fSVenugopal Iyer mrp->mrp_cpu[cpu_cnt++]); 937da14cebeSEric Cheng for (i = 0; i < fanout_cnt_per_srs; i++) { 9380dc2366fSVenugopal Iyer srs_cpu->mc_rx_fanout_cpus[i] = 939da14cebeSEric Cheng mrp->mrp_cpu[cpu_cnt++]; 940da14cebeSEric Cheng } 941da14cebeSEric Cheng ASSERT(cpu_cnt <= no_of_cpus); 942da14cebeSEric Cheng } 943da14cebeSEric Cheng } 9440dc2366fSVenugopal Iyer goto tx_cpu_init; 945da14cebeSEric Cheng } 946da14cebeSEric Cheng 947da14cebeSEric Cheng /* 948da14cebeSEric Cheng * Sub-optimal case. 949da14cebeSEric Cheng * We have the following information: 950da14cebeSEric Cheng * no_of_cpus - no. of cpus that user passed. 951da14cebeSEric Cheng * rx_srs_cnt - no. of rx rings. 952da14cebeSEric Cheng * reqd_rx_cpu_cnt = mac_latency_optimize?rx_srs_cnt*2:rx_srs_cnt*3 953da14cebeSEric Cheng * reqd_tx_cpu_cnt - no. of cpus reqd. for Tx side. 954da14cebeSEric Cheng * reqd_poll_worker_cnt = mac_latency_optimize?rx_srs_cnt:rx_srs_cnt*2 955da14cebeSEric Cheng */ 956da14cebeSEric Cheng /* 957da14cebeSEric Cheng * If we bind the Rx fanout soft rings to the same CPUs 958da14cebeSEric Cheng * as poll/worker, would that be enough? 959da14cebeSEric Cheng */ 960da14cebeSEric Cheng if (no_of_cpus >= (rx_srs_cnt + reqd_tx_cpu_cnt)) { 961da14cebeSEric Cheng boolean_t worker_assign = B_FALSE; 962da14cebeSEric Cheng 963da14cebeSEric Cheng /* 964da14cebeSEric Cheng * If mac_latency_optimize is not set, are there 965da14cebeSEric Cheng * enough CPUs to assign a CPU for worker also? 966da14cebeSEric Cheng */ 967da14cebeSEric Cheng if (no_of_cpus >= (reqd_poll_worker_cnt + reqd_tx_cpu_cnt)) 968da14cebeSEric Cheng worker_assign = B_TRUE; 969da14cebeSEric Cheng /* 970da14cebeSEric Cheng * Zero'th Rx SRS is the default Rx ring. It is not 971da14cebeSEric Cheng * associated with h/w Rx ring. 972da14cebeSEric Cheng */ 973da14cebeSEric Cheng rx_srs = flent->fe_rx_srs[0]; 974da14cebeSEric Cheng ASSERT(rx_srs->srs_ring == NULL); 975da14cebeSEric Cheng if (rx_srs->srs_fanout_state == SRS_FANOUT_INIT) 976da14cebeSEric Cheng rx_srs->srs_fanout_state = SRS_FANOUT_REINIT; 977da14cebeSEric Cheng cpu_cnt = 0; 978da14cebeSEric Cheng srs_cpu = &rx_srs->srs_cpu; 979da14cebeSEric Cheng srs_cpu->mc_ncpus = no_of_cpus; 980da14cebeSEric Cheng bcopy(mrp->mrp_cpu, 981da14cebeSEric Cheng srs_cpu->mc_cpus, sizeof (srs_cpu->mc_cpus)); 9820dc2366fSVenugopal Iyer srs_cpu->mc_rx_fanout_cnt = 1; 9830dc2366fSVenugopal Iyer srs_cpu->mc_rx_pollid = mrp->mrp_cpu[cpu_cnt++]; 9840dc2366fSVenugopal Iyer /* Retarget the interrupt to the same CPU as the poll */ 9850dc2366fSVenugopal Iyer srs_cpu->mc_rx_intr_cpu = srs_cpu->mc_rx_pollid; 9860dc2366fSVenugopal Iyer srs_cpu->mc_rx_workerid = 9870dc2366fSVenugopal Iyer ((!mac_latency_optimize && worker_assign) ? 9880dc2366fSVenugopal Iyer mrp->mrp_cpu[cpu_cnt++] : srs_cpu->mc_rx_pollid); 9890dc2366fSVenugopal Iyer 9900dc2366fSVenugopal Iyer srs_cpu->mc_rx_fanout_cpus[0] = mrp->mrp_cpu[cpu_cnt]; 991da14cebeSEric Cheng 992da14cebeSEric Cheng /* Do CPU bindings for SRSes having h/w Rx rings */ 993da14cebeSEric Cheng if (flent->fe_rx_srs_cnt > 1) { 994da14cebeSEric Cheng cpu_cnt = 0; 995da14cebeSEric Cheng for (srs_cnt = 1; 996da14cebeSEric Cheng srs_cnt < flent->fe_rx_srs_cnt; srs_cnt++) { 997da14cebeSEric Cheng rx_srs = flent->fe_rx_srs[srs_cnt]; 998da14cebeSEric Cheng ASSERT(rx_srs->srs_ring != NULL); 999da14cebeSEric Cheng if (rx_srs->srs_fanout_state == 1000da14cebeSEric Cheng SRS_FANOUT_INIT) { 1001da14cebeSEric Cheng rx_srs->srs_fanout_state = 1002da14cebeSEric Cheng SRS_FANOUT_REINIT; 1003da14cebeSEric Cheng } 1004da14cebeSEric Cheng srs_cpu = &rx_srs->srs_cpu; 1005da14cebeSEric Cheng srs_cpu->mc_ncpus = no_of_cpus; 1006da14cebeSEric Cheng bcopy(mrp->mrp_cpu, srs_cpu->mc_cpus, 1007da14cebeSEric Cheng sizeof (srs_cpu->mc_cpus)); 10080dc2366fSVenugopal Iyer srs_cpu->mc_rx_pollid = 1009da14cebeSEric Cheng mrp->mrp_cpu[cpu_cnt]; 10100dc2366fSVenugopal Iyer srs_cpu->mc_rx_intr_cpu = srs_cpu->mc_rx_pollid; 10110dc2366fSVenugopal Iyer srs_cpu->mc_rx_workerid = 10120dc2366fSVenugopal Iyer ((!mac_latency_optimize && worker_assign) ? 10130dc2366fSVenugopal Iyer mrp->mrp_cpu[++cpu_cnt] : 10140dc2366fSVenugopal Iyer srs_cpu->mc_rx_pollid); 10150dc2366fSVenugopal Iyer srs_cpu->mc_rx_fanout_cnt = 1; 10160dc2366fSVenugopal Iyer srs_cpu->mc_rx_fanout_cpus[0] = 1017da14cebeSEric Cheng mrp->mrp_cpu[cpu_cnt]; 1018da14cebeSEric Cheng cpu_cnt++; 1019da14cebeSEric Cheng ASSERT(cpu_cnt <= no_of_cpus); 1020da14cebeSEric Cheng } 1021da14cebeSEric Cheng } 10220dc2366fSVenugopal Iyer goto tx_cpu_init; 1023da14cebeSEric Cheng } 1024da14cebeSEric Cheng 1025da14cebeSEric Cheng /* 1026da14cebeSEric Cheng * Real sub-optimal case. Not enough CPUs for poll and 1027da14cebeSEric Cheng * Tx soft rings. Do a round robin assignment where 1028da14cebeSEric Cheng * each Rx SRS will get the same CPU for poll, worker 1029da14cebeSEric Cheng * and fanout soft ring. 1030da14cebeSEric Cheng */ 1031da14cebeSEric Cheng cpu_cnt = 0; 1032da14cebeSEric Cheng for (srs_cnt = 0; srs_cnt < flent->fe_rx_srs_cnt; srs_cnt++) { 1033da14cebeSEric Cheng rx_srs = flent->fe_rx_srs[srs_cnt]; 1034da14cebeSEric Cheng srs_cpu = &rx_srs->srs_cpu; 1035da14cebeSEric Cheng if (rx_srs->srs_fanout_state == SRS_FANOUT_INIT) 1036da14cebeSEric Cheng rx_srs->srs_fanout_state = SRS_FANOUT_REINIT; 1037da14cebeSEric Cheng srs_cpu->mc_ncpus = no_of_cpus; 1038da14cebeSEric Cheng bcopy(mrp->mrp_cpu, 1039da14cebeSEric Cheng srs_cpu->mc_cpus, sizeof (srs_cpu->mc_cpus)); 10400dc2366fSVenugopal Iyer srs_cpu->mc_rx_fanout_cnt = 1; 10410dc2366fSVenugopal Iyer srs_cpu->mc_rx_pollid = mrp->mrp_cpu[cpu_cnt]; 10420dc2366fSVenugopal Iyer /* Retarget the interrupt to the same CPU as the poll */ 10430dc2366fSVenugopal Iyer srs_cpu->mc_rx_intr_cpu = srs_cpu->mc_rx_pollid; 10440dc2366fSVenugopal Iyer srs_cpu->mc_rx_workerid = mrp->mrp_cpu[cpu_cnt]; 10450dc2366fSVenugopal Iyer srs_cpu->mc_rx_fanout_cpus[0] = mrp->mrp_cpu[cpu_cnt]; 1046da14cebeSEric Cheng if (++cpu_cnt >= no_of_cpus) 1047da14cebeSEric Cheng cpu_cnt = 0; 1048da14cebeSEric Cheng } 10490dc2366fSVenugopal Iyer 10500dc2366fSVenugopal Iyer tx_cpu_init: 10510dc2366fSVenugopal Iyer mac_tx_cpu_init(flent, mrp, NULL); 10520dc2366fSVenugopal Iyer 10530dc2366fSVenugopal Iyer /* 10540dc2366fSVenugopal Iyer * Copy the user specified CPUs to the effective CPUs 10550dc2366fSVenugopal Iyer */ 10560dc2366fSVenugopal Iyer for (i = 0; i < mrp->mrp_ncpus; i++) { 10570dc2366fSVenugopal Iyer emrp->mrp_cpu[i] = mrp->mrp_cpu[i]; 10580dc2366fSVenugopal Iyer } 10590dc2366fSVenugopal Iyer emrp->mrp_ncpus = mrp->mrp_ncpus; 10600dc2366fSVenugopal Iyer emrp->mrp_mask = mrp->mrp_mask; 10610dc2366fSVenugopal Iyer bzero(emrp->mrp_pool, MAXPATHLEN); 1062da14cebeSEric Cheng } 1063da14cebeSEric Cheng 1064da14cebeSEric Cheng /* 1065da14cebeSEric Cheng * mac_flow_cpu_init(): 1066da14cebeSEric Cheng * 1067da14cebeSEric Cheng * Each SRS has a mac_cpu_t structure, srs_cpu. This routine fills in 1068da14cebeSEric Cheng * the CPU binding information in srs_cpu for all Rx SRSes associated 1069da14cebeSEric Cheng * with a flent. 1070da14cebeSEric Cheng */ 1071da14cebeSEric Cheng static void 10720dc2366fSVenugopal Iyer mac_flow_cpu_init(flow_entry_t *flent, cpupart_t *cpupart) 1073da14cebeSEric Cheng { 1074da14cebeSEric Cheng mac_soft_ring_set_t *rx_srs; 1075da14cebeSEric Cheng processorid_t cpuid; 1076*3714f7beSPaul Winder int i, j, k, srs_cnt, maxcpus, soft_ring_cnt = 0; 1077da14cebeSEric Cheng mac_cpus_t *srs_cpu; 10780dc2366fSVenugopal Iyer mac_resource_props_t *emrp = &flent->fe_effective_props; 1079da14cebeSEric Cheng 10800dc2366fSVenugopal Iyer /* 10810dc2366fSVenugopal Iyer * The maximum number of CPUs available can either be 10820dc2366fSVenugopal Iyer * the number of CPUs in the pool or the number of CPUs 10830dc2366fSVenugopal Iyer * in the system. 10840dc2366fSVenugopal Iyer */ 10850dc2366fSVenugopal Iyer maxcpus = (cpupart != NULL) ? cpupart->cp_ncpus : ncpus; 1086*3714f7beSPaul Winder /* 1087*3714f7beSPaul Winder * We cannot exceed the hard limit imposed by data structures. 1088*3714f7beSPaul Winder * Leave space for polling CPU and the SRS worker thread when 1089*3714f7beSPaul Winder * "mac_latency_optimize" is not set. 1090*3714f7beSPaul Winder */ 1091*3714f7beSPaul Winder maxcpus = MIN(maxcpus, MRP_NCPUS - 2); 10920dc2366fSVenugopal Iyer 1093da14cebeSEric Cheng /* 1094da14cebeSEric Cheng * Compute the number of soft rings needed on top for each Rx 1095da14cebeSEric Cheng * SRS. "rx_srs_cnt-1" indicates the number of Rx SRS 1096da14cebeSEric Cheng * associated with h/w Rx rings. Soft ring count needed for 1097da14cebeSEric Cheng * each h/w Rx SRS is computed and the same is applied to 1098da14cebeSEric Cheng * software classified Rx SRS. The first Rx SRS in fe_rx_srs[] 1099da14cebeSEric Cheng * is the software classified Rx SRS. 1100da14cebeSEric Cheng */ 1101da14cebeSEric Cheng soft_ring_cnt = mac_compute_soft_ring_count(flent, 11020dc2366fSVenugopal Iyer flent->fe_rx_srs_cnt - 1, maxcpus); 1103da14cebeSEric Cheng if (soft_ring_cnt == 0) { 1104da14cebeSEric Cheng /* 1105da14cebeSEric Cheng * Even when soft_ring_cnt is 0, we still need 1106da14cebeSEric Cheng * to create a soft ring for TCP, UDP and 1107da14cebeSEric Cheng * OTHER. So set it to 1. 1108da14cebeSEric Cheng */ 1109da14cebeSEric Cheng soft_ring_cnt = 1; 1110da14cebeSEric Cheng } 1111*3714f7beSPaul Winder 1112*3714f7beSPaul Winder emrp->mrp_ncpus = 0; 1113*3714f7beSPaul Winder for (srs_cnt = 0; srs_cnt < flent->fe_rx_srs_cnt && 1114*3714f7beSPaul Winder emrp->mrp_ncpus < MRP_NCPUS; srs_cnt++) { 1115da14cebeSEric Cheng rx_srs = flent->fe_rx_srs[srs_cnt]; 1116da14cebeSEric Cheng srs_cpu = &rx_srs->srs_cpu; 11170dc2366fSVenugopal Iyer if (rx_srs->srs_fanout_state == SRS_FANOUT_INIT) 1118da14cebeSEric Cheng rx_srs->srs_fanout_state = SRS_FANOUT_REINIT; 1119da14cebeSEric Cheng srs_cpu->mc_ncpus = soft_ring_cnt; 11200dc2366fSVenugopal Iyer srs_cpu->mc_rx_fanout_cnt = soft_ring_cnt; 1121da14cebeSEric Cheng mutex_enter(&cpu_lock); 1122da14cebeSEric Cheng for (j = 0; j < soft_ring_cnt; j++) { 11230dc2366fSVenugopal Iyer cpuid = mac_next_bind_cpu(cpupart); 1124da14cebeSEric Cheng srs_cpu->mc_cpus[j] = cpuid; 11250dc2366fSVenugopal Iyer srs_cpu->mc_rx_fanout_cpus[j] = cpuid; 1126da14cebeSEric Cheng } 11270dc2366fSVenugopal Iyer cpuid = mac_next_bind_cpu(cpupart); 11280dc2366fSVenugopal Iyer srs_cpu->mc_rx_pollid = cpuid; 11290dc2366fSVenugopal Iyer srs_cpu->mc_rx_intr_cpu = (mac_rx_intr_retarget ? 11300dc2366fSVenugopal Iyer srs_cpu->mc_rx_pollid : -1); 1131da14cebeSEric Cheng /* increment ncpus to account for polling cpu */ 1132da14cebeSEric Cheng srs_cpu->mc_ncpus++; 1133da14cebeSEric Cheng srs_cpu->mc_cpus[j++] = cpuid; 1134da14cebeSEric Cheng if (!mac_latency_optimize) { 11350dc2366fSVenugopal Iyer cpuid = mac_next_bind_cpu(cpupart); 1136da14cebeSEric Cheng srs_cpu->mc_ncpus++; 1137da14cebeSEric Cheng srs_cpu->mc_cpus[j++] = cpuid; 1138da14cebeSEric Cheng } 11390dc2366fSVenugopal Iyer srs_cpu->mc_rx_workerid = cpuid; 1140da14cebeSEric Cheng mutex_exit(&cpu_lock); 11410dc2366fSVenugopal Iyer 11420dc2366fSVenugopal Iyer /* 1143*3714f7beSPaul Winder * Copy fanout CPUs to fe_effective_props without duplicates. 11440dc2366fSVenugopal Iyer */ 1145*3714f7beSPaul Winder for (i = 0; i < srs_cpu->mc_ncpus && 1146*3714f7beSPaul Winder emrp->mrp_ncpus < MRP_NCPUS; i++) { 1147*3714f7beSPaul Winder for (j = 0; j < emrp->mrp_ncpus; j++) { 1148*3714f7beSPaul Winder if (emrp->mrp_cpu[j] == srs_cpu->mc_cpus[i]) 11490dc2366fSVenugopal Iyer break; 11500dc2366fSVenugopal Iyer } 1151*3714f7beSPaul Winder if (j == emrp->mrp_ncpus) { 1152*3714f7beSPaul Winder emrp->mrp_cpu[emrp->mrp_ncpus++] = 1153*3714f7beSPaul Winder srs_cpu->mc_cpus[i]; 11540dc2366fSVenugopal Iyer } 1155*3714f7beSPaul Winder } 1156*3714f7beSPaul Winder } 11570dc2366fSVenugopal Iyer 11580dc2366fSVenugopal Iyer mac_tx_cpu_init(flent, NULL, cpupart); 11590dc2366fSVenugopal Iyer } 11600dc2366fSVenugopal Iyer 1161da14cebeSEric Cheng /* 1162da14cebeSEric Cheng * DATAPATH SETUP ROUTINES 1163da14cebeSEric Cheng * (setup SRS and set/update FANOUT, B/W and PRIORITY) 1164da14cebeSEric Cheng */ 1165da14cebeSEric Cheng 11660dc2366fSVenugopal Iyer /* 11670dc2366fSVenugopal Iyer * mac_srs_fanout_list_alloc: 11680dc2366fSVenugopal Iyer * 11690dc2366fSVenugopal Iyer * The underlying device can expose upto MAX_RINGS_PER_GROUP worth of 11700dc2366fSVenugopal Iyer * rings to a client. In such a case, MAX_RINGS_PER_GROUP worth of 11710dc2366fSVenugopal Iyer * array space is needed to store Tx soft rings. Thus we allocate so 11720dc2366fSVenugopal Iyer * much array space for srs_tx_soft_rings. 11730dc2366fSVenugopal Iyer * 11740dc2366fSVenugopal Iyer * And when it is an aggr, again we allocate MAX_RINGS_PER_GROUP worth 11750dc2366fSVenugopal Iyer * of space to st_soft_rings. This array is used for quick access to 11760dc2366fSVenugopal Iyer * soft ring associated with a pseudo Tx ring based on the pseudo 11770dc2366fSVenugopal Iyer * ring's index (mr_index). 11780dc2366fSVenugopal Iyer */ 1179da14cebeSEric Cheng static void 1180da14cebeSEric Cheng mac_srs_fanout_list_alloc(mac_soft_ring_set_t *mac_srs) 1181da14cebeSEric Cheng { 11820dc2366fSVenugopal Iyer mac_client_impl_t *mcip = mac_srs->srs_mcip; 11830dc2366fSVenugopal Iyer 11840dc2366fSVenugopal Iyer if (mac_srs->srs_type & SRST_TX) { 11850dc2366fSVenugopal Iyer mac_srs->srs_tx_soft_rings = (mac_soft_ring_t **) 11860dc2366fSVenugopal Iyer kmem_zalloc(sizeof (mac_soft_ring_t *) * 11870dc2366fSVenugopal Iyer MAX_RINGS_PER_GROUP, KM_SLEEP); 118884de666eSRyan Zezeski if (mcip->mci_state_flags & MCIS_IS_AGGR_CLIENT) { 11890dc2366fSVenugopal Iyer mac_srs_tx_t *tx = &mac_srs->srs_tx; 11900dc2366fSVenugopal Iyer 11910dc2366fSVenugopal Iyer tx->st_soft_rings = (mac_soft_ring_t **) 11920dc2366fSVenugopal Iyer kmem_zalloc(sizeof (mac_soft_ring_t *) * 11930dc2366fSVenugopal Iyer MAX_RINGS_PER_GROUP, KM_SLEEP); 11940dc2366fSVenugopal Iyer } 11950dc2366fSVenugopal Iyer } else { 1196da14cebeSEric Cheng mac_srs->srs_tcp_soft_rings = (mac_soft_ring_t **) 11970dc2366fSVenugopal Iyer kmem_zalloc(sizeof (mac_soft_ring_t *) * MAX_SR_FANOUT, 11980dc2366fSVenugopal Iyer KM_SLEEP); 1199da14cebeSEric Cheng mac_srs->srs_udp_soft_rings = (mac_soft_ring_t **) 12000dc2366fSVenugopal Iyer kmem_zalloc(sizeof (mac_soft_ring_t *) * MAX_SR_FANOUT, 12010dc2366fSVenugopal Iyer KM_SLEEP); 1202da14cebeSEric Cheng mac_srs->srs_oth_soft_rings = (mac_soft_ring_t **) 12030dc2366fSVenugopal Iyer kmem_zalloc(sizeof (mac_soft_ring_t *) * MAX_SR_FANOUT, 12040dc2366fSVenugopal Iyer KM_SLEEP); 12050dc2366fSVenugopal Iyer } 1206da14cebeSEric Cheng } 1207da14cebeSEric Cheng 1208da14cebeSEric Cheng static void 1209da14cebeSEric Cheng mac_srs_worker_bind(mac_soft_ring_set_t *mac_srs, processorid_t cpuid) 1210da14cebeSEric Cheng { 1211da14cebeSEric Cheng cpu_t *cp; 1212da14cebeSEric Cheng boolean_t clear = B_FALSE; 1213da14cebeSEric Cheng 1214da14cebeSEric Cheng ASSERT(MUTEX_HELD(&cpu_lock)); 1215da14cebeSEric Cheng 1216da14cebeSEric Cheng if (!mac_srs_thread_bind) 1217da14cebeSEric Cheng return; 1218da14cebeSEric Cheng 1219da14cebeSEric Cheng cp = cpu_get(cpuid); 1220da14cebeSEric Cheng if (cp == NULL || !cpu_is_online(cp)) 1221da14cebeSEric Cheng return; 1222da14cebeSEric Cheng 1223da14cebeSEric Cheng mutex_enter(&mac_srs->srs_lock); 1224da14cebeSEric Cheng mac_srs->srs_state |= SRS_WORKER_BOUND; 1225da14cebeSEric Cheng if (mac_srs->srs_worker_cpuid != -1) 1226da14cebeSEric Cheng clear = B_TRUE; 1227da14cebeSEric Cheng mac_srs->srs_worker_cpuid = cpuid; 1228da14cebeSEric Cheng mutex_exit(&mac_srs->srs_lock); 1229da14cebeSEric Cheng 1230da14cebeSEric Cheng if (clear) 1231da14cebeSEric Cheng thread_affinity_clear(mac_srs->srs_worker); 1232da14cebeSEric Cheng 1233da14cebeSEric Cheng thread_affinity_set(mac_srs->srs_worker, cpuid); 1234da14cebeSEric Cheng DTRACE_PROBE1(worker__CPU, processorid_t, cpuid); 1235da14cebeSEric Cheng } 1236da14cebeSEric Cheng 1237da14cebeSEric Cheng static void 1238da14cebeSEric Cheng mac_srs_poll_bind(mac_soft_ring_set_t *mac_srs, processorid_t cpuid) 1239da14cebeSEric Cheng { 1240da14cebeSEric Cheng cpu_t *cp; 1241da14cebeSEric Cheng boolean_t clear = B_FALSE; 1242da14cebeSEric Cheng 1243da14cebeSEric Cheng ASSERT(MUTEX_HELD(&cpu_lock)); 1244da14cebeSEric Cheng 1245da14cebeSEric Cheng if (!mac_srs_thread_bind || mac_srs->srs_poll_thr == NULL) 1246da14cebeSEric Cheng return; 1247da14cebeSEric Cheng 1248da14cebeSEric Cheng cp = cpu_get(cpuid); 1249da14cebeSEric Cheng if (cp == NULL || !cpu_is_online(cp)) 1250da14cebeSEric Cheng return; 1251da14cebeSEric Cheng 1252da14cebeSEric Cheng mutex_enter(&mac_srs->srs_lock); 1253da14cebeSEric Cheng mac_srs->srs_state |= SRS_POLL_BOUND; 1254da14cebeSEric Cheng if (mac_srs->srs_poll_cpuid != -1) 1255da14cebeSEric Cheng clear = B_TRUE; 1256da14cebeSEric Cheng mac_srs->srs_poll_cpuid = cpuid; 1257da14cebeSEric Cheng mutex_exit(&mac_srs->srs_lock); 1258da14cebeSEric Cheng 1259da14cebeSEric Cheng if (clear) 1260da14cebeSEric Cheng thread_affinity_clear(mac_srs->srs_poll_thr); 1261da14cebeSEric Cheng 1262da14cebeSEric Cheng thread_affinity_set(mac_srs->srs_poll_thr, cpuid); 1263da14cebeSEric Cheng DTRACE_PROBE1(poll__CPU, processorid_t, cpuid); 1264da14cebeSEric Cheng } 1265da14cebeSEric Cheng 1266da14cebeSEric Cheng /* 12670dc2366fSVenugopal Iyer * Re-target interrupt to the passed CPU. If re-target is successful, 12680dc2366fSVenugopal Iyer * set mc_rx_intr_cpu to the re-targeted CPU. Otherwise set it to -1. 12690dc2366fSVenugopal Iyer */ 12700dc2366fSVenugopal Iyer void 12710dc2366fSVenugopal Iyer mac_rx_srs_retarget_intr(mac_soft_ring_set_t *mac_srs, processorid_t cpuid) 12720dc2366fSVenugopal Iyer { 12730dc2366fSVenugopal Iyer cpu_t *cp; 12740dc2366fSVenugopal Iyer mac_ring_t *ring = mac_srs->srs_ring; 12750dc2366fSVenugopal Iyer mac_intr_t *mintr = &ring->mr_info.mri_intr; 12760dc2366fSVenugopal Iyer flow_entry_t *flent = mac_srs->srs_flent; 12770dc2366fSVenugopal Iyer boolean_t primary = mac_is_primary_client(mac_srs->srs_mcip); 12780dc2366fSVenugopal Iyer 12790dc2366fSVenugopal Iyer ASSERT(MUTEX_HELD(&cpu_lock)); 12800dc2366fSVenugopal Iyer 12810dc2366fSVenugopal Iyer /* 12820dc2366fSVenugopal Iyer * Don't re-target the interrupt for these cases: 12830dc2366fSVenugopal Iyer * 1) ring is NULL 12840dc2366fSVenugopal Iyer * 2) the interrupt is shared (mi_ddi_shared) 12850dc2366fSVenugopal Iyer * 3) ddi_handle is NULL and !primary 12860dc2366fSVenugopal Iyer * 4) primary, ddi_handle is NULL but fe_rx_srs_cnt > 2 12870dc2366fSVenugopal Iyer * Case 3 & 4 are because of mac_client_intr_cpu() routine. 12880dc2366fSVenugopal Iyer * This routine will re-target fixed interrupt for primary 12890dc2366fSVenugopal Iyer * mac client if the client has only one ring. In that 12900dc2366fSVenugopal Iyer * case, mc_rx_intr_cpu will already have the correct value. 12910dc2366fSVenugopal Iyer */ 12920dc2366fSVenugopal Iyer if (ring == NULL || mintr->mi_ddi_shared || cpuid == -1 || 12930dc2366fSVenugopal Iyer (mintr->mi_ddi_handle == NULL && !primary) || (primary && 12940dc2366fSVenugopal Iyer mintr->mi_ddi_handle == NULL && flent->fe_rx_srs_cnt > 2)) { 12950dc2366fSVenugopal Iyer mac_srs->srs_cpu.mc_rx_intr_cpu = -1; 12960dc2366fSVenugopal Iyer return; 12970dc2366fSVenugopal Iyer } 12980dc2366fSVenugopal Iyer 12990dc2366fSVenugopal Iyer if (mintr->mi_ddi_handle == NULL) 13000dc2366fSVenugopal Iyer return; 13010dc2366fSVenugopal Iyer 13020dc2366fSVenugopal Iyer cp = cpu_get(cpuid); 13030dc2366fSVenugopal Iyer if (cp == NULL || !cpu_is_online(cp)) 13040dc2366fSVenugopal Iyer return; 13050dc2366fSVenugopal Iyer 1306657f87deSgongtian zhao - Sun Microsystems - Beijing China /* Drop the cpu_lock as set_intr_affinity() holds it */ 13070dc2366fSVenugopal Iyer mutex_exit(&cpu_lock); 1308657f87deSgongtian zhao - Sun Microsystems - Beijing China if (set_intr_affinity(mintr->mi_ddi_handle, cpuid) == DDI_SUCCESS) 13090dc2366fSVenugopal Iyer mac_srs->srs_cpu.mc_rx_intr_cpu = cpuid; 13100dc2366fSVenugopal Iyer else 13110dc2366fSVenugopal Iyer mac_srs->srs_cpu.mc_rx_intr_cpu = -1; 13120dc2366fSVenugopal Iyer mutex_enter(&cpu_lock); 13130dc2366fSVenugopal Iyer } 13140dc2366fSVenugopal Iyer 13150dc2366fSVenugopal Iyer /* 13160dc2366fSVenugopal Iyer * Re-target Tx interrupts 13170dc2366fSVenugopal Iyer */ 13180dc2366fSVenugopal Iyer void 13190dc2366fSVenugopal Iyer mac_tx_srs_retarget_intr(mac_soft_ring_set_t *mac_srs) 13200dc2366fSVenugopal Iyer { 13210dc2366fSVenugopal Iyer cpu_t *cp; 13220dc2366fSVenugopal Iyer mac_ring_t *ring; 13230dc2366fSVenugopal Iyer mac_intr_t *mintr; 13240dc2366fSVenugopal Iyer mac_soft_ring_t *sringp; 13250dc2366fSVenugopal Iyer mac_srs_tx_t *srs_tx; 13260dc2366fSVenugopal Iyer mac_cpus_t *srs_cpu; 13270dc2366fSVenugopal Iyer processorid_t cpuid; 13280dc2366fSVenugopal Iyer int i; 13290dc2366fSVenugopal Iyer 13300dc2366fSVenugopal Iyer ASSERT(MUTEX_HELD(&cpu_lock)); 13310dc2366fSVenugopal Iyer 13320dc2366fSVenugopal Iyer srs_cpu = &mac_srs->srs_cpu; 13330dc2366fSVenugopal Iyer if (MAC_TX_SOFT_RINGS(mac_srs)) { 13340dc2366fSVenugopal Iyer for (i = 0; i < mac_srs->srs_tx_ring_count; i++) { 13350dc2366fSVenugopal Iyer sringp = mac_srs->srs_tx_soft_rings[i]; 13360dc2366fSVenugopal Iyer ring = (mac_ring_t *)sringp->s_ring_tx_arg2; 13370dc2366fSVenugopal Iyer cpuid = srs_cpu->mc_tx_intr_cpu[i]; 13380dc2366fSVenugopal Iyer cp = cpu_get(cpuid); 13390dc2366fSVenugopal Iyer if (cp == NULL || !cpu_is_online(cp) || 13400dc2366fSVenugopal Iyer !MAC_RING_RETARGETABLE(ring)) { 13410dc2366fSVenugopal Iyer srs_cpu->mc_tx_retargeted_cpu[i] = -1; 13420dc2366fSVenugopal Iyer continue; 13430dc2366fSVenugopal Iyer } 13440dc2366fSVenugopal Iyer mintr = &ring->mr_info.mri_intr; 13450dc2366fSVenugopal Iyer /* 1346657f87deSgongtian zhao - Sun Microsystems - Beijing China * Drop the cpu_lock as set_intr_affinity() 13470dc2366fSVenugopal Iyer * holds it 13480dc2366fSVenugopal Iyer */ 13490dc2366fSVenugopal Iyer mutex_exit(&cpu_lock); 1350657f87deSgongtian zhao - Sun Microsystems - Beijing China if (set_intr_affinity(mintr->mi_ddi_handle, 13510dc2366fSVenugopal Iyer cpuid) == DDI_SUCCESS) { 13520dc2366fSVenugopal Iyer srs_cpu->mc_tx_retargeted_cpu[i] = cpuid; 13530dc2366fSVenugopal Iyer } else { 13540dc2366fSVenugopal Iyer srs_cpu->mc_tx_retargeted_cpu[i] = -1; 13550dc2366fSVenugopal Iyer } 13560dc2366fSVenugopal Iyer mutex_enter(&cpu_lock); 13570dc2366fSVenugopal Iyer } 13580dc2366fSVenugopal Iyer } else { 13590dc2366fSVenugopal Iyer cpuid = srs_cpu->mc_tx_intr_cpu[0]; 13600dc2366fSVenugopal Iyer cp = cpu_get(cpuid); 13610dc2366fSVenugopal Iyer if (cp == NULL || !cpu_is_online(cp)) { 13620dc2366fSVenugopal Iyer srs_cpu->mc_tx_retargeted_cpu[0] = -1; 13630dc2366fSVenugopal Iyer return; 13640dc2366fSVenugopal Iyer } 13650dc2366fSVenugopal Iyer srs_tx = &mac_srs->srs_tx; 13660dc2366fSVenugopal Iyer ring = (mac_ring_t *)srs_tx->st_arg2; 13670dc2366fSVenugopal Iyer if (MAC_RING_RETARGETABLE(ring)) { 13680dc2366fSVenugopal Iyer mintr = &ring->mr_info.mri_intr; 13690dc2366fSVenugopal Iyer mutex_exit(&cpu_lock); 1370657f87deSgongtian zhao - Sun Microsystems - Beijing China if ((set_intr_affinity(mintr->mi_ddi_handle, 13710dc2366fSVenugopal Iyer cpuid) == DDI_SUCCESS)) { 13720dc2366fSVenugopal Iyer srs_cpu->mc_tx_retargeted_cpu[0] = cpuid; 13730dc2366fSVenugopal Iyer } else { 13740dc2366fSVenugopal Iyer srs_cpu->mc_tx_retargeted_cpu[0] = -1; 13750dc2366fSVenugopal Iyer } 13760dc2366fSVenugopal Iyer mutex_enter(&cpu_lock); 13770dc2366fSVenugopal Iyer } 13780dc2366fSVenugopal Iyer } 13790dc2366fSVenugopal Iyer } 13800dc2366fSVenugopal Iyer 13810dc2366fSVenugopal Iyer /* 1382da14cebeSEric Cheng * When a CPU comes back online, bind the MAC kernel threads which 1383da14cebeSEric Cheng * were previously bound to that CPU, and had to be unbound because 1384da14cebeSEric Cheng * the CPU was going away. 1385da14cebeSEric Cheng * 1386da14cebeSEric Cheng * These functions are called with cpu_lock held and hence we can't 1387da14cebeSEric Cheng * cv_wait to grab the mac perimeter. Since these functions walk the soft 1388da14cebeSEric Cheng * ring list of an SRS without being in the perimeter, the list itself 1389da14cebeSEric Cheng * is protected by the SRS lock. 1390da14cebeSEric Cheng */ 1391da14cebeSEric Cheng static void 1392da14cebeSEric Cheng mac_walk_srs_and_bind(int cpuid) 1393da14cebeSEric Cheng { 1394da14cebeSEric Cheng mac_soft_ring_set_t *mac_srs; 1395da14cebeSEric Cheng mac_soft_ring_t *soft_ring; 1396da14cebeSEric Cheng 1397da14cebeSEric Cheng rw_enter(&mac_srs_g_lock, RW_READER); 1398da14cebeSEric Cheng 1399da14cebeSEric Cheng if ((mac_srs = mac_srs_g_list) == NULL) 1400da14cebeSEric Cheng goto done; 1401da14cebeSEric Cheng 1402da14cebeSEric Cheng for (; mac_srs != NULL; mac_srs = mac_srs->srs_next) { 1403da14cebeSEric Cheng if (mac_srs->srs_worker_cpuid == -1 && 1404da14cebeSEric Cheng mac_srs->srs_worker_cpuid_save == cpuid) { 1405da14cebeSEric Cheng mac_srs->srs_worker_cpuid_save = -1; 1406da14cebeSEric Cheng mac_srs_worker_bind(mac_srs, cpuid); 1407da14cebeSEric Cheng } 1408da14cebeSEric Cheng 1409da14cebeSEric Cheng if (!(mac_srs->srs_type & SRST_TX)) { 1410da14cebeSEric Cheng if (mac_srs->srs_poll_cpuid == -1 && 1411da14cebeSEric Cheng mac_srs->srs_poll_cpuid_save == cpuid) { 1412da14cebeSEric Cheng mac_srs->srs_poll_cpuid_save = -1; 1413da14cebeSEric Cheng mac_srs_poll_bind(mac_srs, cpuid); 1414da14cebeSEric Cheng } 1415da14cebeSEric Cheng } 1416da14cebeSEric Cheng 1417da14cebeSEric Cheng /* Next tackle the soft rings associated with the srs */ 1418da14cebeSEric Cheng mutex_enter(&mac_srs->srs_lock); 1419da14cebeSEric Cheng for (soft_ring = mac_srs->srs_soft_ring_head; soft_ring != NULL; 1420da14cebeSEric Cheng soft_ring = soft_ring->s_ring_next) { 1421da14cebeSEric Cheng if (soft_ring->s_ring_cpuid == -1 && 1422da14cebeSEric Cheng soft_ring->s_ring_cpuid_save == cpuid) { 1423da14cebeSEric Cheng soft_ring->s_ring_cpuid_save = -1; 1424da14cebeSEric Cheng (void) mac_soft_ring_bind(soft_ring, cpuid); 1425da14cebeSEric Cheng } 1426da14cebeSEric Cheng } 1427da14cebeSEric Cheng mutex_exit(&mac_srs->srs_lock); 1428da14cebeSEric Cheng } 1429da14cebeSEric Cheng done: 1430da14cebeSEric Cheng rw_exit(&mac_srs_g_lock); 1431da14cebeSEric Cheng } 1432da14cebeSEric Cheng 1433da14cebeSEric Cheng /* 1434da14cebeSEric Cheng * Change the priority of the SRS's poll and worker thread. Additionally, 1435da14cebeSEric Cheng * update the priority of the worker threads for the SRS's soft rings. 1436da14cebeSEric Cheng * Need to modify any associated squeue threads. 1437da14cebeSEric Cheng */ 1438da14cebeSEric Cheng void 1439da14cebeSEric Cheng mac_update_srs_priority(mac_soft_ring_set_t *mac_srs, pri_t prival) 1440da14cebeSEric Cheng { 1441da14cebeSEric Cheng mac_soft_ring_t *ringp; 1442da14cebeSEric Cheng 1443da14cebeSEric Cheng mac_srs->srs_pri = prival; 1444da14cebeSEric Cheng thread_lock(mac_srs->srs_worker); 1445da14cebeSEric Cheng (void) thread_change_pri(mac_srs->srs_worker, mac_srs->srs_pri, 0); 1446da14cebeSEric Cheng thread_unlock(mac_srs->srs_worker); 1447da14cebeSEric Cheng if (mac_srs->srs_poll_thr != NULL) { 1448da14cebeSEric Cheng thread_lock(mac_srs->srs_poll_thr); 1449da14cebeSEric Cheng (void) thread_change_pri(mac_srs->srs_poll_thr, 1450da14cebeSEric Cheng mac_srs->srs_pri, 0); 1451da14cebeSEric Cheng thread_unlock(mac_srs->srs_poll_thr); 1452da14cebeSEric Cheng } 1453da14cebeSEric Cheng if ((ringp = mac_srs->srs_soft_ring_head) == NULL) 1454da14cebeSEric Cheng return; 1455da14cebeSEric Cheng while (ringp != mac_srs->srs_soft_ring_tail) { 1456da14cebeSEric Cheng thread_lock(ringp->s_ring_worker); 1457da14cebeSEric Cheng (void) thread_change_pri(ringp->s_ring_worker, 1458da14cebeSEric Cheng mac_srs->srs_pri, 0); 1459da14cebeSEric Cheng thread_unlock(ringp->s_ring_worker); 1460da14cebeSEric Cheng ringp = ringp->s_ring_next; 1461da14cebeSEric Cheng } 1462da14cebeSEric Cheng ASSERT(ringp == mac_srs->srs_soft_ring_tail); 1463da14cebeSEric Cheng thread_lock(ringp->s_ring_worker); 1464da14cebeSEric Cheng (void) thread_change_pri(ringp->s_ring_worker, mac_srs->srs_pri, 0); 1465da14cebeSEric Cheng thread_unlock(ringp->s_ring_worker); 1466da14cebeSEric Cheng } 1467da14cebeSEric Cheng 1468da14cebeSEric Cheng /* 1469da14cebeSEric Cheng * Change the receive bandwidth limit. 1470da14cebeSEric Cheng */ 1471da14cebeSEric Cheng static void 1472da14cebeSEric Cheng mac_rx_srs_update_bwlimit(mac_soft_ring_set_t *srs, mac_resource_props_t *mrp) 1473da14cebeSEric Cheng { 1474da14cebeSEric Cheng mac_soft_ring_t *softring; 1475da14cebeSEric Cheng 1476da14cebeSEric Cheng mutex_enter(&srs->srs_lock); 1477da14cebeSEric Cheng mutex_enter(&srs->srs_bw->mac_bw_lock); 1478da14cebeSEric Cheng 1479da14cebeSEric Cheng if (mrp->mrp_maxbw == MRP_MAXBW_RESETVAL) { 1480da14cebeSEric Cheng /* Reset bandwidth limit */ 1481da14cebeSEric Cheng if (srs->srs_type & SRST_BW_CONTROL) { 1482da14cebeSEric Cheng softring = srs->srs_soft_ring_head; 1483da14cebeSEric Cheng while (softring != NULL) { 1484da14cebeSEric Cheng softring->s_ring_type &= ~ST_RING_BW_CTL; 1485da14cebeSEric Cheng softring = softring->s_ring_next; 1486da14cebeSEric Cheng } 1487da14cebeSEric Cheng srs->srs_type &= ~SRST_BW_CONTROL; 1488da14cebeSEric Cheng srs->srs_drain_func = mac_rx_srs_drain; 1489da14cebeSEric Cheng } 1490da14cebeSEric Cheng } else { 1491da14cebeSEric Cheng /* Set/Modify bandwidth limit */ 1492da14cebeSEric Cheng srs->srs_bw->mac_bw_limit = FLOW_BYTES_PER_TICK(mrp->mrp_maxbw); 1493da14cebeSEric Cheng /* 1494da14cebeSEric Cheng * Give twice the queuing capability before 1495da14cebeSEric Cheng * dropping packets. The unit is bytes/tick. 1496da14cebeSEric Cheng */ 1497da14cebeSEric Cheng srs->srs_bw->mac_bw_drop_threshold = 1498da14cebeSEric Cheng srs->srs_bw->mac_bw_limit << 1; 1499da14cebeSEric Cheng if (!(srs->srs_type & SRST_BW_CONTROL)) { 1500da14cebeSEric Cheng softring = srs->srs_soft_ring_head; 1501da14cebeSEric Cheng while (softring != NULL) { 1502da14cebeSEric Cheng softring->s_ring_type |= ST_RING_BW_CTL; 1503da14cebeSEric Cheng softring = softring->s_ring_next; 1504da14cebeSEric Cheng } 1505da14cebeSEric Cheng srs->srs_type |= SRST_BW_CONTROL; 1506da14cebeSEric Cheng srs->srs_drain_func = mac_rx_srs_drain_bw; 1507da14cebeSEric Cheng } 1508da14cebeSEric Cheng } 1509da14cebeSEric Cheng done: 1510da14cebeSEric Cheng mutex_exit(&srs->srs_bw->mac_bw_lock); 1511da14cebeSEric Cheng mutex_exit(&srs->srs_lock); 1512da14cebeSEric Cheng } 1513da14cebeSEric Cheng 1514da14cebeSEric Cheng /* Change the transmit bandwidth limit */ 1515da14cebeSEric Cheng static void 1516da14cebeSEric Cheng mac_tx_srs_update_bwlimit(mac_soft_ring_set_t *srs, mac_resource_props_t *mrp) 1517da14cebeSEric Cheng { 15180dc2366fSVenugopal Iyer uint32_t tx_mode, ring_info = 0; 1519f94ede51SEric Cheng mac_srs_tx_t *srs_tx = &srs->srs_tx; 1520f94ede51SEric Cheng mac_client_impl_t *mcip = srs->srs_mcip; 1521f94ede51SEric Cheng 1522f94ede51SEric Cheng /* 1523f94ede51SEric Cheng * We need to quiesce/restart the client here because mac_tx() and 1524f94ede51SEric Cheng * srs->srs_tx->st_func do not hold srs->srs_lock while accessing 1525f94ede51SEric Cheng * st_mode and related fields, which are modified by the code below. 1526f94ede51SEric Cheng */ 15270dc2366fSVenugopal Iyer mac_tx_client_quiesce((mac_client_handle_t)mcip); 1528da14cebeSEric Cheng 1529da14cebeSEric Cheng mutex_enter(&srs->srs_lock); 1530da14cebeSEric Cheng mutex_enter(&srs->srs_bw->mac_bw_lock); 1531da14cebeSEric Cheng 1532da14cebeSEric Cheng tx_mode = srs_tx->st_mode; 1533da14cebeSEric Cheng if (mrp->mrp_maxbw == MRP_MAXBW_RESETVAL) { 1534da14cebeSEric Cheng /* Reset bandwidth limit */ 1535da14cebeSEric Cheng if (tx_mode == SRS_TX_BW) { 15360dc2366fSVenugopal Iyer if (srs_tx->st_arg2 != NULL) 15370dc2366fSVenugopal Iyer ring_info = mac_hwring_getinfo(srs_tx->st_arg2); 1538da14cebeSEric Cheng if (mac_tx_serialize || 15390dc2366fSVenugopal Iyer (ring_info & MAC_RING_TX_SERIALIZE)) { 1540da14cebeSEric Cheng srs_tx->st_mode = SRS_TX_SERIALIZE; 1541da14cebeSEric Cheng } else { 1542da14cebeSEric Cheng srs_tx->st_mode = SRS_TX_DEFAULT; 1543da14cebeSEric Cheng } 1544da14cebeSEric Cheng } else if (tx_mode == SRS_TX_BW_FANOUT) { 1545da14cebeSEric Cheng srs_tx->st_mode = SRS_TX_FANOUT; 15460dc2366fSVenugopal Iyer } else if (tx_mode == SRS_TX_BW_AGGR) { 15470dc2366fSVenugopal Iyer srs_tx->st_mode = SRS_TX_AGGR; 1548da14cebeSEric Cheng } 1549da14cebeSEric Cheng srs->srs_type &= ~SRST_BW_CONTROL; 1550da14cebeSEric Cheng } else { 1551da14cebeSEric Cheng /* Set/Modify bandwidth limit */ 1552da14cebeSEric Cheng srs->srs_bw->mac_bw_limit = FLOW_BYTES_PER_TICK(mrp->mrp_maxbw); 1553da14cebeSEric Cheng /* 1554da14cebeSEric Cheng * Give twice the queuing capability before 1555da14cebeSEric Cheng * dropping packets. The unit is bytes/tick. 1556da14cebeSEric Cheng */ 1557da14cebeSEric Cheng srs->srs_bw->mac_bw_drop_threshold = 1558da14cebeSEric Cheng srs->srs_bw->mac_bw_limit << 1; 1559da14cebeSEric Cheng srs->srs_type |= SRST_BW_CONTROL; 15600dc2366fSVenugopal Iyer if (tx_mode != SRS_TX_BW && tx_mode != SRS_TX_BW_FANOUT && 15610dc2366fSVenugopal Iyer tx_mode != SRS_TX_BW_AGGR) { 1562da14cebeSEric Cheng if (tx_mode == SRS_TX_SERIALIZE || 1563da14cebeSEric Cheng tx_mode == SRS_TX_DEFAULT) { 1564da14cebeSEric Cheng srs_tx->st_mode = SRS_TX_BW; 1565da14cebeSEric Cheng } else if (tx_mode == SRS_TX_FANOUT) { 1566da14cebeSEric Cheng srs_tx->st_mode = SRS_TX_BW_FANOUT; 15670dc2366fSVenugopal Iyer } else if (tx_mode == SRS_TX_AGGR) { 15680dc2366fSVenugopal Iyer srs_tx->st_mode = SRS_TX_BW_AGGR; 1569da14cebeSEric Cheng } else { 1570da14cebeSEric Cheng ASSERT(0); 1571da14cebeSEric Cheng } 1572da14cebeSEric Cheng } 1573da14cebeSEric Cheng } 1574da14cebeSEric Cheng done: 1575da14cebeSEric Cheng srs_tx->st_func = mac_tx_get_func(srs_tx->st_mode); 1576da14cebeSEric Cheng mutex_exit(&srs->srs_bw->mac_bw_lock); 1577da14cebeSEric Cheng mutex_exit(&srs->srs_lock); 1578f94ede51SEric Cheng 15790dc2366fSVenugopal Iyer mac_tx_client_restart((mac_client_handle_t)mcip); 1580da14cebeSEric Cheng } 1581da14cebeSEric Cheng 1582da14cebeSEric Cheng /* 1583da14cebeSEric Cheng * The uber function that deals with any update to bandwidth limits. 1584da14cebeSEric Cheng */ 1585da14cebeSEric Cheng void 1586da14cebeSEric Cheng mac_srs_update_bwlimit(flow_entry_t *flent, mac_resource_props_t *mrp) 1587da14cebeSEric Cheng { 1588da14cebeSEric Cheng int count; 1589da14cebeSEric Cheng 1590da14cebeSEric Cheng for (count = 0; count < flent->fe_rx_srs_cnt; count++) 1591da14cebeSEric Cheng mac_rx_srs_update_bwlimit(flent->fe_rx_srs[count], mrp); 1592da14cebeSEric Cheng mac_tx_srs_update_bwlimit(flent->fe_tx_srs, mrp); 1593da14cebeSEric Cheng } 1594da14cebeSEric Cheng 1595da14cebeSEric Cheng /* 1596da14cebeSEric Cheng * When the first sub-flow is added to a link, we disable polling on the 159784de666eSRyan Zezeski * link and also modify the entry point to mac_rx_srs_subflow_process(). 1598da14cebeSEric Cheng * (polling is disabled because with the subflow added, accounting 1599da14cebeSEric Cheng * for polling needs additional logic, it is assumed that when a subflow is 1600da14cebeSEric Cheng * added, we can take some hit as a result of disabling polling rather than 1601da14cebeSEric Cheng * adding more complexity - if this becomes a perf. issue we need to 1602da14cebeSEric Cheng * re-rvaluate this logic). When the last subflow is removed, we turn back 160384de666eSRyan Zezeski * polling and also reset the entry point to mac_rx_srs_process(). 1604da14cebeSEric Cheng * 1605da14cebeSEric Cheng * In the future if there are multiple SRS, we can simply 1606da14cebeSEric Cheng * take one and give it to the flow rather than disabling polling and 1607da14cebeSEric Cheng * resetting the entry point. 1608da14cebeSEric Cheng */ 1609da14cebeSEric Cheng void 1610da14cebeSEric Cheng mac_client_update_classifier(mac_client_impl_t *mcip, boolean_t enable) 1611da14cebeSEric Cheng { 1612da14cebeSEric Cheng flow_entry_t *flent = mcip->mci_flent; 1613da14cebeSEric Cheng int i; 1614da14cebeSEric Cheng mac_impl_t *mip = mcip->mci_mip; 1615da14cebeSEric Cheng mac_rx_func_t rx_func; 1616da14cebeSEric Cheng uint_t rx_srs_cnt; 1617da14cebeSEric Cheng boolean_t enable_classifier; 1618da14cebeSEric Cheng 1619da14cebeSEric Cheng ASSERT(MAC_PERIM_HELD((mac_handle_t)mip)); 1620da14cebeSEric Cheng 1621da14cebeSEric Cheng enable_classifier = !FLOW_TAB_EMPTY(mcip->mci_subflow_tab) && enable; 1622da14cebeSEric Cheng 1623da14cebeSEric Cheng rx_func = enable_classifier ? mac_rx_srs_subflow_process : 1624da14cebeSEric Cheng mac_rx_srs_process; 1625da14cebeSEric Cheng 16264eaa4710SRishi Srivatsavai /* Tell mac_srs_poll_state_change to disable polling if necessary */ 16274eaa4710SRishi Srivatsavai if (mip->mi_state_flags & MIS_POLL_DISABLE) 16284eaa4710SRishi Srivatsavai enable_classifier = B_TRUE; 16294eaa4710SRishi Srivatsavai 1630da14cebeSEric Cheng /* 1631da14cebeSEric Cheng * If receive function has already been configured correctly for 1632da14cebeSEric Cheng * current subflow configuration, do nothing. 1633da14cebeSEric Cheng */ 1634da14cebeSEric Cheng if (flent->fe_cb_fn == (flow_fn_t)rx_func) 1635da14cebeSEric Cheng return; 1636da14cebeSEric Cheng 1637da14cebeSEric Cheng rx_srs_cnt = flent->fe_rx_srs_cnt; 1638da14cebeSEric Cheng for (i = 0; i < rx_srs_cnt; i++) { 1639da14cebeSEric Cheng ASSERT(flent->fe_rx_srs[i] != NULL); 1640da14cebeSEric Cheng mac_srs_poll_state_change(flent->fe_rx_srs[i], 1641da14cebeSEric Cheng enable_classifier, rx_func); 1642da14cebeSEric Cheng } 1643da14cebeSEric Cheng 1644da14cebeSEric Cheng /* 1645da14cebeSEric Cheng * Change the S/W classifier so that we can land in the 1646da14cebeSEric Cheng * correct processing function with correct argument. 1647da14cebeSEric Cheng * If all subflows have been removed we can revert to 164884de666eSRyan Zezeski * mac_rx_srs_process(), else we need mac_rx_srs_subflow_process(). 1649da14cebeSEric Cheng */ 1650da14cebeSEric Cheng mutex_enter(&flent->fe_lock); 1651da14cebeSEric Cheng flent->fe_cb_fn = (flow_fn_t)rx_func; 1652da14cebeSEric Cheng flent->fe_cb_arg1 = (void *)mip; 1653da14cebeSEric Cheng flent->fe_cb_arg2 = flent->fe_rx_srs[0]; 1654da14cebeSEric Cheng mutex_exit(&flent->fe_lock); 1655da14cebeSEric Cheng } 1656da14cebeSEric Cheng 1657da14cebeSEric Cheng static void 1658da14cebeSEric Cheng mac_srs_update_fanout_list(mac_soft_ring_set_t *mac_srs) 1659da14cebeSEric Cheng { 16600dc2366fSVenugopal Iyer int tcp_count = 0, udp_count = 0, oth_count = 0, tx_count = 0; 1661da14cebeSEric Cheng mac_soft_ring_t *softring; 1662da14cebeSEric Cheng 1663da14cebeSEric Cheng softring = mac_srs->srs_soft_ring_head; 1664da14cebeSEric Cheng if (softring == NULL) { 1665da14cebeSEric Cheng ASSERT(mac_srs->srs_soft_ring_count == 0); 1666da14cebeSEric Cheng mac_srs->srs_tcp_ring_count = 0; 1667da14cebeSEric Cheng mac_srs->srs_udp_ring_count = 0; 1668da14cebeSEric Cheng mac_srs->srs_oth_ring_count = 0; 16690dc2366fSVenugopal Iyer mac_srs->srs_tx_ring_count = 0; 1670da14cebeSEric Cheng return; 1671da14cebeSEric Cheng } 1672da14cebeSEric Cheng 1673da14cebeSEric Cheng while (softring != NULL) { 16740dc2366fSVenugopal Iyer if (softring->s_ring_type & ST_RING_TCP) { 1675da14cebeSEric Cheng mac_srs->srs_tcp_soft_rings[tcp_count++] = softring; 16760dc2366fSVenugopal Iyer } else if (softring->s_ring_type & ST_RING_UDP) { 1677da14cebeSEric Cheng mac_srs->srs_udp_soft_rings[udp_count++] = softring; 16780dc2366fSVenugopal Iyer } else if (softring->s_ring_type & ST_RING_OTH) { 1679da14cebeSEric Cheng mac_srs->srs_oth_soft_rings[oth_count++] = softring; 16800dc2366fSVenugopal Iyer } else { 16810dc2366fSVenugopal Iyer ASSERT(softring->s_ring_type & ST_RING_TX); 16820dc2366fSVenugopal Iyer mac_srs->srs_tx_soft_rings[tx_count++] = softring; 16830dc2366fSVenugopal Iyer } 1684da14cebeSEric Cheng softring = softring->s_ring_next; 1685da14cebeSEric Cheng } 1686da14cebeSEric Cheng 1687da14cebeSEric Cheng ASSERT(mac_srs->srs_soft_ring_count == 16880dc2366fSVenugopal Iyer (tcp_count + udp_count + oth_count + tx_count)); 1689da14cebeSEric Cheng mac_srs->srs_tcp_ring_count = tcp_count; 1690da14cebeSEric Cheng mac_srs->srs_udp_ring_count = udp_count; 1691da14cebeSEric Cheng mac_srs->srs_oth_ring_count = oth_count; 16920dc2366fSVenugopal Iyer mac_srs->srs_tx_ring_count = tx_count; 1693da14cebeSEric Cheng } 1694da14cebeSEric Cheng 1695da14cebeSEric Cheng void 16960dc2366fSVenugopal Iyer mac_srs_create_proto_softrings(int id, uint16_t type, pri_t pri, 16970dc2366fSVenugopal Iyer mac_client_impl_t *mcip, mac_soft_ring_set_t *mac_srs, 1698da14cebeSEric Cheng processorid_t cpuid, mac_direct_rx_t rx_func, void *x_arg1, 1699da14cebeSEric Cheng mac_resource_handle_t x_arg2, boolean_t set_bypass) 1700da14cebeSEric Cheng { 1701da14cebeSEric Cheng mac_soft_ring_t *softring; 1702da14cebeSEric Cheng mac_rx_fifo_t mrf; 1703da14cebeSEric Cheng 1704da14cebeSEric Cheng bzero(&mrf, sizeof (mac_rx_fifo_t)); 1705da14cebeSEric Cheng mrf.mrf_type = MAC_RX_FIFO; 1706da14cebeSEric Cheng mrf.mrf_receive = (mac_receive_t)mac_soft_ring_poll; 17072c465844SToomas Soome mrf.mrf_intr_enable = (mac_intr_enable_t)mac_soft_ring_intr_enable; 17082c465844SToomas Soome mrf.mrf_intr_disable = (mac_intr_disable_t)mac_soft_ring_intr_disable; 1709da14cebeSEric Cheng mrf.mrf_flow_priority = pri; 1710da14cebeSEric Cheng 1711da14cebeSEric Cheng softring = mac_soft_ring_create(id, mac_soft_ring_worker_wait, 17120dc2366fSVenugopal Iyer (type|ST_RING_TCP), pri, mcip, mac_srs, 1713da14cebeSEric Cheng cpuid, rx_func, x_arg1, x_arg2); 1714da14cebeSEric Cheng softring->s_ring_rx_arg2 = NULL; 1715da14cebeSEric Cheng 1716da14cebeSEric Cheng /* 1717da14cebeSEric Cheng * TCP and UDP support DLS bypass. In addition TCP 1718da14cebeSEric Cheng * squeue can also poll their corresponding soft rings. 1719da14cebeSEric Cheng */ 1720da14cebeSEric Cheng if (set_bypass && (mcip->mci_resource_arg != NULL)) { 1721da14cebeSEric Cheng mac_soft_ring_dls_bypass(softring, 1722da14cebeSEric Cheng mcip->mci_direct_rx_fn, 1723da14cebeSEric Cheng mcip->mci_direct_rx_arg); 1724da14cebeSEric Cheng 1725da14cebeSEric Cheng mrf.mrf_rx_arg = softring; 1726da14cebeSEric Cheng mrf.mrf_intr_handle = (mac_intr_handle_t)softring; 1727da14cebeSEric Cheng 1728da14cebeSEric Cheng /* 1729da14cebeSEric Cheng * Make a call in IP to get a TCP squeue assigned to 1730da14cebeSEric Cheng * this softring to maintain full CPU locality through 1731da14cebeSEric Cheng * the stack and allow the squeue to be able to poll 1732da14cebeSEric Cheng * the softring so the flow control can be pushed 1733da14cebeSEric Cheng * all the way to H/W. 1734da14cebeSEric Cheng */ 1735da14cebeSEric Cheng softring->s_ring_rx_arg2 = 1736da14cebeSEric Cheng mcip->mci_resource_add((void *)mcip->mci_resource_arg, 1737da14cebeSEric Cheng (mac_resource_t *)&mrf); 1738da14cebeSEric Cheng } 1739da14cebeSEric Cheng 1740da14cebeSEric Cheng /* 1741da14cebeSEric Cheng * Non-TCP protocols don't support squeues. Hence we 1742da14cebeSEric Cheng * don't make any ring addition callbacks for non-TCP 1743da14cebeSEric Cheng * rings. Now create the UDP softring and allow it to 1744da14cebeSEric Cheng * bypass the DLS layer. 1745da14cebeSEric Cheng */ 1746da14cebeSEric Cheng softring = mac_soft_ring_create(id, mac_soft_ring_worker_wait, 17470dc2366fSVenugopal Iyer (type|ST_RING_UDP), pri, mcip, mac_srs, 1748da14cebeSEric Cheng cpuid, rx_func, x_arg1, x_arg2); 1749da14cebeSEric Cheng softring->s_ring_rx_arg2 = NULL; 1750da14cebeSEric Cheng 1751da14cebeSEric Cheng if (set_bypass && (mcip->mci_resource_arg != NULL)) { 1752da14cebeSEric Cheng mac_soft_ring_dls_bypass(softring, 1753da14cebeSEric Cheng mcip->mci_direct_rx_fn, 1754da14cebeSEric Cheng mcip->mci_direct_rx_arg); 1755da14cebeSEric Cheng } 1756da14cebeSEric Cheng 1757da14cebeSEric Cheng /* Create the Oth softrings which has to go through the DLS */ 1758da14cebeSEric Cheng softring = mac_soft_ring_create(id, mac_soft_ring_worker_wait, 17590dc2366fSVenugopal Iyer (type|ST_RING_OTH), pri, mcip, mac_srs, 1760da14cebeSEric Cheng cpuid, rx_func, x_arg1, x_arg2); 1761da14cebeSEric Cheng softring->s_ring_rx_arg2 = NULL; 1762da14cebeSEric Cheng } 1763da14cebeSEric Cheng 1764da14cebeSEric Cheng /* 1765da14cebeSEric Cheng * This routine associates a CPU or a set of CPU to process incoming 1766da14cebeSEric Cheng * traffic from a mac client. If multiple CPUs are specified, then 1767da14cebeSEric Cheng * so many soft rings are created with each soft ring worker thread 1768da14cebeSEric Cheng * bound to a CPU in the set. Each soft ring in turn will be 1769da14cebeSEric Cheng * associated with an squeue and the squeue will be moved to the 1770da14cebeSEric Cheng * same CPU as that of the soft ring's. 1771da14cebeSEric Cheng */ 1772da14cebeSEric Cheng static void 17730dc2366fSVenugopal Iyer mac_srs_fanout_modify(mac_client_impl_t *mcip, mac_direct_rx_t rx_func, 17740dc2366fSVenugopal Iyer void *x_arg1, mac_resource_handle_t x_arg2, 17750dc2366fSVenugopal Iyer mac_soft_ring_set_t *mac_rx_srs, mac_soft_ring_set_t *mac_tx_srs) 1776da14cebeSEric Cheng { 1777da14cebeSEric Cheng mac_soft_ring_t *softring; 1778ae6aa22aSVenugopal Iyer uint32_t soft_ring_flag = 0; 1779da14cebeSEric Cheng processorid_t cpuid = -1; 1780da14cebeSEric Cheng int i, srings_present, new_fanout_cnt; 1781da14cebeSEric Cheng mac_cpus_t *srs_cpu; 1782da14cebeSEric Cheng 1783da14cebeSEric Cheng /* fanout state is REINIT. Set it back to INIT */ 1784da14cebeSEric Cheng ASSERT(mac_rx_srs->srs_fanout_state == SRS_FANOUT_REINIT); 1785da14cebeSEric Cheng mac_rx_srs->srs_fanout_state = SRS_FANOUT_INIT; 1786da14cebeSEric Cheng 1787da14cebeSEric Cheng /* how many are present right now */ 1788da14cebeSEric Cheng srings_present = mac_rx_srs->srs_tcp_ring_count; 1789da14cebeSEric Cheng /* new request */ 1790da14cebeSEric Cheng srs_cpu = &mac_rx_srs->srs_cpu; 17910dc2366fSVenugopal Iyer new_fanout_cnt = srs_cpu->mc_rx_fanout_cnt; 1792da14cebeSEric Cheng 1793da14cebeSEric Cheng mutex_enter(&mac_rx_srs->srs_lock); 1794da14cebeSEric Cheng if (mac_rx_srs->srs_type & SRST_BW_CONTROL) 1795da14cebeSEric Cheng soft_ring_flag |= ST_RING_BW_CTL; 1796da14cebeSEric Cheng mutex_exit(&mac_rx_srs->srs_lock); 1797da14cebeSEric Cheng 1798da14cebeSEric Cheng if (new_fanout_cnt > srings_present) { 1799da14cebeSEric Cheng /* soft rings increased */ 1800da14cebeSEric Cheng mutex_enter(&mac_rx_srs->srs_lock); 1801da14cebeSEric Cheng mac_rx_srs->srs_type |= SRST_FANOUT_SRC_IP; 1802da14cebeSEric Cheng mutex_exit(&mac_rx_srs->srs_lock); 1803da14cebeSEric Cheng 1804da14cebeSEric Cheng for (i = mac_rx_srs->srs_tcp_ring_count; 1805da14cebeSEric Cheng i < new_fanout_cnt; i++) { 1806da14cebeSEric Cheng /* 1807da14cebeSEric Cheng * Create the protocol softrings and set the 1808da14cebeSEric Cheng * DLS bypass where possible. 1809da14cebeSEric Cheng */ 18100dc2366fSVenugopal Iyer mac_srs_create_proto_softrings(i, soft_ring_flag, 1811da14cebeSEric Cheng mac_rx_srs->srs_pri, mcip, mac_rx_srs, cpuid, 1812da14cebeSEric Cheng rx_func, x_arg1, x_arg2, B_TRUE); 1813da14cebeSEric Cheng } 1814da14cebeSEric Cheng mac_srs_update_fanout_list(mac_rx_srs); 1815da14cebeSEric Cheng } else if (new_fanout_cnt < srings_present) { 1816da14cebeSEric Cheng /* soft rings decreased */ 1817da14cebeSEric Cheng if (new_fanout_cnt == 1) { 1818da14cebeSEric Cheng mutex_enter(&mac_rx_srs->srs_lock); 1819da14cebeSEric Cheng mac_rx_srs->srs_type &= ~SRST_FANOUT_SRC_IP; 1820da14cebeSEric Cheng ASSERT(mac_rx_srs->srs_type & SRST_FANOUT_PROTO); 1821da14cebeSEric Cheng mutex_exit(&mac_rx_srs->srs_lock); 1822da14cebeSEric Cheng } 1823da14cebeSEric Cheng /* Get rid of extra soft rings */ 1824da14cebeSEric Cheng for (i = new_fanout_cnt; 1825da14cebeSEric Cheng i < mac_rx_srs->srs_tcp_ring_count; i++) { 1826da14cebeSEric Cheng softring = mac_rx_srs->srs_tcp_soft_rings[i]; 1827da14cebeSEric Cheng if (softring->s_ring_rx_arg2 != NULL) { 1828da14cebeSEric Cheng mcip->mci_resource_remove( 1829da14cebeSEric Cheng (void *)mcip->mci_resource_arg, 1830da14cebeSEric Cheng softring->s_ring_rx_arg2); 1831da14cebeSEric Cheng } 1832da14cebeSEric Cheng mac_soft_ring_remove(mac_rx_srs, 1833da14cebeSEric Cheng mac_rx_srs->srs_tcp_soft_rings[i]); 1834da14cebeSEric Cheng mac_soft_ring_remove(mac_rx_srs, 1835da14cebeSEric Cheng mac_rx_srs->srs_udp_soft_rings[i]); 1836da14cebeSEric Cheng mac_soft_ring_remove(mac_rx_srs, 1837da14cebeSEric Cheng mac_rx_srs->srs_oth_soft_rings[i]); 1838da14cebeSEric Cheng } 1839da14cebeSEric Cheng mac_srs_update_fanout_list(mac_rx_srs); 1840da14cebeSEric Cheng } 1841da14cebeSEric Cheng 1842da14cebeSEric Cheng ASSERT(new_fanout_cnt == mac_rx_srs->srs_tcp_ring_count); 1843da14cebeSEric Cheng mutex_enter(&cpu_lock); 1844da14cebeSEric Cheng for (i = 0; i < mac_rx_srs->srs_tcp_ring_count; i++) { 18450dc2366fSVenugopal Iyer cpuid = srs_cpu->mc_rx_fanout_cpus[i]; 1846da14cebeSEric Cheng (void) mac_soft_ring_bind(mac_rx_srs->srs_udp_soft_rings[i], 1847da14cebeSEric Cheng cpuid); 1848da14cebeSEric Cheng (void) mac_soft_ring_bind(mac_rx_srs->srs_oth_soft_rings[i], 1849da14cebeSEric Cheng cpuid); 1850da14cebeSEric Cheng (void) mac_soft_ring_bind(mac_rx_srs->srs_tcp_soft_rings[i], 1851da14cebeSEric Cheng cpuid); 1852da14cebeSEric Cheng softring = mac_rx_srs->srs_tcp_soft_rings[i]; 1853da14cebeSEric Cheng if (softring->s_ring_rx_arg2 != NULL) { 1854da14cebeSEric Cheng mcip->mci_resource_bind((void *)mcip->mci_resource_arg, 1855da14cebeSEric Cheng softring->s_ring_rx_arg2, cpuid); 1856da14cebeSEric Cheng } 1857da14cebeSEric Cheng } 1858da14cebeSEric Cheng 18590dc2366fSVenugopal Iyer mac_srs_worker_bind(mac_rx_srs, srs_cpu->mc_rx_workerid); 18600dc2366fSVenugopal Iyer mac_srs_poll_bind(mac_rx_srs, srs_cpu->mc_rx_pollid); 18610dc2366fSVenugopal Iyer mac_rx_srs_retarget_intr(mac_rx_srs, srs_cpu->mc_rx_intr_cpu); 1862da14cebeSEric Cheng /* 1863da14cebeSEric Cheng * Bind Tx srs and soft ring threads too. Let's bind tx 1864da14cebeSEric Cheng * srs to the last cpu in mrp list. 1865da14cebeSEric Cheng */ 18660dc2366fSVenugopal Iyer if (mac_tx_srs != NULL) { 1867da14cebeSEric Cheng BIND_TX_SRS_AND_SOFT_RINGS(mac_tx_srs, mrp); 18680dc2366fSVenugopal Iyer mac_tx_srs_retarget_intr(mac_tx_srs); 1869da14cebeSEric Cheng } 1870da14cebeSEric Cheng mutex_exit(&cpu_lock); 1871da14cebeSEric Cheng } 1872da14cebeSEric Cheng 1873da14cebeSEric Cheng /* 1874da14cebeSEric Cheng * Bind SRS threads and soft rings to CPUs/create fanout list. 1875da14cebeSEric Cheng */ 1876da14cebeSEric Cheng void 18770dc2366fSVenugopal Iyer mac_srs_fanout_init(mac_client_impl_t *mcip, mac_resource_props_t *mrp, 18780dc2366fSVenugopal Iyer mac_direct_rx_t rx_func, void *x_arg1, mac_resource_handle_t x_arg2, 18790dc2366fSVenugopal Iyer mac_soft_ring_set_t *mac_rx_srs, mac_soft_ring_set_t *mac_tx_srs, 18800dc2366fSVenugopal Iyer cpupart_t *cpupart) 1881da14cebeSEric Cheng { 1882da14cebeSEric Cheng int i; 18830dc2366fSVenugopal Iyer processorid_t cpuid; 1884ae6aa22aSVenugopal Iyer uint32_t soft_ring_flag = 0; 1885da14cebeSEric Cheng int soft_ring_cnt; 1886da14cebeSEric Cheng mac_cpus_t *srs_cpu = &mac_rx_srs->srs_cpu; 1887da14cebeSEric Cheng 1888da14cebeSEric Cheng /* 1889da14cebeSEric Cheng * Remove the no soft ring flag and we will adjust it 1890da14cebeSEric Cheng * appropriately further down. 1891da14cebeSEric Cheng */ 1892da14cebeSEric Cheng mutex_enter(&mac_rx_srs->srs_lock); 1893da14cebeSEric Cheng mac_rx_srs->srs_type &= ~SRST_NO_SOFT_RINGS; 1894da14cebeSEric Cheng mutex_exit(&mac_rx_srs->srs_lock); 1895da14cebeSEric Cheng 1896da14cebeSEric Cheng ASSERT(mac_rx_srs->srs_soft_ring_head == NULL); 1897da14cebeSEric Cheng 1898da14cebeSEric Cheng if (mac_rx_srs->srs_type & SRST_BW_CONTROL) 1899da14cebeSEric Cheng soft_ring_flag |= ST_RING_BW_CTL; 1900da14cebeSEric Cheng 1901da14cebeSEric Cheng ASSERT(mac_rx_srs->srs_fanout_state == SRS_FANOUT_UNINIT); 1902da14cebeSEric Cheng mac_rx_srs->srs_fanout_state = SRS_FANOUT_INIT; 1903da14cebeSEric Cheng /* 1904da14cebeSEric Cheng * Ring count can be 0 if no fanout is required and no cpu 1905da14cebeSEric Cheng * were specified. Leave the SRS worker and poll thread 1906da14cebeSEric Cheng * unbound 1907da14cebeSEric Cheng */ 1908da14cebeSEric Cheng ASSERT(mrp != NULL); 19090dc2366fSVenugopal Iyer soft_ring_cnt = srs_cpu->mc_rx_fanout_cnt; 1910da14cebeSEric Cheng 1911da14cebeSEric Cheng /* Step 1: bind cpu contains cpu list where threads need to bind */ 1912da14cebeSEric Cheng if (soft_ring_cnt > 0) { 1913da14cebeSEric Cheng mutex_enter(&cpu_lock); 1914da14cebeSEric Cheng for (i = 0; i < soft_ring_cnt; i++) { 19150dc2366fSVenugopal Iyer cpuid = srs_cpu->mc_rx_fanout_cpus[i]; 1916da14cebeSEric Cheng /* Create the protocol softrings */ 19170dc2366fSVenugopal Iyer mac_srs_create_proto_softrings(i, soft_ring_flag, 19180dc2366fSVenugopal Iyer mac_rx_srs->srs_pri, mcip, mac_rx_srs, cpuid, 19190dc2366fSVenugopal Iyer rx_func, x_arg1, x_arg2, B_FALSE); 1920da14cebeSEric Cheng } 19210dc2366fSVenugopal Iyer mac_srs_worker_bind(mac_rx_srs, srs_cpu->mc_rx_workerid); 19220dc2366fSVenugopal Iyer mac_srs_poll_bind(mac_rx_srs, srs_cpu->mc_rx_pollid); 19230dc2366fSVenugopal Iyer mac_rx_srs_retarget_intr(mac_rx_srs, srs_cpu->mc_rx_intr_cpu); 1924da14cebeSEric Cheng /* 1925da14cebeSEric Cheng * Bind Tx srs and soft ring threads too. 1926da14cebeSEric Cheng * Let's bind tx srs to the last cpu in 1927da14cebeSEric Cheng * mrp list. 1928da14cebeSEric Cheng */ 1929da14cebeSEric Cheng if (mac_tx_srs == NULL) { 1930da14cebeSEric Cheng mutex_exit(&cpu_lock); 1931da14cebeSEric Cheng goto alldone; 1932da14cebeSEric Cheng } 1933da14cebeSEric Cheng 1934da14cebeSEric Cheng BIND_TX_SRS_AND_SOFT_RINGS(mac_tx_srs, mrp); 19350dc2366fSVenugopal Iyer mac_tx_srs_retarget_intr(mac_tx_srs); 1936da14cebeSEric Cheng mutex_exit(&cpu_lock); 1937da14cebeSEric Cheng } else { 1938da14cebeSEric Cheng mutex_enter(&cpu_lock); 1939da14cebeSEric Cheng /* 1940da14cebeSEric Cheng * For a subflow, mrp_workerid and mrp_pollid 1941da14cebeSEric Cheng * is not set. 1942da14cebeSEric Cheng */ 19430dc2366fSVenugopal Iyer mac_srs_worker_bind(mac_rx_srs, mrp->mrp_rx_workerid); 19440dc2366fSVenugopal Iyer mac_srs_poll_bind(mac_rx_srs, mrp->mrp_rx_pollid); 1945da14cebeSEric Cheng mutex_exit(&cpu_lock); 1946da14cebeSEric Cheng goto no_softrings; 1947da14cebeSEric Cheng } 1948da14cebeSEric Cheng 1949da14cebeSEric Cheng alldone: 1950da14cebeSEric Cheng if (soft_ring_cnt > 1) 1951da14cebeSEric Cheng mac_rx_srs->srs_type |= SRST_FANOUT_SRC_IP; 1952da14cebeSEric Cheng mac_srs_update_fanout_list(mac_rx_srs); 1953da14cebeSEric Cheng mac_srs_client_poll_enable(mcip, mac_rx_srs); 1954da14cebeSEric Cheng return; 1955da14cebeSEric Cheng 1956da14cebeSEric Cheng no_softrings: 1957da14cebeSEric Cheng if (mac_rx_srs->srs_type & SRST_FANOUT_PROTO) { 1958da14cebeSEric Cheng mutex_enter(&cpu_lock); 19590dc2366fSVenugopal Iyer cpuid = mac_next_bind_cpu(cpupart); 1960da14cebeSEric Cheng /* Create the protocol softrings */ 19610dc2366fSVenugopal Iyer mac_srs_create_proto_softrings(0, soft_ring_flag, 19620dc2366fSVenugopal Iyer mac_rx_srs->srs_pri, mcip, mac_rx_srs, cpuid, 19630dc2366fSVenugopal Iyer rx_func, x_arg1, x_arg2, B_FALSE); 1964da14cebeSEric Cheng mutex_exit(&cpu_lock); 1965da14cebeSEric Cheng } else { 1966da14cebeSEric Cheng /* 1967da14cebeSEric Cheng * This is the case when there is no fanout which is 1968da14cebeSEric Cheng * true for subflows. 1969da14cebeSEric Cheng */ 1970da14cebeSEric Cheng mac_rx_srs->srs_type |= SRST_NO_SOFT_RINGS; 1971da14cebeSEric Cheng } 1972da14cebeSEric Cheng mac_srs_update_fanout_list(mac_rx_srs); 1973da14cebeSEric Cheng mac_srs_client_poll_enable(mcip, mac_rx_srs); 1974da14cebeSEric Cheng } 1975da14cebeSEric Cheng 1976da14cebeSEric Cheng /* 1977da14cebeSEric Cheng * Calls mac_srs_fanout_init() or modify() depending upon whether 1978da14cebeSEric Cheng * the SRS is getting initialized or re-initialized. 1979da14cebeSEric Cheng */ 1980da14cebeSEric Cheng void 1981da14cebeSEric Cheng mac_fanout_setup(mac_client_impl_t *mcip, flow_entry_t *flent, 1982da14cebeSEric Cheng mac_resource_props_t *mrp, mac_direct_rx_t rx_func, void *x_arg1, 19830dc2366fSVenugopal Iyer mac_resource_handle_t x_arg2, cpupart_t *cpupart) 1984da14cebeSEric Cheng { 1985da14cebeSEric Cheng mac_soft_ring_set_t *mac_rx_srs, *mac_tx_srs; 1986da14cebeSEric Cheng int i, rx_srs_cnt; 1987da14cebeSEric Cheng 1988da14cebeSEric Cheng ASSERT(MAC_PERIM_HELD((mac_handle_t)mcip->mci_mip)); 1989da14cebeSEric Cheng 199045948e49SRyan Zezeski /* 199145948e49SRyan Zezeski * Aggr ports do not have SRSes. This function should never be 199245948e49SRyan Zezeski * called on an aggr port. 199345948e49SRyan Zezeski */ 199445948e49SRyan Zezeski ASSERT3U((mcip->mci_state_flags & MCIS_IS_AGGR_PORT), ==, 0); 1995da14cebeSEric Cheng mac_rx_srs = flent->fe_rx_srs[0]; 199645948e49SRyan Zezeski 1997da14cebeSEric Cheng /* 1998da14cebeSEric Cheng * Set up the fanout on the tx side only once, with the 1999da14cebeSEric Cheng * first rx SRS. The CPU binding, fanout, and bandwidth 2000da14cebeSEric Cheng * criteria are common to both RX and TX, so 2001da14cebeSEric Cheng * initializing them along side avoids redundant code. 2002da14cebeSEric Cheng */ 2003da14cebeSEric Cheng mac_tx_srs = flent->fe_tx_srs; 2004da14cebeSEric Cheng rx_srs_cnt = flent->fe_rx_srs_cnt; 2005da14cebeSEric Cheng 2006da14cebeSEric Cheng /* No fanout for subflows */ 2007da14cebeSEric Cheng if (flent->fe_type & FLOW_USER) { 20080dc2366fSVenugopal Iyer mac_srs_fanout_init(mcip, mrp, rx_func, 20090dc2366fSVenugopal Iyer x_arg1, x_arg2, mac_rx_srs, mac_tx_srs, 20100dc2366fSVenugopal Iyer cpupart); 2011da14cebeSEric Cheng return; 2012da14cebeSEric Cheng } 2013da14cebeSEric Cheng 20140dc2366fSVenugopal Iyer if (mrp->mrp_mask & MRP_CPUS_USERSPEC) 20150dc2366fSVenugopal Iyer mac_flow_user_cpu_init(flent, mrp); 20160dc2366fSVenugopal Iyer else 20170dc2366fSVenugopal Iyer mac_flow_cpu_init(flent, cpupart); 20180dc2366fSVenugopal Iyer 20190dc2366fSVenugopal Iyer mrp->mrp_rx_fanout_cnt = mac_rx_srs->srs_cpu.mc_rx_fanout_cnt; 2020da14cebeSEric Cheng 2021da14cebeSEric Cheng /* 2022da14cebeSEric Cheng * Set up fanout for both SW (0th SRS) and HW classified 2023da14cebeSEric Cheng * SRS (the rest of Rx SRSs in flent). 2024da14cebeSEric Cheng */ 2025da14cebeSEric Cheng for (i = 0; i < rx_srs_cnt; i++) { 2026da14cebeSEric Cheng mac_rx_srs = flent->fe_rx_srs[i]; 2027da14cebeSEric Cheng if (i != 0) 2028da14cebeSEric Cheng mac_tx_srs = NULL; 2029da14cebeSEric Cheng switch (mac_rx_srs->srs_fanout_state) { 2030da14cebeSEric Cheng case SRS_FANOUT_UNINIT: 20310dc2366fSVenugopal Iyer mac_srs_fanout_init(mcip, mrp, rx_func, 20320dc2366fSVenugopal Iyer x_arg1, x_arg2, mac_rx_srs, mac_tx_srs, 20330dc2366fSVenugopal Iyer cpupart); 2034da14cebeSEric Cheng break; 2035da14cebeSEric Cheng case SRS_FANOUT_INIT: 2036da14cebeSEric Cheng break; 2037da14cebeSEric Cheng case SRS_FANOUT_REINIT: 2038da14cebeSEric Cheng mac_rx_srs_quiesce(mac_rx_srs, SRS_QUIESCE); 20390dc2366fSVenugopal Iyer mac_srs_fanout_modify(mcip, rx_func, x_arg1, 20400dc2366fSVenugopal Iyer x_arg2, mac_rx_srs, mac_tx_srs); 2041da14cebeSEric Cheng mac_rx_srs_restart(mac_rx_srs); 2042da14cebeSEric Cheng break; 2043da14cebeSEric Cheng default: 2044da14cebeSEric Cheng VERIFY(mac_rx_srs->srs_fanout_state <= 2045da14cebeSEric Cheng SRS_FANOUT_REINIT); 2046da14cebeSEric Cheng break; 2047da14cebeSEric Cheng } 2048da14cebeSEric Cheng } 2049da14cebeSEric Cheng } 2050da14cebeSEric Cheng 2051da14cebeSEric Cheng /* 2052da14cebeSEric Cheng * Create a mac_soft_ring_set_t (SRS). If soft_ring_fanout_type is 2053da14cebeSEric Cheng * SRST_TX, an SRS for Tx side is created. Otherwise an SRS for Rx side 2054da14cebeSEric Cheng * processing is created. 2055da14cebeSEric Cheng * 2056da14cebeSEric Cheng * Details on Rx SRS: 2057da14cebeSEric Cheng * Create a SRS and also add the necessary soft rings for TCP and 2058da14cebeSEric Cheng * non-TCP based on fanout type and count specified. 2059da14cebeSEric Cheng * 2060da14cebeSEric Cheng * mac_soft_ring_fanout, mac_srs_fanout_modify (?), 2061da14cebeSEric Cheng * mac_soft_ring_stop_workers, mac_soft_ring_set_destroy, etc need 2062da14cebeSEric Cheng * to be heavily modified. 2063da14cebeSEric Cheng * 2064da14cebeSEric Cheng * mi_soft_ring_list_size, mi_soft_ring_size, etc need to disappear. 2065da14cebeSEric Cheng */ 2066da14cebeSEric Cheng mac_soft_ring_set_t * 2067da14cebeSEric Cheng mac_srs_create(mac_client_impl_t *mcip, flow_entry_t *flent, uint32_t srs_type, 2068da14cebeSEric Cheng mac_direct_rx_t rx_func, void *x_arg1, mac_resource_handle_t x_arg2, 2069da14cebeSEric Cheng mac_ring_t *ring) 2070da14cebeSEric Cheng { 2071da14cebeSEric Cheng mac_soft_ring_set_t *mac_srs; 2072da14cebeSEric Cheng mac_srs_rx_t *srs_rx; 2073da14cebeSEric Cheng mac_srs_tx_t *srs_tx; 2074da14cebeSEric Cheng mac_bw_ctl_t *mac_bw; 2075da14cebeSEric Cheng mac_resource_props_t *mrp; 2076da14cebeSEric Cheng boolean_t is_tx_srs = ((srs_type & SRST_TX) != 0); 2077da14cebeSEric Cheng 2078da14cebeSEric Cheng mac_srs = kmem_cache_alloc(mac_srs_cache, KM_SLEEP); 2079da14cebeSEric Cheng bzero(mac_srs, sizeof (mac_soft_ring_set_t)); 2080da14cebeSEric Cheng srs_rx = &mac_srs->srs_rx; 2081da14cebeSEric Cheng srs_tx = &mac_srs->srs_tx; 2082da14cebeSEric Cheng 2083da14cebeSEric Cheng mutex_enter(&flent->fe_lock); 2084da14cebeSEric Cheng 2085da14cebeSEric Cheng /* 2086da14cebeSEric Cheng * Get the bandwidth control structure from the flent. Get 2087da14cebeSEric Cheng * rid of any residual values in the control structure for 2088da14cebeSEric Cheng * the tx bw struct and also for the rx, if the rx srs is 2089da14cebeSEric Cheng * the 1st one being brought up (the rx bw ctl struct may 2090da14cebeSEric Cheng * be shared by multiple SRSs) 2091da14cebeSEric Cheng */ 2092da14cebeSEric Cheng if (is_tx_srs) { 2093da14cebeSEric Cheng mac_srs->srs_bw = &flent->fe_tx_bw; 2094da14cebeSEric Cheng bzero(mac_srs->srs_bw, sizeof (mac_bw_ctl_t)); 2095da14cebeSEric Cheng flent->fe_tx_srs = mac_srs; 2096da14cebeSEric Cheng } else { 2097da14cebeSEric Cheng /* 2098da14cebeSEric Cheng * The bw counter (stored in the flent) is shared 2099da14cebeSEric Cheng * by SRS's within an rx group. 2100da14cebeSEric Cheng */ 2101da14cebeSEric Cheng mac_srs->srs_bw = &flent->fe_rx_bw; 2102da14cebeSEric Cheng /* First rx SRS, clear the bw structure */ 2103da14cebeSEric Cheng if (flent->fe_rx_srs_cnt == 0) 2104da14cebeSEric Cheng bzero(mac_srs->srs_bw, sizeof (mac_bw_ctl_t)); 210577e3ef6dSVenugopal Iyer 210677e3ef6dSVenugopal Iyer /* 210777e3ef6dSVenugopal Iyer * It is better to panic here rather than just assert because 210877e3ef6dSVenugopal Iyer * on a non-debug kernel we might end up courrupting memory 210977e3ef6dSVenugopal Iyer * and making it difficult to debug. 211077e3ef6dSVenugopal Iyer */ 211177e3ef6dSVenugopal Iyer if (flent->fe_rx_srs_cnt >= MAX_RINGS_PER_GROUP) { 211277e3ef6dSVenugopal Iyer panic("Array Overrun detected due to MAC client %p " 211377e3ef6dSVenugopal Iyer " having more rings than %d", (void *)mcip, 211477e3ef6dSVenugopal Iyer MAX_RINGS_PER_GROUP); 211577e3ef6dSVenugopal Iyer } 2116da14cebeSEric Cheng flent->fe_rx_srs[flent->fe_rx_srs_cnt] = mac_srs; 2117da14cebeSEric Cheng flent->fe_rx_srs_cnt++; 2118da14cebeSEric Cheng } 2119da14cebeSEric Cheng mac_srs->srs_flent = flent; 2120da14cebeSEric Cheng mutex_exit(&flent->fe_lock); 2121da14cebeSEric Cheng 2122da14cebeSEric Cheng mac_srs->srs_state = 0; 2123da14cebeSEric Cheng mac_srs->srs_type = (srs_type | SRST_NO_SOFT_RINGS); 2124da14cebeSEric Cheng mac_srs->srs_worker_cpuid = mac_srs->srs_worker_cpuid_save = -1; 2125da14cebeSEric Cheng mac_srs->srs_poll_cpuid = mac_srs->srs_poll_cpuid_save = -1; 21260dc2366fSVenugopal Iyer mac_srs->srs_mcip = mcip; 2127da14cebeSEric Cheng mac_srs_fanout_list_alloc(mac_srs); 2128da14cebeSEric Cheng 2129da14cebeSEric Cheng /* 2130da14cebeSEric Cheng * For a flow we use the underlying MAC client's priority range with 2131da14cebeSEric Cheng * the priority value to find an absolute priority value. For a MAC 2132da14cebeSEric Cheng * client we use the MAC client's maximum priority as the value. 2133da14cebeSEric Cheng */ 2134da14cebeSEric Cheng mrp = &flent->fe_effective_props; 2135da14cebeSEric Cheng if ((mac_srs->srs_type & SRST_FLOW) != 0) { 2136da14cebeSEric Cheng mac_srs->srs_pri = FLOW_PRIORITY(mcip->mci_min_pri, 2137da14cebeSEric Cheng mcip->mci_max_pri, mrp->mrp_priority); 2138da14cebeSEric Cheng } else { 2139da14cebeSEric Cheng mac_srs->srs_pri = mcip->mci_max_pri; 2140da14cebeSEric Cheng } 2141da14cebeSEric Cheng /* 2142da14cebeSEric Cheng * We need to insert the SRS in the global list before 2143da14cebeSEric Cheng * binding the SRS and SR threads. Otherwise there is a 2144da14cebeSEric Cheng * is a small window where the cpu reconfig callbacks 2145da14cebeSEric Cheng * may miss the SRS in the list walk and DR could fail 2146da14cebeSEric Cheng * as there are bound threads. 2147da14cebeSEric Cheng */ 2148da14cebeSEric Cheng mac_srs_add_glist(mac_srs); 2149da14cebeSEric Cheng 2150da14cebeSEric Cheng /* Initialize bw limit */ 2151da14cebeSEric Cheng if ((mrp->mrp_mask & MRP_MAXBW) != 0) { 2152da14cebeSEric Cheng mac_srs->srs_drain_func = mac_rx_srs_drain_bw; 2153da14cebeSEric Cheng 2154da14cebeSEric Cheng mac_bw = mac_srs->srs_bw; 2155da14cebeSEric Cheng mutex_enter(&mac_bw->mac_bw_lock); 2156da14cebeSEric Cheng mac_bw->mac_bw_limit = FLOW_BYTES_PER_TICK(mrp->mrp_maxbw); 2157da14cebeSEric Cheng 2158da14cebeSEric Cheng /* 2159da14cebeSEric Cheng * Give twice the queuing capability before 2160da14cebeSEric Cheng * dropping packets. The unit is bytes/tick. 2161da14cebeSEric Cheng */ 2162da14cebeSEric Cheng mac_bw->mac_bw_drop_threshold = mac_bw->mac_bw_limit << 1; 2163da14cebeSEric Cheng mutex_exit(&mac_bw->mac_bw_lock); 2164da14cebeSEric Cheng mac_srs->srs_type |= SRST_BW_CONTROL; 2165da14cebeSEric Cheng } else { 2166da14cebeSEric Cheng mac_srs->srs_drain_func = mac_rx_srs_drain; 2167da14cebeSEric Cheng } 2168da14cebeSEric Cheng 2169da14cebeSEric Cheng /* 2170da14cebeSEric Cheng * We use the following policy to control Receive 2171da14cebeSEric Cheng * Side Dynamic Polling: 2172da14cebeSEric Cheng * 1) We switch to poll mode anytime the processing thread causes 2173da14cebeSEric Cheng * a backlog to build up in SRS and its associated Soft Rings 2174da14cebeSEric Cheng * (sr_poll_pkt_cnt > 0). 2175da14cebeSEric Cheng * 2) As long as the backlog stays under the low water mark 2176da14cebeSEric Cheng * (sr_lowat), we poll the H/W for more packets. 2177da14cebeSEric Cheng * 3) If the backlog (sr_poll_pkt_cnt) exceeds low water mark, we 2178da14cebeSEric Cheng * stay in poll mode but don't poll the H/W for more packets. 2179da14cebeSEric Cheng * 4) Anytime in polling mode, if we poll the H/W for packets and 2180da14cebeSEric Cheng * find nothing plus we have an existing backlog 2181da14cebeSEric Cheng * (sr_poll_pkt_cnt > 0), we stay in polling mode but don't poll 2182da14cebeSEric Cheng * the H/W for packets anymore (let the polling thread go to sleep). 218384de666eSRyan Zezeski * 5) Once the backlog is relieved (packets are processed) we reenable 2184da14cebeSEric Cheng * polling (by signalling the poll thread) only when the backlog 2185da14cebeSEric Cheng * dips below sr_poll_thres. 2186da14cebeSEric Cheng * 6) sr_hiwat is used exclusively when we are not polling capable 2187da14cebeSEric Cheng * and is used to decide when to drop packets so the SRS queue 2188da14cebeSEric Cheng * length doesn't grow infinitely. 2189da14cebeSEric Cheng */ 2190da14cebeSEric Cheng if (!is_tx_srs) { 2191da14cebeSEric Cheng srs_rx->sr_hiwat = mac_soft_ring_max_q_cnt; 2192da14cebeSEric Cheng /* Low water mark needs to be less than high water mark */ 2193da14cebeSEric Cheng srs_rx->sr_lowat = mac_soft_ring_min_q_cnt <= 2194da14cebeSEric Cheng mac_soft_ring_max_q_cnt ? mac_soft_ring_min_q_cnt : 2195da14cebeSEric Cheng (mac_soft_ring_max_q_cnt >> 2); 2196da14cebeSEric Cheng /* Poll threshold need to be half of low water mark or less */ 2197da14cebeSEric Cheng srs_rx->sr_poll_thres = mac_soft_ring_poll_thres <= 2198da14cebeSEric Cheng (srs_rx->sr_lowat >> 1) ? mac_soft_ring_poll_thres : 2199da14cebeSEric Cheng (srs_rx->sr_lowat >> 1); 2200da14cebeSEric Cheng if (mac_latency_optimize) 22013631b19bSRajagopal Kunhappan mac_srs->srs_state |= SRS_LATENCY_OPT; 22023631b19bSRajagopal Kunhappan else 22033631b19bSRajagopal Kunhappan mac_srs->srs_state |= SRS_SOFTRING_QUEUE; 2204da14cebeSEric Cheng } 2205da14cebeSEric Cheng 2206da14cebeSEric Cheng mac_srs->srs_worker = thread_create(NULL, 0, 2207da14cebeSEric Cheng mac_srs_worker, mac_srs, 0, &p0, TS_RUN, mac_srs->srs_pri); 2208da14cebeSEric Cheng 2209da14cebeSEric Cheng if (is_tx_srs) { 2210da14cebeSEric Cheng /* Handle everything about Tx SRS and return */ 2211da14cebeSEric Cheng mac_srs->srs_drain_func = mac_tx_srs_drain; 2212da14cebeSEric Cheng srs_tx->st_max_q_cnt = mac_tx_srs_max_q_cnt; 2213da14cebeSEric Cheng srs_tx->st_hiwat = 2214da14cebeSEric Cheng (mac_tx_srs_hiwat > mac_tx_srs_max_q_cnt) ? 2215da14cebeSEric Cheng mac_tx_srs_max_q_cnt : mac_tx_srs_hiwat; 2216da14cebeSEric Cheng srs_tx->st_arg1 = x_arg1; 2217da14cebeSEric Cheng srs_tx->st_arg2 = x_arg2; 22180dc2366fSVenugopal Iyer goto done; 2219da14cebeSEric Cheng } 2220da14cebeSEric Cheng 2221da14cebeSEric Cheng if ((srs_type & SRST_FLOW) != 0 || 2222da14cebeSEric Cheng FLOW_TAB_EMPTY(mcip->mci_subflow_tab)) 2223da14cebeSEric Cheng srs_rx->sr_lower_proc = mac_rx_srs_process; 2224da14cebeSEric Cheng else 2225da14cebeSEric Cheng srs_rx->sr_lower_proc = mac_rx_srs_subflow_process; 2226da14cebeSEric Cheng 2227da14cebeSEric Cheng srs_rx->sr_func = rx_func; 2228da14cebeSEric Cheng srs_rx->sr_arg1 = x_arg1; 2229da14cebeSEric Cheng srs_rx->sr_arg2 = x_arg2; 2230da14cebeSEric Cheng 2231da14cebeSEric Cheng if (ring != NULL) { 22320dc2366fSVenugopal Iyer uint_t ring_info; 22330dc2366fSVenugopal Iyer 2234da14cebeSEric Cheng /* Is the mac_srs created over the RX default group? */ 2235da14cebeSEric Cheng if (ring->mr_gh == (mac_group_handle_t) 22360dc2366fSVenugopal Iyer MAC_DEFAULT_RX_GROUP(mcip->mci_mip)) { 2237da14cebeSEric Cheng mac_srs->srs_type |= SRST_DEFAULT_GRP; 22380dc2366fSVenugopal Iyer } 2239da14cebeSEric Cheng mac_srs->srs_ring = ring; 2240da14cebeSEric Cheng ring->mr_srs = mac_srs; 2241da14cebeSEric Cheng ring->mr_classify_type = MAC_HW_CLASSIFIER; 2242da14cebeSEric Cheng ring->mr_flag |= MR_INCIPIENT; 2243da14cebeSEric Cheng 22444eaa4710SRishi Srivatsavai if (!(mcip->mci_mip->mi_state_flags & MIS_POLL_DISABLE) && 22454eaa4710SRishi Srivatsavai FLOW_TAB_EMPTY(mcip->mci_subflow_tab) && mac_poll_enable) 2246da14cebeSEric Cheng mac_srs->srs_state |= SRS_POLLING_CAPAB; 2247da14cebeSEric Cheng 2248da14cebeSEric Cheng mac_srs->srs_poll_thr = thread_create(NULL, 0, 2249da14cebeSEric Cheng mac_rx_srs_poll_ring, mac_srs, 0, &p0, TS_RUN, 2250da14cebeSEric Cheng mac_srs->srs_pri); 2251ae6aa22aSVenugopal Iyer /* 2252ae6aa22aSVenugopal Iyer * Some drivers require serialization and don't send 2253ae6aa22aSVenugopal Iyer * packet chains in interrupt context. For such 225484de666eSRyan Zezeski * drivers, we should always queue in the soft ring 225584de666eSRyan Zezeski * so that we get a chance to switch into polling 2256ae6aa22aSVenugopal Iyer * mode under backlog. 2257ae6aa22aSVenugopal Iyer */ 22580dc2366fSVenugopal Iyer ring_info = mac_hwring_getinfo((mac_ring_handle_t)ring); 22590dc2366fSVenugopal Iyer if (ring_info & MAC_RING_RX_ENQUEUE) 2260ae6aa22aSVenugopal Iyer mac_srs->srs_state |= SRS_SOFTRING_QUEUE; 2261da14cebeSEric Cheng } 22620dc2366fSVenugopal Iyer done: 22630dc2366fSVenugopal Iyer mac_srs_stat_create(mac_srs); 2264da14cebeSEric Cheng return (mac_srs); 2265da14cebeSEric Cheng } 2266da14cebeSEric Cheng 2267da14cebeSEric Cheng /* 2268da14cebeSEric Cheng * Figure out the number of soft rings required. Its dependant on 2269da14cebeSEric Cheng * if protocol fanout is required (for LINKs), global settings 2270da14cebeSEric Cheng * require us to do fanout for performance (based on mac_soft_ring_enable), 2271da14cebeSEric Cheng * or user has specifically requested fanout. 2272da14cebeSEric Cheng */ 2273da14cebeSEric Cheng static uint32_t 2274da14cebeSEric Cheng mac_find_fanout(flow_entry_t *flent, uint32_t link_type) 2275da14cebeSEric Cheng { 2276da14cebeSEric Cheng uint32_t fanout_type; 2277da14cebeSEric Cheng mac_resource_props_t *mrp = &flent->fe_effective_props; 2278da14cebeSEric Cheng 2279da14cebeSEric Cheng /* no fanout for subflows */ 2280da14cebeSEric Cheng switch (link_type) { 2281da14cebeSEric Cheng case SRST_FLOW: 2282da14cebeSEric Cheng fanout_type = SRST_NO_SOFT_RINGS; 2283da14cebeSEric Cheng break; 2284da14cebeSEric Cheng case SRST_LINK: 2285da14cebeSEric Cheng fanout_type = SRST_FANOUT_PROTO; 2286da14cebeSEric Cheng break; 2287da14cebeSEric Cheng } 2288da14cebeSEric Cheng 2289da14cebeSEric Cheng /* A primary NIC/link is being plumbed */ 2290da14cebeSEric Cheng if (flent->fe_type & FLOW_PRIMARY_MAC) { 2291da14cebeSEric Cheng if (mac_soft_ring_enable && mac_rx_soft_ring_count > 1) { 2292da14cebeSEric Cheng fanout_type |= SRST_FANOUT_SRC_IP; 2293da14cebeSEric Cheng } 2294da14cebeSEric Cheng } else if (flent->fe_type & FLOW_VNIC) { 2295da14cebeSEric Cheng /* A VNIC is being created */ 2296da14cebeSEric Cheng if (mrp != NULL && mrp->mrp_ncpus > 0) { 2297da14cebeSEric Cheng fanout_type |= SRST_FANOUT_SRC_IP; 2298da14cebeSEric Cheng } 2299da14cebeSEric Cheng } 2300da14cebeSEric Cheng 2301da14cebeSEric Cheng return (fanout_type); 2302da14cebeSEric Cheng } 2303da14cebeSEric Cheng 2304da14cebeSEric Cheng /* 2305da14cebeSEric Cheng * Change a group from h/w to s/w classification. 2306da14cebeSEric Cheng */ 23070dc2366fSVenugopal Iyer void 2308da14cebeSEric Cheng mac_rx_switch_grp_to_sw(mac_group_t *group) 2309da14cebeSEric Cheng { 2310da14cebeSEric Cheng mac_ring_t *ring; 2311da14cebeSEric Cheng mac_soft_ring_set_t *mac_srs; 2312da14cebeSEric Cheng 2313da14cebeSEric Cheng for (ring = group->mrg_rings; ring != NULL; ring = ring->mr_next) { 2314da14cebeSEric Cheng if (ring->mr_classify_type == MAC_HW_CLASSIFIER) { 2315da14cebeSEric Cheng /* 2316da14cebeSEric Cheng * Remove the SRS associated with the HW ring. 2317da14cebeSEric Cheng * As a result, polling will be disabled. 2318da14cebeSEric Cheng */ 2319da14cebeSEric Cheng mac_srs = ring->mr_srs; 2320da14cebeSEric Cheng ASSERT(mac_srs != NULL); 2321da14cebeSEric Cheng mac_rx_srs_remove(mac_srs); 2322da14cebeSEric Cheng ring->mr_srs = NULL; 2323da14cebeSEric Cheng } 2324da14cebeSEric Cheng 2325da14cebeSEric Cheng if (ring->mr_state != MR_INUSE) 2326da14cebeSEric Cheng (void) mac_start_ring(ring); 23270dc2366fSVenugopal Iyer 2328da14cebeSEric Cheng /* 2329da14cebeSEric Cheng * We need to perform SW classification 2330da14cebeSEric Cheng * for packets landing in these rings 2331da14cebeSEric Cheng */ 2332da14cebeSEric Cheng ring->mr_flag = 0; 2333da14cebeSEric Cheng ring->mr_classify_type = MAC_SW_CLASSIFIER; 2334da14cebeSEric Cheng } 2335da14cebeSEric Cheng } 2336da14cebeSEric Cheng 2337da14cebeSEric Cheng /* 2338da14cebeSEric Cheng * Create the Rx SRS for S/W classifier and for each ring in the 2339da14cebeSEric Cheng * group (if exclusive group). Also create the Tx SRS. 2340da14cebeSEric Cheng */ 2341da14cebeSEric Cheng void 2342da14cebeSEric Cheng mac_srs_group_setup(mac_client_impl_t *mcip, flow_entry_t *flent, 23430dc2366fSVenugopal Iyer uint32_t link_type) 23440dc2366fSVenugopal Iyer { 23450dc2366fSVenugopal Iyer cpupart_t *cpupart; 23460dc2366fSVenugopal Iyer mac_resource_props_t *mrp = MCIP_RESOURCE_PROPS(mcip); 23470dc2366fSVenugopal Iyer mac_resource_props_t *emrp = MCIP_EFFECTIVE_PROPS(mcip); 23480dc2366fSVenugopal Iyer boolean_t use_default = B_FALSE; 23490dc2366fSVenugopal Iyer 23500dc2366fSVenugopal Iyer mac_rx_srs_group_setup(mcip, flent, link_type); 23510dc2366fSVenugopal Iyer mac_tx_srs_group_setup(mcip, flent, link_type); 23520dc2366fSVenugopal Iyer 235345948e49SRyan Zezeski /* Aggr ports don't have SRSes; thus there is no soft ring fanout. */ 235445948e49SRyan Zezeski if ((mcip->mci_state_flags & MCIS_IS_AGGR_PORT) != 0) 235545948e49SRyan Zezeski return; 235645948e49SRyan Zezeski 23570dc2366fSVenugopal Iyer pool_lock(); 23580dc2366fSVenugopal Iyer cpupart = mac_pset_find(mrp, &use_default); 23590dc2366fSVenugopal Iyer mac_fanout_setup(mcip, flent, MCIP_RESOURCE_PROPS(mcip), 23600dc2366fSVenugopal Iyer mac_rx_deliver, mcip, NULL, cpupart); 23610dc2366fSVenugopal Iyer mac_set_pool_effective(use_default, cpupart, mrp, emrp); 23620dc2366fSVenugopal Iyer pool_unlock(); 23630dc2366fSVenugopal Iyer } 23640dc2366fSVenugopal Iyer 23650dc2366fSVenugopal Iyer /* 236684de666eSRyan Zezeski * Set up the Rx SRSes. If there is no group associated with the 236784de666eSRyan Zezeski * client, then only setup SW classification. If the client has 236884de666eSRyan Zezeski * exlusive (MAC_GROUP_STATE_RESERVED) use of the group, then create an 236984de666eSRyan Zezeski * SRS for each HW ring. If the client is sharing a group, then make 237084de666eSRyan Zezeski * sure to teardown the HW SRSes. 23710dc2366fSVenugopal Iyer */ 23720dc2366fSVenugopal Iyer void 23730dc2366fSVenugopal Iyer mac_rx_srs_group_setup(mac_client_impl_t *mcip, flow_entry_t *flent, 23740dc2366fSVenugopal Iyer uint32_t link_type) 2375da14cebeSEric Cheng { 2376da14cebeSEric Cheng mac_impl_t *mip = mcip->mci_mip; 2377da14cebeSEric Cheng mac_soft_ring_set_t *mac_srs; 2378da14cebeSEric Cheng mac_ring_t *ring; 2379da14cebeSEric Cheng uint32_t fanout_type; 23800dc2366fSVenugopal Iyer mac_group_t *rx_group = flent->fe_rx_ring_group; 238184de666eSRyan Zezeski boolean_t no_unicast; 2382da14cebeSEric Cheng 238345948e49SRyan Zezeski /* 238445948e49SRyan Zezeski * If this is an an aggr port, then don't setup Rx SRS and Rx 238545948e49SRyan Zezeski * soft rings as they won't be used. However, we still need to 238645948e49SRyan Zezeski * start the rings to receive data on them. 238745948e49SRyan Zezeski */ 238845948e49SRyan Zezeski if (mcip->mci_state_flags & MCIS_IS_AGGR_PORT) { 238945948e49SRyan Zezeski if (rx_group == NULL) 239045948e49SRyan Zezeski return; 239145948e49SRyan Zezeski 239245948e49SRyan Zezeski for (ring = rx_group->mrg_rings; ring != NULL; 239345948e49SRyan Zezeski ring = ring->mr_next) { 239445948e49SRyan Zezeski if (ring->mr_state != MR_INUSE) 239545948e49SRyan Zezeski (void) mac_start_ring(ring); 239645948e49SRyan Zezeski } 239745948e49SRyan Zezeski 239845948e49SRyan Zezeski return; 239945948e49SRyan Zezeski } 240045948e49SRyan Zezeski 240145948e49SRyan Zezeski /* 240245948e49SRyan Zezeski * Aggr ports should never have SRSes. 240345948e49SRyan Zezeski */ 240445948e49SRyan Zezeski ASSERT3U((mcip->mci_state_flags & MCIS_IS_AGGR_PORT), ==, 0); 240545948e49SRyan Zezeski 2406da14cebeSEric Cheng fanout_type = mac_find_fanout(flent, link_type); 240784de666eSRyan Zezeski no_unicast = (mcip->mci_state_flags & MCIS_NO_UNICAST_ADDR) != 0; 2408da14cebeSEric Cheng 240984de666eSRyan Zezeski /* Create the SRS for SW classification if none exists */ 2410da14cebeSEric Cheng if (flent->fe_rx_srs[0] == NULL) { 2411da14cebeSEric Cheng ASSERT(flent->fe_rx_srs_cnt == 0); 2412da14cebeSEric Cheng mac_srs = mac_srs_create(mcip, flent, fanout_type | link_type, 2413da14cebeSEric Cheng mac_rx_deliver, mcip, NULL, NULL); 2414da14cebeSEric Cheng mutex_enter(&flent->fe_lock); 2415da14cebeSEric Cheng flent->fe_cb_fn = (flow_fn_t)mac_srs->srs_rx.sr_lower_proc; 2416da14cebeSEric Cheng flent->fe_cb_arg1 = (void *)mip; 2417da14cebeSEric Cheng flent->fe_cb_arg2 = (void *)mac_srs; 2418da14cebeSEric Cheng mutex_exit(&flent->fe_lock); 2419da14cebeSEric Cheng } 2420da14cebeSEric Cheng 24210dc2366fSVenugopal Iyer if (rx_group == NULL) 2422da14cebeSEric Cheng return; 242384de666eSRyan Zezeski 2424da14cebeSEric Cheng /* 242584de666eSRyan Zezeski * If the group is marked RESERVED then setup an SRS and 242684de666eSRyan Zezeski * fanout for each HW ring. 2427da14cebeSEric Cheng */ 24280dc2366fSVenugopal Iyer switch (rx_group->mrg_state) { 2429da14cebeSEric Cheng case MAC_GROUP_STATE_RESERVED: 24300dc2366fSVenugopal Iyer for (ring = rx_group->mrg_rings; ring != NULL; 2431da14cebeSEric Cheng ring = ring->mr_next) { 243284de666eSRyan Zezeski uint16_t vid = i_mac_flow_vid(mcip->mci_flent); 243384de666eSRyan Zezeski 2434da14cebeSEric Cheng switch (ring->mr_state) { 2435da14cebeSEric Cheng case MR_INUSE: 2436da14cebeSEric Cheng case MR_FREE: 2437da14cebeSEric Cheng if (ring->mr_srs != NULL) 2438da14cebeSEric Cheng break; 2439da14cebeSEric Cheng if (ring->mr_state != MR_INUSE) 2440da14cebeSEric Cheng (void) mac_start_ring(ring); 2441da14cebeSEric Cheng 24420dc2366fSVenugopal Iyer /* 244384de666eSRyan Zezeski * If a client requires SW VLAN 244484de666eSRyan Zezeski * filtering or has no unicast address 244584de666eSRyan Zezeski * then we don't create any HW ring 244684de666eSRyan Zezeski * SRSes. 24470dc2366fSVenugopal Iyer */ 244884de666eSRyan Zezeski if ((!MAC_GROUP_HW_VLAN(rx_group) && 244984de666eSRyan Zezeski vid != VLAN_ID_NONE) || no_unicast) 24500dc2366fSVenugopal Iyer break; 245184de666eSRyan Zezeski 245284de666eSRyan Zezeski /* 245384de666eSRyan Zezeski * When a client has exclusive use of 245484de666eSRyan Zezeski * a group, and that group's traffic 245584de666eSRyan Zezeski * is fully HW classified, we create 245684de666eSRyan Zezeski * an SRS for each HW ring in order to 245784de666eSRyan Zezeski * make use of dynamic polling of said 245884de666eSRyan Zezeski * HW rings. 245984de666eSRyan Zezeski */ 2460da14cebeSEric Cheng mac_srs = mac_srs_create(mcip, flent, 2461da14cebeSEric Cheng fanout_type | link_type, 2462da14cebeSEric Cheng mac_rx_deliver, mcip, NULL, ring); 2463da14cebeSEric Cheng break; 2464da14cebeSEric Cheng default: 24650dc2366fSVenugopal Iyer cmn_err(CE_PANIC, 24660dc2366fSVenugopal Iyer "srs_setup: mcip = %p " 2467da14cebeSEric Cheng "trying to add UNKNOWN ring = %p\n", 2468da14cebeSEric Cheng (void *)mcip, (void *)ring); 2469da14cebeSEric Cheng break; 2470da14cebeSEric Cheng } 2471da14cebeSEric Cheng } 2472da14cebeSEric Cheng break; 2473da14cebeSEric Cheng case MAC_GROUP_STATE_SHARED: 2474da14cebeSEric Cheng /* 247584de666eSRyan Zezeski * When a group is shared by multiple clients, we must 247684de666eSRyan Zezeski * use SW classifiction to ensure packets are 247784de666eSRyan Zezeski * delivered to the correct client. 2478da14cebeSEric Cheng */ 24790dc2366fSVenugopal Iyer mac_rx_switch_grp_to_sw(rx_group); 2480da14cebeSEric Cheng break; 2481da14cebeSEric Cheng default: 2482da14cebeSEric Cheng ASSERT(B_FALSE); 2483da14cebeSEric Cheng break; 2484da14cebeSEric Cheng } 2485da14cebeSEric Cheng } 2486da14cebeSEric Cheng 24870dc2366fSVenugopal Iyer /* 24880dc2366fSVenugopal Iyer * Set up the TX SRS. 24890dc2366fSVenugopal Iyer */ 2490da14cebeSEric Cheng void 24910dc2366fSVenugopal Iyer mac_tx_srs_group_setup(mac_client_impl_t *mcip, flow_entry_t *flent, 2492da14cebeSEric Cheng uint32_t link_type) 2493da14cebeSEric Cheng { 249445948e49SRyan Zezeski /* 249545948e49SRyan Zezeski * If this is an exclusive client (e.g. an aggr port), then 249645948e49SRyan Zezeski * don't setup Tx SRS and Tx soft rings as they won't be used. 249745948e49SRyan Zezeski * However, we still need to start the rings to send data 249845948e49SRyan Zezeski * across them. 249945948e49SRyan Zezeski */ 250045948e49SRyan Zezeski if (mcip->mci_state_flags & MCIS_EXCLUSIVE) { 25010dc2366fSVenugopal Iyer mac_ring_t *ring; 25020dc2366fSVenugopal Iyer mac_group_t *grp; 2503da14cebeSEric Cheng 25040dc2366fSVenugopal Iyer grp = (mac_group_t *)flent->fe_tx_ring_group; 250545948e49SRyan Zezeski 250645948e49SRyan Zezeski if (grp == NULL) 250745948e49SRyan Zezeski return; 250845948e49SRyan Zezeski 250945948e49SRyan Zezeski for (ring = grp->mrg_rings; ring != NULL; 251045948e49SRyan Zezeski ring = ring->mr_next) { 251145948e49SRyan Zezeski if (ring->mr_state != MR_INUSE) 25120dc2366fSVenugopal Iyer (void) mac_start_ring(ring); 25130dc2366fSVenugopal Iyer } 251445948e49SRyan Zezeski 25150dc2366fSVenugopal Iyer return; 25160dc2366fSVenugopal Iyer } 251745948e49SRyan Zezeski 251845948e49SRyan Zezeski /* 251945948e49SRyan Zezeski * Aggr ports should never have SRSes. 252045948e49SRyan Zezeski */ 252145948e49SRyan Zezeski ASSERT3U((mcip->mci_state_flags & MCIS_IS_AGGR_PORT), ==, 0); 252245948e49SRyan Zezeski 25230dc2366fSVenugopal Iyer if (flent->fe_tx_srs == NULL) { 25240dc2366fSVenugopal Iyer (void) mac_srs_create(mcip, flent, SRST_TX | link_type, 25250dc2366fSVenugopal Iyer NULL, mcip, NULL, NULL); 25260dc2366fSVenugopal Iyer } 252745948e49SRyan Zezeski 25280dc2366fSVenugopal Iyer mac_tx_srs_setup(mcip, flent); 25290dc2366fSVenugopal Iyer } 25300dc2366fSVenugopal Iyer 25310dc2366fSVenugopal Iyer /* 253284de666eSRyan Zezeski * Teardown all the Rx SRSes. Unless hwonly is set, then only teardown 253384de666eSRyan Zezeski * the Rx HW SRSes and leave the SW SRS alone. The hwonly flag is set 253484de666eSRyan Zezeski * when we wish to move a MAC client from one group to another. In 253584de666eSRyan Zezeski * that case, we need to release the current HW SRSes but keep the SW 253684de666eSRyan Zezeski * SRS for continued traffic classifiction. 25370dc2366fSVenugopal Iyer */ 25380dc2366fSVenugopal Iyer void 25390dc2366fSVenugopal Iyer mac_rx_srs_group_teardown(flow_entry_t *flent, boolean_t hwonly) 25400dc2366fSVenugopal Iyer { 25410dc2366fSVenugopal Iyer mac_soft_ring_set_t *mac_srs; 25420dc2366fSVenugopal Iyer int i; 25430dc2366fSVenugopal Iyer int count = flent->fe_rx_srs_cnt; 25440dc2366fSVenugopal Iyer 25450dc2366fSVenugopal Iyer for (i = 0; i < count; i++) { 25460dc2366fSVenugopal Iyer if (i == 0 && hwonly) 25470dc2366fSVenugopal Iyer continue; 2548da14cebeSEric Cheng mac_srs = flent->fe_rx_srs[i]; 2549da14cebeSEric Cheng mac_rx_srs_quiesce(mac_srs, SRS_CONDEMNED); 2550da14cebeSEric Cheng mac_srs_free(mac_srs); 2551da14cebeSEric Cheng flent->fe_rx_srs[i] = NULL; 25520dc2366fSVenugopal Iyer flent->fe_rx_srs_cnt--; 2553da14cebeSEric Cheng } 255484de666eSRyan Zezeski 255584de666eSRyan Zezeski /* 255684de666eSRyan Zezeski * If we are only tearing down the HW SRSes then there must be 255784de666eSRyan Zezeski * one SRS left for SW classification. Otherwise we are tearing 255884de666eSRyan Zezeski * down both HW and SW and there should be no SRSes left. 255984de666eSRyan Zezeski */ 256084de666eSRyan Zezeski if (hwonly) 256184de666eSRyan Zezeski VERIFY3S(flent->fe_rx_srs_cnt, ==, 1); 256284de666eSRyan Zezeski else 256384de666eSRyan Zezeski VERIFY3S(flent->fe_rx_srs_cnt, ==, 0); 25640dc2366fSVenugopal Iyer } 2565da14cebeSEric Cheng 25660dc2366fSVenugopal Iyer /* 25670dc2366fSVenugopal Iyer * Remove the TX SRS. 25680dc2366fSVenugopal Iyer */ 25690dc2366fSVenugopal Iyer void 25700dc2366fSVenugopal Iyer mac_tx_srs_group_teardown(mac_client_impl_t *mcip, flow_entry_t *flent, 25710dc2366fSVenugopal Iyer uint32_t link_type) 25720dc2366fSVenugopal Iyer { 25730dc2366fSVenugopal Iyer mac_soft_ring_set_t *tx_srs; 25740dc2366fSVenugopal Iyer mac_srs_tx_t *tx; 25750dc2366fSVenugopal Iyer 25760dc2366fSVenugopal Iyer if ((tx_srs = flent->fe_tx_srs) == NULL) 25770dc2366fSVenugopal Iyer return; 25780dc2366fSVenugopal Iyer 2579da14cebeSEric Cheng tx = &tx_srs->srs_tx; 2580da14cebeSEric Cheng switch (link_type) { 2581da14cebeSEric Cheng case SRST_FLOW: 2582da14cebeSEric Cheng /* 2583da14cebeSEric Cheng * For flows, we need to work with passed 2584da14cebeSEric Cheng * flent to find the Rx/Tx SRS. 2585da14cebeSEric Cheng */ 2586da14cebeSEric Cheng mac_tx_srs_quiesce(tx_srs, SRS_CONDEMNED); 2587da14cebeSEric Cheng break; 2588da14cebeSEric Cheng case SRST_LINK: 25890dc2366fSVenugopal Iyer mac_tx_client_condemn((mac_client_handle_t)mcip); 2590da14cebeSEric Cheng if (tx->st_arg2 != NULL) { 2591da14cebeSEric Cheng ASSERT(tx_srs->srs_type & SRST_TX); 25920dc2366fSVenugopal Iyer /* 25930dc2366fSVenugopal Iyer * The ring itself will be stopped when 25940dc2366fSVenugopal Iyer * we release the group or in the 25950dc2366fSVenugopal Iyer * mac_datapath_teardown (for the default 25960dc2366fSVenugopal Iyer * group) 25970dc2366fSVenugopal Iyer */ 25980dc2366fSVenugopal Iyer tx->st_arg2 = NULL; 2599da14cebeSEric Cheng } 2600da14cebeSEric Cheng break; 2601da14cebeSEric Cheng default: 2602da14cebeSEric Cheng ASSERT(B_FALSE); 2603da14cebeSEric Cheng break; 2604da14cebeSEric Cheng } 2605da14cebeSEric Cheng mac_srs_free(tx_srs); 2606da14cebeSEric Cheng flent->fe_tx_srs = NULL; 2607da14cebeSEric Cheng } 2608da14cebeSEric Cheng 2609da14cebeSEric Cheng /* 26100dc2366fSVenugopal Iyer * This is the group state machine. 26110dc2366fSVenugopal Iyer * 26120dc2366fSVenugopal Iyer * The state of an Rx group is given by 2613da14cebeSEric Cheng * the following table. The default group and its rings are started in 2614da14cebeSEric Cheng * mac_start itself and the default group stays in SHARED state until 2615da14cebeSEric Cheng * mac_stop at which time the group and rings are stopped and and it 2616da14cebeSEric Cheng * reverts to the Registered state. 2617da14cebeSEric Cheng * 2618da14cebeSEric Cheng * Typically this function is called on a group after adding or removing a 2619da14cebeSEric Cheng * client from it, to find out what should be the new state of the group. 2620da14cebeSEric Cheng * If the new state is RESERVED, then the client that owns this group 2621da14cebeSEric Cheng * exclusively is also returned. Note that adding or removing a client from 2622da14cebeSEric Cheng * a group could also impact the default group and the caller needs to 2623da14cebeSEric Cheng * evaluate the effect on the default group. 2624da14cebeSEric Cheng * 2625da14cebeSEric Cheng * Group type # of clients mi_nactiveclients Group State 2626da14cebeSEric Cheng * in the group 2627da14cebeSEric Cheng * 2628da14cebeSEric Cheng * Non-default 0 N.A. REGISTERED 2629da14cebeSEric Cheng * Non-default 1 N.A. RESERVED 2630da14cebeSEric Cheng * 2631da14cebeSEric Cheng * Default 0 N.A. SHARED 2632da14cebeSEric Cheng * Default 1 1 RESERVED 2633da14cebeSEric Cheng * Default 1 > 1 SHARED 2634da14cebeSEric Cheng * Default > 1 N.A. SHARED 26350dc2366fSVenugopal Iyer * 26360dc2366fSVenugopal Iyer * For a TX group, the following is the state table. 26370dc2366fSVenugopal Iyer * 26380dc2366fSVenugopal Iyer * Group type # of clients Group State 26390dc2366fSVenugopal Iyer * in the group 26400dc2366fSVenugopal Iyer * 26410dc2366fSVenugopal Iyer * Non-default 0 REGISTERED 26420dc2366fSVenugopal Iyer * Non-default 1 RESERVED 26430dc2366fSVenugopal Iyer * 26440dc2366fSVenugopal Iyer * Default 0 REGISTERED 26450dc2366fSVenugopal Iyer * Default 1 RESERVED 26460dc2366fSVenugopal Iyer * Default > 1 SHARED 2647da14cebeSEric Cheng */ 2648da14cebeSEric Cheng mac_group_state_t 26490dc2366fSVenugopal Iyer mac_group_next_state(mac_group_t *grp, mac_client_impl_t **group_only_mcip, 26500dc2366fSVenugopal Iyer mac_group_t *defgrp, boolean_t rx_group) 2651da14cebeSEric Cheng { 2652da14cebeSEric Cheng mac_impl_t *mip = (mac_impl_t *)grp->mrg_mh; 2653da14cebeSEric Cheng 2654da14cebeSEric Cheng *group_only_mcip = NULL; 2655da14cebeSEric Cheng 2656da14cebeSEric Cheng /* Non-default group */ 2657da14cebeSEric Cheng 26580dc2366fSVenugopal Iyer if (grp != defgrp) { 26590dc2366fSVenugopal Iyer if (MAC_GROUP_NO_CLIENT(grp)) 2660da14cebeSEric Cheng return (MAC_GROUP_STATE_REGISTERED); 2661da14cebeSEric Cheng 26620dc2366fSVenugopal Iyer *group_only_mcip = MAC_GROUP_ONLY_CLIENT(grp); 2663da14cebeSEric Cheng if (*group_only_mcip != NULL) 2664da14cebeSEric Cheng return (MAC_GROUP_STATE_RESERVED); 2665da14cebeSEric Cheng 2666da14cebeSEric Cheng return (MAC_GROUP_STATE_SHARED); 2667da14cebeSEric Cheng } 2668da14cebeSEric Cheng 2669da14cebeSEric Cheng /* Default group */ 2670da14cebeSEric Cheng 26710dc2366fSVenugopal Iyer if (MAC_GROUP_NO_CLIENT(grp)) { 26720dc2366fSVenugopal Iyer if (rx_group) 26730dc2366fSVenugopal Iyer return (MAC_GROUP_STATE_SHARED); 26740dc2366fSVenugopal Iyer else 26750dc2366fSVenugopal Iyer return (MAC_GROUP_STATE_REGISTERED); 26760dc2366fSVenugopal Iyer } 26770dc2366fSVenugopal Iyer *group_only_mcip = MAC_GROUP_ONLY_CLIENT(grp); 26780dc2366fSVenugopal Iyer if (*group_only_mcip == NULL) 2679da14cebeSEric Cheng return (MAC_GROUP_STATE_SHARED); 2680da14cebeSEric Cheng 26810dc2366fSVenugopal Iyer if (rx_group && mip->mi_nactiveclients != 1) 26820dc2366fSVenugopal Iyer return (MAC_GROUP_STATE_SHARED); 26830dc2366fSVenugopal Iyer 2684da14cebeSEric Cheng ASSERT(*group_only_mcip != NULL); 2685da14cebeSEric Cheng return (MAC_GROUP_STATE_RESERVED); 2686da14cebeSEric Cheng } 2687da14cebeSEric Cheng 2688da14cebeSEric Cheng /* 2689da14cebeSEric Cheng * OVERVIEW NOTES FOR DATAPATH 2690da14cebeSEric Cheng * =========================== 2691da14cebeSEric Cheng * 2692da14cebeSEric Cheng * Create an SRS and setup the corresponding flow function and args. 2693da14cebeSEric Cheng * Add a classification rule for the flow specified by 'flent' and program 2694da14cebeSEric Cheng * the hardware classifier when applicable. 2695da14cebeSEric Cheng * 2696da14cebeSEric Cheng * Rx ring assignment, SRS, polling and B/W enforcement 2697da14cebeSEric Cheng * ---------------------------------------------------- 2698da14cebeSEric Cheng * 2699da14cebeSEric Cheng * We try to use H/W classification on NIC and assign traffic to a 2700da14cebeSEric Cheng * MAC address to a particular Rx ring. There is a 1-1 mapping 2701da14cebeSEric Cheng * between a SRS and a Rx ring. The SRS (short for soft ring set) 2702da14cebeSEric Cheng * dynamically switches the underlying Rx ring between interrupt 2703da14cebeSEric Cheng * and polling mode and enforces any specified B/W control. 2704da14cebeSEric Cheng * 2705da14cebeSEric Cheng * There is always a SRS created and tied to each H/W and S/W rule. 2706da14cebeSEric Cheng * Whenever we create a H/W rule, we always add the the same rule to 2707da14cebeSEric Cheng * S/W classifier and tie a SRS to it. 2708da14cebeSEric Cheng * 2709da14cebeSEric Cheng * In case a B/W control is specified, its broken into bytes 2710da14cebeSEric Cheng * per ticks and as soon as the quota for a tick is exhausted, 2711da14cebeSEric Cheng * the underlying Rx ring is forced into poll mode for remianing 2712da14cebeSEric Cheng * tick. The SRS poll thread only polls for bytes that are 2713da14cebeSEric Cheng * allowed to come in the SRS. We typically let 4x the configured 2714da14cebeSEric Cheng * B/W worth of packets to come in the SRS (to prevent unnecessary 2715da14cebeSEric Cheng * drops due to bursts) but only process the specified amount. 2716da14cebeSEric Cheng * 2717da14cebeSEric Cheng * A Link (primary NIC, VNIC, VLAN or aggr) can have 1 or more 2718da14cebeSEric Cheng * Rx rings (and corresponding SRSs) assigned to it. The SRS 2719da14cebeSEric Cheng * in turn can have softrings to do protocol level fanout or 2720da14cebeSEric Cheng * softrings to do S/W based fanout or both. In case the NIC 2721da14cebeSEric Cheng * has no Rx rings, we do S/W classification to respective SRS. 2722da14cebeSEric Cheng * The S/W classification rule is always setup and ready. This 2723da14cebeSEric Cheng * allows the MAC layer to reassign Rx rings whenever needed 2724da14cebeSEric Cheng * but packets still continue to flow via the default path and 2725da14cebeSEric Cheng * getting S/W classified to correct SRS. 2726da14cebeSEric Cheng * 2727da14cebeSEric Cheng * In other cases where a NIC or VNIC is plumbed, our goal is use 2728da14cebeSEric Cheng * H/W classifier and get two Rx ring assigned for the Link. One 2729da14cebeSEric Cheng * for TCP and one for UDP|SCTP. The respective SRS still do the 2730da14cebeSEric Cheng * polling on the Rx ring. For Link that is plumbed for IP, there 2731da14cebeSEric Cheng * is a TCP squeue which also does polling and can control the 2732da14cebeSEric Cheng * the Rx ring directly (where SRS is just pass through). For 2733da14cebeSEric Cheng * the following cases, the SRS does the polling underneath. 2734da14cebeSEric Cheng * 1) non IP based Links (Links which are not plumbed via ifconfig) 2735da14cebeSEric Cheng * and paths which have no IP squeues (UDP & SCTP) 2736da14cebeSEric Cheng * 2) If B/W control is specified on the Link 2737da14cebeSEric Cheng * 3) If S/W fanout is secified 2738da14cebeSEric Cheng * 2739da14cebeSEric Cheng * Note1: As of current implementation, we try to assign only 1 Rx 2740da14cebeSEric Cheng * ring per Link and more than 1 Rx ring for primary Link for 2741da14cebeSEric Cheng * H/W based fanout. We always create following softrings per SRS: 2742da14cebeSEric Cheng * 1) TCP softring which is polled by TCP squeue where possible 2743da14cebeSEric Cheng * (and also bypasses DLS) 2744da14cebeSEric Cheng * 2) UDP/SCTP based which bypasses DLS 2745da14cebeSEric Cheng * 3) OTH softring which goes via DLS (currently deal with IPv6 2746da14cebeSEric Cheng * and non TCP/UDP/SCTP for IPv4 packets). 2747da14cebeSEric Cheng * 2748da14cebeSEric Cheng * It is necessary to create 3 softrings since SRS has to poll 2749da14cebeSEric Cheng * the single Rx ring underneath and enforce any link level B/W 2750da14cebeSEric Cheng * control (we can't switch the Rx ring in poll mode just based 2751da14cebeSEric Cheng * on TCP squeue if the same Rx ring is sharing UDP and other 2752da14cebeSEric Cheng * traffic as well). Once polling is done and any Link level B/W 2753da14cebeSEric Cheng * control is specified, the packets are assigned to respective 2754da14cebeSEric Cheng * softring based on protocol. Since TCP has IP based squeue 2755da14cebeSEric Cheng * which benefits by polling, we separate TCP packets into 2756da14cebeSEric Cheng * its own softring which can be polled by IP squeue. We need 2757da14cebeSEric Cheng * to separate out UDP/SCTP to UDP softring since it can bypass 2758da14cebeSEric Cheng * the DLS layer which has heavy performance advanatges and we 2759da14cebeSEric Cheng * need a softring (OTH) for rest. 2760da14cebeSEric Cheng * 2761da14cebeSEric Cheng * ToDo: The 3 softrings for protocol are needed only till we can 2762da14cebeSEric Cheng * get rid of DLS from datapath, make IPv4 and IPv6 paths 2763da14cebeSEric Cheng * symmetric (deal with mac_header_info for v6 and polling for 2764da14cebeSEric Cheng * IPv4 TCP - ip_accept_tcp is IPv4 specific although squeues 2765da14cebeSEric Cheng * are generic), and bring SAP based classification to MAC layer 2766da14cebeSEric Cheng * 2767da14cebeSEric Cheng * H/W and S/W based fanout and multiple Rx rings per Link 2768da14cebeSEric Cheng * ------------------------------------------------------- 2769da14cebeSEric Cheng * 2770da14cebeSEric Cheng * In case, fanout is requested (or determined automatically based 2771da14cebeSEric Cheng * on Link speed and processor speed), we try to assign multiple 2772da14cebeSEric Cheng * Rx rings per Link with their respective SRS. In this case 2773da14cebeSEric Cheng * the NIC should be capable of fanning out incoming packets between 2774da14cebeSEric Cheng * the assigned Rx rings (H/W based fanout). All the SRS 2775da14cebeSEric Cheng * individually switch their Rx ring between interrupt and polling 2776da14cebeSEric Cheng * mode but share a common B/W control counter in case of Link 2777da14cebeSEric Cheng * level B/W is specified. 2778da14cebeSEric Cheng * 2779da14cebeSEric Cheng * If S/W based fanout is specified in lieu of H/W based fanout, 2780da14cebeSEric Cheng * the Link SRS creates the specified number of softrings for 2781da14cebeSEric Cheng * each protocol (TCP, UDP, OTH). Incoming packets are fanned 2782da14cebeSEric Cheng * out to the correct softring based on their protocol and 2783da14cebeSEric Cheng * protocol specific hash function. 2784da14cebeSEric Cheng * 2785da14cebeSEric Cheng * Primary and non primary MAC clients 2786da14cebeSEric Cheng * ----------------------------------- 2787da14cebeSEric Cheng * 2788da14cebeSEric Cheng * The NICs, VNICs, Vlans, and Aggrs are typically termed as Links 2789da14cebeSEric Cheng * and are a Layer 2 construct. 2790da14cebeSEric Cheng * 2791da14cebeSEric Cheng * Primary NIC: 2792da14cebeSEric Cheng * The Link that owns the primary MAC address and typically 2793da14cebeSEric Cheng * is used as the data NIC in non virtualized cases. As such 2794da14cebeSEric Cheng * H/W resources are preferntially given to primary NIC. As 2795da14cebeSEric Cheng * far as code is concerned, there is no difference in the 2796da14cebeSEric Cheng * primary NIC vs VNICs. They are all treated as Links. 2797da14cebeSEric Cheng * At the very first call to mac_unicast_add() we program the S/W 2798da14cebeSEric Cheng * classifier for the primary MAC address, get a soft ring set 2799da14cebeSEric Cheng * (and soft rings based on 'ip_soft_ring_cnt') 2800da14cebeSEric Cheng * and a Rx ring assigned for polling to get enabled. 2801da14cebeSEric Cheng * When IP get plumbed and negotiates polling, we can 2802da14cebeSEric Cheng * let squeue do the polling on TCP softring. 2803da14cebeSEric Cheng * 2804da14cebeSEric Cheng * VNICs: 2805da14cebeSEric Cheng * Same as any other Link. As long as the H/W resource assignments 2806da14cebeSEric Cheng * are equal, the data path and setup for all Links is same. 2807da14cebeSEric Cheng * 2808da14cebeSEric Cheng * Flows: 2809da14cebeSEric Cheng * Can be configured on Links. They have their own SRS and the 2810da14cebeSEric Cheng * S/W classifier is programmed appropriately based on the flow. 2811da14cebeSEric Cheng * The flows typically deal with layer 3 and above and 2812da14cebeSEric Cheng * creates a soft ring set specific to the flow. The receive 2813da14cebeSEric Cheng * side function is switched from mac_rx_srs_process to 2814da14cebeSEric Cheng * mac_rx_srs_subflow_process which first tries to assign the 2815da14cebeSEric Cheng * packet to appropriate flow SRS and failing which assigns it 2816da14cebeSEric Cheng * to link SRS. This allows us to avoid the layered approach 2817da14cebeSEric Cheng * which gets complex. 2818da14cebeSEric Cheng * 2819da14cebeSEric Cheng * By the time mac_datapath_setup() completes, we already have the 2820da14cebeSEric Cheng * soft rings set, Rx rings, soft rings, etc figured out and both H/W 2821da14cebeSEric Cheng * and S/W classifiers programmed. IP is not plumbed yet (and might 2822da14cebeSEric Cheng * never be for Virtual Machines guest OS path). When IP is plumbed 2823da14cebeSEric Cheng * (for both NIC and VNIC), we do a capability negotiation for polling 2824da14cebeSEric Cheng * and upcall functions etc. 2825da14cebeSEric Cheng * 2826da14cebeSEric Cheng * Rx ring Assignement NOTES 2827da14cebeSEric Cheng * ------------------------- 2828da14cebeSEric Cheng * 2829da14cebeSEric Cheng * For NICs which have only 1 Rx ring (we treat NICs with no Rx rings 2830da14cebeSEric Cheng * as NIC with a single default ring), we assign the only ring to 28310dc2366fSVenugopal Iyer * primary Link. The primary Link SRS can do polling on it as long as 28320dc2366fSVenugopal Iyer * it is the only link in use and we compare the MAC address for unicast 28330dc2366fSVenugopal Iyer * packets before accepting an incoming packet (there is no need for S/W 28340dc2366fSVenugopal Iyer * classification in this case). We disable polling on the only ring the 28350dc2366fSVenugopal Iyer * moment 2nd link gets created (the polling remains enabled even though 28360dc2366fSVenugopal Iyer * there are broadcast and * multicast flows created). 2837da14cebeSEric Cheng * 2838da14cebeSEric Cheng * If the NIC has more than 1 Rx ring, we assign the default ring (the 2839da14cebeSEric Cheng * 1st ring) to deal with broadcast, multicast and traffic for other 2840da14cebeSEric Cheng * NICs which needs S/W classification. We assign the primary mac 2841da14cebeSEric Cheng * addresses to another ring by specifiying a classification rule for 2842da14cebeSEric Cheng * primary unicast MAC address to the selected ring. The primary Link 2843da14cebeSEric Cheng * (and its SRS) can continue to poll the assigned Rx ring at all times 2844da14cebeSEric Cheng * independantly. 2845da14cebeSEric Cheng * 2846da14cebeSEric Cheng * Note: In future, if no fanout is specified, we try to assign 2 Rx 2847da14cebeSEric Cheng * rings for the primary Link with the primary MAC address + TCP going 2848da14cebeSEric Cheng * to one ring and primary MAC address + UDP|SCTP going to other ring. 2849da14cebeSEric Cheng * Any remaining traffic for primary MAC address can go to the default 2850da14cebeSEric Cheng * Rx ring and get S/W classified. This way the respective SRSs don't 2851da14cebeSEric Cheng * need to do proto fanout and don't need to have softrings at all and 2852da14cebeSEric Cheng * can poll their respective Rx rings. 2853da14cebeSEric Cheng * 2854da14cebeSEric Cheng * As an optimization, when a new NIC or VNIC is created, we can get 2855da14cebeSEric Cheng * only one Rx ring and make it a TCP specific Rx ring and use the 2856da14cebeSEric Cheng * H/W default Rx ring for the rest (this Rx ring is never polled). 28570dc2366fSVenugopal Iyer * 28580dc2366fSVenugopal Iyer * For clients that don't have MAC address, but want to receive and 28590dc2366fSVenugopal Iyer * transmit packets (e.g, bpf, gvrp etc.), we need to setup the datapath. 28600dc2366fSVenugopal Iyer * For such clients (identified by the MCIS_NO_UNICAST_ADDR flag) we 28610dc2366fSVenugopal Iyer * always give the default group and use software classification (i.e. 28620dc2366fSVenugopal Iyer * even if this is the only client in the default group, we will 28630dc2366fSVenugopal Iyer * leave group as shared). 2864da14cebeSEric Cheng */ 286584de666eSRyan Zezeski 2866da14cebeSEric Cheng int 2867da14cebeSEric Cheng mac_datapath_setup(mac_client_impl_t *mcip, flow_entry_t *flent, 2868da14cebeSEric Cheng uint32_t link_type) 2869da14cebeSEric Cheng { 2870da14cebeSEric Cheng mac_impl_t *mip = mcip->mci_mip; 28710dc2366fSVenugopal Iyer mac_group_t *rgroup = NULL; 28720dc2366fSVenugopal Iyer mac_group_t *tgroup = NULL; 28730dc2366fSVenugopal Iyer mac_group_t *default_rgroup; 28740dc2366fSVenugopal Iyer mac_group_t *default_tgroup; 2875da14cebeSEric Cheng int err; 287684de666eSRyan Zezeski uint16_t vid; 2877da14cebeSEric Cheng uint8_t *mac_addr; 2878da14cebeSEric Cheng mac_group_state_t next_state; 2879da14cebeSEric Cheng mac_client_impl_t *group_only_mcip; 28800dc2366fSVenugopal Iyer mac_resource_props_t *mrp = MCIP_RESOURCE_PROPS(mcip); 28810dc2366fSVenugopal Iyer mac_resource_props_t *emrp = MCIP_EFFECTIVE_PROPS(mcip); 28820dc2366fSVenugopal Iyer boolean_t rxhw; 28830dc2366fSVenugopal Iyer boolean_t txhw; 28840dc2366fSVenugopal Iyer boolean_t use_default = B_FALSE; 28850dc2366fSVenugopal Iyer cpupart_t *cpupart; 28860dc2366fSVenugopal Iyer boolean_t no_unicast; 28870dc2366fSVenugopal Iyer boolean_t isprimary = flent->fe_type & FLOW_PRIMARY_MAC; 28880dc2366fSVenugopal Iyer mac_client_impl_t *reloc_pmcip = NULL; 288984de666eSRyan Zezeski boolean_t use_hw; 2890da14cebeSEric Cheng 2891da14cebeSEric Cheng ASSERT(MAC_PERIM_HELD((mac_handle_t)mip)); 2892da14cebeSEric Cheng 2893da14cebeSEric Cheng switch (link_type) { 2894da14cebeSEric Cheng case SRST_FLOW: 28950dc2366fSVenugopal Iyer mac_srs_group_setup(mcip, flent, link_type); 2896da14cebeSEric Cheng return (0); 2897da14cebeSEric Cheng 2898da14cebeSEric Cheng case SRST_LINK: 28990dc2366fSVenugopal Iyer no_unicast = mcip->mci_state_flags & MCIS_NO_UNICAST_ADDR; 2900da14cebeSEric Cheng mac_addr = flent->fe_flow_desc.fd_dst_mac; 2901da14cebeSEric Cheng 29020dc2366fSVenugopal Iyer /* Default RX group */ 29030dc2366fSVenugopal Iyer default_rgroup = MAC_DEFAULT_RX_GROUP(mip); 2904da14cebeSEric Cheng 29050dc2366fSVenugopal Iyer /* Default TX group */ 29060dc2366fSVenugopal Iyer default_tgroup = MAC_DEFAULT_TX_GROUP(mip); 29070dc2366fSVenugopal Iyer 29080dc2366fSVenugopal Iyer if (no_unicast) { 29090dc2366fSVenugopal Iyer rgroup = default_rgroup; 29100dc2366fSVenugopal Iyer tgroup = default_tgroup; 29110dc2366fSVenugopal Iyer goto grp_found; 29120dc2366fSVenugopal Iyer } 29130dc2366fSVenugopal Iyer rxhw = (mrp->mrp_mask & MRP_RX_RINGS) && 29140dc2366fSVenugopal Iyer (mrp->mrp_nrxrings > 0 || 29150dc2366fSVenugopal Iyer (mrp->mrp_mask & MRP_RXRINGS_UNSPEC)); 29160dc2366fSVenugopal Iyer txhw = (mrp->mrp_mask & MRP_TX_RINGS) && 29170dc2366fSVenugopal Iyer (mrp->mrp_ntxrings > 0 || 29180dc2366fSVenugopal Iyer (mrp->mrp_mask & MRP_TXRINGS_UNSPEC)); 29190dc2366fSVenugopal Iyer 29200dc2366fSVenugopal Iyer /* 292184de666eSRyan Zezeski * All the rings initially belong to the default group 292284de666eSRyan Zezeski * under dynamic grouping. The primary client uses the 292384de666eSRyan Zezeski * default group when it is the only client. The 292484de666eSRyan Zezeski * default group is also used as the destination for 292584de666eSRyan Zezeski * all multicast and broadcast traffic of all clients. 292684de666eSRyan Zezeski * Therefore, the primary client loses its ability to 292784de666eSRyan Zezeski * poll the softrings on addition of a second client. 292884de666eSRyan Zezeski * To avoid a performance penalty, MAC will move the 292984de666eSRyan Zezeski * primary client to a dedicated group when it can. 293084de666eSRyan Zezeski * 293184de666eSRyan Zezeski * When using static grouping, the primary client 293284de666eSRyan Zezeski * begins life on a non-default group. There is 293384de666eSRyan Zezeski * no moving needed upon addition of a second client. 29340dc2366fSVenugopal Iyer */ 29350dc2366fSVenugopal Iyer if (!isprimary && mip->mi_nactiveclients == 2 && 29360dc2366fSVenugopal Iyer (group_only_mcip = mac_primary_client_handle(mip)) != 29370dc2366fSVenugopal Iyer NULL && mip->mi_rx_group_type == MAC_GROUP_TYPE_DYNAMIC) { 29380dc2366fSVenugopal Iyer reloc_pmcip = mac_check_primary_relocation( 29390dc2366fSVenugopal Iyer group_only_mcip, rxhw); 29400dc2366fSVenugopal Iyer } 294184de666eSRyan Zezeski 2942da14cebeSEric Cheng /* 2943da14cebeSEric Cheng * Check to see if we can get an exclusive group for 2944da14cebeSEric Cheng * this mac address or if there already exists a 2945da14cebeSEric Cheng * group that has this mac address (case of VLANs). 2946da14cebeSEric Cheng * If no groups are available, use the default group. 2947da14cebeSEric Cheng */ 29480dc2366fSVenugopal Iyer rgroup = mac_reserve_rx_group(mcip, mac_addr, B_FALSE); 29490dc2366fSVenugopal Iyer if (rgroup == NULL && rxhw) { 29500dc2366fSVenugopal Iyer err = ENOSPC; 29510dc2366fSVenugopal Iyer goto setup_failed; 29520dc2366fSVenugopal Iyer } else if (rgroup == NULL) { 29530dc2366fSVenugopal Iyer rgroup = default_rgroup; 2954da14cebeSEric Cheng } 295584de666eSRyan Zezeski 295684de666eSRyan Zezeski /* 295784de666eSRyan Zezeski * If we are adding a second client to a 295884de666eSRyan Zezeski * non-default group then we need to move the 295984de666eSRyan Zezeski * existing client to the default group and 296084de666eSRyan Zezeski * add the new client to the default group as 296184de666eSRyan Zezeski * well. 296284de666eSRyan Zezeski */ 296384de666eSRyan Zezeski if (rgroup != default_rgroup && 296484de666eSRyan Zezeski rgroup->mrg_state == MAC_GROUP_STATE_RESERVED) { 296584de666eSRyan Zezeski group_only_mcip = MAC_GROUP_ONLY_CLIENT(rgroup); 296684de666eSRyan Zezeski err = mac_rx_switch_group(group_only_mcip, rgroup, 296784de666eSRyan Zezeski default_rgroup); 296884de666eSRyan Zezeski 296984de666eSRyan Zezeski if (err != 0) 297084de666eSRyan Zezeski goto setup_failed; 297184de666eSRyan Zezeski 297284de666eSRyan Zezeski rgroup = default_rgroup; 297384de666eSRyan Zezeski } 297484de666eSRyan Zezeski 29750dc2366fSVenugopal Iyer /* 29760dc2366fSVenugopal Iyer * Check to see if we can get an exclusive group for 29770dc2366fSVenugopal Iyer * this mac client. If no groups are available, use 29780dc2366fSVenugopal Iyer * the default group. 29790dc2366fSVenugopal Iyer */ 29800dc2366fSVenugopal Iyer tgroup = mac_reserve_tx_group(mcip, B_FALSE); 29810dc2366fSVenugopal Iyer if (tgroup == NULL && txhw) { 29820dc2366fSVenugopal Iyer if (rgroup != NULL && rgroup != default_rgroup) 29830dc2366fSVenugopal Iyer mac_release_rx_group(mcip, rgroup); 29840dc2366fSVenugopal Iyer err = ENOSPC; 29850dc2366fSVenugopal Iyer goto setup_failed; 29860dc2366fSVenugopal Iyer } else if (tgroup == NULL) { 29870dc2366fSVenugopal Iyer tgroup = default_tgroup; 2988da14cebeSEric Cheng } 2989da14cebeSEric Cheng 2990da14cebeSEric Cheng /* 2991da14cebeSEric Cheng * Some NICs don't support any Rx rings, so there may not 2992da14cebeSEric Cheng * even be a default group. 2993da14cebeSEric Cheng */ 29940dc2366fSVenugopal Iyer grp_found: 29950dc2366fSVenugopal Iyer if (rgroup != NULL) { 29960dc2366fSVenugopal Iyer if (rgroup != default_rgroup && 29970dc2366fSVenugopal Iyer MAC_GROUP_NO_CLIENT(rgroup) && 299836f99a58SToomas Soome (rxhw || mcip->mci_share != 0)) { 29990dc2366fSVenugopal Iyer MAC_RX_GRP_RESERVED(mip); 30000dc2366fSVenugopal Iyer if (mip->mi_rx_group_type == 30010dc2366fSVenugopal Iyer MAC_GROUP_TYPE_DYNAMIC) { 30020dc2366fSVenugopal Iyer MAC_RX_RING_RESERVED(mip, 30030dc2366fSVenugopal Iyer rgroup->mrg_cur_count); 30040dc2366fSVenugopal Iyer } 30050dc2366fSVenugopal Iyer } 300684de666eSRyan Zezeski 30070dc2366fSVenugopal Iyer flent->fe_rx_ring_group = rgroup; 3008da14cebeSEric Cheng /* 300984de666eSRyan Zezeski * Add the client to the group and update the 301084de666eSRyan Zezeski * group's state. If rgroup != default_group 301184de666eSRyan Zezeski * then the rgroup should only ever have one 301284de666eSRyan Zezeski * client and be in the RESERVED state. But no 301384de666eSRyan Zezeski * matter what, the default_rgroup will enter 301484de666eSRyan Zezeski * the SHARED state since it has to receive 301584de666eSRyan Zezeski * all broadcast and multicast traffic. This 301684de666eSRyan Zezeski * case is handled later in the function. 3017da14cebeSEric Cheng */ 30180dc2366fSVenugopal Iyer mac_group_add_client(rgroup, mcip); 30190dc2366fSVenugopal Iyer next_state = mac_group_next_state(rgroup, 30200dc2366fSVenugopal Iyer &group_only_mcip, default_rgroup, B_TRUE); 30210dc2366fSVenugopal Iyer mac_set_group_state(rgroup, next_state); 3022da14cebeSEric Cheng } 3023da14cebeSEric Cheng 30240dc2366fSVenugopal Iyer if (tgroup != NULL) { 30250dc2366fSVenugopal Iyer if (tgroup != default_tgroup && 30260dc2366fSVenugopal Iyer MAC_GROUP_NO_CLIENT(tgroup) && 302736f99a58SToomas Soome (txhw || mcip->mci_share != 0)) { 30280dc2366fSVenugopal Iyer MAC_TX_GRP_RESERVED(mip); 30290dc2366fSVenugopal Iyer if (mip->mi_tx_group_type == 30300dc2366fSVenugopal Iyer MAC_GROUP_TYPE_DYNAMIC) { 30310dc2366fSVenugopal Iyer MAC_TX_RING_RESERVED(mip, 30320dc2366fSVenugopal Iyer tgroup->mrg_cur_count); 30330dc2366fSVenugopal Iyer } 30340dc2366fSVenugopal Iyer } 30350dc2366fSVenugopal Iyer flent->fe_tx_ring_group = tgroup; 30360dc2366fSVenugopal Iyer mac_group_add_client(tgroup, mcip); 30370dc2366fSVenugopal Iyer next_state = mac_group_next_state(tgroup, 30380dc2366fSVenugopal Iyer &group_only_mcip, default_tgroup, B_FALSE); 30390dc2366fSVenugopal Iyer tgroup->mrg_state = next_state; 30400dc2366fSVenugopal Iyer } 3041da14cebeSEric Cheng 30420dc2366fSVenugopal Iyer /* We are setting up minimal datapath only */ 304384de666eSRyan Zezeski if (no_unicast) { 304484de666eSRyan Zezeski mac_srs_group_setup(mcip, flent, link_type); 30450dc2366fSVenugopal Iyer break; 304684de666eSRyan Zezeski } 304784de666eSRyan Zezeski 304884de666eSRyan Zezeski /* Program software classification. */ 3049da14cebeSEric Cheng if ((err = mac_flow_add(mip->mi_flow_tab, flent)) != 0) 3050da14cebeSEric Cheng goto setup_failed; 3051da14cebeSEric Cheng 305284de666eSRyan Zezeski /* Program hardware classification. */ 305384de666eSRyan Zezeski vid = i_mac_flow_vid(flent); 305484de666eSRyan Zezeski use_hw = (mcip->mci_state_flags & MCIS_UNICAST_HW) != 0; 305584de666eSRyan Zezeski err = mac_add_macaddr_vlan(mip, rgroup, mac_addr, vid, use_hw); 305684de666eSRyan Zezeski 305784de666eSRyan Zezeski if (err != 0) 3058da14cebeSEric Cheng goto setup_failed; 305984de666eSRyan Zezeski 3060da14cebeSEric Cheng mcip->mci_unicast = mac_find_macaddr(mip, mac_addr); 306184de666eSRyan Zezeski VERIFY3P(mcip->mci_unicast, !=, NULL); 306284de666eSRyan Zezeski 306384de666eSRyan Zezeski /* 306484de666eSRyan Zezeski * Setup the Rx and Tx SRSes. If the client has a 306584de666eSRyan Zezeski * reserved group, then mac_srs_group_setup() creates 306684de666eSRyan Zezeski * the required SRSes for the HW rings. If we have a 306784de666eSRyan Zezeski * shared group, mac_srs_group_setup() dismantles the 306884de666eSRyan Zezeski * HW SRSes of the previously exclusive group. 306984de666eSRyan Zezeski */ 307084de666eSRyan Zezeski mac_srs_group_setup(mcip, flent, link_type); 307184de666eSRyan Zezeski 3072797f979dSCody Peter Mello /* (Re)init the v6 token & local addr used by link protection */ 3073797f979dSCody Peter Mello mac_protect_update_mac_token(mcip); 3074da14cebeSEric Cheng break; 3075da14cebeSEric Cheng 3076da14cebeSEric Cheng default: 3077da14cebeSEric Cheng ASSERT(B_FALSE); 3078da14cebeSEric Cheng break; 3079da14cebeSEric Cheng } 3080da14cebeSEric Cheng 3081da14cebeSEric Cheng /* 3082da14cebeSEric Cheng * All broadcast and multicast traffic is received only on the default 3083da14cebeSEric Cheng * group. If we have setup the datapath for a non-default group above 3084da14cebeSEric Cheng * then move the default group to shared state to allow distribution of 3085da14cebeSEric Cheng * incoming broadcast traffic to the other groups and dismantle the 3086da14cebeSEric Cheng * SRSes over the default group. 3087da14cebeSEric Cheng */ 30880dc2366fSVenugopal Iyer if (rgroup != NULL) { 30890dc2366fSVenugopal Iyer if (rgroup != default_rgroup) { 30900dc2366fSVenugopal Iyer if (default_rgroup->mrg_state == 3091da14cebeSEric Cheng MAC_GROUP_STATE_RESERVED) { 30920dc2366fSVenugopal Iyer group_only_mcip = MAC_GROUP_ONLY_CLIENT( 30930dc2366fSVenugopal Iyer default_rgroup); 3094da14cebeSEric Cheng ASSERT(group_only_mcip != NULL && 3095da14cebeSEric Cheng mip->mi_nactiveclients > 1); 3096da14cebeSEric Cheng 30970dc2366fSVenugopal Iyer mac_set_group_state(default_rgroup, 3098da14cebeSEric Cheng MAC_GROUP_STATE_SHARED); 30990dc2366fSVenugopal Iyer mac_rx_srs_group_setup(group_only_mcip, 31000dc2366fSVenugopal Iyer group_only_mcip->mci_flent, SRST_LINK); 31010dc2366fSVenugopal Iyer pool_lock(); 31020dc2366fSVenugopal Iyer cpupart = mac_pset_find(mrp, &use_default); 31030dc2366fSVenugopal Iyer mac_fanout_setup(group_only_mcip, 3104da14cebeSEric Cheng group_only_mcip->mci_flent, 31050dc2366fSVenugopal Iyer MCIP_RESOURCE_PROPS(group_only_mcip), 31060dc2366fSVenugopal Iyer mac_rx_deliver, group_only_mcip, NULL, 31070dc2366fSVenugopal Iyer cpupart); 31080dc2366fSVenugopal Iyer mac_set_pool_effective(use_default, cpupart, 31090dc2366fSVenugopal Iyer mrp, emrp); 31100dc2366fSVenugopal Iyer pool_unlock(); 3111da14cebeSEric Cheng } 31120dc2366fSVenugopal Iyer ASSERT(default_rgroup->mrg_state == 3113da14cebeSEric Cheng MAC_GROUP_STATE_SHARED); 3114da14cebeSEric Cheng } 311584de666eSRyan Zezeski 3116da14cebeSEric Cheng /* 311784de666eSRyan Zezeski * A VLAN MAC client on a reserved group still 311884de666eSRyan Zezeski * requires SW classification if the MAC doesn't 311984de666eSRyan Zezeski * provide VLAN HW filtering. 312084de666eSRyan Zezeski * 312184de666eSRyan Zezeski * Clients with no unicast address also require SW 312284de666eSRyan Zezeski * classification. 3123da14cebeSEric Cheng */ 31240dc2366fSVenugopal Iyer if (rgroup->mrg_state == MAC_GROUP_STATE_RESERVED && 312584de666eSRyan Zezeski ((!MAC_GROUP_HW_VLAN(rgroup) && vid != VLAN_ID_NONE) || 312684de666eSRyan Zezeski no_unicast)) { 31270dc2366fSVenugopal Iyer mac_rx_switch_grp_to_sw(rgroup); 3128da14cebeSEric Cheng } 312984de666eSRyan Zezeski 3130da14cebeSEric Cheng } 313184de666eSRyan Zezeski 31320dc2366fSVenugopal Iyer mac_set_rings_effective(mcip); 3133da14cebeSEric Cheng return (0); 3134da14cebeSEric Cheng 3135da14cebeSEric Cheng setup_failed: 31360dc2366fSVenugopal Iyer /* Switch the primary back to default group */ 31370dc2366fSVenugopal Iyer if (reloc_pmcip != NULL) { 31380dc2366fSVenugopal Iyer (void) mac_rx_switch_group(reloc_pmcip, 31390dc2366fSVenugopal Iyer reloc_pmcip->mci_flent->fe_rx_ring_group, default_rgroup); 31400dc2366fSVenugopal Iyer } 3141da14cebeSEric Cheng mac_datapath_teardown(mcip, flent, link_type); 3142da14cebeSEric Cheng return (err); 3143da14cebeSEric Cheng } 3144da14cebeSEric Cheng 3145da14cebeSEric Cheng void 3146da14cebeSEric Cheng mac_datapath_teardown(mac_client_impl_t *mcip, flow_entry_t *flent, 3147da14cebeSEric Cheng uint32_t link_type) 3148da14cebeSEric Cheng { 3149da14cebeSEric Cheng mac_impl_t *mip = mcip->mci_mip; 3150da14cebeSEric Cheng mac_group_t *group = NULL; 3151da14cebeSEric Cheng mac_client_impl_t *grp_only_mcip; 3152da14cebeSEric Cheng flow_entry_t *group_only_flent; 3153da14cebeSEric Cheng mac_group_t *default_group; 3154da14cebeSEric Cheng boolean_t check_default_group = B_FALSE; 3155da14cebeSEric Cheng mac_group_state_t next_state; 31560dc2366fSVenugopal Iyer mac_resource_props_t *mrp = MCIP_RESOURCE_PROPS(mcip); 315784de666eSRyan Zezeski uint16_t vid; 3158da14cebeSEric Cheng 3159da14cebeSEric Cheng ASSERT(MAC_PERIM_HELD((mac_handle_t)mip)); 3160da14cebeSEric Cheng 3161da14cebeSEric Cheng switch (link_type) { 3162da14cebeSEric Cheng case SRST_FLOW: 31630dc2366fSVenugopal Iyer mac_rx_srs_group_teardown(flent, B_FALSE); 31640dc2366fSVenugopal Iyer mac_tx_srs_group_teardown(mcip, flent, SRST_FLOW); 3165da14cebeSEric Cheng return; 3166da14cebeSEric Cheng 3167da14cebeSEric Cheng case SRST_LINK: 3168da14cebeSEric Cheng /* Stop sending packets */ 3169da14cebeSEric Cheng mac_tx_client_block(mcip); 317084de666eSRyan Zezeski group = flent->fe_rx_ring_group; 317184de666eSRyan Zezeski vid = i_mac_flow_vid(flent); 3172da14cebeSEric Cheng 317384de666eSRyan Zezeski /* 317484de666eSRyan Zezeski * Stop the packet flow from the hardware by disabling 317584de666eSRyan Zezeski * any hardware filters assigned to this client. 317684de666eSRyan Zezeski */ 3177da14cebeSEric Cheng if (mcip->mci_unicast != NULL) { 3178da14cebeSEric Cheng int err; 317984de666eSRyan Zezeski 318084de666eSRyan Zezeski err = mac_remove_macaddr_vlan(mcip->mci_unicast, vid); 318184de666eSRyan Zezeski 3182da14cebeSEric Cheng if (err != 0) { 318384de666eSRyan Zezeski cmn_err(CE_WARN, "%s: failed to remove a MAC HW" 318484de666eSRyan Zezeski " filters because of error 0x%x", 3185da14cebeSEric Cheng mip->mi_name, err); 3186da14cebeSEric Cheng } 318784de666eSRyan Zezeski 3188da14cebeSEric Cheng mcip->mci_unicast = NULL; 3189da14cebeSEric Cheng } 3190da14cebeSEric Cheng 3191da14cebeSEric Cheng /* Stop the packets coming from the S/W classifier */ 3192da14cebeSEric Cheng mac_flow_remove(mip->mi_flow_tab, flent, B_FALSE); 3193da14cebeSEric Cheng mac_flow_wait(flent, FLOW_DRIVER_UPCALL); 3194da14cebeSEric Cheng 319545948e49SRyan Zezeski /* Quiesce and destroy all the SRSes. */ 31960dc2366fSVenugopal Iyer mac_rx_srs_group_teardown(flent, B_FALSE); 31970dc2366fSVenugopal Iyer mac_tx_srs_group_teardown(mcip, flent, SRST_LINK); 31980dc2366fSVenugopal Iyer 319945948e49SRyan Zezeski ASSERT3P(mcip->mci_flent, ==, flent); 320045948e49SRyan Zezeski ASSERT3P(flent->fe_next, ==, NULL); 3201da14cebeSEric Cheng 3202da14cebeSEric Cheng /* 3203da14cebeSEric Cheng * Release our hold on the group as well. We need 3204da14cebeSEric Cheng * to check if the shared group has only one client 3205da14cebeSEric Cheng * left who can use it exclusively. Also, if we 3206da14cebeSEric Cheng * were the last client, release the group. 3207da14cebeSEric Cheng */ 32080dc2366fSVenugopal Iyer default_group = MAC_DEFAULT_RX_GROUP(mip); 3209da14cebeSEric Cheng if (group != NULL) { 32100dc2366fSVenugopal Iyer mac_group_remove_client(group, mcip); 32110dc2366fSVenugopal Iyer next_state = mac_group_next_state(group, 32120dc2366fSVenugopal Iyer &grp_only_mcip, default_group, B_TRUE); 321384de666eSRyan Zezeski 3214da14cebeSEric Cheng if (next_state == MAC_GROUP_STATE_RESERVED) { 3215da14cebeSEric Cheng /* 3216da14cebeSEric Cheng * Only one client left on this RX group. 3217da14cebeSEric Cheng */ 321884de666eSRyan Zezeski VERIFY3P(grp_only_mcip, !=, NULL); 32190dc2366fSVenugopal Iyer mac_set_group_state(group, 3220da14cebeSEric Cheng MAC_GROUP_STATE_RESERVED); 3221da14cebeSEric Cheng group_only_flent = grp_only_mcip->mci_flent; 3222da14cebeSEric Cheng 3223da14cebeSEric Cheng /* 3224da14cebeSEric Cheng * The only remaining client has exclusive 3225da14cebeSEric Cheng * access on the group. Allow it to 3226da14cebeSEric Cheng * dynamically poll the H/W rings etc. 3227da14cebeSEric Cheng */ 32280dc2366fSVenugopal Iyer mac_rx_srs_group_setup(grp_only_mcip, 32290dc2366fSVenugopal Iyer group_only_flent, SRST_LINK); 32300dc2366fSVenugopal Iyer mac_fanout_setup(grp_only_mcip, 32310dc2366fSVenugopal Iyer group_only_flent, 32320dc2366fSVenugopal Iyer MCIP_RESOURCE_PROPS(grp_only_mcip), 32330dc2366fSVenugopal Iyer mac_rx_deliver, grp_only_mcip, NULL, NULL); 3234da14cebeSEric Cheng mac_rx_group_unmark(group, MR_INCIPIENT); 32350dc2366fSVenugopal Iyer mac_set_rings_effective(grp_only_mcip); 3236da14cebeSEric Cheng } else if (next_state == MAC_GROUP_STATE_REGISTERED) { 3237da14cebeSEric Cheng /* 3238da14cebeSEric Cheng * This is a non-default group being freed up. 3239da14cebeSEric Cheng * We need to reevaluate the default group 3240da14cebeSEric Cheng * to see if the primary client can get 3241da14cebeSEric Cheng * exclusive access to the default group. 3242da14cebeSEric Cheng */ 324384de666eSRyan Zezeski VERIFY3P(group, !=, MAC_DEFAULT_RX_GROUP(mip)); 32440dc2366fSVenugopal Iyer if (mrp->mrp_mask & MRP_RX_RINGS) { 32450dc2366fSVenugopal Iyer MAC_RX_GRP_RELEASED(mip); 32460dc2366fSVenugopal Iyer if (mip->mi_rx_group_type == 32470dc2366fSVenugopal Iyer MAC_GROUP_TYPE_DYNAMIC) { 32480dc2366fSVenugopal Iyer MAC_RX_RING_RELEASED(mip, 32490dc2366fSVenugopal Iyer group->mrg_cur_count); 32500dc2366fSVenugopal Iyer } 32510dc2366fSVenugopal Iyer } 3252da14cebeSEric Cheng mac_release_rx_group(mcip, group); 32530dc2366fSVenugopal Iyer mac_set_group_state(group, 3254da14cebeSEric Cheng MAC_GROUP_STATE_REGISTERED); 3255da14cebeSEric Cheng check_default_group = B_TRUE; 3256da14cebeSEric Cheng } else { 325784de666eSRyan Zezeski VERIFY3S(next_state, ==, 325884de666eSRyan Zezeski MAC_GROUP_STATE_SHARED); 32590dc2366fSVenugopal Iyer mac_set_group_state(group, 3260da14cebeSEric Cheng MAC_GROUP_STATE_SHARED); 3261da14cebeSEric Cheng mac_rx_group_unmark(group, MR_CONDEMNED); 3262da14cebeSEric Cheng } 3263da14cebeSEric Cheng flent->fe_rx_ring_group = NULL; 3264da14cebeSEric Cheng } 32650dc2366fSVenugopal Iyer /* 32660dc2366fSVenugopal Iyer * Remove the client from the TX group. Additionally, if 32670dc2366fSVenugopal Iyer * this a non-default group, then we also need to release 32680dc2366fSVenugopal Iyer * the group. 32690dc2366fSVenugopal Iyer */ 32700dc2366fSVenugopal Iyer group = flent->fe_tx_ring_group; 32710dc2366fSVenugopal Iyer default_group = MAC_DEFAULT_TX_GROUP(mip); 32720dc2366fSVenugopal Iyer if (group != NULL) { 32730dc2366fSVenugopal Iyer mac_group_remove_client(group, mcip); 32740dc2366fSVenugopal Iyer next_state = mac_group_next_state(group, 32750dc2366fSVenugopal Iyer &grp_only_mcip, default_group, B_FALSE); 32760dc2366fSVenugopal Iyer if (next_state == MAC_GROUP_STATE_REGISTERED) { 32770dc2366fSVenugopal Iyer if (group != default_group) { 32780dc2366fSVenugopal Iyer if (mrp->mrp_mask & MRP_TX_RINGS) { 32790dc2366fSVenugopal Iyer MAC_TX_GRP_RELEASED(mip); 32800dc2366fSVenugopal Iyer if (mip->mi_tx_group_type == 32810dc2366fSVenugopal Iyer MAC_GROUP_TYPE_DYNAMIC) { 32820dc2366fSVenugopal Iyer MAC_TX_RING_RELEASED( 32830dc2366fSVenugopal Iyer mip, group-> 32840dc2366fSVenugopal Iyer mrg_cur_count); 32850dc2366fSVenugopal Iyer } 32860dc2366fSVenugopal Iyer } 32870dc2366fSVenugopal Iyer mac_release_tx_group(mcip, group); 32880dc2366fSVenugopal Iyer /* 32890dc2366fSVenugopal Iyer * If the default group is reserved, 32900dc2366fSVenugopal Iyer * then we need to set the effective 32910dc2366fSVenugopal Iyer * rings as we would have given 32920dc2366fSVenugopal Iyer * back some rings when the group 32930dc2366fSVenugopal Iyer * was released 32940dc2366fSVenugopal Iyer */ 32950dc2366fSVenugopal Iyer if (mip->mi_tx_group_type == 32960dc2366fSVenugopal Iyer MAC_GROUP_TYPE_DYNAMIC && 32970dc2366fSVenugopal Iyer default_group->mrg_state == 32980dc2366fSVenugopal Iyer MAC_GROUP_STATE_RESERVED) { 32990dc2366fSVenugopal Iyer grp_only_mcip = 33000dc2366fSVenugopal Iyer MAC_GROUP_ONLY_CLIENT 33010dc2366fSVenugopal Iyer (default_group); 33020dc2366fSVenugopal Iyer mac_set_rings_effective( 33030dc2366fSVenugopal Iyer grp_only_mcip); 33040dc2366fSVenugopal Iyer } 33050dc2366fSVenugopal Iyer } else { 33060dc2366fSVenugopal Iyer mac_ring_t *ring; 33070dc2366fSVenugopal Iyer int cnt; 33080dc2366fSVenugopal Iyer int ringcnt; 33090dc2366fSVenugopal Iyer 33100dc2366fSVenugopal Iyer /* 33110dc2366fSVenugopal Iyer * Stop all the rings except the 33120dc2366fSVenugopal Iyer * default ring. 33130dc2366fSVenugopal Iyer */ 33140dc2366fSVenugopal Iyer ringcnt = group->mrg_cur_count; 33150dc2366fSVenugopal Iyer ring = group->mrg_rings; 33160dc2366fSVenugopal Iyer for (cnt = 0; cnt < ringcnt; cnt++) { 33170dc2366fSVenugopal Iyer if (ring->mr_state == 33180dc2366fSVenugopal Iyer MR_INUSE && ring != 33190dc2366fSVenugopal Iyer (mac_ring_t *) 33200dc2366fSVenugopal Iyer mip->mi_default_tx_ring) { 33210dc2366fSVenugopal Iyer mac_stop_ring(ring); 33220dc2366fSVenugopal Iyer ring->mr_flag = 0; 33230dc2366fSVenugopal Iyer } 33240dc2366fSVenugopal Iyer ring = ring->mr_next; 33250dc2366fSVenugopal Iyer } 33260dc2366fSVenugopal Iyer } 33270dc2366fSVenugopal Iyer } else if (next_state == MAC_GROUP_STATE_RESERVED) { 33280dc2366fSVenugopal Iyer mac_set_rings_effective(grp_only_mcip); 33290dc2366fSVenugopal Iyer } 33300dc2366fSVenugopal Iyer flent->fe_tx_ring_group = NULL; 33310dc2366fSVenugopal Iyer group->mrg_state = next_state; 33320dc2366fSVenugopal Iyer } 3333da14cebeSEric Cheng break; 3334da14cebeSEric Cheng default: 3335da14cebeSEric Cheng ASSERT(B_FALSE); 3336da14cebeSEric Cheng break; 3337da14cebeSEric Cheng } 3338da14cebeSEric Cheng 3339da14cebeSEric Cheng /* 3340da14cebeSEric Cheng * The mac client using the default group gets exclusive access to the 3341da14cebeSEric Cheng * default group if and only if it is the sole client on the entire 3342da14cebeSEric Cheng * mip. If so set the group state to reserved, and set up the SRSes 3343da14cebeSEric Cheng * over the default group. 3344da14cebeSEric Cheng */ 3345da14cebeSEric Cheng if (check_default_group) { 33460dc2366fSVenugopal Iyer default_group = MAC_DEFAULT_RX_GROUP(mip); 334784de666eSRyan Zezeski VERIFY3S(default_group->mrg_state, ==, MAC_GROUP_STATE_SHARED); 33480dc2366fSVenugopal Iyer next_state = mac_group_next_state(default_group, 33490dc2366fSVenugopal Iyer &grp_only_mcip, default_group, B_TRUE); 3350da14cebeSEric Cheng if (next_state == MAC_GROUP_STATE_RESERVED) { 335184de666eSRyan Zezeski VERIFY3P(grp_only_mcip, !=, NULL); 335284de666eSRyan Zezeski VERIFY3U(mip->mi_nactiveclients, ==, 1); 33530dc2366fSVenugopal Iyer mac_set_group_state(default_group, 3354da14cebeSEric Cheng MAC_GROUP_STATE_RESERVED); 33550dc2366fSVenugopal Iyer mac_rx_srs_group_setup(grp_only_mcip, 33560dc2366fSVenugopal Iyer grp_only_mcip->mci_flent, SRST_LINK); 33570dc2366fSVenugopal Iyer mac_fanout_setup(grp_only_mcip, 3358da14cebeSEric Cheng grp_only_mcip->mci_flent, 33590dc2366fSVenugopal Iyer MCIP_RESOURCE_PROPS(grp_only_mcip), mac_rx_deliver, 33600dc2366fSVenugopal Iyer grp_only_mcip, NULL, NULL); 3361ae6aa22aSVenugopal Iyer mac_rx_group_unmark(default_group, MR_INCIPIENT); 33620dc2366fSVenugopal Iyer mac_set_rings_effective(grp_only_mcip); 3363da14cebeSEric Cheng } 3364da14cebeSEric Cheng } 33650dc2366fSVenugopal Iyer 33660dc2366fSVenugopal Iyer /* 33670dc2366fSVenugopal Iyer * If the primary is the only one left and the MAC supports 33680dc2366fSVenugopal Iyer * dynamic grouping, we need to see if the primary needs to 33690dc2366fSVenugopal Iyer * be moved to the default group so that it can use all the 33700dc2366fSVenugopal Iyer * H/W rings. 33710dc2366fSVenugopal Iyer */ 33720dc2366fSVenugopal Iyer if (!(flent->fe_type & FLOW_PRIMARY_MAC) && 33730dc2366fSVenugopal Iyer mip->mi_nactiveclients == 1 && 33740dc2366fSVenugopal Iyer mip->mi_rx_group_type == MAC_GROUP_TYPE_DYNAMIC) { 33750dc2366fSVenugopal Iyer default_group = MAC_DEFAULT_RX_GROUP(mip); 33760dc2366fSVenugopal Iyer grp_only_mcip = mac_primary_client_handle(mip); 33770dc2366fSVenugopal Iyer if (grp_only_mcip == NULL) 33780dc2366fSVenugopal Iyer return; 33790dc2366fSVenugopal Iyer group_only_flent = grp_only_mcip->mci_flent; 33800dc2366fSVenugopal Iyer mrp = MCIP_RESOURCE_PROPS(grp_only_mcip); 33810dc2366fSVenugopal Iyer /* 33820dc2366fSVenugopal Iyer * If the primary has an explicit property set, leave it 33830dc2366fSVenugopal Iyer * alone. 33840dc2366fSVenugopal Iyer */ 33850dc2366fSVenugopal Iyer if (mrp->mrp_mask & MRP_RX_RINGS) 33860dc2366fSVenugopal Iyer return; 33870dc2366fSVenugopal Iyer /* 33880dc2366fSVenugopal Iyer * Switch the primary to the default group. 33890dc2366fSVenugopal Iyer */ 33900dc2366fSVenugopal Iyer (void) mac_rx_switch_group(grp_only_mcip, 33910dc2366fSVenugopal Iyer group_only_flent->fe_rx_ring_group, default_group); 33920dc2366fSVenugopal Iyer } 3393da14cebeSEric Cheng } 3394da14cebeSEric Cheng 3395da14cebeSEric Cheng /* DATAPATH TEAR DOWN ROUTINES (SRS and FANOUT teardown) */ 3396da14cebeSEric Cheng 3397da14cebeSEric Cheng static void 3398da14cebeSEric Cheng mac_srs_fanout_list_free(mac_soft_ring_set_t *mac_srs) 3399da14cebeSEric Cheng { 34000dc2366fSVenugopal Iyer if (mac_srs->srs_type & SRST_TX) { 34010dc2366fSVenugopal Iyer mac_srs_tx_t *tx; 34020dc2366fSVenugopal Iyer 34030dc2366fSVenugopal Iyer ASSERT(mac_srs->srs_tcp_soft_rings == NULL); 34040dc2366fSVenugopal Iyer ASSERT(mac_srs->srs_udp_soft_rings == NULL); 34050dc2366fSVenugopal Iyer ASSERT(mac_srs->srs_oth_soft_rings == NULL); 34060dc2366fSVenugopal Iyer ASSERT(mac_srs->srs_tx_soft_rings != NULL); 34070dc2366fSVenugopal Iyer kmem_free(mac_srs->srs_tx_soft_rings, 34080dc2366fSVenugopal Iyer sizeof (mac_soft_ring_t *) * MAX_RINGS_PER_GROUP); 34090dc2366fSVenugopal Iyer mac_srs->srs_tx_soft_rings = NULL; 34100dc2366fSVenugopal Iyer tx = &mac_srs->srs_tx; 34110dc2366fSVenugopal Iyer if (tx->st_soft_rings != NULL) { 34120dc2366fSVenugopal Iyer kmem_free(tx->st_soft_rings, 34130dc2366fSVenugopal Iyer sizeof (mac_soft_ring_t *) * MAX_RINGS_PER_GROUP); 34140dc2366fSVenugopal Iyer } 34150dc2366fSVenugopal Iyer } else { 34160dc2366fSVenugopal Iyer ASSERT(mac_srs->srs_tx_soft_rings == NULL); 3417da14cebeSEric Cheng ASSERT(mac_srs->srs_tcp_soft_rings != NULL); 3418da14cebeSEric Cheng kmem_free(mac_srs->srs_tcp_soft_rings, 3419da14cebeSEric Cheng sizeof (mac_soft_ring_t *) * MAX_SR_FANOUT); 3420da14cebeSEric Cheng mac_srs->srs_tcp_soft_rings = NULL; 3421da14cebeSEric Cheng ASSERT(mac_srs->srs_udp_soft_rings != NULL); 3422da14cebeSEric Cheng kmem_free(mac_srs->srs_udp_soft_rings, 3423da14cebeSEric Cheng sizeof (mac_soft_ring_t *) * MAX_SR_FANOUT); 3424da14cebeSEric Cheng mac_srs->srs_udp_soft_rings = NULL; 3425da14cebeSEric Cheng ASSERT(mac_srs->srs_oth_soft_rings != NULL); 3426da14cebeSEric Cheng kmem_free(mac_srs->srs_oth_soft_rings, 3427da14cebeSEric Cheng sizeof (mac_soft_ring_t *) * MAX_SR_FANOUT); 3428da14cebeSEric Cheng mac_srs->srs_oth_soft_rings = NULL; 3429da14cebeSEric Cheng } 34300dc2366fSVenugopal Iyer } 3431da14cebeSEric Cheng 3432da14cebeSEric Cheng /* 3433da14cebeSEric Cheng * An RX SRS is attached to at most one mac_ring. 3434da14cebeSEric Cheng * A TX SRS has no rings. 3435da14cebeSEric Cheng */ 3436da14cebeSEric Cheng static void 3437da14cebeSEric Cheng mac_srs_ring_free(mac_soft_ring_set_t *mac_srs) 3438da14cebeSEric Cheng { 3439da14cebeSEric Cheng mac_client_impl_t *mcip; 3440da14cebeSEric Cheng mac_ring_t *ring; 3441da14cebeSEric Cheng flow_entry_t *flent; 3442da14cebeSEric Cheng 3443da14cebeSEric Cheng ring = mac_srs->srs_ring; 3444da14cebeSEric Cheng if (mac_srs->srs_type & SRST_TX) { 3445da14cebeSEric Cheng ASSERT(ring == NULL); 3446da14cebeSEric Cheng return; 3447da14cebeSEric Cheng } 3448da14cebeSEric Cheng 3449da14cebeSEric Cheng if (ring == NULL) 3450da14cebeSEric Cheng return; 3451da14cebeSEric Cheng 3452da14cebeSEric Cheng /* 3453da14cebeSEric Cheng * Broadcast flows don't have a client impl association, but they 3454da14cebeSEric Cheng * use only soft rings. 3455da14cebeSEric Cheng */ 3456da14cebeSEric Cheng flent = mac_srs->srs_flent; 3457da14cebeSEric Cheng mcip = flent->fe_mcip; 3458da14cebeSEric Cheng ASSERT(mcip != NULL); 3459da14cebeSEric Cheng 3460da14cebeSEric Cheng ring->mr_classify_type = MAC_NO_CLASSIFIER; 3461da14cebeSEric Cheng ring->mr_srs = NULL; 3462da14cebeSEric Cheng } 3463da14cebeSEric Cheng 3464da14cebeSEric Cheng /* 3465da14cebeSEric Cheng * Physical unlink and free of the data structures happen below. This is 3466da14cebeSEric Cheng * driven from mac_flow_destroy(), on the last refrele of a flow. 3467da14cebeSEric Cheng * 3468da14cebeSEric Cheng * Assumes Rx srs is 1-1 mapped with an ring. 3469da14cebeSEric Cheng */ 3470da14cebeSEric Cheng void 3471da14cebeSEric Cheng mac_srs_free(mac_soft_ring_set_t *mac_srs) 3472da14cebeSEric Cheng { 3473da14cebeSEric Cheng ASSERT(mac_srs->srs_mcip == NULL || 3474da14cebeSEric Cheng MAC_PERIM_HELD((mac_handle_t)mac_srs->srs_mcip->mci_mip)); 3475da14cebeSEric Cheng ASSERT((mac_srs->srs_state & (SRS_CONDEMNED | SRS_CONDEMNED_DONE | 3476da14cebeSEric Cheng SRS_PROC | SRS_PROC_FAST)) == (SRS_CONDEMNED | SRS_CONDEMNED_DONE)); 3477da14cebeSEric Cheng 3478c61a1653SRyan Zezeski mac_drop_chain(mac_srs->srs_first, "SRS free"); 3479da14cebeSEric Cheng mac_srs_ring_free(mac_srs); 34800dc2366fSVenugopal Iyer mac_srs_soft_rings_free(mac_srs); 3481da14cebeSEric Cheng mac_srs_fanout_list_free(mac_srs); 3482da14cebeSEric Cheng 3483da14cebeSEric Cheng mac_srs->srs_bw = NULL; 34840dc2366fSVenugopal Iyer mac_srs_stat_delete(mac_srs); 3485da14cebeSEric Cheng kmem_cache_free(mac_srs_cache, mac_srs); 3486da14cebeSEric Cheng } 3487da14cebeSEric Cheng 3488da14cebeSEric Cheng static void 3489da14cebeSEric Cheng mac_srs_soft_rings_quiesce(mac_soft_ring_set_t *mac_srs, uint_t s_ring_flag) 3490da14cebeSEric Cheng { 3491da14cebeSEric Cheng mac_soft_ring_t *softring; 3492da14cebeSEric Cheng 3493da14cebeSEric Cheng ASSERT(MUTEX_HELD(&mac_srs->srs_lock)); 3494da14cebeSEric Cheng 3495da14cebeSEric Cheng mac_srs_soft_rings_signal(mac_srs, s_ring_flag); 3496da14cebeSEric Cheng if (s_ring_flag == S_RING_CONDEMNED) { 3497da14cebeSEric Cheng while (mac_srs->srs_soft_ring_condemned_count != 3498da14cebeSEric Cheng mac_srs->srs_soft_ring_count) 3499da14cebeSEric Cheng cv_wait(&mac_srs->srs_async, &mac_srs->srs_lock); 3500da14cebeSEric Cheng } else { 3501da14cebeSEric Cheng while (mac_srs->srs_soft_ring_quiesced_count != 3502da14cebeSEric Cheng mac_srs->srs_soft_ring_count) 3503da14cebeSEric Cheng cv_wait(&mac_srs->srs_async, &mac_srs->srs_lock); 3504da14cebeSEric Cheng } 3505da14cebeSEric Cheng mutex_exit(&mac_srs->srs_lock); 3506da14cebeSEric Cheng 3507da14cebeSEric Cheng for (softring = mac_srs->srs_soft_ring_head; softring != NULL; 3508da906b4eSRyan Zezeski softring = softring->s_ring_next) { 3509da14cebeSEric Cheng (void) untimeout(softring->s_ring_tid); 3510da906b4eSRyan Zezeski softring->s_ring_tid = NULL; 3511da906b4eSRyan Zezeski } 3512da14cebeSEric Cheng 3513da14cebeSEric Cheng (void) untimeout(mac_srs->srs_tid); 3514da906b4eSRyan Zezeski mac_srs->srs_tid = NULL; 3515da14cebeSEric Cheng 3516da14cebeSEric Cheng mutex_enter(&mac_srs->srs_lock); 3517da14cebeSEric Cheng } 3518da14cebeSEric Cheng 3519da14cebeSEric Cheng /* 3520da14cebeSEric Cheng * The block comment above mac_rx_classify_flow_state_change explains the 3521da14cebeSEric Cheng * background. At this point upcalls from the driver (both hardware classified 3522da14cebeSEric Cheng * and software classified) have been cut off. We now need to quiesce the 3523da14cebeSEric Cheng * SRS worker, poll, and softring threads. The SRS worker thread serves as 3524da14cebeSEric Cheng * the master controller. The steps involved are described below in the function 3525da14cebeSEric Cheng */ 3526da14cebeSEric Cheng void 3527da14cebeSEric Cheng mac_srs_worker_quiesce(mac_soft_ring_set_t *mac_srs) 3528da14cebeSEric Cheng { 3529da14cebeSEric Cheng uint_t s_ring_flag; 3530da14cebeSEric Cheng uint_t srs_poll_wait_flag; 3531da14cebeSEric Cheng 3532da14cebeSEric Cheng ASSERT(MUTEX_HELD(&mac_srs->srs_lock)); 3533da14cebeSEric Cheng ASSERT(mac_srs->srs_state & (SRS_CONDEMNED | SRS_QUIESCE)); 3534da14cebeSEric Cheng 3535da14cebeSEric Cheng if (mac_srs->srs_state & SRS_CONDEMNED) { 3536da14cebeSEric Cheng s_ring_flag = S_RING_CONDEMNED; 3537da14cebeSEric Cheng srs_poll_wait_flag = SRS_POLL_THR_EXITED; 3538da14cebeSEric Cheng } else { 3539da14cebeSEric Cheng s_ring_flag = S_RING_QUIESCE; 3540da14cebeSEric Cheng srs_poll_wait_flag = SRS_POLL_THR_QUIESCED; 3541da14cebeSEric Cheng } 3542da14cebeSEric Cheng 3543da14cebeSEric Cheng /* 3544da14cebeSEric Cheng * In the case of Rx SRS wait till the poll thread is done. 3545da14cebeSEric Cheng */ 3546da14cebeSEric Cheng if ((mac_srs->srs_type & SRST_TX) == 0 && 3547da14cebeSEric Cheng mac_srs->srs_poll_thr != NULL) { 3548da14cebeSEric Cheng while (!(mac_srs->srs_state & srs_poll_wait_flag)) 3549da14cebeSEric Cheng cv_wait(&mac_srs->srs_async, &mac_srs->srs_lock); 3550da14cebeSEric Cheng 3551da14cebeSEric Cheng /* 3552da14cebeSEric Cheng * Turn off polling as part of the quiesce operation. 3553da14cebeSEric Cheng */ 3554da14cebeSEric Cheng MAC_SRS_POLLING_OFF(mac_srs); 3555da14cebeSEric Cheng mac_srs->srs_state &= ~(SRS_POLLING | SRS_GET_PKTS); 3556da14cebeSEric Cheng } 3557da14cebeSEric Cheng 3558da14cebeSEric Cheng /* 3559da14cebeSEric Cheng * Then signal the soft ring worker threads to quiesce or quit 3560da14cebeSEric Cheng * as needed and then wait till that happens. 3561da14cebeSEric Cheng */ 3562da14cebeSEric Cheng mac_srs_soft_rings_quiesce(mac_srs, s_ring_flag); 3563da14cebeSEric Cheng 3564da14cebeSEric Cheng if (mac_srs->srs_state & SRS_CONDEMNED) 3565da14cebeSEric Cheng mac_srs->srs_state |= (SRS_QUIESCE_DONE | SRS_CONDEMNED_DONE); 3566da14cebeSEric Cheng else 3567da14cebeSEric Cheng mac_srs->srs_state |= SRS_QUIESCE_DONE; 3568da14cebeSEric Cheng cv_signal(&mac_srs->srs_quiesce_done_cv); 3569da14cebeSEric Cheng } 3570da14cebeSEric Cheng 3571da14cebeSEric Cheng /* 3572da14cebeSEric Cheng * Signal an SRS to start a temporary quiesce, or permanent removal, or restart 3573da14cebeSEric Cheng * a quiesced SRS by setting the appropriate flags and signaling the SRS worker 3574da14cebeSEric Cheng * or poll thread. This function is internal to the quiescing logic and is 3575da14cebeSEric Cheng * called internally from the SRS quiesce or flow quiesce or client quiesce 3576da14cebeSEric Cheng * higher level functions. 3577da14cebeSEric Cheng */ 3578da14cebeSEric Cheng void 3579da14cebeSEric Cheng mac_srs_signal(mac_soft_ring_set_t *mac_srs, uint_t srs_flag) 3580da14cebeSEric Cheng { 3581da14cebeSEric Cheng mac_ring_t *ring; 3582da14cebeSEric Cheng 3583da14cebeSEric Cheng ring = mac_srs->srs_ring; 3584da14cebeSEric Cheng ASSERT(ring == NULL || ring->mr_refcnt == 0); 3585da14cebeSEric Cheng 3586da14cebeSEric Cheng if (srs_flag == SRS_CONDEMNED) { 3587da14cebeSEric Cheng /* 3588da14cebeSEric Cheng * The SRS is going away. We need to unbind the SRS and SR 3589da14cebeSEric Cheng * threads before removing from the global SRS list. Otherwise 3590da14cebeSEric Cheng * there is a small window where the cpu reconfig callbacks 3591da14cebeSEric Cheng * may miss the SRS in the list walk and DR could fail since 3592da14cebeSEric Cheng * there are still bound threads. 3593da14cebeSEric Cheng */ 3594da14cebeSEric Cheng mac_srs_threads_unbind(mac_srs); 3595da14cebeSEric Cheng mac_srs_remove_glist(mac_srs); 3596da14cebeSEric Cheng } 3597da14cebeSEric Cheng /* 3598da14cebeSEric Cheng * Wakeup the SRS worker and poll threads. 3599da14cebeSEric Cheng */ 3600da14cebeSEric Cheng mutex_enter(&mac_srs->srs_lock); 3601da14cebeSEric Cheng mac_srs->srs_state |= srs_flag; 3602da14cebeSEric Cheng cv_signal(&mac_srs->srs_async); 3603da14cebeSEric Cheng cv_signal(&mac_srs->srs_cv); 3604da14cebeSEric Cheng mutex_exit(&mac_srs->srs_lock); 3605da14cebeSEric Cheng } 3606da14cebeSEric Cheng 3607da14cebeSEric Cheng /* 3608da14cebeSEric Cheng * In the Rx side, the quiescing is done bottom up. After the Rx upcalls 3609da14cebeSEric Cheng * from the driver are done, then the Rx SRS is quiesced and only then can 3610da14cebeSEric Cheng * we signal the soft rings. Thus this function can't be called arbitrarily 3611da14cebeSEric Cheng * without satisfying the prerequisites. On the Tx side, the threads from 3612da14cebeSEric Cheng * top need to quiesced, then the Tx SRS and only then can we signal the 3613da14cebeSEric Cheng * Tx soft rings. 3614da14cebeSEric Cheng */ 3615da14cebeSEric Cheng static void 3616da14cebeSEric Cheng mac_srs_soft_rings_signal(mac_soft_ring_set_t *mac_srs, uint_t sr_flag) 3617da14cebeSEric Cheng { 3618da14cebeSEric Cheng mac_soft_ring_t *softring; 3619da14cebeSEric Cheng 3620da14cebeSEric Cheng for (softring = mac_srs->srs_soft_ring_head; softring != NULL; 3621da14cebeSEric Cheng softring = softring->s_ring_next) 3622da14cebeSEric Cheng mac_soft_ring_signal(softring, sr_flag); 3623da14cebeSEric Cheng } 3624da14cebeSEric Cheng 3625da14cebeSEric Cheng /* 3626da14cebeSEric Cheng * The block comment above mac_rx_classify_flow_state_change explains the 3627da14cebeSEric Cheng * background. At this point the SRS is quiesced and we need to restart the 3628da14cebeSEric Cheng * SRS worker, poll, and softring threads. The SRS worker thread serves as 3629da14cebeSEric Cheng * the master controller. The steps involved are described below in the function 3630da14cebeSEric Cheng */ 3631da14cebeSEric Cheng void 3632da14cebeSEric Cheng mac_srs_worker_restart(mac_soft_ring_set_t *mac_srs) 3633da14cebeSEric Cheng { 3634da14cebeSEric Cheng boolean_t iam_rx_srs; 3635da14cebeSEric Cheng mac_soft_ring_t *softring; 3636da14cebeSEric Cheng 3637da14cebeSEric Cheng ASSERT(MUTEX_HELD(&mac_srs->srs_lock)); 3638da14cebeSEric Cheng if ((mac_srs->srs_type & SRST_TX) != 0) { 3639da14cebeSEric Cheng iam_rx_srs = B_FALSE; 3640da14cebeSEric Cheng ASSERT((mac_srs->srs_state & 3641da14cebeSEric Cheng (SRS_POLL_THR_QUIESCED | SRS_QUIESCE_DONE | SRS_QUIESCE)) == 3642da14cebeSEric Cheng (SRS_QUIESCE_DONE | SRS_QUIESCE)); 3643da14cebeSEric Cheng } else { 3644da14cebeSEric Cheng iam_rx_srs = B_TRUE; 3645da14cebeSEric Cheng ASSERT((mac_srs->srs_state & 3646da14cebeSEric Cheng (SRS_QUIESCE_DONE | SRS_QUIESCE)) == 3647da14cebeSEric Cheng (SRS_QUIESCE_DONE | SRS_QUIESCE)); 3648da14cebeSEric Cheng if (mac_srs->srs_poll_thr != NULL) { 3649da14cebeSEric Cheng ASSERT((mac_srs->srs_state & SRS_POLL_THR_QUIESCED) == 3650da14cebeSEric Cheng SRS_POLL_THR_QUIESCED); 3651da14cebeSEric Cheng } 3652da14cebeSEric Cheng } 3653da14cebeSEric Cheng 3654da14cebeSEric Cheng /* 3655da14cebeSEric Cheng * Signal any quiesced soft ring workers to restart and wait for the 3656da14cebeSEric Cheng * soft ring down count to come down to zero. 3657da14cebeSEric Cheng */ 3658da14cebeSEric Cheng if (mac_srs->srs_soft_ring_quiesced_count != 0) { 3659da14cebeSEric Cheng for (softring = mac_srs->srs_soft_ring_head; softring != NULL; 3660da14cebeSEric Cheng softring = softring->s_ring_next) { 3661da14cebeSEric Cheng if (!(softring->s_ring_state & S_RING_QUIESCE)) 3662da14cebeSEric Cheng continue; 3663da14cebeSEric Cheng mac_soft_ring_signal(softring, S_RING_RESTART); 3664da14cebeSEric Cheng } 3665da14cebeSEric Cheng while (mac_srs->srs_soft_ring_quiesced_count != 0) 3666da14cebeSEric Cheng cv_wait(&mac_srs->srs_async, &mac_srs->srs_lock); 3667da14cebeSEric Cheng } 3668da14cebeSEric Cheng 3669da14cebeSEric Cheng mac_srs->srs_state &= ~(SRS_QUIESCE_DONE | SRS_QUIESCE | SRS_RESTART); 3670da14cebeSEric Cheng if (iam_rx_srs && mac_srs->srs_poll_thr != NULL) { 3671da14cebeSEric Cheng /* 3672da14cebeSEric Cheng * Signal the poll thread and ask it to restart. Wait till it 3673da14cebeSEric Cheng * actually restarts and the SRS_POLL_THR_QUIESCED flag gets 3674da14cebeSEric Cheng * cleared. 3675da14cebeSEric Cheng */ 3676da14cebeSEric Cheng mac_srs->srs_state |= SRS_POLL_THR_RESTART; 3677da14cebeSEric Cheng cv_signal(&mac_srs->srs_cv); 3678da14cebeSEric Cheng while (mac_srs->srs_state & SRS_POLL_THR_QUIESCED) 3679da14cebeSEric Cheng cv_wait(&mac_srs->srs_async, &mac_srs->srs_lock); 3680da14cebeSEric Cheng ASSERT(!(mac_srs->srs_state & SRS_POLL_THR_RESTART)); 3681da14cebeSEric Cheng } 3682da14cebeSEric Cheng /* Wake up any waiter waiting for the restart to complete */ 3683da14cebeSEric Cheng mac_srs->srs_state |= SRS_RESTART_DONE; 3684da14cebeSEric Cheng cv_signal(&mac_srs->srs_quiesce_done_cv); 3685da14cebeSEric Cheng } 3686da14cebeSEric Cheng 3687da14cebeSEric Cheng static void 3688da14cebeSEric Cheng mac_srs_worker_unbind(mac_soft_ring_set_t *mac_srs) 3689da14cebeSEric Cheng { 3690da14cebeSEric Cheng mutex_enter(&mac_srs->srs_lock); 3691da14cebeSEric Cheng if (!(mac_srs->srs_state & SRS_WORKER_BOUND)) { 3692da14cebeSEric Cheng ASSERT(mac_srs->srs_worker_cpuid == -1); 3693da14cebeSEric Cheng mutex_exit(&mac_srs->srs_lock); 3694da14cebeSEric Cheng return; 3695da14cebeSEric Cheng } 3696da14cebeSEric Cheng 3697da14cebeSEric Cheng mac_srs->srs_worker_cpuid = -1; 3698da14cebeSEric Cheng mac_srs->srs_state &= ~SRS_WORKER_BOUND; 3699da14cebeSEric Cheng thread_affinity_clear(mac_srs->srs_worker); 3700da14cebeSEric Cheng mutex_exit(&mac_srs->srs_lock); 3701da14cebeSEric Cheng } 3702da14cebeSEric Cheng 3703da14cebeSEric Cheng static void 3704da14cebeSEric Cheng mac_srs_poll_unbind(mac_soft_ring_set_t *mac_srs) 3705da14cebeSEric Cheng { 3706da14cebeSEric Cheng mutex_enter(&mac_srs->srs_lock); 3707da14cebeSEric Cheng if (mac_srs->srs_poll_thr == NULL || 3708da14cebeSEric Cheng (mac_srs->srs_state & SRS_POLL_BOUND) == 0) { 3709da14cebeSEric Cheng ASSERT(mac_srs->srs_poll_cpuid == -1); 3710da14cebeSEric Cheng mutex_exit(&mac_srs->srs_lock); 3711da14cebeSEric Cheng return; 3712da14cebeSEric Cheng } 3713da14cebeSEric Cheng 3714da14cebeSEric Cheng mac_srs->srs_poll_cpuid = -1; 3715da14cebeSEric Cheng mac_srs->srs_state &= ~SRS_POLL_BOUND; 3716da14cebeSEric Cheng thread_affinity_clear(mac_srs->srs_poll_thr); 3717da14cebeSEric Cheng mutex_exit(&mac_srs->srs_lock); 3718da14cebeSEric Cheng } 3719da14cebeSEric Cheng 3720da14cebeSEric Cheng static void 3721da14cebeSEric Cheng mac_srs_threads_unbind(mac_soft_ring_set_t *mac_srs) 3722da14cebeSEric Cheng { 3723da14cebeSEric Cheng mac_soft_ring_t *soft_ring; 3724da14cebeSEric Cheng 3725da14cebeSEric Cheng ASSERT(MAC_PERIM_HELD((mac_handle_t)mac_srs->srs_mcip->mci_mip)); 3726da14cebeSEric Cheng 3727da14cebeSEric Cheng mutex_enter(&cpu_lock); 3728da14cebeSEric Cheng mac_srs_worker_unbind(mac_srs); 3729da14cebeSEric Cheng if (!(mac_srs->srs_type & SRST_TX)) 3730da14cebeSEric Cheng mac_srs_poll_unbind(mac_srs); 3731da14cebeSEric Cheng 3732da14cebeSEric Cheng for (soft_ring = mac_srs->srs_soft_ring_head; soft_ring != NULL; 3733da14cebeSEric Cheng soft_ring = soft_ring->s_ring_next) { 3734da14cebeSEric Cheng mac_soft_ring_unbind(soft_ring); 3735da14cebeSEric Cheng } 3736da14cebeSEric Cheng mutex_exit(&cpu_lock); 3737da14cebeSEric Cheng } 3738da14cebeSEric Cheng 3739da14cebeSEric Cheng /* 3740da14cebeSEric Cheng * When a CPU is going away, unbind all MAC threads which are bound 3741da14cebeSEric Cheng * to that CPU. The affinity of the thread to the CPU is saved to allow 3742da14cebeSEric Cheng * the thread to be rebound to the CPU if it comes back online. 3743da14cebeSEric Cheng */ 3744da14cebeSEric Cheng static void 3745da14cebeSEric Cheng mac_walk_srs_and_unbind(int cpuid) 3746da14cebeSEric Cheng { 3747da14cebeSEric Cheng mac_soft_ring_set_t *mac_srs; 3748da14cebeSEric Cheng mac_soft_ring_t *soft_ring; 3749da14cebeSEric Cheng 3750da14cebeSEric Cheng rw_enter(&mac_srs_g_lock, RW_READER); 3751da14cebeSEric Cheng 3752da14cebeSEric Cheng if ((mac_srs = mac_srs_g_list) == NULL) 3753da14cebeSEric Cheng goto done; 3754da14cebeSEric Cheng 3755da14cebeSEric Cheng for (; mac_srs != NULL; mac_srs = mac_srs->srs_next) { 3756da14cebeSEric Cheng if (mac_srs->srs_worker_cpuid == cpuid) { 3757da14cebeSEric Cheng mac_srs->srs_worker_cpuid_save = cpuid; 3758da14cebeSEric Cheng mac_srs_worker_unbind(mac_srs); 3759da14cebeSEric Cheng } 3760da14cebeSEric Cheng 3761da14cebeSEric Cheng if (!(mac_srs->srs_type & SRST_TX)) { 3762da14cebeSEric Cheng if (mac_srs->srs_poll_cpuid == cpuid) { 3763da14cebeSEric Cheng mac_srs->srs_poll_cpuid_save = cpuid; 3764da14cebeSEric Cheng mac_srs_poll_unbind(mac_srs); 3765da14cebeSEric Cheng } 3766da14cebeSEric Cheng } 3767da14cebeSEric Cheng 3768da14cebeSEric Cheng /* Next tackle the soft rings associated with the srs */ 3769da14cebeSEric Cheng mutex_enter(&mac_srs->srs_lock); 3770da14cebeSEric Cheng for (soft_ring = mac_srs->srs_soft_ring_head; soft_ring != NULL; 3771da14cebeSEric Cheng soft_ring = soft_ring->s_ring_next) { 3772da14cebeSEric Cheng if (soft_ring->s_ring_cpuid == cpuid) { 3773da14cebeSEric Cheng soft_ring->s_ring_cpuid_save = cpuid; 3774da14cebeSEric Cheng mac_soft_ring_unbind(soft_ring); 3775da14cebeSEric Cheng } 3776da14cebeSEric Cheng } 3777da14cebeSEric Cheng mutex_exit(&mac_srs->srs_lock); 3778da14cebeSEric Cheng } 3779da14cebeSEric Cheng done: 3780da14cebeSEric Cheng rw_exit(&mac_srs_g_lock); 3781da14cebeSEric Cheng } 3782da14cebeSEric Cheng 3783da14cebeSEric Cheng /* TX SETUP and TEARDOWN ROUTINES */ 3784da14cebeSEric Cheng 3785da14cebeSEric Cheng /* 3786da14cebeSEric Cheng * XXXHIO need to make sure the two mac_tx_srs_{add,del}_ring() 3787da14cebeSEric Cheng * handle the case where the number of rings is one. I.e. there is 3788da14cebeSEric Cheng * a ring pointed to by mac_srs->srs_tx_arg2. 3789da14cebeSEric Cheng */ 3790da14cebeSEric Cheng void 3791da14cebeSEric Cheng mac_tx_srs_add_ring(mac_soft_ring_set_t *mac_srs, mac_ring_t *tx_ring) 3792da14cebeSEric Cheng { 3793da14cebeSEric Cheng mac_client_impl_t *mcip = mac_srs->srs_mcip; 3794da14cebeSEric Cheng mac_soft_ring_t *soft_ring; 37950dc2366fSVenugopal Iyer int count = mac_srs->srs_tx_ring_count; 37960dc2366fSVenugopal Iyer uint32_t soft_ring_type = ST_RING_TX; 37970dc2366fSVenugopal Iyer uint_t ring_info; 3798da14cebeSEric Cheng 3799da14cebeSEric Cheng ASSERT(mac_srs->srs_state & SRS_QUIESCE); 38000dc2366fSVenugopal Iyer ring_info = mac_hwring_getinfo((mac_ring_handle_t)tx_ring); 38010dc2366fSVenugopal Iyer if (mac_tx_serialize || (ring_info & MAC_RING_TX_SERIALIZE)) 38020dc2366fSVenugopal Iyer soft_ring_type |= ST_RING_WORKER_ONLY; 38030dc2366fSVenugopal Iyer soft_ring = mac_soft_ring_create(count, 0, 38040dc2366fSVenugopal Iyer soft_ring_type, maxclsyspri, mcip, mac_srs, -1, 3805da14cebeSEric Cheng NULL, mcip, (mac_resource_handle_t)tx_ring); 38060dc2366fSVenugopal Iyer mac_srs->srs_tx_ring_count++; 38070dc2366fSVenugopal Iyer mac_srs_update_fanout_list(mac_srs); 3808da14cebeSEric Cheng /* 3809da14cebeSEric Cheng * put this soft ring in quiesce mode too so when we restart 3810da14cebeSEric Cheng * all soft rings in the srs are in the same state. 3811da14cebeSEric Cheng */ 3812da14cebeSEric Cheng mac_soft_ring_signal(soft_ring, S_RING_QUIESCE); 3813da14cebeSEric Cheng } 3814da14cebeSEric Cheng 3815da14cebeSEric Cheng static void 3816da14cebeSEric Cheng mac_soft_ring_remove(mac_soft_ring_set_t *mac_srs, mac_soft_ring_t *softring) 3817da14cebeSEric Cheng { 3818da14cebeSEric Cheng int sringcnt; 3819da14cebeSEric Cheng 3820da14cebeSEric Cheng mutex_enter(&mac_srs->srs_lock); 3821da14cebeSEric Cheng sringcnt = mac_srs->srs_soft_ring_count; 3822da14cebeSEric Cheng ASSERT(sringcnt > 0); 3823da14cebeSEric Cheng mac_soft_ring_signal(softring, S_RING_CONDEMNED); 3824da14cebeSEric Cheng 3825da14cebeSEric Cheng ASSERT(mac_srs->srs_soft_ring_condemned_count == 0); 3826da14cebeSEric Cheng while (mac_srs->srs_soft_ring_condemned_count != 1) 3827da14cebeSEric Cheng cv_wait(&mac_srs->srs_async, &mac_srs->srs_lock); 3828da14cebeSEric Cheng 3829da14cebeSEric Cheng if (softring == mac_srs->srs_soft_ring_head) { 3830da14cebeSEric Cheng mac_srs->srs_soft_ring_head = softring->s_ring_next; 3831da14cebeSEric Cheng if (mac_srs->srs_soft_ring_head != NULL) { 3832da14cebeSEric Cheng mac_srs->srs_soft_ring_head->s_ring_prev = NULL; 3833da14cebeSEric Cheng } else { 3834da14cebeSEric Cheng mac_srs->srs_soft_ring_tail = NULL; 3835da14cebeSEric Cheng } 3836da14cebeSEric Cheng } else { 3837da14cebeSEric Cheng softring->s_ring_prev->s_ring_next = 3838da14cebeSEric Cheng softring->s_ring_next; 3839da14cebeSEric Cheng if (softring->s_ring_next != NULL) { 3840da14cebeSEric Cheng softring->s_ring_next->s_ring_prev = 3841da14cebeSEric Cheng softring->s_ring_prev; 3842da14cebeSEric Cheng } else { 3843da14cebeSEric Cheng mac_srs->srs_soft_ring_tail = 3844da14cebeSEric Cheng softring->s_ring_prev; 3845da14cebeSEric Cheng } 3846da14cebeSEric Cheng } 3847da14cebeSEric Cheng mac_srs->srs_soft_ring_count--; 3848da14cebeSEric Cheng 3849da14cebeSEric Cheng mac_srs->srs_soft_ring_condemned_count--; 3850da14cebeSEric Cheng mutex_exit(&mac_srs->srs_lock); 3851da14cebeSEric Cheng 38520dc2366fSVenugopal Iyer mac_soft_ring_free(softring); 3853da14cebeSEric Cheng } 3854da14cebeSEric Cheng 3855da14cebeSEric Cheng void 3856da14cebeSEric Cheng mac_tx_srs_del_ring(mac_soft_ring_set_t *mac_srs, mac_ring_t *tx_ring) 3857da14cebeSEric Cheng { 3858da14cebeSEric Cheng int i; 3859da14cebeSEric Cheng mac_soft_ring_t *soft_ring, *remove_sring; 38600dc2366fSVenugopal Iyer mac_client_impl_t *mcip = mac_srs->srs_mcip; 3861da14cebeSEric Cheng 3862da14cebeSEric Cheng mutex_enter(&mac_srs->srs_lock); 38630dc2366fSVenugopal Iyer for (i = 0; i < mac_srs->srs_tx_ring_count; i++) { 38640dc2366fSVenugopal Iyer soft_ring = mac_srs->srs_tx_soft_rings[i]; 3865da14cebeSEric Cheng if (soft_ring->s_ring_tx_arg2 == tx_ring) 3866da14cebeSEric Cheng break; 3867da14cebeSEric Cheng } 3868da14cebeSEric Cheng mutex_exit(&mac_srs->srs_lock); 38690dc2366fSVenugopal Iyer ASSERT(i < mac_srs->srs_tx_ring_count); 3870da14cebeSEric Cheng remove_sring = soft_ring; 38710dc2366fSVenugopal Iyer /* 38720dc2366fSVenugopal Iyer * In the case of aggr, the soft ring associated with a Tx ring 38730dc2366fSVenugopal Iyer * is also stored in st_soft_rings[] array. That entry should 38740dc2366fSVenugopal Iyer * be removed. 38750dc2366fSVenugopal Iyer */ 387684de666eSRyan Zezeski if (mcip->mci_state_flags & MCIS_IS_AGGR_CLIENT) { 38770dc2366fSVenugopal Iyer mac_srs_tx_t *tx = &mac_srs->srs_tx; 38780dc2366fSVenugopal Iyer 38790dc2366fSVenugopal Iyer ASSERT(tx->st_soft_rings[tx_ring->mr_index] == remove_sring); 38800dc2366fSVenugopal Iyer tx->st_soft_rings[tx_ring->mr_index] = NULL; 38810dc2366fSVenugopal Iyer } 3882da14cebeSEric Cheng mac_soft_ring_remove(mac_srs, remove_sring); 3883da14cebeSEric Cheng mac_srs_update_fanout_list(mac_srs); 3884da14cebeSEric Cheng } 3885da14cebeSEric Cheng 3886da14cebeSEric Cheng /* 3887da14cebeSEric Cheng * mac_tx_srs_setup(): 3888da14cebeSEric Cheng * Used to setup Tx rings. If no free Tx ring is available, then default 3889da14cebeSEric Cheng * Tx ring is used. 3890da14cebeSEric Cheng */ 3891da14cebeSEric Cheng void 38920dc2366fSVenugopal Iyer mac_tx_srs_setup(mac_client_impl_t *mcip, flow_entry_t *flent) 3893da14cebeSEric Cheng { 3894da14cebeSEric Cheng mac_impl_t *mip = mcip->mci_mip; 38950dc2366fSVenugopal Iyer mac_soft_ring_set_t *tx_srs = flent->fe_tx_srs; 38960dc2366fSVenugopal Iyer int i; 38970dc2366fSVenugopal Iyer int tx_ring_count = 0; 3898da14cebeSEric Cheng uint32_t soft_ring_type; 3899da14cebeSEric Cheng mac_group_t *grp = NULL; 3900da14cebeSEric Cheng mac_ring_t *ring; 39010dc2366fSVenugopal Iyer mac_srs_tx_t *tx = &tx_srs->srs_tx; 39020dc2366fSVenugopal Iyer boolean_t is_aggr; 39030dc2366fSVenugopal Iyer uint_t ring_info = 0; 3904da14cebeSEric Cheng 390584de666eSRyan Zezeski is_aggr = (mcip->mci_state_flags & MCIS_IS_AGGR_CLIENT) != 0; 39060dc2366fSVenugopal Iyer grp = flent->fe_tx_ring_group; 39070dc2366fSVenugopal Iyer if (grp == NULL) { 39080dc2366fSVenugopal Iyer ring = (mac_ring_t *)mip->mi_default_tx_ring; 39090dc2366fSVenugopal Iyer goto no_group; 39100dc2366fSVenugopal Iyer } 3911da14cebeSEric Cheng tx_ring_count = grp->mrg_cur_count; 3912da14cebeSEric Cheng ring = grp->mrg_rings; 3913da14cebeSEric Cheng /* 3914da14cebeSEric Cheng * An attempt is made to reserve 'tx_ring_count' number 3915da14cebeSEric Cheng * of Tx rings. If tx_ring_count is 0, default Tx ring 3916da14cebeSEric Cheng * is used. If it is 1, an attempt is made to reserve one 3917da14cebeSEric Cheng * Tx ring. In both the cases, the ring information is 3918da14cebeSEric Cheng * stored in Tx SRS. If multiple Tx rings are specified, 3919da14cebeSEric Cheng * then each Tx ring will have a Tx-side soft ring. All 3920da14cebeSEric Cheng * these soft rings will be hang off Tx SRS. 3921da14cebeSEric Cheng */ 39220dc2366fSVenugopal Iyer switch (grp->mrg_state) { 39230dc2366fSVenugopal Iyer case MAC_GROUP_STATE_SHARED: 39240dc2366fSVenugopal Iyer case MAC_GROUP_STATE_RESERVED: 39250dc2366fSVenugopal Iyer if (tx_ring_count <= 1 && !is_aggr) { 39260dc2366fSVenugopal Iyer no_group: 39270dc2366fSVenugopal Iyer if (ring != NULL && 39280dc2366fSVenugopal Iyer ring->mr_state != MR_INUSE) { 39290dc2366fSVenugopal Iyer (void) mac_start_ring(ring); 39300dc2366fSVenugopal Iyer ring_info = mac_hwring_getinfo( 39310dc2366fSVenugopal Iyer (mac_ring_handle_t)ring); 39320dc2366fSVenugopal Iyer } 39330dc2366fSVenugopal Iyer tx->st_arg2 = (void *)ring; 39340dc2366fSVenugopal Iyer mac_tx_srs_stat_recreate(tx_srs, B_FALSE); 39350dc2366fSVenugopal Iyer if (tx_srs->srs_type & SRST_BW_CONTROL) { 39360dc2366fSVenugopal Iyer tx->st_mode = SRS_TX_BW; 39370dc2366fSVenugopal Iyer } else if (mac_tx_serialize || 39380dc2366fSVenugopal Iyer (ring_info & MAC_RING_TX_SERIALIZE)) { 39390dc2366fSVenugopal Iyer tx->st_mode = SRS_TX_SERIALIZE; 3940da14cebeSEric Cheng } else { 39410dc2366fSVenugopal Iyer tx->st_mode = SRS_TX_DEFAULT; 39420dc2366fSVenugopal Iyer } 3943da14cebeSEric Cheng break; 3944da14cebeSEric Cheng } 39450dc2366fSVenugopal Iyer soft_ring_type = ST_RING_TX; 3946da14cebeSEric Cheng if (tx_srs->srs_type & SRST_BW_CONTROL) { 39470dc2366fSVenugopal Iyer tx->st_mode = is_aggr ? 39480dc2366fSVenugopal Iyer SRS_TX_BW_AGGR : SRS_TX_BW_FANOUT; 3949da14cebeSEric Cheng } else { 39500dc2366fSVenugopal Iyer tx->st_mode = is_aggr ? SRS_TX_AGGR : 39510dc2366fSVenugopal Iyer SRS_TX_FANOUT; 3952da14cebeSEric Cheng } 39530dc2366fSVenugopal Iyer for (i = 0; i < tx_ring_count; i++) { 39540dc2366fSVenugopal Iyer ASSERT(ring != NULL); 39550dc2366fSVenugopal Iyer switch (ring->mr_state) { 39560dc2366fSVenugopal Iyer case MR_INUSE: 39570dc2366fSVenugopal Iyer case MR_FREE: 39580dc2366fSVenugopal Iyer ASSERT(ring->mr_srs == NULL); 39590dc2366fSVenugopal Iyer 39600dc2366fSVenugopal Iyer if (ring->mr_state != MR_INUSE) 39610dc2366fSVenugopal Iyer (void) mac_start_ring(ring); 39620dc2366fSVenugopal Iyer ring_info = mac_hwring_getinfo( 39630dc2366fSVenugopal Iyer (mac_ring_handle_t)ring); 39640dc2366fSVenugopal Iyer if (mac_tx_serialize || (ring_info & 39650dc2366fSVenugopal Iyer MAC_RING_TX_SERIALIZE)) { 39660dc2366fSVenugopal Iyer soft_ring_type |= 39670dc2366fSVenugopal Iyer ST_RING_WORKER_ONLY; 39680dc2366fSVenugopal Iyer } 39690dc2366fSVenugopal Iyer (void) mac_soft_ring_create(i, 0, 39700dc2366fSVenugopal Iyer soft_ring_type, maxclsyspri, 39710dc2366fSVenugopal Iyer mcip, tx_srs, -1, NULL, mcip, 39720dc2366fSVenugopal Iyer (mac_resource_handle_t)ring); 39730dc2366fSVenugopal Iyer break; 39740dc2366fSVenugopal Iyer default: 39750dc2366fSVenugopal Iyer cmn_err(CE_PANIC, 39760dc2366fSVenugopal Iyer "srs_setup: mcip = %p " 39770dc2366fSVenugopal Iyer "trying to add UNKNOWN ring = %p\n", 39780dc2366fSVenugopal Iyer (void *)mcip, (void *)ring); 39790dc2366fSVenugopal Iyer break; 39800dc2366fSVenugopal Iyer } 39810dc2366fSVenugopal Iyer ring = ring->mr_next; 3982da14cebeSEric Cheng } 3983da14cebeSEric Cheng mac_srs_update_fanout_list(tx_srs); 39840dc2366fSVenugopal Iyer break; 39850dc2366fSVenugopal Iyer default: 39860dc2366fSVenugopal Iyer ASSERT(B_FALSE); 39870dc2366fSVenugopal Iyer break; 3988da14cebeSEric Cheng } 3989da14cebeSEric Cheng tx->st_func = mac_tx_get_func(tx->st_mode); 39900dc2366fSVenugopal Iyer if (is_aggr) { 39910dc2366fSVenugopal Iyer VERIFY(i_mac_capab_get((mac_handle_t)mip, 39920dc2366fSVenugopal Iyer MAC_CAPAB_AGGR, &tx->st_capab_aggr)); 3993da14cebeSEric Cheng } 39940dc2366fSVenugopal Iyer DTRACE_PROBE3(tx__srs___setup__return, mac_soft_ring_set_t *, tx_srs, 39950dc2366fSVenugopal Iyer int, tx->st_mode, int, tx_srs->srs_tx_ring_count); 3996da14cebeSEric Cheng } 3997da14cebeSEric Cheng 3998da14cebeSEric Cheng /* 399908ac1c49SNicolas Droux * Update the fanout of a client if its recorded link speed doesn't match 400008ac1c49SNicolas Droux * its current link speed. 400108ac1c49SNicolas Droux */ 400208ac1c49SNicolas Droux void 40030dc2366fSVenugopal Iyer mac_fanout_recompute_client(mac_client_impl_t *mcip, cpupart_t *cpupart) 400408ac1c49SNicolas Droux { 400508ac1c49SNicolas Droux uint64_t link_speed; 400608ac1c49SNicolas Droux mac_resource_props_t *mcip_mrp; 40070dc2366fSVenugopal Iyer flow_entry_t *flent = mcip->mci_flent; 40080dc2366fSVenugopal Iyer mac_soft_ring_set_t *rx_srs; 40090dc2366fSVenugopal Iyer mac_cpus_t *srs_cpu; 40100dc2366fSVenugopal Iyer int soft_ring_count, maxcpus; 401108ac1c49SNicolas Droux 401208ac1c49SNicolas Droux ASSERT(MAC_PERIM_HELD((mac_handle_t)mcip->mci_mip)); 401308ac1c49SNicolas Droux 401408ac1c49SNicolas Droux link_speed = mac_client_stat_get(mcip->mci_flent->fe_mcip, 401508ac1c49SNicolas Droux MAC_STAT_IFSPEED); 401608ac1c49SNicolas Droux 401708ac1c49SNicolas Droux if ((link_speed != 0) && 401808ac1c49SNicolas Droux (link_speed != mcip->mci_flent->fe_nic_speed)) { 401908ac1c49SNicolas Droux mcip_mrp = MCIP_RESOURCE_PROPS(mcip); 40200dc2366fSVenugopal Iyer /* 40210dc2366fSVenugopal Iyer * Before calling mac_fanout_setup(), check to see if 40220dc2366fSVenugopal Iyer * the SRSes already have the right number of soft 40230dc2366fSVenugopal Iyer * rings. mac_fanout_setup() is a heavy duty operation 40240dc2366fSVenugopal Iyer * where new cpu bindings are done for SRS and soft 40250dc2366fSVenugopal Iyer * ring threads and interrupts re-targeted. 40260dc2366fSVenugopal Iyer */ 40270dc2366fSVenugopal Iyer maxcpus = (cpupart != NULL) ? cpupart->cp_ncpus : ncpus; 40280dc2366fSVenugopal Iyer soft_ring_count = mac_compute_soft_ring_count(flent, 40290dc2366fSVenugopal Iyer flent->fe_rx_srs_cnt - 1, maxcpus); 40300dc2366fSVenugopal Iyer /* 40310dc2366fSVenugopal Iyer * If soft_ring_count returned by 40320dc2366fSVenugopal Iyer * mac_compute_soft_ring_count() is 0, bump it 40330dc2366fSVenugopal Iyer * up by 1 because we always have atleast one 40340dc2366fSVenugopal Iyer * TCP, UDP, and OTH soft ring associated with 40350dc2366fSVenugopal Iyer * an SRS. 40360dc2366fSVenugopal Iyer */ 40370dc2366fSVenugopal Iyer soft_ring_count = (soft_ring_count == 0) ? 40380dc2366fSVenugopal Iyer 1 : soft_ring_count; 40390dc2366fSVenugopal Iyer rx_srs = flent->fe_rx_srs[0]; 40400dc2366fSVenugopal Iyer srs_cpu = &rx_srs->srs_cpu; 40410dc2366fSVenugopal Iyer if (soft_ring_count != srs_cpu->mc_rx_fanout_cnt) { 40420dc2366fSVenugopal Iyer mac_fanout_setup(mcip, flent, mcip_mrp, 40430dc2366fSVenugopal Iyer mac_rx_deliver, mcip, NULL, cpupart); 40440dc2366fSVenugopal Iyer } 404508ac1c49SNicolas Droux } 404608ac1c49SNicolas Droux } 404708ac1c49SNicolas Droux 404808ac1c49SNicolas Droux /* 404945948e49SRyan Zezeski * Walk through the list of MAC clients for the MAC. 405045948e49SRyan Zezeski * For each active MAC client, recompute the number of soft rings 4051da14cebeSEric Cheng * associated with every client, only if current speed is different 4052da14cebeSEric Cheng * from the speed that was previously used for soft ring computation. 4053da14cebeSEric Cheng * If the cable is disconnected whlie the NIC is started, we would get 4054da14cebeSEric Cheng * notification with speed set to 0. We do not recompute in that case. 4055da14cebeSEric Cheng */ 4056da14cebeSEric Cheng void 4057da14cebeSEric Cheng mac_fanout_recompute(mac_impl_t *mip) 4058da14cebeSEric Cheng { 4059da14cebeSEric Cheng mac_client_impl_t *mcip; 40600dc2366fSVenugopal Iyer cpupart_t *cpupart; 40610dc2366fSVenugopal Iyer boolean_t use_default; 40620dc2366fSVenugopal Iyer mac_resource_props_t *mrp, *emrp; 406308ac1c49SNicolas Droux 4064da14cebeSEric Cheng i_mac_perim_enter(mip); 40652c4ec682SEric Cheng if ((mip->mi_state_flags & MIS_IS_VNIC) != 0 || 40662c4ec682SEric Cheng mip->mi_linkstate != LINK_STATE_UP) { 4067da14cebeSEric Cheng i_mac_perim_exit(mip); 4068da14cebeSEric Cheng return; 4069da14cebeSEric Cheng } 4070da14cebeSEric Cheng 4071da14cebeSEric Cheng for (mcip = mip->mi_clients_list; mcip != NULL; 4072da14cebeSEric Cheng mcip = mcip->mci_client_next) { 407345948e49SRyan Zezeski /* Aggr port clients don't have SRSes. */ 407445948e49SRyan Zezeski if ((mcip->mci_state_flags & MCIS_IS_AGGR_PORT) != 0) 407545948e49SRyan Zezeski continue; 407645948e49SRyan Zezeski 407708ac1c49SNicolas Droux if ((mcip->mci_state_flags & MCIS_SHARE_BOUND) != 0 || 407808ac1c49SNicolas Droux !MCIP_DATAPATH_SETUP(mcip)) 4079da14cebeSEric Cheng continue; 40800dc2366fSVenugopal Iyer mrp = MCIP_RESOURCE_PROPS(mcip); 40810dc2366fSVenugopal Iyer emrp = MCIP_EFFECTIVE_PROPS(mcip); 40820dc2366fSVenugopal Iyer use_default = B_FALSE; 40830dc2366fSVenugopal Iyer pool_lock(); 40840dc2366fSVenugopal Iyer cpupart = mac_pset_find(mrp, &use_default); 40850dc2366fSVenugopal Iyer mac_fanout_recompute_client(mcip, cpupart); 40860dc2366fSVenugopal Iyer mac_set_pool_effective(use_default, cpupart, mrp, emrp); 40870dc2366fSVenugopal Iyer pool_unlock(); 4088da14cebeSEric Cheng } 408945948e49SRyan Zezeski 4090da14cebeSEric Cheng i_mac_perim_exit(mip); 4091da14cebeSEric Cheng } 40924eaa4710SRishi Srivatsavai 40934eaa4710SRishi Srivatsavai /* 40944eaa4710SRishi Srivatsavai * Given a MAC, change the polling state for all its MAC clients. 'enable' is 40954eaa4710SRishi Srivatsavai * B_TRUE to enable polling or B_FALSE to disable. Polling is enabled by 40964eaa4710SRishi Srivatsavai * default. 40974eaa4710SRishi Srivatsavai */ 40984eaa4710SRishi Srivatsavai void 40994eaa4710SRishi Srivatsavai mac_poll_state_change(mac_handle_t mh, boolean_t enable) 41004eaa4710SRishi Srivatsavai { 41014eaa4710SRishi Srivatsavai mac_impl_t *mip = (mac_impl_t *)mh; 41024eaa4710SRishi Srivatsavai mac_client_impl_t *mcip; 41034eaa4710SRishi Srivatsavai 41044eaa4710SRishi Srivatsavai i_mac_perim_enter(mip); 41054eaa4710SRishi Srivatsavai if (enable) 41064eaa4710SRishi Srivatsavai mip->mi_state_flags &= ~MIS_POLL_DISABLE; 41074eaa4710SRishi Srivatsavai else 41084eaa4710SRishi Srivatsavai mip->mi_state_flags |= MIS_POLL_DISABLE; 41094eaa4710SRishi Srivatsavai for (mcip = mip->mi_clients_list; mcip != NULL; 41104eaa4710SRishi Srivatsavai mcip = mcip->mci_client_next) 41114eaa4710SRishi Srivatsavai mac_client_update_classifier(mcip, B_TRUE); 41124eaa4710SRishi Srivatsavai i_mac_perim_exit(mip); 41134eaa4710SRishi Srivatsavai } 4114