1da14cebeSEric Cheng /* 2da14cebeSEric Cheng * CDDL HEADER START 3da14cebeSEric Cheng * 4da14cebeSEric Cheng * The contents of this file are subject to the terms of the 5da14cebeSEric Cheng * Common Development and Distribution License (the "License"). 6da14cebeSEric Cheng * You may not use this file except in compliance with the License. 7da14cebeSEric Cheng * 8da14cebeSEric Cheng * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE 9da14cebeSEric Cheng * or http://www.opensolaris.org/os/licensing. 10da14cebeSEric Cheng * See the License for the specific language governing permissions 11da14cebeSEric Cheng * and limitations under the License. 12da14cebeSEric Cheng * 13da14cebeSEric Cheng * When distributing Covered Code, include this CDDL HEADER in each 14da14cebeSEric Cheng * file and include the License file at usr/src/OPENSOLARIS.LICENSE. 15da14cebeSEric Cheng * If applicable, add the following below this CDDL HEADER, with the 16da14cebeSEric Cheng * fields enclosed by brackets "[]" replaced with your own identifying 17da14cebeSEric Cheng * information: Portions Copyright [yyyy] [name of copyright owner] 18da14cebeSEric Cheng * 19da14cebeSEric Cheng * CDDL HEADER END 20da14cebeSEric Cheng */ 21da14cebeSEric Cheng /* 225adf34bdSRajagopal Kunhappan * Copyright (c) 2008, 2010, Oracle and/or its affiliates. All rights reserved. 2384de666eSRyan Zezeski * Copyright 2018 Joyent, Inc. 24da14cebeSEric Cheng */ 25da14cebeSEric Cheng 26da14cebeSEric Cheng #include <sys/types.h> 27da14cebeSEric Cheng #include <sys/callb.h> 280dc2366fSVenugopal Iyer #include <sys/cpupart.h> 290dc2366fSVenugopal Iyer #include <sys/pool.h> 300dc2366fSVenugopal Iyer #include <sys/pool_pset.h> 31da14cebeSEric Cheng #include <sys/sdt.h> 32da14cebeSEric Cheng #include <sys/strsubr.h> 33da14cebeSEric Cheng #include <sys/strsun.h> 34da14cebeSEric Cheng #include <sys/vlan.h> 35da14cebeSEric Cheng #include <inet/ipsec_impl.h> 36da14cebeSEric Cheng #include <inet/ip_impl.h> 37da14cebeSEric Cheng #include <inet/sadb.h> 38da14cebeSEric Cheng #include <inet/ipsecesp.h> 39da14cebeSEric Cheng #include <inet/ipsecah.h> 40da14cebeSEric Cheng 41da14cebeSEric Cheng #include <sys/mac_impl.h> 42da14cebeSEric Cheng #include <sys/mac_client_impl.h> 43da14cebeSEric Cheng #include <sys/mac_client_priv.h> 44da14cebeSEric Cheng #include <sys/mac_soft_ring.h> 45da14cebeSEric Cheng #include <sys/mac_flow_impl.h> 460dc2366fSVenugopal Iyer #include <sys/mac_stat.h> 47da14cebeSEric Cheng 48da14cebeSEric Cheng static void mac_srs_soft_rings_signal(mac_soft_ring_set_t *, uint_t); 49da14cebeSEric Cheng static void mac_srs_update_fanout_list(mac_soft_ring_set_t *); 50da14cebeSEric Cheng static void mac_srs_poll_unbind(mac_soft_ring_set_t *); 51da14cebeSEric Cheng static void mac_srs_worker_unbind(mac_soft_ring_set_t *); 52da14cebeSEric Cheng static void mac_srs_soft_rings_quiesce(mac_soft_ring_set_t *, uint_t); 53da14cebeSEric Cheng 54da14cebeSEric Cheng static int mac_srs_cpu_setup(cpu_setup_t, int, void *); 55da14cebeSEric Cheng static void mac_srs_worker_bind(mac_soft_ring_set_t *, processorid_t); 56da14cebeSEric Cheng static void mac_srs_poll_bind(mac_soft_ring_set_t *, processorid_t); 57da14cebeSEric Cheng static void mac_srs_threads_unbind(mac_soft_ring_set_t *); 58da14cebeSEric Cheng static void mac_srs_add_glist(mac_soft_ring_set_t *); 59da14cebeSEric Cheng static void mac_srs_remove_glist(mac_soft_ring_set_t *); 60da14cebeSEric Cheng static void mac_srs_fanout_list_free(mac_soft_ring_set_t *); 61da14cebeSEric Cheng static void mac_soft_ring_remove(mac_soft_ring_set_t *, mac_soft_ring_t *); 62da14cebeSEric Cheng 630dc2366fSVenugopal Iyer static int mac_compute_soft_ring_count(flow_entry_t *, int, int); 64da14cebeSEric Cheng static void mac_walk_srs_and_bind(int); 65da14cebeSEric Cheng static void mac_walk_srs_and_unbind(int); 66da14cebeSEric Cheng 67da14cebeSEric Cheng extern boolean_t mac_latency_optimize; 68da14cebeSEric Cheng 69da14cebeSEric Cheng static kmem_cache_t *mac_srs_cache; 70da14cebeSEric Cheng kmem_cache_t *mac_soft_ring_cache; 71da14cebeSEric Cheng 72da14cebeSEric Cheng /* 73da14cebeSEric Cheng * The duration in msec we wait before signalling the soft ring 74da14cebeSEric Cheng * worker thread in case packets get queued. 75da14cebeSEric Cheng */ 76ae6aa22aSVenugopal Iyer uint32_t mac_soft_ring_worker_wait = 0; 77ae6aa22aSVenugopal Iyer 78ae6aa22aSVenugopal Iyer /* 79ae6aa22aSVenugopal Iyer * A global tunable for turning polling on/off. By default, dynamic 80ae6aa22aSVenugopal Iyer * polling is always on and is always very beneficial. It should be 81ae6aa22aSVenugopal Iyer * turned off with absolute care and for the rare workload (very 82ae6aa22aSVenugopal Iyer * low latency sensitive traffic). 83ae6aa22aSVenugopal Iyer */ 84ae6aa22aSVenugopal Iyer int mac_poll_enable = B_TRUE; 85da14cebeSEric Cheng 86da14cebeSEric Cheng /* 87da14cebeSEric Cheng * Need to set mac_soft_ring_max_q_cnt based on bandwidth and perhaps latency. 88da14cebeSEric Cheng * Large values could end up in consuming lot of system memory and cause 89da14cebeSEric Cheng * system hang. 90da14cebeSEric Cheng */ 91ae6aa22aSVenugopal Iyer int mac_soft_ring_max_q_cnt = 1024; 92ae6aa22aSVenugopal Iyer int mac_soft_ring_min_q_cnt = 256; 93ae6aa22aSVenugopal Iyer int mac_soft_ring_poll_thres = 16; 94da14cebeSEric Cheng 95ae6aa22aSVenugopal Iyer boolean_t mac_tx_serialize = B_FALSE; 96da14cebeSEric Cheng 97da14cebeSEric Cheng /* 98da14cebeSEric Cheng * mac_tx_srs_hiwat is the queue depth threshold at which callers of 99da14cebeSEric Cheng * mac_tx() will be notified of flow control condition. 100da14cebeSEric Cheng * 101da14cebeSEric Cheng * TCP does not honour flow control condition sent up by mac_tx(). 102da14cebeSEric Cheng * Thus provision is made for TCP to allow more packets to be queued 103da14cebeSEric Cheng * in SRS upto a maximum of mac_tx_srs_max_q_cnt. 104da14cebeSEric Cheng * 105da14cebeSEric Cheng * Note that mac_tx_srs_hiwat is always be lesser than 106da14cebeSEric Cheng * mac_tx_srs_max_q_cnt. 107da14cebeSEric Cheng */ 108ae6aa22aSVenugopal Iyer uint32_t mac_tx_srs_max_q_cnt = 100000; 109ae6aa22aSVenugopal Iyer uint32_t mac_tx_srs_hiwat = 1000; 110da14cebeSEric Cheng 111da14cebeSEric Cheng /* 112da14cebeSEric Cheng * mac_rx_soft_ring_count, mac_soft_ring_10gig_count: 113da14cebeSEric Cheng * 114da14cebeSEric Cheng * Global tunables that determines the number of soft rings to be used for 115da14cebeSEric Cheng * fanning out incoming traffic on a link. These count will be used only 116da14cebeSEric Cheng * when no explicit set of CPUs was assigned to the data-links. 117da14cebeSEric Cheng * 118da14cebeSEric Cheng * mac_rx_soft_ring_count tunable will come into effect only if 119da14cebeSEric Cheng * mac_soft_ring_enable is set. mac_soft_ring_enable is turned on by 120da14cebeSEric Cheng * default only for sun4v platforms. 121da14cebeSEric Cheng * 122da14cebeSEric Cheng * mac_rx_soft_ring_10gig_count will come into effect if you are running on a 123da14cebeSEric Cheng * 10Gbps link and is not dependent upon mac_soft_ring_enable. 124da14cebeSEric Cheng * 125da14cebeSEric Cheng * The number of soft rings for fanout for a link or a flow is determined 126da14cebeSEric Cheng * by mac_compute_soft_ring_count() routine. This routine will take into 127da14cebeSEric Cheng * account mac_soft_ring_enable, mac_rx_soft_ring_count and 128da14cebeSEric Cheng * mac_rx_soft_ring_10gig_count to determine the soft ring count for a link. 129da14cebeSEric Cheng * 130da14cebeSEric Cheng * If a bandwidth is specified, the determination of the number of soft 131da14cebeSEric Cheng * rings is based on specified bandwidth, CPU speed and number of CPUs in 132da14cebeSEric Cheng * the system. 133da14cebeSEric Cheng */ 134ae6aa22aSVenugopal Iyer uint_t mac_rx_soft_ring_count = 8; 135ae6aa22aSVenugopal Iyer uint_t mac_rx_soft_ring_10gig_count = 8; 136da14cebeSEric Cheng 137da14cebeSEric Cheng /* 138da14cebeSEric Cheng * Every Tx and Rx mac_soft_ring_set_t (mac_srs) created gets added 139da14cebeSEric Cheng * to mac_srs_g_list and mac_srs_g_lock protects mac_srs_g_list. The 140da14cebeSEric Cheng * list is used to walk the list of all MAC threads when a CPU is 141da14cebeSEric Cheng * coming online or going offline. 142da14cebeSEric Cheng */ 143da14cebeSEric Cheng static mac_soft_ring_set_t *mac_srs_g_list = NULL; 144da14cebeSEric Cheng static krwlock_t mac_srs_g_lock; 145da14cebeSEric Cheng 146da14cebeSEric Cheng /* 147da14cebeSEric Cheng * Whether the SRS threads should be bound, or not. 148da14cebeSEric Cheng */ 149ae6aa22aSVenugopal Iyer boolean_t mac_srs_thread_bind = B_TRUE; 150da14cebeSEric Cheng 151da14cebeSEric Cheng /* 1520dc2366fSVenugopal Iyer * Whether Rx/Tx interrupts should be re-targeted. Disabled by default. 1530dc2366fSVenugopal Iyer * dladm command would override this. 154da14cebeSEric Cheng */ 1550dc2366fSVenugopal Iyer boolean_t mac_tx_intr_retarget = B_FALSE; 1560dc2366fSVenugopal Iyer boolean_t mac_rx_intr_retarget = B_FALSE; 157da14cebeSEric Cheng 158da14cebeSEric Cheng /* 159da14cebeSEric Cheng * If cpu bindings are specified by user, then Tx SRS and its soft 160da14cebeSEric Cheng * rings should also be bound to the CPUs specified by user. The 161da14cebeSEric Cheng * CPUs for Tx bindings are at the end of the cpu list provided by 162da14cebeSEric Cheng * the user. If enough CPUs are not available (for Tx and Rx 163da14cebeSEric Cheng * SRSes), then the CPUs are shared by both Tx and Rx SRSes. 164da14cebeSEric Cheng */ 165da14cebeSEric Cheng #define BIND_TX_SRS_AND_SOFT_RINGS(mac_tx_srs, mrp) { \ 166da14cebeSEric Cheng processorid_t cpuid; \ 1670dc2366fSVenugopal Iyer int i; \ 168da14cebeSEric Cheng mac_soft_ring_t *softring; \ 1690dc2366fSVenugopal Iyer mac_cpus_t *srs_cpu; \ 170da14cebeSEric Cheng \ 1710dc2366fSVenugopal Iyer srs_cpu = &mac_tx_srs->srs_cpu; \ 1720dc2366fSVenugopal Iyer cpuid = srs_cpu->mc_tx_fanout_cpus[0]; \ 173da14cebeSEric Cheng mac_srs_worker_bind(mac_tx_srs, cpuid); \ 1740dc2366fSVenugopal Iyer if (MAC_TX_SOFT_RINGS(mac_tx_srs)) { \ 1750dc2366fSVenugopal Iyer for (i = 0; i < mac_tx_srs->srs_tx_ring_count; i++) { \ 1760dc2366fSVenugopal Iyer cpuid = srs_cpu->mc_tx_fanout_cpus[i]; \ 1770dc2366fSVenugopal Iyer softring = mac_tx_srs->srs_tx_soft_rings[i]; \ 1780dc2366fSVenugopal Iyer if (cpuid != -1) { \ 1790dc2366fSVenugopal Iyer (void) mac_soft_ring_bind(softring, \ 1800dc2366fSVenugopal Iyer cpuid); \ 1810dc2366fSVenugopal Iyer } \ 182da14cebeSEric Cheng } \ 183da14cebeSEric Cheng } \ 184da14cebeSEric Cheng } 185da14cebeSEric Cheng 1860dc2366fSVenugopal Iyer /* 1870dc2366fSVenugopal Iyer * Re-targeting is allowed only for exclusive group or for primary. 1880dc2366fSVenugopal Iyer */ 1890dc2366fSVenugopal Iyer #define RETARGETABLE_CLIENT(group, mcip) \ 1900dc2366fSVenugopal Iyer ((((group) != NULL) && \ 1910dc2366fSVenugopal Iyer ((group)->mrg_state == MAC_GROUP_STATE_RESERVED)) || \ 1920dc2366fSVenugopal Iyer mac_is_primary_client(mcip)) 1930dc2366fSVenugopal Iyer 1940dc2366fSVenugopal Iyer #define MAC_RING_RETARGETABLE(ring) \ 1950dc2366fSVenugopal Iyer (((ring) != NULL) && \ 1960dc2366fSVenugopal Iyer ((ring)->mr_info.mri_intr.mi_ddi_handle != NULL) && \ 1970dc2366fSVenugopal Iyer !((ring)->mr_info.mri_intr.mi_ddi_shared)) 1980dc2366fSVenugopal Iyer 1990dc2366fSVenugopal Iyer 200da14cebeSEric Cheng /* INIT and FINI ROUTINES */ 201da14cebeSEric Cheng 202da14cebeSEric Cheng void 203da14cebeSEric Cheng mac_soft_ring_init(void) 204da14cebeSEric Cheng { 205da14cebeSEric Cheng mac_soft_ring_cache = kmem_cache_create("mac_soft_ring_cache", 206da14cebeSEric Cheng sizeof (mac_soft_ring_t), 64, NULL, NULL, NULL, NULL, NULL, 0); 207da14cebeSEric Cheng 208da14cebeSEric Cheng mac_srs_cache = kmem_cache_create("mac_srs_cache", 209da14cebeSEric Cheng sizeof (mac_soft_ring_set_t), 210da14cebeSEric Cheng 64, NULL, NULL, NULL, NULL, NULL, 0); 211da14cebeSEric Cheng 212da14cebeSEric Cheng rw_init(&mac_srs_g_lock, NULL, RW_DEFAULT, NULL); 213da14cebeSEric Cheng mutex_enter(&cpu_lock); 214da14cebeSEric Cheng register_cpu_setup_func(mac_srs_cpu_setup, NULL); 215da14cebeSEric Cheng mutex_exit(&cpu_lock); 216da14cebeSEric Cheng } 217da14cebeSEric Cheng 218da14cebeSEric Cheng void 219da14cebeSEric Cheng mac_soft_ring_finish(void) 220da14cebeSEric Cheng { 221da14cebeSEric Cheng mutex_enter(&cpu_lock); 222da14cebeSEric Cheng unregister_cpu_setup_func(mac_srs_cpu_setup, NULL); 223da14cebeSEric Cheng mutex_exit(&cpu_lock); 224da14cebeSEric Cheng rw_destroy(&mac_srs_g_lock); 225da14cebeSEric Cheng kmem_cache_destroy(mac_soft_ring_cache); 226da14cebeSEric Cheng kmem_cache_destroy(mac_srs_cache); 227da14cebeSEric Cheng } 228da14cebeSEric Cheng 229da14cebeSEric Cheng static void 2300dc2366fSVenugopal Iyer mac_srs_soft_rings_free(mac_soft_ring_set_t *mac_srs) 231da14cebeSEric Cheng { 232da14cebeSEric Cheng mac_soft_ring_t *softring, *next, *head; 233da14cebeSEric Cheng 234da14cebeSEric Cheng /* 235da14cebeSEric Cheng * Synchronize with mac_walk_srs_bind/unbind which are callbacks from 236da14cebeSEric Cheng * DR. The callbacks from DR are called with cpu_lock held, and hence 237da14cebeSEric Cheng * can't wait to grab the mac perimeter. The soft ring list is hence 238da14cebeSEric Cheng * protected for read access by srs_lock. Changing the soft ring list 239da14cebeSEric Cheng * needs the mac perimeter and the srs_lock. 240da14cebeSEric Cheng */ 241da14cebeSEric Cheng mutex_enter(&mac_srs->srs_lock); 242da14cebeSEric Cheng 243da14cebeSEric Cheng head = mac_srs->srs_soft_ring_head; 244da14cebeSEric Cheng mac_srs->srs_soft_ring_head = NULL; 245da14cebeSEric Cheng mac_srs->srs_soft_ring_tail = NULL; 246da14cebeSEric Cheng mac_srs->srs_soft_ring_count = 0; 247da14cebeSEric Cheng 248da14cebeSEric Cheng mutex_exit(&mac_srs->srs_lock); 249da14cebeSEric Cheng 250da14cebeSEric Cheng for (softring = head; softring != NULL; softring = next) { 251da14cebeSEric Cheng next = softring->s_ring_next; 2520dc2366fSVenugopal Iyer mac_soft_ring_free(softring); 253da14cebeSEric Cheng } 254da14cebeSEric Cheng } 255da14cebeSEric Cheng 256da14cebeSEric Cheng static void 257da14cebeSEric Cheng mac_srs_add_glist(mac_soft_ring_set_t *mac_srs) 258da14cebeSEric Cheng { 259da14cebeSEric Cheng ASSERT(mac_srs->srs_next == NULL && mac_srs->srs_prev == NULL); 260da14cebeSEric Cheng ASSERT(MAC_PERIM_HELD((mac_handle_t)mac_srs->srs_mcip->mci_mip)); 261da14cebeSEric Cheng 262da14cebeSEric Cheng rw_enter(&mac_srs_g_lock, RW_WRITER); 263da14cebeSEric Cheng mutex_enter(&mac_srs->srs_lock); 264da14cebeSEric Cheng 265da14cebeSEric Cheng ASSERT((mac_srs->srs_state & SRS_IN_GLIST) == 0); 266da14cebeSEric Cheng 267da14cebeSEric Cheng if (mac_srs_g_list == NULL) { 268da14cebeSEric Cheng mac_srs_g_list = mac_srs; 269da14cebeSEric Cheng } else { 270da14cebeSEric Cheng mac_srs->srs_next = mac_srs_g_list; 271da14cebeSEric Cheng mac_srs_g_list->srs_prev = mac_srs; 272da14cebeSEric Cheng mac_srs->srs_prev = NULL; 273da14cebeSEric Cheng mac_srs_g_list = mac_srs; 274da14cebeSEric Cheng } 275da14cebeSEric Cheng mac_srs->srs_state |= SRS_IN_GLIST; 276da14cebeSEric Cheng 277da14cebeSEric Cheng mutex_exit(&mac_srs->srs_lock); 278da14cebeSEric Cheng rw_exit(&mac_srs_g_lock); 279da14cebeSEric Cheng } 280da14cebeSEric Cheng 281da14cebeSEric Cheng static void 282da14cebeSEric Cheng mac_srs_remove_glist(mac_soft_ring_set_t *mac_srs) 283da14cebeSEric Cheng { 284da14cebeSEric Cheng ASSERT(MAC_PERIM_HELD((mac_handle_t)mac_srs->srs_mcip->mci_mip)); 285da14cebeSEric Cheng 286da14cebeSEric Cheng rw_enter(&mac_srs_g_lock, RW_WRITER); 287da14cebeSEric Cheng mutex_enter(&mac_srs->srs_lock); 288da14cebeSEric Cheng 289da14cebeSEric Cheng ASSERT((mac_srs->srs_state & SRS_IN_GLIST) != 0); 290da14cebeSEric Cheng 291da14cebeSEric Cheng if (mac_srs == mac_srs_g_list) { 292da14cebeSEric Cheng mac_srs_g_list = mac_srs->srs_next; 293da14cebeSEric Cheng if (mac_srs_g_list != NULL) 294da14cebeSEric Cheng mac_srs_g_list->srs_prev = NULL; 295da14cebeSEric Cheng } else { 296da14cebeSEric Cheng mac_srs->srs_prev->srs_next = mac_srs->srs_next; 297da14cebeSEric Cheng if (mac_srs->srs_next != NULL) 298da14cebeSEric Cheng mac_srs->srs_next->srs_prev = mac_srs->srs_prev; 299da14cebeSEric Cheng } 300da14cebeSEric Cheng mac_srs->srs_state &= ~SRS_IN_GLIST; 301da14cebeSEric Cheng 302da14cebeSEric Cheng mutex_exit(&mac_srs->srs_lock); 303da14cebeSEric Cheng rw_exit(&mac_srs_g_lock); 304da14cebeSEric Cheng } 305da14cebeSEric Cheng 306da14cebeSEric Cheng /* POLLING SETUP AND TEAR DOWN ROUTINES */ 307da14cebeSEric Cheng 308da14cebeSEric Cheng /* 309da14cebeSEric Cheng * mac_srs_client_poll_quiesce and mac_srs_client_poll_restart 310da14cebeSEric Cheng * 311da14cebeSEric Cheng * These routines are used to call back into the upper layer 312da14cebeSEric Cheng * (primarily TCP squeue) to stop polling the soft rings or 313da14cebeSEric Cheng * restart polling. 314da14cebeSEric Cheng */ 315da14cebeSEric Cheng void 316da14cebeSEric Cheng mac_srs_client_poll_quiesce(mac_client_impl_t *mcip, 317da14cebeSEric Cheng mac_soft_ring_set_t *mac_srs) 318da14cebeSEric Cheng { 319da14cebeSEric Cheng mac_soft_ring_t *softring; 320da14cebeSEric Cheng 321da14cebeSEric Cheng ASSERT(MAC_PERIM_HELD((mac_handle_t)mcip->mci_mip)); 322da14cebeSEric Cheng 323da14cebeSEric Cheng if (!(mac_srs->srs_type & SRST_CLIENT_POLL_ENABLED)) { 324da14cebeSEric Cheng ASSERT(!(mac_srs->srs_type & SRST_DLS_BYPASS)); 325da14cebeSEric Cheng return; 326da14cebeSEric Cheng } 327da14cebeSEric Cheng 328da14cebeSEric Cheng for (softring = mac_srs->srs_soft_ring_head; 329da14cebeSEric Cheng softring != NULL; softring = softring->s_ring_next) { 330da14cebeSEric Cheng if ((softring->s_ring_type & ST_RING_TCP) && 331da14cebeSEric Cheng (softring->s_ring_rx_arg2 != NULL)) { 332da14cebeSEric Cheng mcip->mci_resource_quiesce(mcip->mci_resource_arg, 333da14cebeSEric Cheng softring->s_ring_rx_arg2); 334da14cebeSEric Cheng } 335da14cebeSEric Cheng } 336da14cebeSEric Cheng } 337da14cebeSEric Cheng 338da14cebeSEric Cheng void 339da14cebeSEric Cheng mac_srs_client_poll_restart(mac_client_impl_t *mcip, 340da14cebeSEric Cheng mac_soft_ring_set_t *mac_srs) 341da14cebeSEric Cheng { 342da14cebeSEric Cheng mac_soft_ring_t *softring; 343da14cebeSEric Cheng 344da14cebeSEric Cheng ASSERT(MAC_PERIM_HELD((mac_handle_t)mcip->mci_mip)); 345da14cebeSEric Cheng 346da14cebeSEric Cheng if (!(mac_srs->srs_type & SRST_CLIENT_POLL_ENABLED)) { 347da14cebeSEric Cheng ASSERT(!(mac_srs->srs_type & SRST_DLS_BYPASS)); 348da14cebeSEric Cheng return; 349da14cebeSEric Cheng } 350da14cebeSEric Cheng 351da14cebeSEric Cheng for (softring = mac_srs->srs_soft_ring_head; 352da14cebeSEric Cheng softring != NULL; softring = softring->s_ring_next) { 353da14cebeSEric Cheng if ((softring->s_ring_type & ST_RING_TCP) && 354da14cebeSEric Cheng (softring->s_ring_rx_arg2 != NULL)) { 355da14cebeSEric Cheng mcip->mci_resource_restart(mcip->mci_resource_arg, 356da14cebeSEric Cheng softring->s_ring_rx_arg2); 357da14cebeSEric Cheng } 358da14cebeSEric Cheng } 359da14cebeSEric Cheng } 360da14cebeSEric Cheng 361da14cebeSEric Cheng /* 362da14cebeSEric Cheng * Register the given SRS and associated soft rings with the consumer and 363da14cebeSEric Cheng * enable the polling interface used by the consumer.(i.e IP) over this 364da14cebeSEric Cheng * SRS and associated soft rings. 365da14cebeSEric Cheng */ 366da14cebeSEric Cheng void 367da14cebeSEric Cheng mac_srs_client_poll_enable(mac_client_impl_t *mcip, 368da14cebeSEric Cheng mac_soft_ring_set_t *mac_srs) 369da14cebeSEric Cheng { 370da14cebeSEric Cheng mac_rx_fifo_t mrf; 371da14cebeSEric Cheng mac_soft_ring_t *softring; 372da14cebeSEric Cheng 373da14cebeSEric Cheng ASSERT(mac_srs->srs_mcip == mcip); 374da14cebeSEric Cheng ASSERT(MAC_PERIM_HELD((mac_handle_t)mcip->mci_mip)); 375da14cebeSEric Cheng 376da14cebeSEric Cheng if (!(mcip->mci_state_flags & MCIS_CLIENT_POLL_CAPABLE)) 377da14cebeSEric Cheng return; 378da14cebeSEric Cheng 379da14cebeSEric Cheng bzero(&mrf, sizeof (mac_rx_fifo_t)); 380da14cebeSEric Cheng mrf.mrf_type = MAC_RX_FIFO; 381da14cebeSEric Cheng 382da14cebeSEric Cheng /* 383da14cebeSEric Cheng * A SRS is capable of acting as a soft ring for cases 384da14cebeSEric Cheng * where no fanout is needed. This is the case for userland 385da14cebeSEric Cheng * flows. 386da14cebeSEric Cheng */ 387da14cebeSEric Cheng if (mac_srs->srs_type & SRST_NO_SOFT_RINGS) 388da14cebeSEric Cheng return; 389da14cebeSEric Cheng 390da14cebeSEric Cheng mrf.mrf_receive = (mac_receive_t)mac_soft_ring_poll; 391da14cebeSEric Cheng mrf.mrf_intr_enable = (mac_intr_enable_t)mac_soft_ring_intr_enable; 392da14cebeSEric Cheng mrf.mrf_intr_disable = (mac_intr_disable_t)mac_soft_ring_intr_disable; 393da14cebeSEric Cheng mac_srs->srs_type |= SRST_CLIENT_POLL_ENABLED; 394da14cebeSEric Cheng 395da14cebeSEric Cheng softring = mac_srs->srs_soft_ring_head; 396da14cebeSEric Cheng while (softring != NULL) { 397da14cebeSEric Cheng if (softring->s_ring_type & (ST_RING_TCP | ST_RING_UDP)) { 398da14cebeSEric Cheng /* 399da14cebeSEric Cheng * TCP and UDP support DLS bypass. Squeue polling 400da14cebeSEric Cheng * support implies DLS bypass since the squeue poll 401da14cebeSEric Cheng * path does not have DLS processing. 402da14cebeSEric Cheng */ 403da14cebeSEric Cheng mac_soft_ring_dls_bypass(softring, 404da14cebeSEric Cheng mcip->mci_direct_rx_fn, mcip->mci_direct_rx_arg); 405da14cebeSEric Cheng } 406da14cebeSEric Cheng /* 407da14cebeSEric Cheng * Non-TCP protocols don't support squeues. Hence we don't 408da14cebeSEric Cheng * make any ring addition callbacks for non-TCP rings 409da14cebeSEric Cheng */ 410da14cebeSEric Cheng if (!(softring->s_ring_type & ST_RING_TCP)) { 411da14cebeSEric Cheng softring->s_ring_rx_arg2 = NULL; 412da14cebeSEric Cheng softring = softring->s_ring_next; 413da14cebeSEric Cheng continue; 414da14cebeSEric Cheng } 415da14cebeSEric Cheng mrf.mrf_rx_arg = softring; 416da14cebeSEric Cheng mrf.mrf_intr_handle = (mac_intr_handle_t)softring; 417da14cebeSEric Cheng mrf.mrf_cpu_id = softring->s_ring_cpuid; 418da14cebeSEric Cheng mrf.mrf_flow_priority = mac_srs->srs_pri; 419da14cebeSEric Cheng 420da14cebeSEric Cheng softring->s_ring_rx_arg2 = mcip->mci_resource_add( 421da14cebeSEric Cheng mcip->mci_resource_arg, (mac_resource_t *)&mrf); 422da14cebeSEric Cheng 423da14cebeSEric Cheng softring = softring->s_ring_next; 424da14cebeSEric Cheng } 425da14cebeSEric Cheng } 426da14cebeSEric Cheng 427da14cebeSEric Cheng /* 428da14cebeSEric Cheng * Unregister the given SRS and associated soft rings with the consumer and 429da14cebeSEric Cheng * disable the polling interface used by the consumer.(i.e IP) over this 430da14cebeSEric Cheng * SRS and associated soft rings. 431da14cebeSEric Cheng */ 432da14cebeSEric Cheng void 433da14cebeSEric Cheng mac_srs_client_poll_disable(mac_client_impl_t *mcip, 434da14cebeSEric Cheng mac_soft_ring_set_t *mac_srs) 435da14cebeSEric Cheng { 436da14cebeSEric Cheng mac_soft_ring_t *softring; 437da14cebeSEric Cheng 438da14cebeSEric Cheng ASSERT(MAC_PERIM_HELD((mac_handle_t)mcip->mci_mip)); 439da14cebeSEric Cheng 440da14cebeSEric Cheng /* 441da14cebeSEric Cheng * A SRS is capable of acting as a soft ring for cases 442da14cebeSEric Cheng * where no protocol fanout is needed. This is the case 443da14cebeSEric Cheng * for userland flows. Nothing to do here. 444da14cebeSEric Cheng */ 445da14cebeSEric Cheng if (mac_srs->srs_type & SRST_NO_SOFT_RINGS) 446da14cebeSEric Cheng return; 447da14cebeSEric Cheng 448da14cebeSEric Cheng mutex_enter(&mac_srs->srs_lock); 449da14cebeSEric Cheng if (!(mac_srs->srs_type & SRST_CLIENT_POLL_ENABLED)) { 450da14cebeSEric Cheng ASSERT(!(mac_srs->srs_type & SRST_DLS_BYPASS)); 451da14cebeSEric Cheng mutex_exit(&mac_srs->srs_lock); 452da14cebeSEric Cheng return; 453da14cebeSEric Cheng } 454da14cebeSEric Cheng mac_srs->srs_type &= ~(SRST_CLIENT_POLL_ENABLED | SRST_DLS_BYPASS); 455da14cebeSEric Cheng mutex_exit(&mac_srs->srs_lock); 456da14cebeSEric Cheng 457da14cebeSEric Cheng /* 458da14cebeSEric Cheng * DLS bypass is now disabled in the case of both TCP and UDP. 459da14cebeSEric Cheng * Reset the soft ring callbacks to the standard 'mac_rx_deliver' 460da14cebeSEric Cheng * callback. In addition, in the case of TCP, invoke IP's callback 461da14cebeSEric Cheng * for ring removal. 462da14cebeSEric Cheng */ 463da14cebeSEric Cheng for (softring = mac_srs->srs_soft_ring_head; 464da14cebeSEric Cheng softring != NULL; softring = softring->s_ring_next) { 465da14cebeSEric Cheng if (!(softring->s_ring_type & (ST_RING_UDP | ST_RING_TCP))) 466da14cebeSEric Cheng continue; 467da14cebeSEric Cheng 468da14cebeSEric Cheng if ((softring->s_ring_type & ST_RING_TCP) && 469da14cebeSEric Cheng softring->s_ring_rx_arg2 != NULL) { 470da14cebeSEric Cheng mcip->mci_resource_remove(mcip->mci_resource_arg, 471da14cebeSEric Cheng softring->s_ring_rx_arg2); 472da14cebeSEric Cheng } 473da14cebeSEric Cheng 474da14cebeSEric Cheng mutex_enter(&softring->s_ring_lock); 475da14cebeSEric Cheng while (softring->s_ring_state & S_RING_PROC) { 476da14cebeSEric Cheng softring->s_ring_state |= S_RING_CLIENT_WAIT; 477da14cebeSEric Cheng cv_wait(&softring->s_ring_client_cv, 478da14cebeSEric Cheng &softring->s_ring_lock); 479da14cebeSEric Cheng } 480da14cebeSEric Cheng softring->s_ring_state &= ~S_RING_CLIENT_WAIT; 481da14cebeSEric Cheng softring->s_ring_rx_arg2 = NULL; 482da14cebeSEric Cheng softring->s_ring_rx_func = mac_rx_deliver; 483da14cebeSEric Cheng softring->s_ring_rx_arg1 = mcip; 484da14cebeSEric Cheng mutex_exit(&softring->s_ring_lock); 485da14cebeSEric Cheng } 486da14cebeSEric Cheng } 487da14cebeSEric Cheng 488da14cebeSEric Cheng /* 489da14cebeSEric Cheng * Enable or disable poll capability of the SRS on the underlying Rx ring. 490da14cebeSEric Cheng * 491da14cebeSEric Cheng * There is a need to enable or disable the poll capability of an SRS over an 492da14cebeSEric Cheng * Rx ring depending on the number of mac clients sharing the ring and also 493da14cebeSEric Cheng * whether user flows are configured on it. However the poll state is actively 494da14cebeSEric Cheng * manipulated by the SRS worker and poll threads and uncoordinated changes by 495da14cebeSEric Cheng * yet another thread to the underlying capability can surprise them leading 496da14cebeSEric Cheng * to assert failures. Instead we quiesce the SRS, make the changes and then 497da14cebeSEric Cheng * restart the SRS. 498da14cebeSEric Cheng */ 499da14cebeSEric Cheng static void 500da14cebeSEric Cheng mac_srs_poll_state_change(mac_soft_ring_set_t *mac_srs, 501da14cebeSEric Cheng boolean_t turn_off_poll_capab, mac_rx_func_t rx_func) 502da14cebeSEric Cheng { 503da14cebeSEric Cheng boolean_t need_restart = B_FALSE; 504da14cebeSEric Cheng mac_srs_rx_t *srs_rx = &mac_srs->srs_rx; 505da14cebeSEric Cheng mac_ring_t *ring; 506da14cebeSEric Cheng 507da14cebeSEric Cheng if (!SRS_QUIESCED(mac_srs)) { 508da14cebeSEric Cheng mac_rx_srs_quiesce(mac_srs, SRS_QUIESCE); 509da14cebeSEric Cheng need_restart = B_TRUE; 510da14cebeSEric Cheng } 511da14cebeSEric Cheng 512da14cebeSEric Cheng ring = mac_srs->srs_ring; 513da14cebeSEric Cheng if ((ring != NULL) && 514da14cebeSEric Cheng (ring->mr_classify_type == MAC_HW_CLASSIFIER)) { 515da14cebeSEric Cheng if (turn_off_poll_capab) 516da14cebeSEric Cheng mac_srs->srs_state &= ~SRS_POLLING_CAPAB; 517ae6aa22aSVenugopal Iyer else if (mac_poll_enable) 518da14cebeSEric Cheng mac_srs->srs_state |= SRS_POLLING_CAPAB; 519da14cebeSEric Cheng } 520da14cebeSEric Cheng srs_rx->sr_lower_proc = rx_func; 521da14cebeSEric Cheng 522da14cebeSEric Cheng if (need_restart) 523da14cebeSEric Cheng mac_rx_srs_restart(mac_srs); 524da14cebeSEric Cheng } 525da14cebeSEric Cheng 526da14cebeSEric Cheng /* CPU RECONFIGURATION AND FANOUT COMPUTATION ROUTINES */ 527da14cebeSEric Cheng 528da14cebeSEric Cheng /* 529da14cebeSEric Cheng * Return the next CPU to be used to bind a MAC kernel thread. 5300dc2366fSVenugopal Iyer * If a cpupart is specified, the cpu chosen must be from that 5310dc2366fSVenugopal Iyer * cpu partition. 532da14cebeSEric Cheng */ 533da14cebeSEric Cheng static processorid_t 5340dc2366fSVenugopal Iyer mac_next_bind_cpu(cpupart_t *cpupart) 535da14cebeSEric Cheng { 5360dc2366fSVenugopal Iyer static cpu_t *cp = NULL; 5370dc2366fSVenugopal Iyer cpu_t *cp_start; 538da14cebeSEric Cheng 539da14cebeSEric Cheng ASSERT(MUTEX_HELD(&cpu_lock)); 540da14cebeSEric Cheng 5410dc2366fSVenugopal Iyer if (cp == NULL) 5420dc2366fSVenugopal Iyer cp = cpu_list; 543da14cebeSEric Cheng 5440dc2366fSVenugopal Iyer cp = cp->cpu_next_onln; 5450dc2366fSVenugopal Iyer cp_start = cp; 5460dc2366fSVenugopal Iyer 5470dc2366fSVenugopal Iyer do { 5480dc2366fSVenugopal Iyer if ((cpupart == NULL) || (cp->cpu_part == cpupart)) 5490dc2366fSVenugopal Iyer return (cp->cpu_id); 5500dc2366fSVenugopal Iyer 5510dc2366fSVenugopal Iyer } while ((cp = cp->cpu_next_onln) != cp_start); 5520dc2366fSVenugopal Iyer 55336f99a58SToomas Soome return (-1); /* No matching CPU found online */ 554da14cebeSEric Cheng } 555da14cebeSEric Cheng 556da14cebeSEric Cheng /* ARGSUSED */ 557da14cebeSEric Cheng static int 558da14cebeSEric Cheng mac_srs_cpu_setup(cpu_setup_t what, int id, void *arg) 559da14cebeSEric Cheng { 560da14cebeSEric Cheng ASSERT(MUTEX_HELD(&cpu_lock)); 561da14cebeSEric Cheng switch (what) { 562da14cebeSEric Cheng case CPU_CONFIG: 563da14cebeSEric Cheng case CPU_ON: 564da14cebeSEric Cheng case CPU_CPUPART_IN: 565da14cebeSEric Cheng mac_walk_srs_and_bind(id); 566da14cebeSEric Cheng break; 567da14cebeSEric Cheng 568da14cebeSEric Cheng case CPU_UNCONFIG: 569da14cebeSEric Cheng case CPU_OFF: 570da14cebeSEric Cheng case CPU_CPUPART_OUT: 571da14cebeSEric Cheng mac_walk_srs_and_unbind(id); 572da14cebeSEric Cheng break; 573da14cebeSEric Cheng 574da14cebeSEric Cheng default: 575da14cebeSEric Cheng break; 576da14cebeSEric Cheng } 577da14cebeSEric Cheng return (0); 578da14cebeSEric Cheng } 579da14cebeSEric Cheng 580da14cebeSEric Cheng /* 581da14cebeSEric Cheng * mac_compute_soft_ring_count(): 582da14cebeSEric Cheng * 583da14cebeSEric Cheng * This routine computes the number of soft rings needed to handle incoming 584da14cebeSEric Cheng * load given a flow_entry. 585da14cebeSEric Cheng * 586da14cebeSEric Cheng * The routine does the following: 587da14cebeSEric Cheng * 1) soft rings will be created if mac_soft_ring_enable is set. 588da14cebeSEric Cheng * 2) If the underlying link is a 10Gbps link, then soft rings will be 589da14cebeSEric Cheng * created even if mac_soft_ring_enable is not set. The number of soft 590da14cebeSEric Cheng * rings, so created, will equal mac_rx_soft_ring_10gig_count. 591da14cebeSEric Cheng * 3) On a sun4v platform (i.e., mac_soft_ring_enable is set), 2 times the 592da14cebeSEric Cheng * mac_rx_soft_ring_10gig_count number of soft rings will be created for a 593da14cebeSEric Cheng * 10Gbps link. 594da14cebeSEric Cheng * 595da14cebeSEric Cheng * If a bandwidth limit is specified, the number that gets computed is 596da14cebeSEric Cheng * dependent upon CPU speed, the number of Rx rings configured, and 597da14cebeSEric Cheng * the bandwidth limit. 598da14cebeSEric Cheng * If more Rx rings are available, less number of soft rings is needed. 599da14cebeSEric Cheng * 600da14cebeSEric Cheng * mac_use_bw_heuristic is another "hidden" variable that can be used to 601da14cebeSEric Cheng * override the default use of soft ring count computation. Depending upon 602da14cebeSEric Cheng * the usefulness of it, mac_use_bw_heuristic can later be made into a 603da14cebeSEric Cheng * data-link property or removed altogether. 604da14cebeSEric Cheng * 605da14cebeSEric Cheng * TODO: Cleanup and tighten some of the assumptions. 606da14cebeSEric Cheng */ 607da14cebeSEric Cheng boolean_t mac_use_bw_heuristic = B_TRUE; 608da14cebeSEric Cheng static int 6090dc2366fSVenugopal Iyer mac_compute_soft_ring_count(flow_entry_t *flent, int rx_srs_cnt, int maxcpus) 610da14cebeSEric Cheng { 611da14cebeSEric Cheng uint64_t cpu_speed, bw = 0; 612da14cebeSEric Cheng int srings = 0; 613da14cebeSEric Cheng boolean_t bw_enabled = B_FALSE; 614da14cebeSEric Cheng 615da14cebeSEric Cheng ASSERT(!(flent->fe_type & FLOW_USER)); 616da14cebeSEric Cheng if (flent->fe_resource_props.mrp_mask & MRP_MAXBW && 617da14cebeSEric Cheng mac_use_bw_heuristic) { 618da14cebeSEric Cheng /* bandwidth enabled */ 619da14cebeSEric Cheng bw_enabled = B_TRUE; 620da14cebeSEric Cheng bw = flent->fe_resource_props.mrp_maxbw; 621da14cebeSEric Cheng } 622da14cebeSEric Cheng if (!bw_enabled) { 623da14cebeSEric Cheng /* No bandwidth enabled */ 624da14cebeSEric Cheng if (mac_soft_ring_enable) 625da14cebeSEric Cheng srings = mac_rx_soft_ring_count; 626da14cebeSEric Cheng 627da14cebeSEric Cheng /* Is this a 10Gig link? */ 628da14cebeSEric Cheng flent->fe_nic_speed = mac_client_stat_get(flent->fe_mcip, 629da14cebeSEric Cheng MAC_STAT_IFSPEED); 630da14cebeSEric Cheng /* convert to Mbps */ 631da14cebeSEric Cheng if (((flent->fe_nic_speed)/1000000) > 1000 && 632da14cebeSEric Cheng mac_rx_soft_ring_10gig_count > 0) { 633da14cebeSEric Cheng /* This is a 10Gig link */ 634da14cebeSEric Cheng srings = mac_rx_soft_ring_10gig_count; 635da14cebeSEric Cheng /* 636da14cebeSEric Cheng * Use 2 times mac_rx_soft_ring_10gig_count for 637da14cebeSEric Cheng * sun4v systems. 638da14cebeSEric Cheng */ 639da14cebeSEric Cheng if (mac_soft_ring_enable) 640da14cebeSEric Cheng srings = srings * 2; 641da14cebeSEric Cheng } 642da14cebeSEric Cheng } else { 643da14cebeSEric Cheng /* 644da14cebeSEric Cheng * Soft ring computation using CPU speed and specified 645da14cebeSEric Cheng * bandwidth limit. 646da14cebeSEric Cheng */ 647da14cebeSEric Cheng /* Assumption: all CPUs have the same frequency */ 648da14cebeSEric Cheng cpu_speed = (uint64_t)CPU->cpu_type_info.pi_clock; 649da14cebeSEric Cheng 650da14cebeSEric Cheng /* cpu_speed is in MHz; make bw in units of Mbps. */ 651da14cebeSEric Cheng bw = bw/1000000; 652da14cebeSEric Cheng 653da14cebeSEric Cheng if (bw >= 1000) { 654da14cebeSEric Cheng /* 655da14cebeSEric Cheng * bw is greater than or equal to 1Gbps. 656da14cebeSEric Cheng * The number of soft rings required is a function 657da14cebeSEric Cheng * of bandwidth and CPU speed. To keep this simple, 658da14cebeSEric Cheng * let's use this rule: 1GHz CPU can handle 1Gbps. 659da14cebeSEric Cheng * If bw is less than 1 Gbps, then there is no need 660da14cebeSEric Cheng * for soft rings. Assumption is that CPU speeds 661da14cebeSEric Cheng * (on modern systems) are at least 1GHz. 662da14cebeSEric Cheng */ 663da14cebeSEric Cheng srings = bw/cpu_speed; 664da14cebeSEric Cheng if (srings <= 1 && mac_soft_ring_enable) { 665da14cebeSEric Cheng /* 666da14cebeSEric Cheng * Give at least 2 soft rings 667da14cebeSEric Cheng * for sun4v systems 668da14cebeSEric Cheng */ 669da14cebeSEric Cheng srings = 2; 670da14cebeSEric Cheng } 671da14cebeSEric Cheng } 672da14cebeSEric Cheng } 673da14cebeSEric Cheng /* 674da14cebeSEric Cheng * If the flent has multiple Rx SRSs, then each SRS need not 675da14cebeSEric Cheng * have that many soft rings on top of it. The number of 676da14cebeSEric Cheng * soft rings for each Rx SRS is found by dividing srings by 677da14cebeSEric Cheng * rx_srs_cnt. 678da14cebeSEric Cheng */ 679da14cebeSEric Cheng if (rx_srs_cnt > 1) { 680da14cebeSEric Cheng int remainder; 681da14cebeSEric Cheng 682da14cebeSEric Cheng remainder = srings%rx_srs_cnt; 683da14cebeSEric Cheng srings = srings/rx_srs_cnt; 684da14cebeSEric Cheng if (remainder != 0) 685da14cebeSEric Cheng srings++; 686da14cebeSEric Cheng /* 687da14cebeSEric Cheng * Fanning out to 1 soft ring is not very useful. 688da14cebeSEric Cheng * Set it as well to 0 and mac_srs_fanout_init() 689da14cebeSEric Cheng * will take care of creating a single soft ring 690da14cebeSEric Cheng * for proto fanout. 691da14cebeSEric Cheng */ 692da14cebeSEric Cheng if (srings == 1) 693da14cebeSEric Cheng srings = 0; 694da14cebeSEric Cheng } 695da14cebeSEric Cheng /* Do some more massaging */ 6960dc2366fSVenugopal Iyer srings = min(srings, maxcpus); 697da14cebeSEric Cheng srings = min(srings, MAX_SR_FANOUT); 698da14cebeSEric Cheng return (srings); 699da14cebeSEric Cheng } 700da14cebeSEric Cheng 701da14cebeSEric Cheng /* 7020dc2366fSVenugopal Iyer * mac_tx_cpu_init: 7030dc2366fSVenugopal Iyer * set up CPUs for Tx interrupt re-targeting and Tx worker 7040dc2366fSVenugopal Iyer * thread binding 7050dc2366fSVenugopal Iyer */ 7060dc2366fSVenugopal Iyer static void 7070dc2366fSVenugopal Iyer mac_tx_cpu_init(flow_entry_t *flent, mac_resource_props_t *mrp, 7080dc2366fSVenugopal Iyer cpupart_t *cpupart) 7090dc2366fSVenugopal Iyer { 7100dc2366fSVenugopal Iyer mac_soft_ring_set_t *tx_srs = flent->fe_tx_srs; 7110dc2366fSVenugopal Iyer mac_srs_tx_t *srs_tx = &tx_srs->srs_tx; 7120dc2366fSVenugopal Iyer mac_cpus_t *srs_cpu = &tx_srs->srs_cpu; 7130dc2366fSVenugopal Iyer mac_soft_ring_t *sringp; 7140dc2366fSVenugopal Iyer mac_ring_t *ring; 7150dc2366fSVenugopal Iyer processorid_t worker_cpuid; 7160dc2366fSVenugopal Iyer boolean_t retargetable_client = B_FALSE; 7170dc2366fSVenugopal Iyer int i, j; 7180dc2366fSVenugopal Iyer 7190dc2366fSVenugopal Iyer if (RETARGETABLE_CLIENT((mac_group_t *)flent->fe_tx_ring_group, 7200dc2366fSVenugopal Iyer flent->fe_mcip)) { 7210dc2366fSVenugopal Iyer retargetable_client = B_TRUE; 7220dc2366fSVenugopal Iyer } 7230dc2366fSVenugopal Iyer 7240dc2366fSVenugopal Iyer if (MAC_TX_SOFT_RINGS(tx_srs)) { 7250dc2366fSVenugopal Iyer if (mrp != NULL) 7260dc2366fSVenugopal Iyer j = mrp->mrp_ncpus - 1; 7270dc2366fSVenugopal Iyer for (i = 0; i < tx_srs->srs_tx_ring_count; i++) { 7280dc2366fSVenugopal Iyer if (mrp != NULL) { 7290dc2366fSVenugopal Iyer if (j < 0) 7300dc2366fSVenugopal Iyer j = mrp->mrp_ncpus - 1; 7310dc2366fSVenugopal Iyer worker_cpuid = mrp->mrp_cpu[j]; 7320dc2366fSVenugopal Iyer } else { 7330dc2366fSVenugopal Iyer /* 7340dc2366fSVenugopal Iyer * Bind interrupt to the next CPU available 7350dc2366fSVenugopal Iyer * and leave the worker unbound. 7360dc2366fSVenugopal Iyer */ 7370dc2366fSVenugopal Iyer worker_cpuid = -1; 7380dc2366fSVenugopal Iyer } 7390dc2366fSVenugopal Iyer sringp = tx_srs->srs_tx_soft_rings[i]; 7400dc2366fSVenugopal Iyer ring = (mac_ring_t *)sringp->s_ring_tx_arg2; 7410dc2366fSVenugopal Iyer srs_cpu->mc_tx_fanout_cpus[i] = worker_cpuid; 7420dc2366fSVenugopal Iyer if (MAC_RING_RETARGETABLE(ring) && 7430dc2366fSVenugopal Iyer retargetable_client) { 7440dc2366fSVenugopal Iyer mutex_enter(&cpu_lock); 7450dc2366fSVenugopal Iyer srs_cpu->mc_tx_intr_cpu[i] = 7460dc2366fSVenugopal Iyer (mrp != NULL) ? mrp->mrp_cpu[j] : 7470dc2366fSVenugopal Iyer (mac_tx_intr_retarget ? 7480dc2366fSVenugopal Iyer mac_next_bind_cpu(cpupart) : -1); 7490dc2366fSVenugopal Iyer mutex_exit(&cpu_lock); 7500dc2366fSVenugopal Iyer } else { 7510dc2366fSVenugopal Iyer srs_cpu->mc_tx_intr_cpu[i] = -1; 7520dc2366fSVenugopal Iyer } 7530dc2366fSVenugopal Iyer if (mrp != NULL) 7540dc2366fSVenugopal Iyer j--; 7550dc2366fSVenugopal Iyer } 7560dc2366fSVenugopal Iyer } else { 7570dc2366fSVenugopal Iyer /* Tx mac_ring_handle_t is stored in st_arg2 */ 7580dc2366fSVenugopal Iyer srs_cpu->mc_tx_fanout_cpus[0] = 7590dc2366fSVenugopal Iyer (mrp != NULL) ? mrp->mrp_cpu[mrp->mrp_ncpus - 1] : -1; 7600dc2366fSVenugopal Iyer ring = (mac_ring_t *)srs_tx->st_arg2; 7610dc2366fSVenugopal Iyer if (MAC_RING_RETARGETABLE(ring) && retargetable_client) { 7620dc2366fSVenugopal Iyer mutex_enter(&cpu_lock); 7630dc2366fSVenugopal Iyer srs_cpu->mc_tx_intr_cpu[0] = (mrp != NULL) ? 7640dc2366fSVenugopal Iyer mrp->mrp_cpu[mrp->mrp_ncpus - 1] : 7650dc2366fSVenugopal Iyer (mac_tx_intr_retarget ? 7660dc2366fSVenugopal Iyer mac_next_bind_cpu(cpupart) : -1); 7670dc2366fSVenugopal Iyer mutex_exit(&cpu_lock); 7680dc2366fSVenugopal Iyer } else { 7690dc2366fSVenugopal Iyer srs_cpu->mc_tx_intr_cpu[0] = -1; 7700dc2366fSVenugopal Iyer } 7710dc2366fSVenugopal Iyer } 7720dc2366fSVenugopal Iyer } 7730dc2366fSVenugopal Iyer 7740dc2366fSVenugopal Iyer /* 775da14cebeSEric Cheng * Assignment of user specified CPUs to a link. 776da14cebeSEric Cheng * 777da14cebeSEric Cheng * Minimum CPUs required to get an optimal assignmet: 778da14cebeSEric Cheng * For each Rx SRS, atleast two CPUs are needed if mac_latency_optimize 779da14cebeSEric Cheng * flag is set -- one for polling, one for fanout soft ring. 780da14cebeSEric Cheng * If mac_latency_optimize is not set, then 3 CPUs are needed -- one 781da14cebeSEric Cheng * for polling, one for SRS worker thread and one for fanout soft ring. 782da14cebeSEric Cheng * 783da14cebeSEric Cheng * The CPUs needed for Tx side is equal to the number of Tx rings 784da14cebeSEric Cheng * the link is using. 785da14cebeSEric Cheng * 786da14cebeSEric Cheng * mac_flow_user_cpu_init() categorizes the CPU assignment depending 787da14cebeSEric Cheng * upon the number of CPUs in 3 different buckets. 788da14cebeSEric Cheng * 789da14cebeSEric Cheng * In the first bucket, the most optimal case is handled. The user has 790da14cebeSEric Cheng * passed enough number of CPUs and every thread gets its own CPU. 791da14cebeSEric Cheng * 792da14cebeSEric Cheng * The second and third are the sub-optimal cases. Enough CPUs are not 793da14cebeSEric Cheng * available. 794da14cebeSEric Cheng * 795da14cebeSEric Cheng * The second bucket handles the case where atleast one distinct CPU is 796da14cebeSEric Cheng * is available for each of the Rx rings (Rx SRSes) and Tx rings (Tx 797da14cebeSEric Cheng * SRS or soft rings). 798da14cebeSEric Cheng * 799da14cebeSEric Cheng * In the third case (worst case scenario), specified CPU count is less 800da14cebeSEric Cheng * than the Rx rings configured for the link. In this case, we round 801da14cebeSEric Cheng * robin the CPUs among the Rx SRSes and Tx SRS/soft rings. 802da14cebeSEric Cheng */ 803da14cebeSEric Cheng static void 804da14cebeSEric Cheng mac_flow_user_cpu_init(flow_entry_t *flent, mac_resource_props_t *mrp) 805da14cebeSEric Cheng { 806da14cebeSEric Cheng mac_soft_ring_set_t *rx_srs, *tx_srs; 807da14cebeSEric Cheng int i, srs_cnt; 808da14cebeSEric Cheng mac_cpus_t *srs_cpu; 809da14cebeSEric Cheng int no_of_cpus, cpu_cnt; 810da14cebeSEric Cheng int rx_srs_cnt, reqd_rx_cpu_cnt; 811da14cebeSEric Cheng int fanout_cpu_cnt, reqd_tx_cpu_cnt; 812da14cebeSEric Cheng int reqd_poll_worker_cnt, fanout_cnt_per_srs; 8130dc2366fSVenugopal Iyer mac_resource_props_t *emrp = &flent->fe_effective_props; 814da14cebeSEric Cheng 815da14cebeSEric Cheng ASSERT(mrp->mrp_fanout_mode == MCM_CPUS); 816da14cebeSEric Cheng /* 817da14cebeSEric Cheng * The check for nbc_ncpus to be within limits for 818da14cebeSEric Cheng * the user specified case was done earlier and if 819da14cebeSEric Cheng * not within limits, an error would have been 820da14cebeSEric Cheng * returned to the user. 821da14cebeSEric Cheng */ 8225adf34bdSRajagopal Kunhappan ASSERT(mrp->mrp_ncpus > 0); 823da14cebeSEric Cheng 824da14cebeSEric Cheng no_of_cpus = mrp->mrp_ncpus; 825da14cebeSEric Cheng 8260dc2366fSVenugopal Iyer if (mrp->mrp_rx_intr_cpu != -1) { 827da14cebeSEric Cheng /* 828da14cebeSEric Cheng * interrupt has been re-targetted. Poll 829da14cebeSEric Cheng * thread needs to be bound to interrupt 8300dc2366fSVenugopal Iyer * CPU. 831da14cebeSEric Cheng * 832da14cebeSEric Cheng * Find where in the list is the intr 833da14cebeSEric Cheng * CPU and swap it with the first one. 834da14cebeSEric Cheng * We will be using the first CPU in the 835da14cebeSEric Cheng * list for poll. 836da14cebeSEric Cheng */ 837da14cebeSEric Cheng for (i = 0; i < no_of_cpus; i++) { 8380dc2366fSVenugopal Iyer if (mrp->mrp_cpu[i] == mrp->mrp_rx_intr_cpu) 839da14cebeSEric Cheng break; 840da14cebeSEric Cheng } 841da14cebeSEric Cheng mrp->mrp_cpu[i] = mrp->mrp_cpu[0]; 8420dc2366fSVenugopal Iyer mrp->mrp_cpu[0] = mrp->mrp_rx_intr_cpu; 843da14cebeSEric Cheng } 844da14cebeSEric Cheng 845da14cebeSEric Cheng /* 846da14cebeSEric Cheng * Requirements: 847da14cebeSEric Cheng * The number of CPUs that each Rx ring needs is dependent 848da14cebeSEric Cheng * upon mac_latency_optimize flag. 849da14cebeSEric Cheng * 1) If set, atleast 2 CPUs are needed -- one for 850da14cebeSEric Cheng * polling, one for fanout soft ring. 851da14cebeSEric Cheng * 2) If not set, then atleast 3 CPUs are needed -- one 852da14cebeSEric Cheng * for polling, one for srs worker thread, and one for 853da14cebeSEric Cheng * fanout soft ring. 854da14cebeSEric Cheng */ 855da14cebeSEric Cheng rx_srs_cnt = (flent->fe_rx_srs_cnt > 1) ? 856da14cebeSEric Cheng (flent->fe_rx_srs_cnt - 1) : flent->fe_rx_srs_cnt; 857da14cebeSEric Cheng reqd_rx_cpu_cnt = mac_latency_optimize ? 858da14cebeSEric Cheng (rx_srs_cnt * 2) : (rx_srs_cnt * 3); 859da14cebeSEric Cheng 860da14cebeSEric Cheng /* How many CPUs are needed for Tx side? */ 861da14cebeSEric Cheng tx_srs = flent->fe_tx_srs; 8620dc2366fSVenugopal Iyer reqd_tx_cpu_cnt = MAC_TX_SOFT_RINGS(tx_srs) ? 8630dc2366fSVenugopal Iyer tx_srs->srs_tx_ring_count : 1; 864da14cebeSEric Cheng 865da14cebeSEric Cheng /* CPUs needed for Rx SRSes poll and worker threads */ 866da14cebeSEric Cheng reqd_poll_worker_cnt = mac_latency_optimize ? 867da14cebeSEric Cheng rx_srs_cnt : rx_srs_cnt * 2; 868da14cebeSEric Cheng 869da14cebeSEric Cheng /* Has the user provided enough CPUs? */ 870da14cebeSEric Cheng if (no_of_cpus >= (reqd_rx_cpu_cnt + reqd_tx_cpu_cnt)) { 871da14cebeSEric Cheng /* 872da14cebeSEric Cheng * Best case scenario. There is enough CPUs. All 873da14cebeSEric Cheng * Rx rings will get their own set of CPUs plus 874da14cebeSEric Cheng * Tx soft rings will get their own. 875da14cebeSEric Cheng */ 876da14cebeSEric Cheng /* 877da14cebeSEric Cheng * fanout_cpu_cnt is the number of CPUs available 878da14cebeSEric Cheng * for Rx side fanout soft rings. 879da14cebeSEric Cheng */ 880da14cebeSEric Cheng fanout_cpu_cnt = no_of_cpus - 881da14cebeSEric Cheng reqd_poll_worker_cnt - reqd_tx_cpu_cnt; 882da14cebeSEric Cheng 883da14cebeSEric Cheng /* 884da14cebeSEric Cheng * Divide fanout_cpu_cnt by rx_srs_cnt to find 885da14cebeSEric Cheng * out how many fanout soft rings each Rx SRS 886da14cebeSEric Cheng * can have. 887da14cebeSEric Cheng */ 888da14cebeSEric Cheng fanout_cnt_per_srs = fanout_cpu_cnt/rx_srs_cnt; 889da14cebeSEric Cheng 8905adf34bdSRajagopal Kunhappan /* fanout_cnt_per_srs should not be > MAX_SR_FANOUT */ 8915adf34bdSRajagopal Kunhappan fanout_cnt_per_srs = min(fanout_cnt_per_srs, MAX_SR_FANOUT); 8925adf34bdSRajagopal Kunhappan 893da14cebeSEric Cheng /* Do the assignment for the default Rx ring */ 894da14cebeSEric Cheng cpu_cnt = 0; 895da14cebeSEric Cheng rx_srs = flent->fe_rx_srs[0]; 896da14cebeSEric Cheng ASSERT(rx_srs->srs_ring == NULL); 897da14cebeSEric Cheng if (rx_srs->srs_fanout_state == SRS_FANOUT_INIT) 898da14cebeSEric Cheng rx_srs->srs_fanout_state = SRS_FANOUT_REINIT; 899da14cebeSEric Cheng srs_cpu = &rx_srs->srs_cpu; 900da14cebeSEric Cheng srs_cpu->mc_ncpus = no_of_cpus; 901da14cebeSEric Cheng bcopy(mrp->mrp_cpu, 902da14cebeSEric Cheng srs_cpu->mc_cpus, sizeof (srs_cpu->mc_cpus)); 9030dc2366fSVenugopal Iyer srs_cpu->mc_rx_fanout_cnt = fanout_cnt_per_srs; 9040dc2366fSVenugopal Iyer srs_cpu->mc_rx_pollid = mrp->mrp_cpu[cpu_cnt++]; 9050dc2366fSVenugopal Iyer /* Retarget the interrupt to the same CPU as the poll */ 9060dc2366fSVenugopal Iyer srs_cpu->mc_rx_intr_cpu = srs_cpu->mc_rx_pollid; 9070dc2366fSVenugopal Iyer srs_cpu->mc_rx_workerid = (mac_latency_optimize ? 9080dc2366fSVenugopal Iyer srs_cpu->mc_rx_pollid : mrp->mrp_cpu[cpu_cnt++]); 909da14cebeSEric Cheng for (i = 0; i < fanout_cnt_per_srs; i++) 9100dc2366fSVenugopal Iyer srs_cpu->mc_rx_fanout_cpus[i] = mrp->mrp_cpu[cpu_cnt++]; 911da14cebeSEric Cheng 912da14cebeSEric Cheng /* Do the assignment for h/w Rx SRSes */ 913da14cebeSEric Cheng if (flent->fe_rx_srs_cnt > 1) { 914da14cebeSEric Cheng cpu_cnt = 0; 915da14cebeSEric Cheng for (srs_cnt = 1; 916da14cebeSEric Cheng srs_cnt < flent->fe_rx_srs_cnt; srs_cnt++) { 917da14cebeSEric Cheng rx_srs = flent->fe_rx_srs[srs_cnt]; 918da14cebeSEric Cheng ASSERT(rx_srs->srs_ring != NULL); 919da14cebeSEric Cheng if (rx_srs->srs_fanout_state == 920da14cebeSEric Cheng SRS_FANOUT_INIT) { 921da14cebeSEric Cheng rx_srs->srs_fanout_state = 922da14cebeSEric Cheng SRS_FANOUT_REINIT; 923da14cebeSEric Cheng } 924da14cebeSEric Cheng srs_cpu = &rx_srs->srs_cpu; 925da14cebeSEric Cheng srs_cpu->mc_ncpus = no_of_cpus; 926da14cebeSEric Cheng bcopy(mrp->mrp_cpu, srs_cpu->mc_cpus, 927da14cebeSEric Cheng sizeof (srs_cpu->mc_cpus)); 9280dc2366fSVenugopal Iyer srs_cpu->mc_rx_fanout_cnt = fanout_cnt_per_srs; 929da14cebeSEric Cheng /* The first CPU in the list is the intr CPU */ 9300dc2366fSVenugopal Iyer srs_cpu->mc_rx_pollid = mrp->mrp_cpu[cpu_cnt++]; 9310dc2366fSVenugopal Iyer srs_cpu->mc_rx_intr_cpu = srs_cpu->mc_rx_pollid; 9320dc2366fSVenugopal Iyer srs_cpu->mc_rx_workerid = 9330dc2366fSVenugopal Iyer (mac_latency_optimize ? 9340dc2366fSVenugopal Iyer srs_cpu->mc_rx_pollid : 9350dc2366fSVenugopal Iyer mrp->mrp_cpu[cpu_cnt++]); 936da14cebeSEric Cheng for (i = 0; i < fanout_cnt_per_srs; i++) { 9370dc2366fSVenugopal Iyer srs_cpu->mc_rx_fanout_cpus[i] = 938da14cebeSEric Cheng mrp->mrp_cpu[cpu_cnt++]; 939da14cebeSEric Cheng } 940da14cebeSEric Cheng ASSERT(cpu_cnt <= no_of_cpus); 941da14cebeSEric Cheng } 942da14cebeSEric Cheng } 9430dc2366fSVenugopal Iyer goto tx_cpu_init; 944da14cebeSEric Cheng } 945da14cebeSEric Cheng 946da14cebeSEric Cheng /* 947da14cebeSEric Cheng * Sub-optimal case. 948da14cebeSEric Cheng * We have the following information: 949da14cebeSEric Cheng * no_of_cpus - no. of cpus that user passed. 950da14cebeSEric Cheng * rx_srs_cnt - no. of rx rings. 951da14cebeSEric Cheng * reqd_rx_cpu_cnt = mac_latency_optimize?rx_srs_cnt*2:rx_srs_cnt*3 952da14cebeSEric Cheng * reqd_tx_cpu_cnt - no. of cpus reqd. for Tx side. 953da14cebeSEric Cheng * reqd_poll_worker_cnt = mac_latency_optimize?rx_srs_cnt:rx_srs_cnt*2 954da14cebeSEric Cheng */ 955da14cebeSEric Cheng /* 956da14cebeSEric Cheng * If we bind the Rx fanout soft rings to the same CPUs 957da14cebeSEric Cheng * as poll/worker, would that be enough? 958da14cebeSEric Cheng */ 959da14cebeSEric Cheng if (no_of_cpus >= (rx_srs_cnt + reqd_tx_cpu_cnt)) { 960da14cebeSEric Cheng boolean_t worker_assign = B_FALSE; 961da14cebeSEric Cheng 962da14cebeSEric Cheng /* 963da14cebeSEric Cheng * If mac_latency_optimize is not set, are there 964da14cebeSEric Cheng * enough CPUs to assign a CPU for worker also? 965da14cebeSEric Cheng */ 966da14cebeSEric Cheng if (no_of_cpus >= (reqd_poll_worker_cnt + reqd_tx_cpu_cnt)) 967da14cebeSEric Cheng worker_assign = B_TRUE; 968da14cebeSEric Cheng /* 969da14cebeSEric Cheng * Zero'th Rx SRS is the default Rx ring. It is not 970da14cebeSEric Cheng * associated with h/w Rx ring. 971da14cebeSEric Cheng */ 972da14cebeSEric Cheng rx_srs = flent->fe_rx_srs[0]; 973da14cebeSEric Cheng ASSERT(rx_srs->srs_ring == NULL); 974da14cebeSEric Cheng if (rx_srs->srs_fanout_state == SRS_FANOUT_INIT) 975da14cebeSEric Cheng rx_srs->srs_fanout_state = SRS_FANOUT_REINIT; 976da14cebeSEric Cheng cpu_cnt = 0; 977da14cebeSEric Cheng srs_cpu = &rx_srs->srs_cpu; 978da14cebeSEric Cheng srs_cpu->mc_ncpus = no_of_cpus; 979da14cebeSEric Cheng bcopy(mrp->mrp_cpu, 980da14cebeSEric Cheng srs_cpu->mc_cpus, sizeof (srs_cpu->mc_cpus)); 9810dc2366fSVenugopal Iyer srs_cpu->mc_rx_fanout_cnt = 1; 9820dc2366fSVenugopal Iyer srs_cpu->mc_rx_pollid = mrp->mrp_cpu[cpu_cnt++]; 9830dc2366fSVenugopal Iyer /* Retarget the interrupt to the same CPU as the poll */ 9840dc2366fSVenugopal Iyer srs_cpu->mc_rx_intr_cpu = srs_cpu->mc_rx_pollid; 9850dc2366fSVenugopal Iyer srs_cpu->mc_rx_workerid = 9860dc2366fSVenugopal Iyer ((!mac_latency_optimize && worker_assign) ? 9870dc2366fSVenugopal Iyer mrp->mrp_cpu[cpu_cnt++] : srs_cpu->mc_rx_pollid); 9880dc2366fSVenugopal Iyer 9890dc2366fSVenugopal Iyer srs_cpu->mc_rx_fanout_cpus[0] = mrp->mrp_cpu[cpu_cnt]; 990da14cebeSEric Cheng 991da14cebeSEric Cheng /* Do CPU bindings for SRSes having h/w Rx rings */ 992da14cebeSEric Cheng if (flent->fe_rx_srs_cnt > 1) { 993da14cebeSEric Cheng cpu_cnt = 0; 994da14cebeSEric Cheng for (srs_cnt = 1; 995da14cebeSEric Cheng srs_cnt < flent->fe_rx_srs_cnt; srs_cnt++) { 996da14cebeSEric Cheng rx_srs = flent->fe_rx_srs[srs_cnt]; 997da14cebeSEric Cheng ASSERT(rx_srs->srs_ring != NULL); 998da14cebeSEric Cheng if (rx_srs->srs_fanout_state == 999da14cebeSEric Cheng SRS_FANOUT_INIT) { 1000da14cebeSEric Cheng rx_srs->srs_fanout_state = 1001da14cebeSEric Cheng SRS_FANOUT_REINIT; 1002da14cebeSEric Cheng } 1003da14cebeSEric Cheng srs_cpu = &rx_srs->srs_cpu; 1004da14cebeSEric Cheng srs_cpu->mc_ncpus = no_of_cpus; 1005da14cebeSEric Cheng bcopy(mrp->mrp_cpu, srs_cpu->mc_cpus, 1006da14cebeSEric Cheng sizeof (srs_cpu->mc_cpus)); 10070dc2366fSVenugopal Iyer srs_cpu->mc_rx_pollid = 1008da14cebeSEric Cheng mrp->mrp_cpu[cpu_cnt]; 10090dc2366fSVenugopal Iyer srs_cpu->mc_rx_intr_cpu = srs_cpu->mc_rx_pollid; 10100dc2366fSVenugopal Iyer srs_cpu->mc_rx_workerid = 10110dc2366fSVenugopal Iyer ((!mac_latency_optimize && worker_assign) ? 10120dc2366fSVenugopal Iyer mrp->mrp_cpu[++cpu_cnt] : 10130dc2366fSVenugopal Iyer srs_cpu->mc_rx_pollid); 10140dc2366fSVenugopal Iyer srs_cpu->mc_rx_fanout_cnt = 1; 10150dc2366fSVenugopal Iyer srs_cpu->mc_rx_fanout_cpus[0] = 1016da14cebeSEric Cheng mrp->mrp_cpu[cpu_cnt]; 1017da14cebeSEric Cheng cpu_cnt++; 1018da14cebeSEric Cheng ASSERT(cpu_cnt <= no_of_cpus); 1019da14cebeSEric Cheng } 1020da14cebeSEric Cheng } 10210dc2366fSVenugopal Iyer goto tx_cpu_init; 1022da14cebeSEric Cheng } 1023da14cebeSEric Cheng 1024da14cebeSEric Cheng /* 1025da14cebeSEric Cheng * Real sub-optimal case. Not enough CPUs for poll and 1026da14cebeSEric Cheng * Tx soft rings. Do a round robin assignment where 1027da14cebeSEric Cheng * each Rx SRS will get the same CPU for poll, worker 1028da14cebeSEric Cheng * and fanout soft ring. 1029da14cebeSEric Cheng */ 1030da14cebeSEric Cheng cpu_cnt = 0; 1031da14cebeSEric Cheng for (srs_cnt = 0; srs_cnt < flent->fe_rx_srs_cnt; srs_cnt++) { 1032da14cebeSEric Cheng rx_srs = flent->fe_rx_srs[srs_cnt]; 1033da14cebeSEric Cheng srs_cpu = &rx_srs->srs_cpu; 1034da14cebeSEric Cheng if (rx_srs->srs_fanout_state == SRS_FANOUT_INIT) 1035da14cebeSEric Cheng rx_srs->srs_fanout_state = SRS_FANOUT_REINIT; 1036da14cebeSEric Cheng srs_cpu->mc_ncpus = no_of_cpus; 1037da14cebeSEric Cheng bcopy(mrp->mrp_cpu, 1038da14cebeSEric Cheng srs_cpu->mc_cpus, sizeof (srs_cpu->mc_cpus)); 10390dc2366fSVenugopal Iyer srs_cpu->mc_rx_fanout_cnt = 1; 10400dc2366fSVenugopal Iyer srs_cpu->mc_rx_pollid = mrp->mrp_cpu[cpu_cnt]; 10410dc2366fSVenugopal Iyer /* Retarget the interrupt to the same CPU as the poll */ 10420dc2366fSVenugopal Iyer srs_cpu->mc_rx_intr_cpu = srs_cpu->mc_rx_pollid; 10430dc2366fSVenugopal Iyer srs_cpu->mc_rx_workerid = mrp->mrp_cpu[cpu_cnt]; 10440dc2366fSVenugopal Iyer srs_cpu->mc_rx_fanout_cpus[0] = mrp->mrp_cpu[cpu_cnt]; 1045da14cebeSEric Cheng if (++cpu_cnt >= no_of_cpus) 1046da14cebeSEric Cheng cpu_cnt = 0; 1047da14cebeSEric Cheng } 10480dc2366fSVenugopal Iyer 10490dc2366fSVenugopal Iyer tx_cpu_init: 10500dc2366fSVenugopal Iyer mac_tx_cpu_init(flent, mrp, NULL); 10510dc2366fSVenugopal Iyer 10520dc2366fSVenugopal Iyer /* 10530dc2366fSVenugopal Iyer * Copy the user specified CPUs to the effective CPUs 10540dc2366fSVenugopal Iyer */ 10550dc2366fSVenugopal Iyer for (i = 0; i < mrp->mrp_ncpus; i++) { 10560dc2366fSVenugopal Iyer emrp->mrp_cpu[i] = mrp->mrp_cpu[i]; 10570dc2366fSVenugopal Iyer } 10580dc2366fSVenugopal Iyer emrp->mrp_ncpus = mrp->mrp_ncpus; 10590dc2366fSVenugopal Iyer emrp->mrp_mask = mrp->mrp_mask; 10600dc2366fSVenugopal Iyer bzero(emrp->mrp_pool, MAXPATHLEN); 1061da14cebeSEric Cheng } 1062da14cebeSEric Cheng 1063da14cebeSEric Cheng /* 1064da14cebeSEric Cheng * mac_flow_cpu_init(): 1065da14cebeSEric Cheng * 1066da14cebeSEric Cheng * Each SRS has a mac_cpu_t structure, srs_cpu. This routine fills in 1067da14cebeSEric Cheng * the CPU binding information in srs_cpu for all Rx SRSes associated 1068da14cebeSEric Cheng * with a flent. 1069da14cebeSEric Cheng */ 1070da14cebeSEric Cheng static void 10710dc2366fSVenugopal Iyer mac_flow_cpu_init(flow_entry_t *flent, cpupart_t *cpupart) 1072da14cebeSEric Cheng { 1073da14cebeSEric Cheng mac_soft_ring_set_t *rx_srs; 1074da14cebeSEric Cheng processorid_t cpuid; 10750dc2366fSVenugopal Iyer int i, j, k, srs_cnt, nscpus, maxcpus, soft_ring_cnt = 0; 1076da14cebeSEric Cheng mac_cpus_t *srs_cpu; 10770dc2366fSVenugopal Iyer mac_resource_props_t *emrp = &flent->fe_effective_props; 10780dc2366fSVenugopal Iyer uint32_t cpus[MRP_NCPUS]; 1079da14cebeSEric Cheng 10800dc2366fSVenugopal Iyer /* 10810dc2366fSVenugopal Iyer * The maximum number of CPUs available can either be 10820dc2366fSVenugopal Iyer * the number of CPUs in the pool or the number of CPUs 10830dc2366fSVenugopal Iyer * in the system. 10840dc2366fSVenugopal Iyer */ 10850dc2366fSVenugopal Iyer maxcpus = (cpupart != NULL) ? cpupart->cp_ncpus : ncpus; 10860dc2366fSVenugopal Iyer 1087da14cebeSEric Cheng /* 1088da14cebeSEric Cheng * Compute the number of soft rings needed on top for each Rx 1089da14cebeSEric Cheng * SRS. "rx_srs_cnt-1" indicates the number of Rx SRS 1090da14cebeSEric Cheng * associated with h/w Rx rings. Soft ring count needed for 1091da14cebeSEric Cheng * each h/w Rx SRS is computed and the same is applied to 1092da14cebeSEric Cheng * software classified Rx SRS. The first Rx SRS in fe_rx_srs[] 1093da14cebeSEric Cheng * is the software classified Rx SRS. 1094da14cebeSEric Cheng */ 1095da14cebeSEric Cheng soft_ring_cnt = mac_compute_soft_ring_count(flent, 10960dc2366fSVenugopal Iyer flent->fe_rx_srs_cnt - 1, maxcpus); 1097da14cebeSEric Cheng if (soft_ring_cnt == 0) { 1098da14cebeSEric Cheng /* 1099da14cebeSEric Cheng * Even when soft_ring_cnt is 0, we still need 1100da14cebeSEric Cheng * to create a soft ring for TCP, UDP and 1101da14cebeSEric Cheng * OTHER. So set it to 1. 1102da14cebeSEric Cheng */ 1103da14cebeSEric Cheng soft_ring_cnt = 1; 1104da14cebeSEric Cheng } 1105da14cebeSEric Cheng for (srs_cnt = 0; srs_cnt < flent->fe_rx_srs_cnt; srs_cnt++) { 1106da14cebeSEric Cheng rx_srs = flent->fe_rx_srs[srs_cnt]; 1107da14cebeSEric Cheng srs_cpu = &rx_srs->srs_cpu; 11080dc2366fSVenugopal Iyer if (rx_srs->srs_fanout_state == SRS_FANOUT_INIT) 1109da14cebeSEric Cheng rx_srs->srs_fanout_state = SRS_FANOUT_REINIT; 1110da14cebeSEric Cheng srs_cpu->mc_ncpus = soft_ring_cnt; 11110dc2366fSVenugopal Iyer srs_cpu->mc_rx_fanout_cnt = soft_ring_cnt; 1112da14cebeSEric Cheng mutex_enter(&cpu_lock); 1113da14cebeSEric Cheng for (j = 0; j < soft_ring_cnt; j++) { 11140dc2366fSVenugopal Iyer cpuid = mac_next_bind_cpu(cpupart); 1115da14cebeSEric Cheng srs_cpu->mc_cpus[j] = cpuid; 11160dc2366fSVenugopal Iyer srs_cpu->mc_rx_fanout_cpus[j] = cpuid; 1117da14cebeSEric Cheng } 11180dc2366fSVenugopal Iyer cpuid = mac_next_bind_cpu(cpupart); 11190dc2366fSVenugopal Iyer srs_cpu->mc_rx_pollid = cpuid; 11200dc2366fSVenugopal Iyer srs_cpu->mc_rx_intr_cpu = (mac_rx_intr_retarget ? 11210dc2366fSVenugopal Iyer srs_cpu->mc_rx_pollid : -1); 1122da14cebeSEric Cheng /* increment ncpus to account for polling cpu */ 1123da14cebeSEric Cheng srs_cpu->mc_ncpus++; 1124da14cebeSEric Cheng srs_cpu->mc_cpus[j++] = cpuid; 1125da14cebeSEric Cheng if (!mac_latency_optimize) { 11260dc2366fSVenugopal Iyer cpuid = mac_next_bind_cpu(cpupart); 1127da14cebeSEric Cheng srs_cpu->mc_ncpus++; 1128da14cebeSEric Cheng srs_cpu->mc_cpus[j++] = cpuid; 1129da14cebeSEric Cheng } 11300dc2366fSVenugopal Iyer srs_cpu->mc_rx_workerid = cpuid; 1131da14cebeSEric Cheng mutex_exit(&cpu_lock); 1132da14cebeSEric Cheng } 11330dc2366fSVenugopal Iyer 11340dc2366fSVenugopal Iyer nscpus = 0; 11350dc2366fSVenugopal Iyer for (srs_cnt = 0; srs_cnt < flent->fe_rx_srs_cnt; srs_cnt++) { 11360dc2366fSVenugopal Iyer rx_srs = flent->fe_rx_srs[srs_cnt]; 11370dc2366fSVenugopal Iyer srs_cpu = &rx_srs->srs_cpu; 11380dc2366fSVenugopal Iyer for (j = 0; j < srs_cpu->mc_ncpus; j++) { 11390dc2366fSVenugopal Iyer cpus[nscpus++] = srs_cpu->mc_cpus[j]; 1140da14cebeSEric Cheng } 1141da14cebeSEric Cheng } 1142da14cebeSEric Cheng 11430dc2366fSVenugopal Iyer 11440dc2366fSVenugopal Iyer /* 11450dc2366fSVenugopal Iyer * Copy cpu list to fe_effective_props 11460dc2366fSVenugopal Iyer * without duplicates. 11470dc2366fSVenugopal Iyer */ 11480dc2366fSVenugopal Iyer k = 0; 11490dc2366fSVenugopal Iyer for (i = 0; i < nscpus; i++) { 11500dc2366fSVenugopal Iyer for (j = 0; j < k; j++) { 11510dc2366fSVenugopal Iyer if (emrp->mrp_cpu[j] == cpus[i]) 11520dc2366fSVenugopal Iyer break; 11530dc2366fSVenugopal Iyer } 11540dc2366fSVenugopal Iyer if (j == k) 11550dc2366fSVenugopal Iyer emrp->mrp_cpu[k++] = cpus[i]; 11560dc2366fSVenugopal Iyer } 11570dc2366fSVenugopal Iyer emrp->mrp_ncpus = k; 11580dc2366fSVenugopal Iyer 11590dc2366fSVenugopal Iyer mac_tx_cpu_init(flent, NULL, cpupart); 11600dc2366fSVenugopal Iyer } 11610dc2366fSVenugopal Iyer 1162da14cebeSEric Cheng /* 1163da14cebeSEric Cheng * DATAPATH SETUP ROUTINES 1164da14cebeSEric Cheng * (setup SRS and set/update FANOUT, B/W and PRIORITY) 1165da14cebeSEric Cheng */ 1166da14cebeSEric Cheng 11670dc2366fSVenugopal Iyer /* 11680dc2366fSVenugopal Iyer * mac_srs_fanout_list_alloc: 11690dc2366fSVenugopal Iyer * 11700dc2366fSVenugopal Iyer * The underlying device can expose upto MAX_RINGS_PER_GROUP worth of 11710dc2366fSVenugopal Iyer * rings to a client. In such a case, MAX_RINGS_PER_GROUP worth of 11720dc2366fSVenugopal Iyer * array space is needed to store Tx soft rings. Thus we allocate so 11730dc2366fSVenugopal Iyer * much array space for srs_tx_soft_rings. 11740dc2366fSVenugopal Iyer * 11750dc2366fSVenugopal Iyer * And when it is an aggr, again we allocate MAX_RINGS_PER_GROUP worth 11760dc2366fSVenugopal Iyer * of space to st_soft_rings. This array is used for quick access to 11770dc2366fSVenugopal Iyer * soft ring associated with a pseudo Tx ring based on the pseudo 11780dc2366fSVenugopal Iyer * ring's index (mr_index). 11790dc2366fSVenugopal Iyer */ 1180da14cebeSEric Cheng static void 1181da14cebeSEric Cheng mac_srs_fanout_list_alloc(mac_soft_ring_set_t *mac_srs) 1182da14cebeSEric Cheng { 11830dc2366fSVenugopal Iyer mac_client_impl_t *mcip = mac_srs->srs_mcip; 11840dc2366fSVenugopal Iyer 11850dc2366fSVenugopal Iyer if (mac_srs->srs_type & SRST_TX) { 11860dc2366fSVenugopal Iyer mac_srs->srs_tx_soft_rings = (mac_soft_ring_t **) 11870dc2366fSVenugopal Iyer kmem_zalloc(sizeof (mac_soft_ring_t *) * 11880dc2366fSVenugopal Iyer MAX_RINGS_PER_GROUP, KM_SLEEP); 118984de666eSRyan Zezeski if (mcip->mci_state_flags & MCIS_IS_AGGR_CLIENT) { 11900dc2366fSVenugopal Iyer mac_srs_tx_t *tx = &mac_srs->srs_tx; 11910dc2366fSVenugopal Iyer 11920dc2366fSVenugopal Iyer tx->st_soft_rings = (mac_soft_ring_t **) 11930dc2366fSVenugopal Iyer kmem_zalloc(sizeof (mac_soft_ring_t *) * 11940dc2366fSVenugopal Iyer MAX_RINGS_PER_GROUP, KM_SLEEP); 11950dc2366fSVenugopal Iyer } 11960dc2366fSVenugopal Iyer } else { 1197da14cebeSEric Cheng mac_srs->srs_tcp_soft_rings = (mac_soft_ring_t **) 11980dc2366fSVenugopal Iyer kmem_zalloc(sizeof (mac_soft_ring_t *) * MAX_SR_FANOUT, 11990dc2366fSVenugopal Iyer KM_SLEEP); 1200da14cebeSEric Cheng mac_srs->srs_udp_soft_rings = (mac_soft_ring_t **) 12010dc2366fSVenugopal Iyer kmem_zalloc(sizeof (mac_soft_ring_t *) * MAX_SR_FANOUT, 12020dc2366fSVenugopal Iyer KM_SLEEP); 1203da14cebeSEric Cheng mac_srs->srs_oth_soft_rings = (mac_soft_ring_t **) 12040dc2366fSVenugopal Iyer kmem_zalloc(sizeof (mac_soft_ring_t *) * MAX_SR_FANOUT, 12050dc2366fSVenugopal Iyer KM_SLEEP); 12060dc2366fSVenugopal Iyer } 1207da14cebeSEric Cheng } 1208da14cebeSEric Cheng 1209da14cebeSEric Cheng static void 1210da14cebeSEric Cheng mac_srs_worker_bind(mac_soft_ring_set_t *mac_srs, processorid_t cpuid) 1211da14cebeSEric Cheng { 1212da14cebeSEric Cheng cpu_t *cp; 1213da14cebeSEric Cheng boolean_t clear = B_FALSE; 1214da14cebeSEric Cheng 1215da14cebeSEric Cheng ASSERT(MUTEX_HELD(&cpu_lock)); 1216da14cebeSEric Cheng 1217da14cebeSEric Cheng if (!mac_srs_thread_bind) 1218da14cebeSEric Cheng return; 1219da14cebeSEric Cheng 1220da14cebeSEric Cheng cp = cpu_get(cpuid); 1221da14cebeSEric Cheng if (cp == NULL || !cpu_is_online(cp)) 1222da14cebeSEric Cheng return; 1223da14cebeSEric Cheng 1224da14cebeSEric Cheng mutex_enter(&mac_srs->srs_lock); 1225da14cebeSEric Cheng mac_srs->srs_state |= SRS_WORKER_BOUND; 1226da14cebeSEric Cheng if (mac_srs->srs_worker_cpuid != -1) 1227da14cebeSEric Cheng clear = B_TRUE; 1228da14cebeSEric Cheng mac_srs->srs_worker_cpuid = cpuid; 1229da14cebeSEric Cheng mutex_exit(&mac_srs->srs_lock); 1230da14cebeSEric Cheng 1231da14cebeSEric Cheng if (clear) 1232da14cebeSEric Cheng thread_affinity_clear(mac_srs->srs_worker); 1233da14cebeSEric Cheng 1234da14cebeSEric Cheng thread_affinity_set(mac_srs->srs_worker, cpuid); 1235da14cebeSEric Cheng DTRACE_PROBE1(worker__CPU, processorid_t, cpuid); 1236da14cebeSEric Cheng } 1237da14cebeSEric Cheng 1238da14cebeSEric Cheng static void 1239da14cebeSEric Cheng mac_srs_poll_bind(mac_soft_ring_set_t *mac_srs, processorid_t cpuid) 1240da14cebeSEric Cheng { 1241da14cebeSEric Cheng cpu_t *cp; 1242da14cebeSEric Cheng boolean_t clear = B_FALSE; 1243da14cebeSEric Cheng 1244da14cebeSEric Cheng ASSERT(MUTEX_HELD(&cpu_lock)); 1245da14cebeSEric Cheng 1246da14cebeSEric Cheng if (!mac_srs_thread_bind || mac_srs->srs_poll_thr == NULL) 1247da14cebeSEric Cheng return; 1248da14cebeSEric Cheng 1249da14cebeSEric Cheng cp = cpu_get(cpuid); 1250da14cebeSEric Cheng if (cp == NULL || !cpu_is_online(cp)) 1251da14cebeSEric Cheng return; 1252da14cebeSEric Cheng 1253da14cebeSEric Cheng mutex_enter(&mac_srs->srs_lock); 1254da14cebeSEric Cheng mac_srs->srs_state |= SRS_POLL_BOUND; 1255da14cebeSEric Cheng if (mac_srs->srs_poll_cpuid != -1) 1256da14cebeSEric Cheng clear = B_TRUE; 1257da14cebeSEric Cheng mac_srs->srs_poll_cpuid = cpuid; 1258da14cebeSEric Cheng mutex_exit(&mac_srs->srs_lock); 1259da14cebeSEric Cheng 1260da14cebeSEric Cheng if (clear) 1261da14cebeSEric Cheng thread_affinity_clear(mac_srs->srs_poll_thr); 1262da14cebeSEric Cheng 1263da14cebeSEric Cheng thread_affinity_set(mac_srs->srs_poll_thr, cpuid); 1264da14cebeSEric Cheng DTRACE_PROBE1(poll__CPU, processorid_t, cpuid); 1265da14cebeSEric Cheng } 1266da14cebeSEric Cheng 1267da14cebeSEric Cheng /* 12680dc2366fSVenugopal Iyer * Re-target interrupt to the passed CPU. If re-target is successful, 12690dc2366fSVenugopal Iyer * set mc_rx_intr_cpu to the re-targeted CPU. Otherwise set it to -1. 12700dc2366fSVenugopal Iyer */ 12710dc2366fSVenugopal Iyer void 12720dc2366fSVenugopal Iyer mac_rx_srs_retarget_intr(mac_soft_ring_set_t *mac_srs, processorid_t cpuid) 12730dc2366fSVenugopal Iyer { 12740dc2366fSVenugopal Iyer cpu_t *cp; 12750dc2366fSVenugopal Iyer mac_ring_t *ring = mac_srs->srs_ring; 12760dc2366fSVenugopal Iyer mac_intr_t *mintr = &ring->mr_info.mri_intr; 12770dc2366fSVenugopal Iyer flow_entry_t *flent = mac_srs->srs_flent; 12780dc2366fSVenugopal Iyer boolean_t primary = mac_is_primary_client(mac_srs->srs_mcip); 12790dc2366fSVenugopal Iyer 12800dc2366fSVenugopal Iyer ASSERT(MUTEX_HELD(&cpu_lock)); 12810dc2366fSVenugopal Iyer 12820dc2366fSVenugopal Iyer /* 12830dc2366fSVenugopal Iyer * Don't re-target the interrupt for these cases: 12840dc2366fSVenugopal Iyer * 1) ring is NULL 12850dc2366fSVenugopal Iyer * 2) the interrupt is shared (mi_ddi_shared) 12860dc2366fSVenugopal Iyer * 3) ddi_handle is NULL and !primary 12870dc2366fSVenugopal Iyer * 4) primary, ddi_handle is NULL but fe_rx_srs_cnt > 2 12880dc2366fSVenugopal Iyer * Case 3 & 4 are because of mac_client_intr_cpu() routine. 12890dc2366fSVenugopal Iyer * This routine will re-target fixed interrupt for primary 12900dc2366fSVenugopal Iyer * mac client if the client has only one ring. In that 12910dc2366fSVenugopal Iyer * case, mc_rx_intr_cpu will already have the correct value. 12920dc2366fSVenugopal Iyer */ 12930dc2366fSVenugopal Iyer if (ring == NULL || mintr->mi_ddi_shared || cpuid == -1 || 12940dc2366fSVenugopal Iyer (mintr->mi_ddi_handle == NULL && !primary) || (primary && 12950dc2366fSVenugopal Iyer mintr->mi_ddi_handle == NULL && flent->fe_rx_srs_cnt > 2)) { 12960dc2366fSVenugopal Iyer mac_srs->srs_cpu.mc_rx_intr_cpu = -1; 12970dc2366fSVenugopal Iyer return; 12980dc2366fSVenugopal Iyer } 12990dc2366fSVenugopal Iyer 13000dc2366fSVenugopal Iyer if (mintr->mi_ddi_handle == NULL) 13010dc2366fSVenugopal Iyer return; 13020dc2366fSVenugopal Iyer 13030dc2366fSVenugopal Iyer cp = cpu_get(cpuid); 13040dc2366fSVenugopal Iyer if (cp == NULL || !cpu_is_online(cp)) 13050dc2366fSVenugopal Iyer return; 13060dc2366fSVenugopal Iyer 1307657f87deSgongtian zhao - Sun Microsystems - Beijing China /* Drop the cpu_lock as set_intr_affinity() holds it */ 13080dc2366fSVenugopal Iyer mutex_exit(&cpu_lock); 1309657f87deSgongtian zhao - Sun Microsystems - Beijing China if (set_intr_affinity(mintr->mi_ddi_handle, cpuid) == DDI_SUCCESS) 13100dc2366fSVenugopal Iyer mac_srs->srs_cpu.mc_rx_intr_cpu = cpuid; 13110dc2366fSVenugopal Iyer else 13120dc2366fSVenugopal Iyer mac_srs->srs_cpu.mc_rx_intr_cpu = -1; 13130dc2366fSVenugopal Iyer mutex_enter(&cpu_lock); 13140dc2366fSVenugopal Iyer } 13150dc2366fSVenugopal Iyer 13160dc2366fSVenugopal Iyer /* 13170dc2366fSVenugopal Iyer * Re-target Tx interrupts 13180dc2366fSVenugopal Iyer */ 13190dc2366fSVenugopal Iyer void 13200dc2366fSVenugopal Iyer mac_tx_srs_retarget_intr(mac_soft_ring_set_t *mac_srs) 13210dc2366fSVenugopal Iyer { 13220dc2366fSVenugopal Iyer cpu_t *cp; 13230dc2366fSVenugopal Iyer mac_ring_t *ring; 13240dc2366fSVenugopal Iyer mac_intr_t *mintr; 13250dc2366fSVenugopal Iyer mac_soft_ring_t *sringp; 13260dc2366fSVenugopal Iyer mac_srs_tx_t *srs_tx; 13270dc2366fSVenugopal Iyer mac_cpus_t *srs_cpu; 13280dc2366fSVenugopal Iyer processorid_t cpuid; 13290dc2366fSVenugopal Iyer int i; 13300dc2366fSVenugopal Iyer 13310dc2366fSVenugopal Iyer ASSERT(MUTEX_HELD(&cpu_lock)); 13320dc2366fSVenugopal Iyer 13330dc2366fSVenugopal Iyer srs_cpu = &mac_srs->srs_cpu; 13340dc2366fSVenugopal Iyer if (MAC_TX_SOFT_RINGS(mac_srs)) { 13350dc2366fSVenugopal Iyer for (i = 0; i < mac_srs->srs_tx_ring_count; i++) { 13360dc2366fSVenugopal Iyer sringp = mac_srs->srs_tx_soft_rings[i]; 13370dc2366fSVenugopal Iyer ring = (mac_ring_t *)sringp->s_ring_tx_arg2; 13380dc2366fSVenugopal Iyer cpuid = srs_cpu->mc_tx_intr_cpu[i]; 13390dc2366fSVenugopal Iyer cp = cpu_get(cpuid); 13400dc2366fSVenugopal Iyer if (cp == NULL || !cpu_is_online(cp) || 13410dc2366fSVenugopal Iyer !MAC_RING_RETARGETABLE(ring)) { 13420dc2366fSVenugopal Iyer srs_cpu->mc_tx_retargeted_cpu[i] = -1; 13430dc2366fSVenugopal Iyer continue; 13440dc2366fSVenugopal Iyer } 13450dc2366fSVenugopal Iyer mintr = &ring->mr_info.mri_intr; 13460dc2366fSVenugopal Iyer /* 1347657f87deSgongtian zhao - Sun Microsystems - Beijing China * Drop the cpu_lock as set_intr_affinity() 13480dc2366fSVenugopal Iyer * holds it 13490dc2366fSVenugopal Iyer */ 13500dc2366fSVenugopal Iyer mutex_exit(&cpu_lock); 1351657f87deSgongtian zhao - Sun Microsystems - Beijing China if (set_intr_affinity(mintr->mi_ddi_handle, 13520dc2366fSVenugopal Iyer cpuid) == DDI_SUCCESS) { 13530dc2366fSVenugopal Iyer srs_cpu->mc_tx_retargeted_cpu[i] = cpuid; 13540dc2366fSVenugopal Iyer } else { 13550dc2366fSVenugopal Iyer srs_cpu->mc_tx_retargeted_cpu[i] = -1; 13560dc2366fSVenugopal Iyer } 13570dc2366fSVenugopal Iyer mutex_enter(&cpu_lock); 13580dc2366fSVenugopal Iyer } 13590dc2366fSVenugopal Iyer } else { 13600dc2366fSVenugopal Iyer cpuid = srs_cpu->mc_tx_intr_cpu[0]; 13610dc2366fSVenugopal Iyer cp = cpu_get(cpuid); 13620dc2366fSVenugopal Iyer if (cp == NULL || !cpu_is_online(cp)) { 13630dc2366fSVenugopal Iyer srs_cpu->mc_tx_retargeted_cpu[0] = -1; 13640dc2366fSVenugopal Iyer return; 13650dc2366fSVenugopal Iyer } 13660dc2366fSVenugopal Iyer srs_tx = &mac_srs->srs_tx; 13670dc2366fSVenugopal Iyer ring = (mac_ring_t *)srs_tx->st_arg2; 13680dc2366fSVenugopal Iyer if (MAC_RING_RETARGETABLE(ring)) { 13690dc2366fSVenugopal Iyer mintr = &ring->mr_info.mri_intr; 13700dc2366fSVenugopal Iyer mutex_exit(&cpu_lock); 1371657f87deSgongtian zhao - Sun Microsystems - Beijing China if ((set_intr_affinity(mintr->mi_ddi_handle, 13720dc2366fSVenugopal Iyer cpuid) == DDI_SUCCESS)) { 13730dc2366fSVenugopal Iyer srs_cpu->mc_tx_retargeted_cpu[0] = cpuid; 13740dc2366fSVenugopal Iyer } else { 13750dc2366fSVenugopal Iyer srs_cpu->mc_tx_retargeted_cpu[0] = -1; 13760dc2366fSVenugopal Iyer } 13770dc2366fSVenugopal Iyer mutex_enter(&cpu_lock); 13780dc2366fSVenugopal Iyer } 13790dc2366fSVenugopal Iyer } 13800dc2366fSVenugopal Iyer } 13810dc2366fSVenugopal Iyer 13820dc2366fSVenugopal Iyer /* 1383da14cebeSEric Cheng * When a CPU comes back online, bind the MAC kernel threads which 1384da14cebeSEric Cheng * were previously bound to that CPU, and had to be unbound because 1385da14cebeSEric Cheng * the CPU was going away. 1386da14cebeSEric Cheng * 1387da14cebeSEric Cheng * These functions are called with cpu_lock held and hence we can't 1388da14cebeSEric Cheng * cv_wait to grab the mac perimeter. Since these functions walk the soft 1389da14cebeSEric Cheng * ring list of an SRS without being in the perimeter, the list itself 1390da14cebeSEric Cheng * is protected by the SRS lock. 1391da14cebeSEric Cheng */ 1392da14cebeSEric Cheng static void 1393da14cebeSEric Cheng mac_walk_srs_and_bind(int cpuid) 1394da14cebeSEric Cheng { 1395da14cebeSEric Cheng mac_soft_ring_set_t *mac_srs; 1396da14cebeSEric Cheng mac_soft_ring_t *soft_ring; 1397da14cebeSEric Cheng 1398da14cebeSEric Cheng rw_enter(&mac_srs_g_lock, RW_READER); 1399da14cebeSEric Cheng 1400da14cebeSEric Cheng if ((mac_srs = mac_srs_g_list) == NULL) 1401da14cebeSEric Cheng goto done; 1402da14cebeSEric Cheng 1403da14cebeSEric Cheng for (; mac_srs != NULL; mac_srs = mac_srs->srs_next) { 1404da14cebeSEric Cheng if (mac_srs->srs_worker_cpuid == -1 && 1405da14cebeSEric Cheng mac_srs->srs_worker_cpuid_save == cpuid) { 1406da14cebeSEric Cheng mac_srs->srs_worker_cpuid_save = -1; 1407da14cebeSEric Cheng mac_srs_worker_bind(mac_srs, cpuid); 1408da14cebeSEric Cheng } 1409da14cebeSEric Cheng 1410da14cebeSEric Cheng if (!(mac_srs->srs_type & SRST_TX)) { 1411da14cebeSEric Cheng if (mac_srs->srs_poll_cpuid == -1 && 1412da14cebeSEric Cheng mac_srs->srs_poll_cpuid_save == cpuid) { 1413da14cebeSEric Cheng mac_srs->srs_poll_cpuid_save = -1; 1414da14cebeSEric Cheng mac_srs_poll_bind(mac_srs, cpuid); 1415da14cebeSEric Cheng } 1416da14cebeSEric Cheng } 1417da14cebeSEric Cheng 1418da14cebeSEric Cheng /* Next tackle the soft rings associated with the srs */ 1419da14cebeSEric Cheng mutex_enter(&mac_srs->srs_lock); 1420da14cebeSEric Cheng for (soft_ring = mac_srs->srs_soft_ring_head; soft_ring != NULL; 1421da14cebeSEric Cheng soft_ring = soft_ring->s_ring_next) { 1422da14cebeSEric Cheng if (soft_ring->s_ring_cpuid == -1 && 1423da14cebeSEric Cheng soft_ring->s_ring_cpuid_save == cpuid) { 1424da14cebeSEric Cheng soft_ring->s_ring_cpuid_save = -1; 1425da14cebeSEric Cheng (void) mac_soft_ring_bind(soft_ring, cpuid); 1426da14cebeSEric Cheng } 1427da14cebeSEric Cheng } 1428da14cebeSEric Cheng mutex_exit(&mac_srs->srs_lock); 1429da14cebeSEric Cheng } 1430da14cebeSEric Cheng done: 1431da14cebeSEric Cheng rw_exit(&mac_srs_g_lock); 1432da14cebeSEric Cheng } 1433da14cebeSEric Cheng 1434da14cebeSEric Cheng /* 1435da14cebeSEric Cheng * Change the priority of the SRS's poll and worker thread. Additionally, 1436da14cebeSEric Cheng * update the priority of the worker threads for the SRS's soft rings. 1437da14cebeSEric Cheng * Need to modify any associated squeue threads. 1438da14cebeSEric Cheng */ 1439da14cebeSEric Cheng void 1440da14cebeSEric Cheng mac_update_srs_priority(mac_soft_ring_set_t *mac_srs, pri_t prival) 1441da14cebeSEric Cheng { 1442da14cebeSEric Cheng mac_soft_ring_t *ringp; 1443da14cebeSEric Cheng 1444da14cebeSEric Cheng mac_srs->srs_pri = prival; 1445da14cebeSEric Cheng thread_lock(mac_srs->srs_worker); 1446da14cebeSEric Cheng (void) thread_change_pri(mac_srs->srs_worker, mac_srs->srs_pri, 0); 1447da14cebeSEric Cheng thread_unlock(mac_srs->srs_worker); 1448da14cebeSEric Cheng if (mac_srs->srs_poll_thr != NULL) { 1449da14cebeSEric Cheng thread_lock(mac_srs->srs_poll_thr); 1450da14cebeSEric Cheng (void) thread_change_pri(mac_srs->srs_poll_thr, 1451da14cebeSEric Cheng mac_srs->srs_pri, 0); 1452da14cebeSEric Cheng thread_unlock(mac_srs->srs_poll_thr); 1453da14cebeSEric Cheng } 1454da14cebeSEric Cheng if ((ringp = mac_srs->srs_soft_ring_head) == NULL) 1455da14cebeSEric Cheng return; 1456da14cebeSEric Cheng while (ringp != mac_srs->srs_soft_ring_tail) { 1457da14cebeSEric Cheng thread_lock(ringp->s_ring_worker); 1458da14cebeSEric Cheng (void) thread_change_pri(ringp->s_ring_worker, 1459da14cebeSEric Cheng mac_srs->srs_pri, 0); 1460da14cebeSEric Cheng thread_unlock(ringp->s_ring_worker); 1461da14cebeSEric Cheng ringp = ringp->s_ring_next; 1462da14cebeSEric Cheng } 1463da14cebeSEric Cheng ASSERT(ringp == mac_srs->srs_soft_ring_tail); 1464da14cebeSEric Cheng thread_lock(ringp->s_ring_worker); 1465da14cebeSEric Cheng (void) thread_change_pri(ringp->s_ring_worker, mac_srs->srs_pri, 0); 1466da14cebeSEric Cheng thread_unlock(ringp->s_ring_worker); 1467da14cebeSEric Cheng } 1468da14cebeSEric Cheng 1469da14cebeSEric Cheng /* 1470da14cebeSEric Cheng * Change the receive bandwidth limit. 1471da14cebeSEric Cheng */ 1472da14cebeSEric Cheng static void 1473da14cebeSEric Cheng mac_rx_srs_update_bwlimit(mac_soft_ring_set_t *srs, mac_resource_props_t *mrp) 1474da14cebeSEric Cheng { 1475da14cebeSEric Cheng mac_soft_ring_t *softring; 1476da14cebeSEric Cheng 1477da14cebeSEric Cheng mutex_enter(&srs->srs_lock); 1478da14cebeSEric Cheng mutex_enter(&srs->srs_bw->mac_bw_lock); 1479da14cebeSEric Cheng 1480da14cebeSEric Cheng if (mrp->mrp_maxbw == MRP_MAXBW_RESETVAL) { 1481da14cebeSEric Cheng /* Reset bandwidth limit */ 1482da14cebeSEric Cheng if (srs->srs_type & SRST_BW_CONTROL) { 1483da14cebeSEric Cheng softring = srs->srs_soft_ring_head; 1484da14cebeSEric Cheng while (softring != NULL) { 1485da14cebeSEric Cheng softring->s_ring_type &= ~ST_RING_BW_CTL; 1486da14cebeSEric Cheng softring = softring->s_ring_next; 1487da14cebeSEric Cheng } 1488da14cebeSEric Cheng srs->srs_type &= ~SRST_BW_CONTROL; 1489da14cebeSEric Cheng srs->srs_drain_func = mac_rx_srs_drain; 1490da14cebeSEric Cheng } 1491da14cebeSEric Cheng } else { 1492da14cebeSEric Cheng /* Set/Modify bandwidth limit */ 1493da14cebeSEric Cheng srs->srs_bw->mac_bw_limit = FLOW_BYTES_PER_TICK(mrp->mrp_maxbw); 1494da14cebeSEric Cheng /* 1495da14cebeSEric Cheng * Give twice the queuing capability before 1496da14cebeSEric Cheng * dropping packets. The unit is bytes/tick. 1497da14cebeSEric Cheng */ 1498da14cebeSEric Cheng srs->srs_bw->mac_bw_drop_threshold = 1499da14cebeSEric Cheng srs->srs_bw->mac_bw_limit << 1; 1500da14cebeSEric Cheng if (!(srs->srs_type & SRST_BW_CONTROL)) { 1501da14cebeSEric Cheng softring = srs->srs_soft_ring_head; 1502da14cebeSEric Cheng while (softring != NULL) { 1503da14cebeSEric Cheng softring->s_ring_type |= ST_RING_BW_CTL; 1504da14cebeSEric Cheng softring = softring->s_ring_next; 1505da14cebeSEric Cheng } 1506da14cebeSEric Cheng srs->srs_type |= SRST_BW_CONTROL; 1507da14cebeSEric Cheng srs->srs_drain_func = mac_rx_srs_drain_bw; 1508da14cebeSEric Cheng } 1509da14cebeSEric Cheng } 1510da14cebeSEric Cheng done: 1511da14cebeSEric Cheng mutex_exit(&srs->srs_bw->mac_bw_lock); 1512da14cebeSEric Cheng mutex_exit(&srs->srs_lock); 1513da14cebeSEric Cheng } 1514da14cebeSEric Cheng 1515da14cebeSEric Cheng /* Change the transmit bandwidth limit */ 1516da14cebeSEric Cheng static void 1517da14cebeSEric Cheng mac_tx_srs_update_bwlimit(mac_soft_ring_set_t *srs, mac_resource_props_t *mrp) 1518da14cebeSEric Cheng { 15190dc2366fSVenugopal Iyer uint32_t tx_mode, ring_info = 0; 1520f94ede51SEric Cheng mac_srs_tx_t *srs_tx = &srs->srs_tx; 1521f94ede51SEric Cheng mac_client_impl_t *mcip = srs->srs_mcip; 1522f94ede51SEric Cheng 1523f94ede51SEric Cheng /* 1524f94ede51SEric Cheng * We need to quiesce/restart the client here because mac_tx() and 1525f94ede51SEric Cheng * srs->srs_tx->st_func do not hold srs->srs_lock while accessing 1526f94ede51SEric Cheng * st_mode and related fields, which are modified by the code below. 1527f94ede51SEric Cheng */ 15280dc2366fSVenugopal Iyer mac_tx_client_quiesce((mac_client_handle_t)mcip); 1529da14cebeSEric Cheng 1530da14cebeSEric Cheng mutex_enter(&srs->srs_lock); 1531da14cebeSEric Cheng mutex_enter(&srs->srs_bw->mac_bw_lock); 1532da14cebeSEric Cheng 1533da14cebeSEric Cheng tx_mode = srs_tx->st_mode; 1534da14cebeSEric Cheng if (mrp->mrp_maxbw == MRP_MAXBW_RESETVAL) { 1535da14cebeSEric Cheng /* Reset bandwidth limit */ 1536da14cebeSEric Cheng if (tx_mode == SRS_TX_BW) { 15370dc2366fSVenugopal Iyer if (srs_tx->st_arg2 != NULL) 15380dc2366fSVenugopal Iyer ring_info = mac_hwring_getinfo(srs_tx->st_arg2); 1539da14cebeSEric Cheng if (mac_tx_serialize || 15400dc2366fSVenugopal Iyer (ring_info & MAC_RING_TX_SERIALIZE)) { 1541da14cebeSEric Cheng srs_tx->st_mode = SRS_TX_SERIALIZE; 1542da14cebeSEric Cheng } else { 1543da14cebeSEric Cheng srs_tx->st_mode = SRS_TX_DEFAULT; 1544da14cebeSEric Cheng } 1545da14cebeSEric Cheng } else if (tx_mode == SRS_TX_BW_FANOUT) { 1546da14cebeSEric Cheng srs_tx->st_mode = SRS_TX_FANOUT; 15470dc2366fSVenugopal Iyer } else if (tx_mode == SRS_TX_BW_AGGR) { 15480dc2366fSVenugopal Iyer srs_tx->st_mode = SRS_TX_AGGR; 1549da14cebeSEric Cheng } 1550da14cebeSEric Cheng srs->srs_type &= ~SRST_BW_CONTROL; 1551da14cebeSEric Cheng } else { 1552da14cebeSEric Cheng /* Set/Modify bandwidth limit */ 1553da14cebeSEric Cheng srs->srs_bw->mac_bw_limit = FLOW_BYTES_PER_TICK(mrp->mrp_maxbw); 1554da14cebeSEric Cheng /* 1555da14cebeSEric Cheng * Give twice the queuing capability before 1556da14cebeSEric Cheng * dropping packets. The unit is bytes/tick. 1557da14cebeSEric Cheng */ 1558da14cebeSEric Cheng srs->srs_bw->mac_bw_drop_threshold = 1559da14cebeSEric Cheng srs->srs_bw->mac_bw_limit << 1; 1560da14cebeSEric Cheng srs->srs_type |= SRST_BW_CONTROL; 15610dc2366fSVenugopal Iyer if (tx_mode != SRS_TX_BW && tx_mode != SRS_TX_BW_FANOUT && 15620dc2366fSVenugopal Iyer tx_mode != SRS_TX_BW_AGGR) { 1563da14cebeSEric Cheng if (tx_mode == SRS_TX_SERIALIZE || 1564da14cebeSEric Cheng tx_mode == SRS_TX_DEFAULT) { 1565da14cebeSEric Cheng srs_tx->st_mode = SRS_TX_BW; 1566da14cebeSEric Cheng } else if (tx_mode == SRS_TX_FANOUT) { 1567da14cebeSEric Cheng srs_tx->st_mode = SRS_TX_BW_FANOUT; 15680dc2366fSVenugopal Iyer } else if (tx_mode == SRS_TX_AGGR) { 15690dc2366fSVenugopal Iyer srs_tx->st_mode = SRS_TX_BW_AGGR; 1570da14cebeSEric Cheng } else { 1571da14cebeSEric Cheng ASSERT(0); 1572da14cebeSEric Cheng } 1573da14cebeSEric Cheng } 1574da14cebeSEric Cheng } 1575da14cebeSEric Cheng done: 1576da14cebeSEric Cheng srs_tx->st_func = mac_tx_get_func(srs_tx->st_mode); 1577da14cebeSEric Cheng mutex_exit(&srs->srs_bw->mac_bw_lock); 1578da14cebeSEric Cheng mutex_exit(&srs->srs_lock); 1579f94ede51SEric Cheng 15800dc2366fSVenugopal Iyer mac_tx_client_restart((mac_client_handle_t)mcip); 1581da14cebeSEric Cheng } 1582da14cebeSEric Cheng 1583da14cebeSEric Cheng /* 1584da14cebeSEric Cheng * The uber function that deals with any update to bandwidth limits. 1585da14cebeSEric Cheng */ 1586da14cebeSEric Cheng void 1587da14cebeSEric Cheng mac_srs_update_bwlimit(flow_entry_t *flent, mac_resource_props_t *mrp) 1588da14cebeSEric Cheng { 1589da14cebeSEric Cheng int count; 1590da14cebeSEric Cheng 1591da14cebeSEric Cheng for (count = 0; count < flent->fe_rx_srs_cnt; count++) 1592da14cebeSEric Cheng mac_rx_srs_update_bwlimit(flent->fe_rx_srs[count], mrp); 1593da14cebeSEric Cheng mac_tx_srs_update_bwlimit(flent->fe_tx_srs, mrp); 1594da14cebeSEric Cheng } 1595da14cebeSEric Cheng 1596da14cebeSEric Cheng /* 1597da14cebeSEric Cheng * When the first sub-flow is added to a link, we disable polling on the 159884de666eSRyan Zezeski * link and also modify the entry point to mac_rx_srs_subflow_process(). 1599da14cebeSEric Cheng * (polling is disabled because with the subflow added, accounting 1600da14cebeSEric Cheng * for polling needs additional logic, it is assumed that when a subflow is 1601da14cebeSEric Cheng * added, we can take some hit as a result of disabling polling rather than 1602da14cebeSEric Cheng * adding more complexity - if this becomes a perf. issue we need to 1603da14cebeSEric Cheng * re-rvaluate this logic). When the last subflow is removed, we turn back 160484de666eSRyan Zezeski * polling and also reset the entry point to mac_rx_srs_process(). 1605da14cebeSEric Cheng * 1606da14cebeSEric Cheng * In the future if there are multiple SRS, we can simply 1607da14cebeSEric Cheng * take one and give it to the flow rather than disabling polling and 1608da14cebeSEric Cheng * resetting the entry point. 1609da14cebeSEric Cheng */ 1610da14cebeSEric Cheng void 1611da14cebeSEric Cheng mac_client_update_classifier(mac_client_impl_t *mcip, boolean_t enable) 1612da14cebeSEric Cheng { 1613da14cebeSEric Cheng flow_entry_t *flent = mcip->mci_flent; 1614da14cebeSEric Cheng int i; 1615da14cebeSEric Cheng mac_impl_t *mip = mcip->mci_mip; 1616da14cebeSEric Cheng mac_rx_func_t rx_func; 1617da14cebeSEric Cheng uint_t rx_srs_cnt; 1618da14cebeSEric Cheng boolean_t enable_classifier; 1619da14cebeSEric Cheng 1620da14cebeSEric Cheng ASSERT(MAC_PERIM_HELD((mac_handle_t)mip)); 1621da14cebeSEric Cheng 1622da14cebeSEric Cheng enable_classifier = !FLOW_TAB_EMPTY(mcip->mci_subflow_tab) && enable; 1623da14cebeSEric Cheng 1624da14cebeSEric Cheng rx_func = enable_classifier ? mac_rx_srs_subflow_process : 1625da14cebeSEric Cheng mac_rx_srs_process; 1626da14cebeSEric Cheng 16274eaa4710SRishi Srivatsavai /* Tell mac_srs_poll_state_change to disable polling if necessary */ 16284eaa4710SRishi Srivatsavai if (mip->mi_state_flags & MIS_POLL_DISABLE) 16294eaa4710SRishi Srivatsavai enable_classifier = B_TRUE; 16304eaa4710SRishi Srivatsavai 1631da14cebeSEric Cheng /* 1632da14cebeSEric Cheng * If receive function has already been configured correctly for 1633da14cebeSEric Cheng * current subflow configuration, do nothing. 1634da14cebeSEric Cheng */ 1635da14cebeSEric Cheng if (flent->fe_cb_fn == (flow_fn_t)rx_func) 1636da14cebeSEric Cheng return; 1637da14cebeSEric Cheng 1638da14cebeSEric Cheng rx_srs_cnt = flent->fe_rx_srs_cnt; 1639da14cebeSEric Cheng for (i = 0; i < rx_srs_cnt; i++) { 1640da14cebeSEric Cheng ASSERT(flent->fe_rx_srs[i] != NULL); 1641da14cebeSEric Cheng mac_srs_poll_state_change(flent->fe_rx_srs[i], 1642da14cebeSEric Cheng enable_classifier, rx_func); 1643da14cebeSEric Cheng } 1644da14cebeSEric Cheng 1645da14cebeSEric Cheng /* 1646da14cebeSEric Cheng * Change the S/W classifier so that we can land in the 1647da14cebeSEric Cheng * correct processing function with correct argument. 1648da14cebeSEric Cheng * If all subflows have been removed we can revert to 164984de666eSRyan Zezeski * mac_rx_srs_process(), else we need mac_rx_srs_subflow_process(). 1650da14cebeSEric Cheng */ 1651da14cebeSEric Cheng mutex_enter(&flent->fe_lock); 1652da14cebeSEric Cheng flent->fe_cb_fn = (flow_fn_t)rx_func; 1653da14cebeSEric Cheng flent->fe_cb_arg1 = (void *)mip; 1654da14cebeSEric Cheng flent->fe_cb_arg2 = flent->fe_rx_srs[0]; 1655da14cebeSEric Cheng mutex_exit(&flent->fe_lock); 1656da14cebeSEric Cheng } 1657da14cebeSEric Cheng 1658da14cebeSEric Cheng static void 1659da14cebeSEric Cheng mac_srs_update_fanout_list(mac_soft_ring_set_t *mac_srs) 1660da14cebeSEric Cheng { 16610dc2366fSVenugopal Iyer int tcp_count = 0, udp_count = 0, oth_count = 0, tx_count = 0; 1662da14cebeSEric Cheng mac_soft_ring_t *softring; 1663da14cebeSEric Cheng 1664da14cebeSEric Cheng softring = mac_srs->srs_soft_ring_head; 1665da14cebeSEric Cheng if (softring == NULL) { 1666da14cebeSEric Cheng ASSERT(mac_srs->srs_soft_ring_count == 0); 1667da14cebeSEric Cheng mac_srs->srs_tcp_ring_count = 0; 1668da14cebeSEric Cheng mac_srs->srs_udp_ring_count = 0; 1669da14cebeSEric Cheng mac_srs->srs_oth_ring_count = 0; 16700dc2366fSVenugopal Iyer mac_srs->srs_tx_ring_count = 0; 1671da14cebeSEric Cheng return; 1672da14cebeSEric Cheng } 1673da14cebeSEric Cheng 1674da14cebeSEric Cheng while (softring != NULL) { 16750dc2366fSVenugopal Iyer if (softring->s_ring_type & ST_RING_TCP) { 1676da14cebeSEric Cheng mac_srs->srs_tcp_soft_rings[tcp_count++] = softring; 16770dc2366fSVenugopal Iyer } else if (softring->s_ring_type & ST_RING_UDP) { 1678da14cebeSEric Cheng mac_srs->srs_udp_soft_rings[udp_count++] = softring; 16790dc2366fSVenugopal Iyer } else if (softring->s_ring_type & ST_RING_OTH) { 1680da14cebeSEric Cheng mac_srs->srs_oth_soft_rings[oth_count++] = softring; 16810dc2366fSVenugopal Iyer } else { 16820dc2366fSVenugopal Iyer ASSERT(softring->s_ring_type & ST_RING_TX); 16830dc2366fSVenugopal Iyer mac_srs->srs_tx_soft_rings[tx_count++] = softring; 16840dc2366fSVenugopal Iyer } 1685da14cebeSEric Cheng softring = softring->s_ring_next; 1686da14cebeSEric Cheng } 1687da14cebeSEric Cheng 1688da14cebeSEric Cheng ASSERT(mac_srs->srs_soft_ring_count == 16890dc2366fSVenugopal Iyer (tcp_count + udp_count + oth_count + tx_count)); 1690da14cebeSEric Cheng mac_srs->srs_tcp_ring_count = tcp_count; 1691da14cebeSEric Cheng mac_srs->srs_udp_ring_count = udp_count; 1692da14cebeSEric Cheng mac_srs->srs_oth_ring_count = oth_count; 16930dc2366fSVenugopal Iyer mac_srs->srs_tx_ring_count = tx_count; 1694da14cebeSEric Cheng } 1695da14cebeSEric Cheng 1696da14cebeSEric Cheng void 16970dc2366fSVenugopal Iyer mac_srs_create_proto_softrings(int id, uint16_t type, pri_t pri, 16980dc2366fSVenugopal Iyer mac_client_impl_t *mcip, mac_soft_ring_set_t *mac_srs, 1699da14cebeSEric Cheng processorid_t cpuid, mac_direct_rx_t rx_func, void *x_arg1, 1700da14cebeSEric Cheng mac_resource_handle_t x_arg2, boolean_t set_bypass) 1701da14cebeSEric Cheng { 1702da14cebeSEric Cheng mac_soft_ring_t *softring; 1703da14cebeSEric Cheng mac_rx_fifo_t mrf; 1704da14cebeSEric Cheng 1705da14cebeSEric Cheng bzero(&mrf, sizeof (mac_rx_fifo_t)); 1706da14cebeSEric Cheng mrf.mrf_type = MAC_RX_FIFO; 1707da14cebeSEric Cheng mrf.mrf_receive = (mac_receive_t)mac_soft_ring_poll; 17082c465844SToomas Soome mrf.mrf_intr_enable = (mac_intr_enable_t)mac_soft_ring_intr_enable; 17092c465844SToomas Soome mrf.mrf_intr_disable = (mac_intr_disable_t)mac_soft_ring_intr_disable; 1710da14cebeSEric Cheng mrf.mrf_flow_priority = pri; 1711da14cebeSEric Cheng 1712da14cebeSEric Cheng softring = mac_soft_ring_create(id, mac_soft_ring_worker_wait, 17130dc2366fSVenugopal Iyer (type|ST_RING_TCP), pri, mcip, mac_srs, 1714da14cebeSEric Cheng cpuid, rx_func, x_arg1, x_arg2); 1715da14cebeSEric Cheng softring->s_ring_rx_arg2 = NULL; 1716da14cebeSEric Cheng 1717da14cebeSEric Cheng /* 1718da14cebeSEric Cheng * TCP and UDP support DLS bypass. In addition TCP 1719da14cebeSEric Cheng * squeue can also poll their corresponding soft rings. 1720da14cebeSEric Cheng */ 1721da14cebeSEric Cheng if (set_bypass && (mcip->mci_resource_arg != NULL)) { 1722da14cebeSEric Cheng mac_soft_ring_dls_bypass(softring, 1723da14cebeSEric Cheng mcip->mci_direct_rx_fn, 1724da14cebeSEric Cheng mcip->mci_direct_rx_arg); 1725da14cebeSEric Cheng 1726da14cebeSEric Cheng mrf.mrf_rx_arg = softring; 1727da14cebeSEric Cheng mrf.mrf_intr_handle = (mac_intr_handle_t)softring; 1728da14cebeSEric Cheng 1729da14cebeSEric Cheng /* 1730da14cebeSEric Cheng * Make a call in IP to get a TCP squeue assigned to 1731da14cebeSEric Cheng * this softring to maintain full CPU locality through 1732da14cebeSEric Cheng * the stack and allow the squeue to be able to poll 1733da14cebeSEric Cheng * the softring so the flow control can be pushed 1734da14cebeSEric Cheng * all the way to H/W. 1735da14cebeSEric Cheng */ 1736da14cebeSEric Cheng softring->s_ring_rx_arg2 = 1737da14cebeSEric Cheng mcip->mci_resource_add((void *)mcip->mci_resource_arg, 1738da14cebeSEric Cheng (mac_resource_t *)&mrf); 1739da14cebeSEric Cheng } 1740da14cebeSEric Cheng 1741da14cebeSEric Cheng /* 1742da14cebeSEric Cheng * Non-TCP protocols don't support squeues. Hence we 1743da14cebeSEric Cheng * don't make any ring addition callbacks for non-TCP 1744da14cebeSEric Cheng * rings. Now create the UDP softring and allow it to 1745da14cebeSEric Cheng * bypass the DLS layer. 1746da14cebeSEric Cheng */ 1747da14cebeSEric Cheng softring = mac_soft_ring_create(id, mac_soft_ring_worker_wait, 17480dc2366fSVenugopal Iyer (type|ST_RING_UDP), pri, mcip, mac_srs, 1749da14cebeSEric Cheng cpuid, rx_func, x_arg1, x_arg2); 1750da14cebeSEric Cheng softring->s_ring_rx_arg2 = NULL; 1751da14cebeSEric Cheng 1752da14cebeSEric Cheng if (set_bypass && (mcip->mci_resource_arg != NULL)) { 1753da14cebeSEric Cheng mac_soft_ring_dls_bypass(softring, 1754da14cebeSEric Cheng mcip->mci_direct_rx_fn, 1755da14cebeSEric Cheng mcip->mci_direct_rx_arg); 1756da14cebeSEric Cheng } 1757da14cebeSEric Cheng 1758da14cebeSEric Cheng /* Create the Oth softrings which has to go through the DLS */ 1759da14cebeSEric Cheng softring = mac_soft_ring_create(id, mac_soft_ring_worker_wait, 17600dc2366fSVenugopal Iyer (type|ST_RING_OTH), pri, mcip, mac_srs, 1761da14cebeSEric Cheng cpuid, rx_func, x_arg1, x_arg2); 1762da14cebeSEric Cheng softring->s_ring_rx_arg2 = NULL; 1763da14cebeSEric Cheng } 1764da14cebeSEric Cheng 1765da14cebeSEric Cheng /* 1766da14cebeSEric Cheng * This routine associates a CPU or a set of CPU to process incoming 1767da14cebeSEric Cheng * traffic from a mac client. If multiple CPUs are specified, then 1768da14cebeSEric Cheng * so many soft rings are created with each soft ring worker thread 1769da14cebeSEric Cheng * bound to a CPU in the set. Each soft ring in turn will be 1770da14cebeSEric Cheng * associated with an squeue and the squeue will be moved to the 1771da14cebeSEric Cheng * same CPU as that of the soft ring's. 1772da14cebeSEric Cheng */ 1773da14cebeSEric Cheng static void 17740dc2366fSVenugopal Iyer mac_srs_fanout_modify(mac_client_impl_t *mcip, mac_direct_rx_t rx_func, 17750dc2366fSVenugopal Iyer void *x_arg1, mac_resource_handle_t x_arg2, 17760dc2366fSVenugopal Iyer mac_soft_ring_set_t *mac_rx_srs, mac_soft_ring_set_t *mac_tx_srs) 1777da14cebeSEric Cheng { 1778da14cebeSEric Cheng mac_soft_ring_t *softring; 1779ae6aa22aSVenugopal Iyer uint32_t soft_ring_flag = 0; 1780da14cebeSEric Cheng processorid_t cpuid = -1; 1781da14cebeSEric Cheng int i, srings_present, new_fanout_cnt; 1782da14cebeSEric Cheng mac_cpus_t *srs_cpu; 1783da14cebeSEric Cheng 1784da14cebeSEric Cheng /* fanout state is REINIT. Set it back to INIT */ 1785da14cebeSEric Cheng ASSERT(mac_rx_srs->srs_fanout_state == SRS_FANOUT_REINIT); 1786da14cebeSEric Cheng mac_rx_srs->srs_fanout_state = SRS_FANOUT_INIT; 1787da14cebeSEric Cheng 1788da14cebeSEric Cheng /* how many are present right now */ 1789da14cebeSEric Cheng srings_present = mac_rx_srs->srs_tcp_ring_count; 1790da14cebeSEric Cheng /* new request */ 1791da14cebeSEric Cheng srs_cpu = &mac_rx_srs->srs_cpu; 17920dc2366fSVenugopal Iyer new_fanout_cnt = srs_cpu->mc_rx_fanout_cnt; 1793da14cebeSEric Cheng 1794da14cebeSEric Cheng mutex_enter(&mac_rx_srs->srs_lock); 1795da14cebeSEric Cheng if (mac_rx_srs->srs_type & SRST_BW_CONTROL) 1796da14cebeSEric Cheng soft_ring_flag |= ST_RING_BW_CTL; 1797da14cebeSEric Cheng mutex_exit(&mac_rx_srs->srs_lock); 1798da14cebeSEric Cheng 1799da14cebeSEric Cheng if (new_fanout_cnt > srings_present) { 1800da14cebeSEric Cheng /* soft rings increased */ 1801da14cebeSEric Cheng mutex_enter(&mac_rx_srs->srs_lock); 1802da14cebeSEric Cheng mac_rx_srs->srs_type |= SRST_FANOUT_SRC_IP; 1803da14cebeSEric Cheng mutex_exit(&mac_rx_srs->srs_lock); 1804da14cebeSEric Cheng 1805da14cebeSEric Cheng for (i = mac_rx_srs->srs_tcp_ring_count; 1806da14cebeSEric Cheng i < new_fanout_cnt; i++) { 1807da14cebeSEric Cheng /* 1808da14cebeSEric Cheng * Create the protocol softrings and set the 1809da14cebeSEric Cheng * DLS bypass where possible. 1810da14cebeSEric Cheng */ 18110dc2366fSVenugopal Iyer mac_srs_create_proto_softrings(i, soft_ring_flag, 1812da14cebeSEric Cheng mac_rx_srs->srs_pri, mcip, mac_rx_srs, cpuid, 1813da14cebeSEric Cheng rx_func, x_arg1, x_arg2, B_TRUE); 1814da14cebeSEric Cheng } 1815da14cebeSEric Cheng mac_srs_update_fanout_list(mac_rx_srs); 1816da14cebeSEric Cheng } else if (new_fanout_cnt < srings_present) { 1817da14cebeSEric Cheng /* soft rings decreased */ 1818da14cebeSEric Cheng if (new_fanout_cnt == 1) { 1819da14cebeSEric Cheng mutex_enter(&mac_rx_srs->srs_lock); 1820da14cebeSEric Cheng mac_rx_srs->srs_type &= ~SRST_FANOUT_SRC_IP; 1821da14cebeSEric Cheng ASSERT(mac_rx_srs->srs_type & SRST_FANOUT_PROTO); 1822da14cebeSEric Cheng mutex_exit(&mac_rx_srs->srs_lock); 1823da14cebeSEric Cheng } 1824da14cebeSEric Cheng /* Get rid of extra soft rings */ 1825da14cebeSEric Cheng for (i = new_fanout_cnt; 1826da14cebeSEric Cheng i < mac_rx_srs->srs_tcp_ring_count; i++) { 1827da14cebeSEric Cheng softring = mac_rx_srs->srs_tcp_soft_rings[i]; 1828da14cebeSEric Cheng if (softring->s_ring_rx_arg2 != NULL) { 1829da14cebeSEric Cheng mcip->mci_resource_remove( 1830da14cebeSEric Cheng (void *)mcip->mci_resource_arg, 1831da14cebeSEric Cheng softring->s_ring_rx_arg2); 1832da14cebeSEric Cheng } 1833da14cebeSEric Cheng mac_soft_ring_remove(mac_rx_srs, 1834da14cebeSEric Cheng mac_rx_srs->srs_tcp_soft_rings[i]); 1835da14cebeSEric Cheng mac_soft_ring_remove(mac_rx_srs, 1836da14cebeSEric Cheng mac_rx_srs->srs_udp_soft_rings[i]); 1837da14cebeSEric Cheng mac_soft_ring_remove(mac_rx_srs, 1838da14cebeSEric Cheng mac_rx_srs->srs_oth_soft_rings[i]); 1839da14cebeSEric Cheng } 1840da14cebeSEric Cheng mac_srs_update_fanout_list(mac_rx_srs); 1841da14cebeSEric Cheng } 1842da14cebeSEric Cheng 1843da14cebeSEric Cheng ASSERT(new_fanout_cnt == mac_rx_srs->srs_tcp_ring_count); 1844da14cebeSEric Cheng mutex_enter(&cpu_lock); 1845da14cebeSEric Cheng for (i = 0; i < mac_rx_srs->srs_tcp_ring_count; i++) { 18460dc2366fSVenugopal Iyer cpuid = srs_cpu->mc_rx_fanout_cpus[i]; 1847da14cebeSEric Cheng (void) mac_soft_ring_bind(mac_rx_srs->srs_udp_soft_rings[i], 1848da14cebeSEric Cheng cpuid); 1849da14cebeSEric Cheng (void) mac_soft_ring_bind(mac_rx_srs->srs_oth_soft_rings[i], 1850da14cebeSEric Cheng cpuid); 1851da14cebeSEric Cheng (void) mac_soft_ring_bind(mac_rx_srs->srs_tcp_soft_rings[i], 1852da14cebeSEric Cheng cpuid); 1853da14cebeSEric Cheng softring = mac_rx_srs->srs_tcp_soft_rings[i]; 1854da14cebeSEric Cheng if (softring->s_ring_rx_arg2 != NULL) { 1855da14cebeSEric Cheng mcip->mci_resource_bind((void *)mcip->mci_resource_arg, 1856da14cebeSEric Cheng softring->s_ring_rx_arg2, cpuid); 1857da14cebeSEric Cheng } 1858da14cebeSEric Cheng } 1859da14cebeSEric Cheng 18600dc2366fSVenugopal Iyer mac_srs_worker_bind(mac_rx_srs, srs_cpu->mc_rx_workerid); 18610dc2366fSVenugopal Iyer mac_srs_poll_bind(mac_rx_srs, srs_cpu->mc_rx_pollid); 18620dc2366fSVenugopal Iyer mac_rx_srs_retarget_intr(mac_rx_srs, srs_cpu->mc_rx_intr_cpu); 1863da14cebeSEric Cheng /* 1864da14cebeSEric Cheng * Bind Tx srs and soft ring threads too. Let's bind tx 1865da14cebeSEric Cheng * srs to the last cpu in mrp list. 1866da14cebeSEric Cheng */ 18670dc2366fSVenugopal Iyer if (mac_tx_srs != NULL) { 1868da14cebeSEric Cheng BIND_TX_SRS_AND_SOFT_RINGS(mac_tx_srs, mrp); 18690dc2366fSVenugopal Iyer mac_tx_srs_retarget_intr(mac_tx_srs); 1870da14cebeSEric Cheng } 1871da14cebeSEric Cheng mutex_exit(&cpu_lock); 1872da14cebeSEric Cheng } 1873da14cebeSEric Cheng 1874da14cebeSEric Cheng /* 1875da14cebeSEric Cheng * Bind SRS threads and soft rings to CPUs/create fanout list. 1876da14cebeSEric Cheng */ 1877da14cebeSEric Cheng void 18780dc2366fSVenugopal Iyer mac_srs_fanout_init(mac_client_impl_t *mcip, mac_resource_props_t *mrp, 18790dc2366fSVenugopal Iyer mac_direct_rx_t rx_func, void *x_arg1, mac_resource_handle_t x_arg2, 18800dc2366fSVenugopal Iyer mac_soft_ring_set_t *mac_rx_srs, mac_soft_ring_set_t *mac_tx_srs, 18810dc2366fSVenugopal Iyer cpupart_t *cpupart) 1882da14cebeSEric Cheng { 1883da14cebeSEric Cheng int i; 18840dc2366fSVenugopal Iyer processorid_t cpuid; 1885ae6aa22aSVenugopal Iyer uint32_t soft_ring_flag = 0; 1886da14cebeSEric Cheng int soft_ring_cnt; 1887da14cebeSEric Cheng mac_cpus_t *srs_cpu = &mac_rx_srs->srs_cpu; 1888da14cebeSEric Cheng 1889da14cebeSEric Cheng /* 1890da14cebeSEric Cheng * Remove the no soft ring flag and we will adjust it 1891da14cebeSEric Cheng * appropriately further down. 1892da14cebeSEric Cheng */ 1893da14cebeSEric Cheng mutex_enter(&mac_rx_srs->srs_lock); 1894da14cebeSEric Cheng mac_rx_srs->srs_type &= ~SRST_NO_SOFT_RINGS; 1895da14cebeSEric Cheng mutex_exit(&mac_rx_srs->srs_lock); 1896da14cebeSEric Cheng 1897da14cebeSEric Cheng ASSERT(mac_rx_srs->srs_soft_ring_head == NULL); 1898da14cebeSEric Cheng 1899da14cebeSEric Cheng if (mac_rx_srs->srs_type & SRST_BW_CONTROL) 1900da14cebeSEric Cheng soft_ring_flag |= ST_RING_BW_CTL; 1901da14cebeSEric Cheng 1902da14cebeSEric Cheng ASSERT(mac_rx_srs->srs_fanout_state == SRS_FANOUT_UNINIT); 1903da14cebeSEric Cheng mac_rx_srs->srs_fanout_state = SRS_FANOUT_INIT; 1904da14cebeSEric Cheng /* 1905da14cebeSEric Cheng * Ring count can be 0 if no fanout is required and no cpu 1906da14cebeSEric Cheng * were specified. Leave the SRS worker and poll thread 1907da14cebeSEric Cheng * unbound 1908da14cebeSEric Cheng */ 1909da14cebeSEric Cheng ASSERT(mrp != NULL); 19100dc2366fSVenugopal Iyer soft_ring_cnt = srs_cpu->mc_rx_fanout_cnt; 1911da14cebeSEric Cheng 1912da14cebeSEric Cheng /* Step 1: bind cpu contains cpu list where threads need to bind */ 1913da14cebeSEric Cheng if (soft_ring_cnt > 0) { 1914da14cebeSEric Cheng mutex_enter(&cpu_lock); 1915da14cebeSEric Cheng for (i = 0; i < soft_ring_cnt; i++) { 19160dc2366fSVenugopal Iyer cpuid = srs_cpu->mc_rx_fanout_cpus[i]; 1917da14cebeSEric Cheng /* Create the protocol softrings */ 19180dc2366fSVenugopal Iyer mac_srs_create_proto_softrings(i, soft_ring_flag, 19190dc2366fSVenugopal Iyer mac_rx_srs->srs_pri, mcip, mac_rx_srs, cpuid, 19200dc2366fSVenugopal Iyer rx_func, x_arg1, x_arg2, B_FALSE); 1921da14cebeSEric Cheng } 19220dc2366fSVenugopal Iyer mac_srs_worker_bind(mac_rx_srs, srs_cpu->mc_rx_workerid); 19230dc2366fSVenugopal Iyer mac_srs_poll_bind(mac_rx_srs, srs_cpu->mc_rx_pollid); 19240dc2366fSVenugopal Iyer mac_rx_srs_retarget_intr(mac_rx_srs, srs_cpu->mc_rx_intr_cpu); 1925da14cebeSEric Cheng /* 1926da14cebeSEric Cheng * Bind Tx srs and soft ring threads too. 1927da14cebeSEric Cheng * Let's bind tx srs to the last cpu in 1928da14cebeSEric Cheng * mrp list. 1929da14cebeSEric Cheng */ 1930da14cebeSEric Cheng if (mac_tx_srs == NULL) { 1931da14cebeSEric Cheng mutex_exit(&cpu_lock); 1932da14cebeSEric Cheng goto alldone; 1933da14cebeSEric Cheng } 1934da14cebeSEric Cheng 1935da14cebeSEric Cheng BIND_TX_SRS_AND_SOFT_RINGS(mac_tx_srs, mrp); 19360dc2366fSVenugopal Iyer mac_tx_srs_retarget_intr(mac_tx_srs); 1937da14cebeSEric Cheng mutex_exit(&cpu_lock); 1938da14cebeSEric Cheng } else { 1939da14cebeSEric Cheng mutex_enter(&cpu_lock); 1940da14cebeSEric Cheng /* 1941da14cebeSEric Cheng * For a subflow, mrp_workerid and mrp_pollid 1942da14cebeSEric Cheng * is not set. 1943da14cebeSEric Cheng */ 19440dc2366fSVenugopal Iyer mac_srs_worker_bind(mac_rx_srs, mrp->mrp_rx_workerid); 19450dc2366fSVenugopal Iyer mac_srs_poll_bind(mac_rx_srs, mrp->mrp_rx_pollid); 1946da14cebeSEric Cheng mutex_exit(&cpu_lock); 1947da14cebeSEric Cheng goto no_softrings; 1948da14cebeSEric Cheng } 1949da14cebeSEric Cheng 1950da14cebeSEric Cheng alldone: 1951da14cebeSEric Cheng if (soft_ring_cnt > 1) 1952da14cebeSEric Cheng mac_rx_srs->srs_type |= SRST_FANOUT_SRC_IP; 1953da14cebeSEric Cheng mac_srs_update_fanout_list(mac_rx_srs); 1954da14cebeSEric Cheng mac_srs_client_poll_enable(mcip, mac_rx_srs); 1955da14cebeSEric Cheng return; 1956da14cebeSEric Cheng 1957da14cebeSEric Cheng no_softrings: 1958da14cebeSEric Cheng if (mac_rx_srs->srs_type & SRST_FANOUT_PROTO) { 1959da14cebeSEric Cheng mutex_enter(&cpu_lock); 19600dc2366fSVenugopal Iyer cpuid = mac_next_bind_cpu(cpupart); 1961da14cebeSEric Cheng /* Create the protocol softrings */ 19620dc2366fSVenugopal Iyer mac_srs_create_proto_softrings(0, soft_ring_flag, 19630dc2366fSVenugopal Iyer mac_rx_srs->srs_pri, mcip, mac_rx_srs, cpuid, 19640dc2366fSVenugopal Iyer rx_func, x_arg1, x_arg2, B_FALSE); 1965da14cebeSEric Cheng mutex_exit(&cpu_lock); 1966da14cebeSEric Cheng } else { 1967da14cebeSEric Cheng /* 1968da14cebeSEric Cheng * This is the case when there is no fanout which is 1969da14cebeSEric Cheng * true for subflows. 1970da14cebeSEric Cheng */ 1971da14cebeSEric Cheng mac_rx_srs->srs_type |= SRST_NO_SOFT_RINGS; 1972da14cebeSEric Cheng } 1973da14cebeSEric Cheng mac_srs_update_fanout_list(mac_rx_srs); 1974da14cebeSEric Cheng mac_srs_client_poll_enable(mcip, mac_rx_srs); 1975da14cebeSEric Cheng } 1976da14cebeSEric Cheng 1977da14cebeSEric Cheng /* 1978da14cebeSEric Cheng * Calls mac_srs_fanout_init() or modify() depending upon whether 1979da14cebeSEric Cheng * the SRS is getting initialized or re-initialized. 1980da14cebeSEric Cheng */ 1981da14cebeSEric Cheng void 1982da14cebeSEric Cheng mac_fanout_setup(mac_client_impl_t *mcip, flow_entry_t *flent, 1983da14cebeSEric Cheng mac_resource_props_t *mrp, mac_direct_rx_t rx_func, void *x_arg1, 19840dc2366fSVenugopal Iyer mac_resource_handle_t x_arg2, cpupart_t *cpupart) 1985da14cebeSEric Cheng { 1986da14cebeSEric Cheng mac_soft_ring_set_t *mac_rx_srs, *mac_tx_srs; 1987da14cebeSEric Cheng int i, rx_srs_cnt; 1988da14cebeSEric Cheng 1989da14cebeSEric Cheng ASSERT(MAC_PERIM_HELD((mac_handle_t)mcip->mci_mip)); 1990da14cebeSEric Cheng 199145948e49SRyan Zezeski /* 199245948e49SRyan Zezeski * Aggr ports do not have SRSes. This function should never be 199345948e49SRyan Zezeski * called on an aggr port. 199445948e49SRyan Zezeski */ 199545948e49SRyan Zezeski ASSERT3U((mcip->mci_state_flags & MCIS_IS_AGGR_PORT), ==, 0); 1996da14cebeSEric Cheng mac_rx_srs = flent->fe_rx_srs[0]; 199745948e49SRyan Zezeski 1998da14cebeSEric Cheng /* 1999da14cebeSEric Cheng * Set up the fanout on the tx side only once, with the 2000da14cebeSEric Cheng * first rx SRS. The CPU binding, fanout, and bandwidth 2001da14cebeSEric Cheng * criteria are common to both RX and TX, so 2002da14cebeSEric Cheng * initializing them along side avoids redundant code. 2003da14cebeSEric Cheng */ 2004da14cebeSEric Cheng mac_tx_srs = flent->fe_tx_srs; 2005da14cebeSEric Cheng rx_srs_cnt = flent->fe_rx_srs_cnt; 2006da14cebeSEric Cheng 2007da14cebeSEric Cheng /* No fanout for subflows */ 2008da14cebeSEric Cheng if (flent->fe_type & FLOW_USER) { 20090dc2366fSVenugopal Iyer mac_srs_fanout_init(mcip, mrp, rx_func, 20100dc2366fSVenugopal Iyer x_arg1, x_arg2, mac_rx_srs, mac_tx_srs, 20110dc2366fSVenugopal Iyer cpupart); 2012da14cebeSEric Cheng return; 2013da14cebeSEric Cheng } 2014da14cebeSEric Cheng 20150dc2366fSVenugopal Iyer if (mrp->mrp_mask & MRP_CPUS_USERSPEC) 20160dc2366fSVenugopal Iyer mac_flow_user_cpu_init(flent, mrp); 20170dc2366fSVenugopal Iyer else 20180dc2366fSVenugopal Iyer mac_flow_cpu_init(flent, cpupart); 20190dc2366fSVenugopal Iyer 20200dc2366fSVenugopal Iyer mrp->mrp_rx_fanout_cnt = mac_rx_srs->srs_cpu.mc_rx_fanout_cnt; 2021da14cebeSEric Cheng 2022da14cebeSEric Cheng /* 2023da14cebeSEric Cheng * Set up fanout for both SW (0th SRS) and HW classified 2024da14cebeSEric Cheng * SRS (the rest of Rx SRSs in flent). 2025da14cebeSEric Cheng */ 2026da14cebeSEric Cheng for (i = 0; i < rx_srs_cnt; i++) { 2027da14cebeSEric Cheng mac_rx_srs = flent->fe_rx_srs[i]; 2028da14cebeSEric Cheng if (i != 0) 2029da14cebeSEric Cheng mac_tx_srs = NULL; 2030da14cebeSEric Cheng switch (mac_rx_srs->srs_fanout_state) { 2031da14cebeSEric Cheng case SRS_FANOUT_UNINIT: 20320dc2366fSVenugopal Iyer mac_srs_fanout_init(mcip, mrp, rx_func, 20330dc2366fSVenugopal Iyer x_arg1, x_arg2, mac_rx_srs, mac_tx_srs, 20340dc2366fSVenugopal Iyer cpupart); 2035da14cebeSEric Cheng break; 2036da14cebeSEric Cheng case SRS_FANOUT_INIT: 2037da14cebeSEric Cheng break; 2038da14cebeSEric Cheng case SRS_FANOUT_REINIT: 2039da14cebeSEric Cheng mac_rx_srs_quiesce(mac_rx_srs, SRS_QUIESCE); 20400dc2366fSVenugopal Iyer mac_srs_fanout_modify(mcip, rx_func, x_arg1, 20410dc2366fSVenugopal Iyer x_arg2, mac_rx_srs, mac_tx_srs); 2042da14cebeSEric Cheng mac_rx_srs_restart(mac_rx_srs); 2043da14cebeSEric Cheng break; 2044da14cebeSEric Cheng default: 2045da14cebeSEric Cheng VERIFY(mac_rx_srs->srs_fanout_state <= 2046da14cebeSEric Cheng SRS_FANOUT_REINIT); 2047da14cebeSEric Cheng break; 2048da14cebeSEric Cheng } 2049da14cebeSEric Cheng } 2050da14cebeSEric Cheng } 2051da14cebeSEric Cheng 2052da14cebeSEric Cheng /* 2053da14cebeSEric Cheng * Create a mac_soft_ring_set_t (SRS). If soft_ring_fanout_type is 2054da14cebeSEric Cheng * SRST_TX, an SRS for Tx side is created. Otherwise an SRS for Rx side 2055da14cebeSEric Cheng * processing is created. 2056da14cebeSEric Cheng * 2057da14cebeSEric Cheng * Details on Rx SRS: 2058da14cebeSEric Cheng * Create a SRS and also add the necessary soft rings for TCP and 2059da14cebeSEric Cheng * non-TCP based on fanout type and count specified. 2060da14cebeSEric Cheng * 2061da14cebeSEric Cheng * mac_soft_ring_fanout, mac_srs_fanout_modify (?), 2062da14cebeSEric Cheng * mac_soft_ring_stop_workers, mac_soft_ring_set_destroy, etc need 2063da14cebeSEric Cheng * to be heavily modified. 2064da14cebeSEric Cheng * 2065da14cebeSEric Cheng * mi_soft_ring_list_size, mi_soft_ring_size, etc need to disappear. 2066da14cebeSEric Cheng */ 2067da14cebeSEric Cheng mac_soft_ring_set_t * 2068da14cebeSEric Cheng mac_srs_create(mac_client_impl_t *mcip, flow_entry_t *flent, uint32_t srs_type, 2069da14cebeSEric Cheng mac_direct_rx_t rx_func, void *x_arg1, mac_resource_handle_t x_arg2, 2070da14cebeSEric Cheng mac_ring_t *ring) 2071da14cebeSEric Cheng { 2072da14cebeSEric Cheng mac_soft_ring_set_t *mac_srs; 2073da14cebeSEric Cheng mac_srs_rx_t *srs_rx; 2074da14cebeSEric Cheng mac_srs_tx_t *srs_tx; 2075da14cebeSEric Cheng mac_bw_ctl_t *mac_bw; 2076da14cebeSEric Cheng mac_resource_props_t *mrp; 2077da14cebeSEric Cheng boolean_t is_tx_srs = ((srs_type & SRST_TX) != 0); 2078da14cebeSEric Cheng 2079da14cebeSEric Cheng mac_srs = kmem_cache_alloc(mac_srs_cache, KM_SLEEP); 2080da14cebeSEric Cheng bzero(mac_srs, sizeof (mac_soft_ring_set_t)); 2081da14cebeSEric Cheng srs_rx = &mac_srs->srs_rx; 2082da14cebeSEric Cheng srs_tx = &mac_srs->srs_tx; 2083da14cebeSEric Cheng 2084da14cebeSEric Cheng mutex_enter(&flent->fe_lock); 2085da14cebeSEric Cheng 2086da14cebeSEric Cheng /* 2087da14cebeSEric Cheng * Get the bandwidth control structure from the flent. Get 2088da14cebeSEric Cheng * rid of any residual values in the control structure for 2089da14cebeSEric Cheng * the tx bw struct and also for the rx, if the rx srs is 2090da14cebeSEric Cheng * the 1st one being brought up (the rx bw ctl struct may 2091da14cebeSEric Cheng * be shared by multiple SRSs) 2092da14cebeSEric Cheng */ 2093da14cebeSEric Cheng if (is_tx_srs) { 2094da14cebeSEric Cheng mac_srs->srs_bw = &flent->fe_tx_bw; 2095da14cebeSEric Cheng bzero(mac_srs->srs_bw, sizeof (mac_bw_ctl_t)); 2096da14cebeSEric Cheng flent->fe_tx_srs = mac_srs; 2097da14cebeSEric Cheng } else { 2098da14cebeSEric Cheng /* 2099da14cebeSEric Cheng * The bw counter (stored in the flent) is shared 2100da14cebeSEric Cheng * by SRS's within an rx group. 2101da14cebeSEric Cheng */ 2102da14cebeSEric Cheng mac_srs->srs_bw = &flent->fe_rx_bw; 2103da14cebeSEric Cheng /* First rx SRS, clear the bw structure */ 2104da14cebeSEric Cheng if (flent->fe_rx_srs_cnt == 0) 2105da14cebeSEric Cheng bzero(mac_srs->srs_bw, sizeof (mac_bw_ctl_t)); 210677e3ef6dSVenugopal Iyer 210777e3ef6dSVenugopal Iyer /* 210877e3ef6dSVenugopal Iyer * It is better to panic here rather than just assert because 210977e3ef6dSVenugopal Iyer * on a non-debug kernel we might end up courrupting memory 211077e3ef6dSVenugopal Iyer * and making it difficult to debug. 211177e3ef6dSVenugopal Iyer */ 211277e3ef6dSVenugopal Iyer if (flent->fe_rx_srs_cnt >= MAX_RINGS_PER_GROUP) { 211377e3ef6dSVenugopal Iyer panic("Array Overrun detected due to MAC client %p " 211477e3ef6dSVenugopal Iyer " having more rings than %d", (void *)mcip, 211577e3ef6dSVenugopal Iyer MAX_RINGS_PER_GROUP); 211677e3ef6dSVenugopal Iyer } 2117da14cebeSEric Cheng flent->fe_rx_srs[flent->fe_rx_srs_cnt] = mac_srs; 2118da14cebeSEric Cheng flent->fe_rx_srs_cnt++; 2119da14cebeSEric Cheng } 2120da14cebeSEric Cheng mac_srs->srs_flent = flent; 2121da14cebeSEric Cheng mutex_exit(&flent->fe_lock); 2122da14cebeSEric Cheng 2123da14cebeSEric Cheng mac_srs->srs_state = 0; 2124da14cebeSEric Cheng mac_srs->srs_type = (srs_type | SRST_NO_SOFT_RINGS); 2125da14cebeSEric Cheng mac_srs->srs_worker_cpuid = mac_srs->srs_worker_cpuid_save = -1; 2126da14cebeSEric Cheng mac_srs->srs_poll_cpuid = mac_srs->srs_poll_cpuid_save = -1; 21270dc2366fSVenugopal Iyer mac_srs->srs_mcip = mcip; 2128da14cebeSEric Cheng mac_srs_fanout_list_alloc(mac_srs); 2129da14cebeSEric Cheng 2130da14cebeSEric Cheng /* 2131da14cebeSEric Cheng * For a flow we use the underlying MAC client's priority range with 2132da14cebeSEric Cheng * the priority value to find an absolute priority value. For a MAC 2133da14cebeSEric Cheng * client we use the MAC client's maximum priority as the value. 2134da14cebeSEric Cheng */ 2135da14cebeSEric Cheng mrp = &flent->fe_effective_props; 2136da14cebeSEric Cheng if ((mac_srs->srs_type & SRST_FLOW) != 0) { 2137da14cebeSEric Cheng mac_srs->srs_pri = FLOW_PRIORITY(mcip->mci_min_pri, 2138da14cebeSEric Cheng mcip->mci_max_pri, mrp->mrp_priority); 2139da14cebeSEric Cheng } else { 2140da14cebeSEric Cheng mac_srs->srs_pri = mcip->mci_max_pri; 2141da14cebeSEric Cheng } 2142da14cebeSEric Cheng /* 2143da14cebeSEric Cheng * We need to insert the SRS in the global list before 2144da14cebeSEric Cheng * binding the SRS and SR threads. Otherwise there is a 2145da14cebeSEric Cheng * is a small window where the cpu reconfig callbacks 2146da14cebeSEric Cheng * may miss the SRS in the list walk and DR could fail 2147da14cebeSEric Cheng * as there are bound threads. 2148da14cebeSEric Cheng */ 2149da14cebeSEric Cheng mac_srs_add_glist(mac_srs); 2150da14cebeSEric Cheng 2151da14cebeSEric Cheng /* Initialize bw limit */ 2152da14cebeSEric Cheng if ((mrp->mrp_mask & MRP_MAXBW) != 0) { 2153da14cebeSEric Cheng mac_srs->srs_drain_func = mac_rx_srs_drain_bw; 2154da14cebeSEric Cheng 2155da14cebeSEric Cheng mac_bw = mac_srs->srs_bw; 2156da14cebeSEric Cheng mutex_enter(&mac_bw->mac_bw_lock); 2157da14cebeSEric Cheng mac_bw->mac_bw_limit = FLOW_BYTES_PER_TICK(mrp->mrp_maxbw); 2158da14cebeSEric Cheng 2159da14cebeSEric Cheng /* 2160da14cebeSEric Cheng * Give twice the queuing capability before 2161da14cebeSEric Cheng * dropping packets. The unit is bytes/tick. 2162da14cebeSEric Cheng */ 2163da14cebeSEric Cheng mac_bw->mac_bw_drop_threshold = mac_bw->mac_bw_limit << 1; 2164da14cebeSEric Cheng mutex_exit(&mac_bw->mac_bw_lock); 2165da14cebeSEric Cheng mac_srs->srs_type |= SRST_BW_CONTROL; 2166da14cebeSEric Cheng } else { 2167da14cebeSEric Cheng mac_srs->srs_drain_func = mac_rx_srs_drain; 2168da14cebeSEric Cheng } 2169da14cebeSEric Cheng 2170da14cebeSEric Cheng /* 2171da14cebeSEric Cheng * We use the following policy to control Receive 2172da14cebeSEric Cheng * Side Dynamic Polling: 2173da14cebeSEric Cheng * 1) We switch to poll mode anytime the processing thread causes 2174da14cebeSEric Cheng * a backlog to build up in SRS and its associated Soft Rings 2175da14cebeSEric Cheng * (sr_poll_pkt_cnt > 0). 2176da14cebeSEric Cheng * 2) As long as the backlog stays under the low water mark 2177da14cebeSEric Cheng * (sr_lowat), we poll the H/W for more packets. 2178da14cebeSEric Cheng * 3) If the backlog (sr_poll_pkt_cnt) exceeds low water mark, we 2179da14cebeSEric Cheng * stay in poll mode but don't poll the H/W for more packets. 2180da14cebeSEric Cheng * 4) Anytime in polling mode, if we poll the H/W for packets and 2181da14cebeSEric Cheng * find nothing plus we have an existing backlog 2182da14cebeSEric Cheng * (sr_poll_pkt_cnt > 0), we stay in polling mode but don't poll 2183da14cebeSEric Cheng * the H/W for packets anymore (let the polling thread go to sleep). 218484de666eSRyan Zezeski * 5) Once the backlog is relieved (packets are processed) we reenable 2185da14cebeSEric Cheng * polling (by signalling the poll thread) only when the backlog 2186da14cebeSEric Cheng * dips below sr_poll_thres. 2187da14cebeSEric Cheng * 6) sr_hiwat is used exclusively when we are not polling capable 2188da14cebeSEric Cheng * and is used to decide when to drop packets so the SRS queue 2189da14cebeSEric Cheng * length doesn't grow infinitely. 2190da14cebeSEric Cheng */ 2191da14cebeSEric Cheng if (!is_tx_srs) { 2192da14cebeSEric Cheng srs_rx->sr_hiwat = mac_soft_ring_max_q_cnt; 2193da14cebeSEric Cheng /* Low water mark needs to be less than high water mark */ 2194da14cebeSEric Cheng srs_rx->sr_lowat = mac_soft_ring_min_q_cnt <= 2195da14cebeSEric Cheng mac_soft_ring_max_q_cnt ? mac_soft_ring_min_q_cnt : 2196da14cebeSEric Cheng (mac_soft_ring_max_q_cnt >> 2); 2197da14cebeSEric Cheng /* Poll threshold need to be half of low water mark or less */ 2198da14cebeSEric Cheng srs_rx->sr_poll_thres = mac_soft_ring_poll_thres <= 2199da14cebeSEric Cheng (srs_rx->sr_lowat >> 1) ? mac_soft_ring_poll_thres : 2200da14cebeSEric Cheng (srs_rx->sr_lowat >> 1); 2201da14cebeSEric Cheng if (mac_latency_optimize) 22023631b19bSRajagopal Kunhappan mac_srs->srs_state |= SRS_LATENCY_OPT; 22033631b19bSRajagopal Kunhappan else 22043631b19bSRajagopal Kunhappan mac_srs->srs_state |= SRS_SOFTRING_QUEUE; 2205da14cebeSEric Cheng } 2206da14cebeSEric Cheng 2207da14cebeSEric Cheng mac_srs->srs_worker = thread_create(NULL, 0, 2208da14cebeSEric Cheng mac_srs_worker, mac_srs, 0, &p0, TS_RUN, mac_srs->srs_pri); 2209da14cebeSEric Cheng 2210da14cebeSEric Cheng if (is_tx_srs) { 2211da14cebeSEric Cheng /* Handle everything about Tx SRS and return */ 2212da14cebeSEric Cheng mac_srs->srs_drain_func = mac_tx_srs_drain; 2213da14cebeSEric Cheng srs_tx->st_max_q_cnt = mac_tx_srs_max_q_cnt; 2214da14cebeSEric Cheng srs_tx->st_hiwat = 2215da14cebeSEric Cheng (mac_tx_srs_hiwat > mac_tx_srs_max_q_cnt) ? 2216da14cebeSEric Cheng mac_tx_srs_max_q_cnt : mac_tx_srs_hiwat; 2217da14cebeSEric Cheng srs_tx->st_arg1 = x_arg1; 2218da14cebeSEric Cheng srs_tx->st_arg2 = x_arg2; 22190dc2366fSVenugopal Iyer goto done; 2220da14cebeSEric Cheng } 2221da14cebeSEric Cheng 2222da14cebeSEric Cheng if ((srs_type & SRST_FLOW) != 0 || 2223da14cebeSEric Cheng FLOW_TAB_EMPTY(mcip->mci_subflow_tab)) 2224da14cebeSEric Cheng srs_rx->sr_lower_proc = mac_rx_srs_process; 2225da14cebeSEric Cheng else 2226da14cebeSEric Cheng srs_rx->sr_lower_proc = mac_rx_srs_subflow_process; 2227da14cebeSEric Cheng 2228da14cebeSEric Cheng srs_rx->sr_func = rx_func; 2229da14cebeSEric Cheng srs_rx->sr_arg1 = x_arg1; 2230da14cebeSEric Cheng srs_rx->sr_arg2 = x_arg2; 2231da14cebeSEric Cheng 2232da14cebeSEric Cheng if (ring != NULL) { 22330dc2366fSVenugopal Iyer uint_t ring_info; 22340dc2366fSVenugopal Iyer 2235da14cebeSEric Cheng /* Is the mac_srs created over the RX default group? */ 2236da14cebeSEric Cheng if (ring->mr_gh == (mac_group_handle_t) 22370dc2366fSVenugopal Iyer MAC_DEFAULT_RX_GROUP(mcip->mci_mip)) { 2238da14cebeSEric Cheng mac_srs->srs_type |= SRST_DEFAULT_GRP; 22390dc2366fSVenugopal Iyer } 2240da14cebeSEric Cheng mac_srs->srs_ring = ring; 2241da14cebeSEric Cheng ring->mr_srs = mac_srs; 2242da14cebeSEric Cheng ring->mr_classify_type = MAC_HW_CLASSIFIER; 2243da14cebeSEric Cheng ring->mr_flag |= MR_INCIPIENT; 2244da14cebeSEric Cheng 22454eaa4710SRishi Srivatsavai if (!(mcip->mci_mip->mi_state_flags & MIS_POLL_DISABLE) && 22464eaa4710SRishi Srivatsavai FLOW_TAB_EMPTY(mcip->mci_subflow_tab) && mac_poll_enable) 2247da14cebeSEric Cheng mac_srs->srs_state |= SRS_POLLING_CAPAB; 2248da14cebeSEric Cheng 2249da14cebeSEric Cheng mac_srs->srs_poll_thr = thread_create(NULL, 0, 2250da14cebeSEric Cheng mac_rx_srs_poll_ring, mac_srs, 0, &p0, TS_RUN, 2251da14cebeSEric Cheng mac_srs->srs_pri); 2252ae6aa22aSVenugopal Iyer /* 2253ae6aa22aSVenugopal Iyer * Some drivers require serialization and don't send 2254ae6aa22aSVenugopal Iyer * packet chains in interrupt context. For such 225584de666eSRyan Zezeski * drivers, we should always queue in the soft ring 225684de666eSRyan Zezeski * so that we get a chance to switch into polling 2257ae6aa22aSVenugopal Iyer * mode under backlog. 2258ae6aa22aSVenugopal Iyer */ 22590dc2366fSVenugopal Iyer ring_info = mac_hwring_getinfo((mac_ring_handle_t)ring); 22600dc2366fSVenugopal Iyer if (ring_info & MAC_RING_RX_ENQUEUE) 2261ae6aa22aSVenugopal Iyer mac_srs->srs_state |= SRS_SOFTRING_QUEUE; 2262da14cebeSEric Cheng } 22630dc2366fSVenugopal Iyer done: 22640dc2366fSVenugopal Iyer mac_srs_stat_create(mac_srs); 2265da14cebeSEric Cheng return (mac_srs); 2266da14cebeSEric Cheng } 2267da14cebeSEric Cheng 2268da14cebeSEric Cheng /* 2269da14cebeSEric Cheng * Figure out the number of soft rings required. Its dependant on 2270da14cebeSEric Cheng * if protocol fanout is required (for LINKs), global settings 2271da14cebeSEric Cheng * require us to do fanout for performance (based on mac_soft_ring_enable), 2272da14cebeSEric Cheng * or user has specifically requested fanout. 2273da14cebeSEric Cheng */ 2274da14cebeSEric Cheng static uint32_t 2275da14cebeSEric Cheng mac_find_fanout(flow_entry_t *flent, uint32_t link_type) 2276da14cebeSEric Cheng { 2277da14cebeSEric Cheng uint32_t fanout_type; 2278da14cebeSEric Cheng mac_resource_props_t *mrp = &flent->fe_effective_props; 2279da14cebeSEric Cheng 2280da14cebeSEric Cheng /* no fanout for subflows */ 2281da14cebeSEric Cheng switch (link_type) { 2282da14cebeSEric Cheng case SRST_FLOW: 2283da14cebeSEric Cheng fanout_type = SRST_NO_SOFT_RINGS; 2284da14cebeSEric Cheng break; 2285da14cebeSEric Cheng case SRST_LINK: 2286da14cebeSEric Cheng fanout_type = SRST_FANOUT_PROTO; 2287da14cebeSEric Cheng break; 2288da14cebeSEric Cheng } 2289da14cebeSEric Cheng 2290da14cebeSEric Cheng /* A primary NIC/link is being plumbed */ 2291da14cebeSEric Cheng if (flent->fe_type & FLOW_PRIMARY_MAC) { 2292da14cebeSEric Cheng if (mac_soft_ring_enable && mac_rx_soft_ring_count > 1) { 2293da14cebeSEric Cheng fanout_type |= SRST_FANOUT_SRC_IP; 2294da14cebeSEric Cheng } 2295da14cebeSEric Cheng } else if (flent->fe_type & FLOW_VNIC) { 2296da14cebeSEric Cheng /* A VNIC is being created */ 2297da14cebeSEric Cheng if (mrp != NULL && mrp->mrp_ncpus > 0) { 2298da14cebeSEric Cheng fanout_type |= SRST_FANOUT_SRC_IP; 2299da14cebeSEric Cheng } 2300da14cebeSEric Cheng } 2301da14cebeSEric Cheng 2302da14cebeSEric Cheng return (fanout_type); 2303da14cebeSEric Cheng } 2304da14cebeSEric Cheng 2305da14cebeSEric Cheng /* 2306da14cebeSEric Cheng * Change a group from h/w to s/w classification. 2307da14cebeSEric Cheng */ 23080dc2366fSVenugopal Iyer void 2309da14cebeSEric Cheng mac_rx_switch_grp_to_sw(mac_group_t *group) 2310da14cebeSEric Cheng { 2311da14cebeSEric Cheng mac_ring_t *ring; 2312da14cebeSEric Cheng mac_soft_ring_set_t *mac_srs; 2313da14cebeSEric Cheng 2314da14cebeSEric Cheng for (ring = group->mrg_rings; ring != NULL; ring = ring->mr_next) { 2315da14cebeSEric Cheng if (ring->mr_classify_type == MAC_HW_CLASSIFIER) { 2316da14cebeSEric Cheng /* 2317da14cebeSEric Cheng * Remove the SRS associated with the HW ring. 2318da14cebeSEric Cheng * As a result, polling will be disabled. 2319da14cebeSEric Cheng */ 2320da14cebeSEric Cheng mac_srs = ring->mr_srs; 2321da14cebeSEric Cheng ASSERT(mac_srs != NULL); 2322da14cebeSEric Cheng mac_rx_srs_remove(mac_srs); 2323da14cebeSEric Cheng ring->mr_srs = NULL; 2324da14cebeSEric Cheng } 2325da14cebeSEric Cheng 2326da14cebeSEric Cheng if (ring->mr_state != MR_INUSE) 2327da14cebeSEric Cheng (void) mac_start_ring(ring); 23280dc2366fSVenugopal Iyer 2329da14cebeSEric Cheng /* 2330da14cebeSEric Cheng * We need to perform SW classification 2331da14cebeSEric Cheng * for packets landing in these rings 2332da14cebeSEric Cheng */ 2333da14cebeSEric Cheng ring->mr_flag = 0; 2334da14cebeSEric Cheng ring->mr_classify_type = MAC_SW_CLASSIFIER; 2335da14cebeSEric Cheng } 2336da14cebeSEric Cheng } 2337da14cebeSEric Cheng 2338da14cebeSEric Cheng /* 2339da14cebeSEric Cheng * Create the Rx SRS for S/W classifier and for each ring in the 2340da14cebeSEric Cheng * group (if exclusive group). Also create the Tx SRS. 2341da14cebeSEric Cheng */ 2342da14cebeSEric Cheng void 2343da14cebeSEric Cheng mac_srs_group_setup(mac_client_impl_t *mcip, flow_entry_t *flent, 23440dc2366fSVenugopal Iyer uint32_t link_type) 23450dc2366fSVenugopal Iyer { 23460dc2366fSVenugopal Iyer cpupart_t *cpupart; 23470dc2366fSVenugopal Iyer mac_resource_props_t *mrp = MCIP_RESOURCE_PROPS(mcip); 23480dc2366fSVenugopal Iyer mac_resource_props_t *emrp = MCIP_EFFECTIVE_PROPS(mcip); 23490dc2366fSVenugopal Iyer boolean_t use_default = B_FALSE; 23500dc2366fSVenugopal Iyer 23510dc2366fSVenugopal Iyer mac_rx_srs_group_setup(mcip, flent, link_type); 23520dc2366fSVenugopal Iyer mac_tx_srs_group_setup(mcip, flent, link_type); 23530dc2366fSVenugopal Iyer 235445948e49SRyan Zezeski /* Aggr ports don't have SRSes; thus there is no soft ring fanout. */ 235545948e49SRyan Zezeski if ((mcip->mci_state_flags & MCIS_IS_AGGR_PORT) != 0) 235645948e49SRyan Zezeski return; 235745948e49SRyan Zezeski 23580dc2366fSVenugopal Iyer pool_lock(); 23590dc2366fSVenugopal Iyer cpupart = mac_pset_find(mrp, &use_default); 23600dc2366fSVenugopal Iyer mac_fanout_setup(mcip, flent, MCIP_RESOURCE_PROPS(mcip), 23610dc2366fSVenugopal Iyer mac_rx_deliver, mcip, NULL, cpupart); 23620dc2366fSVenugopal Iyer mac_set_pool_effective(use_default, cpupart, mrp, emrp); 23630dc2366fSVenugopal Iyer pool_unlock(); 23640dc2366fSVenugopal Iyer } 23650dc2366fSVenugopal Iyer 23660dc2366fSVenugopal Iyer /* 236784de666eSRyan Zezeski * Set up the Rx SRSes. If there is no group associated with the 236884de666eSRyan Zezeski * client, then only setup SW classification. If the client has 236984de666eSRyan Zezeski * exlusive (MAC_GROUP_STATE_RESERVED) use of the group, then create an 237084de666eSRyan Zezeski * SRS for each HW ring. If the client is sharing a group, then make 237184de666eSRyan Zezeski * sure to teardown the HW SRSes. 23720dc2366fSVenugopal Iyer */ 23730dc2366fSVenugopal Iyer void 23740dc2366fSVenugopal Iyer mac_rx_srs_group_setup(mac_client_impl_t *mcip, flow_entry_t *flent, 23750dc2366fSVenugopal Iyer uint32_t link_type) 2376da14cebeSEric Cheng { 2377da14cebeSEric Cheng mac_impl_t *mip = mcip->mci_mip; 2378da14cebeSEric Cheng mac_soft_ring_set_t *mac_srs; 2379da14cebeSEric Cheng mac_ring_t *ring; 2380da14cebeSEric Cheng uint32_t fanout_type; 23810dc2366fSVenugopal Iyer mac_group_t *rx_group = flent->fe_rx_ring_group; 238284de666eSRyan Zezeski boolean_t no_unicast; 2383da14cebeSEric Cheng 238445948e49SRyan Zezeski /* 238545948e49SRyan Zezeski * If this is an an aggr port, then don't setup Rx SRS and Rx 238645948e49SRyan Zezeski * soft rings as they won't be used. However, we still need to 238745948e49SRyan Zezeski * start the rings to receive data on them. 238845948e49SRyan Zezeski */ 238945948e49SRyan Zezeski if (mcip->mci_state_flags & MCIS_IS_AGGR_PORT) { 239045948e49SRyan Zezeski if (rx_group == NULL) 239145948e49SRyan Zezeski return; 239245948e49SRyan Zezeski 239345948e49SRyan Zezeski for (ring = rx_group->mrg_rings; ring != NULL; 239445948e49SRyan Zezeski ring = ring->mr_next) { 239545948e49SRyan Zezeski if (ring->mr_state != MR_INUSE) 239645948e49SRyan Zezeski (void) mac_start_ring(ring); 239745948e49SRyan Zezeski } 239845948e49SRyan Zezeski 239945948e49SRyan Zezeski return; 240045948e49SRyan Zezeski } 240145948e49SRyan Zezeski 240245948e49SRyan Zezeski /* 240345948e49SRyan Zezeski * Aggr ports should never have SRSes. 240445948e49SRyan Zezeski */ 240545948e49SRyan Zezeski ASSERT3U((mcip->mci_state_flags & MCIS_IS_AGGR_PORT), ==, 0); 240645948e49SRyan Zezeski 2407da14cebeSEric Cheng fanout_type = mac_find_fanout(flent, link_type); 240884de666eSRyan Zezeski no_unicast = (mcip->mci_state_flags & MCIS_NO_UNICAST_ADDR) != 0; 2409da14cebeSEric Cheng 241084de666eSRyan Zezeski /* Create the SRS for SW classification if none exists */ 2411da14cebeSEric Cheng if (flent->fe_rx_srs[0] == NULL) { 2412da14cebeSEric Cheng ASSERT(flent->fe_rx_srs_cnt == 0); 2413da14cebeSEric Cheng mac_srs = mac_srs_create(mcip, flent, fanout_type | link_type, 2414da14cebeSEric Cheng mac_rx_deliver, mcip, NULL, NULL); 2415da14cebeSEric Cheng mutex_enter(&flent->fe_lock); 2416da14cebeSEric Cheng flent->fe_cb_fn = (flow_fn_t)mac_srs->srs_rx.sr_lower_proc; 2417da14cebeSEric Cheng flent->fe_cb_arg1 = (void *)mip; 2418da14cebeSEric Cheng flent->fe_cb_arg2 = (void *)mac_srs; 2419da14cebeSEric Cheng mutex_exit(&flent->fe_lock); 2420da14cebeSEric Cheng } 2421da14cebeSEric Cheng 24220dc2366fSVenugopal Iyer if (rx_group == NULL) 2423da14cebeSEric Cheng return; 242484de666eSRyan Zezeski 2425da14cebeSEric Cheng /* 242684de666eSRyan Zezeski * If the group is marked RESERVED then setup an SRS and 242784de666eSRyan Zezeski * fanout for each HW ring. 2428da14cebeSEric Cheng */ 24290dc2366fSVenugopal Iyer switch (rx_group->mrg_state) { 2430da14cebeSEric Cheng case MAC_GROUP_STATE_RESERVED: 24310dc2366fSVenugopal Iyer for (ring = rx_group->mrg_rings; ring != NULL; 2432da14cebeSEric Cheng ring = ring->mr_next) { 243384de666eSRyan Zezeski uint16_t vid = i_mac_flow_vid(mcip->mci_flent); 243484de666eSRyan Zezeski 2435da14cebeSEric Cheng switch (ring->mr_state) { 2436da14cebeSEric Cheng case MR_INUSE: 2437da14cebeSEric Cheng case MR_FREE: 2438da14cebeSEric Cheng if (ring->mr_srs != NULL) 2439da14cebeSEric Cheng break; 2440da14cebeSEric Cheng if (ring->mr_state != MR_INUSE) 2441da14cebeSEric Cheng (void) mac_start_ring(ring); 2442da14cebeSEric Cheng 24430dc2366fSVenugopal Iyer /* 244484de666eSRyan Zezeski * If a client requires SW VLAN 244584de666eSRyan Zezeski * filtering or has no unicast address 244684de666eSRyan Zezeski * then we don't create any HW ring 244784de666eSRyan Zezeski * SRSes. 24480dc2366fSVenugopal Iyer */ 244984de666eSRyan Zezeski if ((!MAC_GROUP_HW_VLAN(rx_group) && 245084de666eSRyan Zezeski vid != VLAN_ID_NONE) || no_unicast) 24510dc2366fSVenugopal Iyer break; 245284de666eSRyan Zezeski 245384de666eSRyan Zezeski /* 245484de666eSRyan Zezeski * When a client has exclusive use of 245584de666eSRyan Zezeski * a group, and that group's traffic 245684de666eSRyan Zezeski * is fully HW classified, we create 245784de666eSRyan Zezeski * an SRS for each HW ring in order to 245884de666eSRyan Zezeski * make use of dynamic polling of said 245984de666eSRyan Zezeski * HW rings. 246084de666eSRyan Zezeski */ 2461da14cebeSEric Cheng mac_srs = mac_srs_create(mcip, flent, 2462da14cebeSEric Cheng fanout_type | link_type, 2463da14cebeSEric Cheng mac_rx_deliver, mcip, NULL, ring); 2464da14cebeSEric Cheng break; 2465da14cebeSEric Cheng default: 24660dc2366fSVenugopal Iyer cmn_err(CE_PANIC, 24670dc2366fSVenugopal Iyer "srs_setup: mcip = %p " 2468da14cebeSEric Cheng "trying to add UNKNOWN ring = %p\n", 2469da14cebeSEric Cheng (void *)mcip, (void *)ring); 2470da14cebeSEric Cheng break; 2471da14cebeSEric Cheng } 2472da14cebeSEric Cheng } 2473da14cebeSEric Cheng break; 2474da14cebeSEric Cheng case MAC_GROUP_STATE_SHARED: 2475da14cebeSEric Cheng /* 247684de666eSRyan Zezeski * When a group is shared by multiple clients, we must 247784de666eSRyan Zezeski * use SW classifiction to ensure packets are 247884de666eSRyan Zezeski * delivered to the correct client. 2479da14cebeSEric Cheng */ 24800dc2366fSVenugopal Iyer mac_rx_switch_grp_to_sw(rx_group); 2481da14cebeSEric Cheng break; 2482da14cebeSEric Cheng default: 2483da14cebeSEric Cheng ASSERT(B_FALSE); 2484da14cebeSEric Cheng break; 2485da14cebeSEric Cheng } 2486da14cebeSEric Cheng } 2487da14cebeSEric Cheng 24880dc2366fSVenugopal Iyer /* 24890dc2366fSVenugopal Iyer * Set up the TX SRS. 24900dc2366fSVenugopal Iyer */ 2491da14cebeSEric Cheng void 24920dc2366fSVenugopal Iyer mac_tx_srs_group_setup(mac_client_impl_t *mcip, flow_entry_t *flent, 2493da14cebeSEric Cheng uint32_t link_type) 2494da14cebeSEric Cheng { 249545948e49SRyan Zezeski /* 249645948e49SRyan Zezeski * If this is an exclusive client (e.g. an aggr port), then 249745948e49SRyan Zezeski * don't setup Tx SRS and Tx soft rings as they won't be used. 249845948e49SRyan Zezeski * However, we still need to start the rings to send data 249945948e49SRyan Zezeski * across them. 250045948e49SRyan Zezeski */ 250145948e49SRyan Zezeski if (mcip->mci_state_flags & MCIS_EXCLUSIVE) { 25020dc2366fSVenugopal Iyer mac_ring_t *ring; 25030dc2366fSVenugopal Iyer mac_group_t *grp; 2504da14cebeSEric Cheng 25050dc2366fSVenugopal Iyer grp = (mac_group_t *)flent->fe_tx_ring_group; 250645948e49SRyan Zezeski 250745948e49SRyan Zezeski if (grp == NULL) 250845948e49SRyan Zezeski return; 250945948e49SRyan Zezeski 251045948e49SRyan Zezeski for (ring = grp->mrg_rings; ring != NULL; 251145948e49SRyan Zezeski ring = ring->mr_next) { 251245948e49SRyan Zezeski if (ring->mr_state != MR_INUSE) 25130dc2366fSVenugopal Iyer (void) mac_start_ring(ring); 25140dc2366fSVenugopal Iyer } 251545948e49SRyan Zezeski 25160dc2366fSVenugopal Iyer return; 25170dc2366fSVenugopal Iyer } 251845948e49SRyan Zezeski 251945948e49SRyan Zezeski /* 252045948e49SRyan Zezeski * Aggr ports should never have SRSes. 252145948e49SRyan Zezeski */ 252245948e49SRyan Zezeski ASSERT3U((mcip->mci_state_flags & MCIS_IS_AGGR_PORT), ==, 0); 252345948e49SRyan Zezeski 25240dc2366fSVenugopal Iyer if (flent->fe_tx_srs == NULL) { 25250dc2366fSVenugopal Iyer (void) mac_srs_create(mcip, flent, SRST_TX | link_type, 25260dc2366fSVenugopal Iyer NULL, mcip, NULL, NULL); 25270dc2366fSVenugopal Iyer } 252845948e49SRyan Zezeski 25290dc2366fSVenugopal Iyer mac_tx_srs_setup(mcip, flent); 25300dc2366fSVenugopal Iyer } 25310dc2366fSVenugopal Iyer 25320dc2366fSVenugopal Iyer /* 253384de666eSRyan Zezeski * Teardown all the Rx SRSes. Unless hwonly is set, then only teardown 253484de666eSRyan Zezeski * the Rx HW SRSes and leave the SW SRS alone. The hwonly flag is set 253584de666eSRyan Zezeski * when we wish to move a MAC client from one group to another. In 253684de666eSRyan Zezeski * that case, we need to release the current HW SRSes but keep the SW 253784de666eSRyan Zezeski * SRS for continued traffic classifiction. 25380dc2366fSVenugopal Iyer */ 25390dc2366fSVenugopal Iyer void 25400dc2366fSVenugopal Iyer mac_rx_srs_group_teardown(flow_entry_t *flent, boolean_t hwonly) 25410dc2366fSVenugopal Iyer { 25420dc2366fSVenugopal Iyer mac_soft_ring_set_t *mac_srs; 25430dc2366fSVenugopal Iyer int i; 25440dc2366fSVenugopal Iyer int count = flent->fe_rx_srs_cnt; 25450dc2366fSVenugopal Iyer 25460dc2366fSVenugopal Iyer for (i = 0; i < count; i++) { 25470dc2366fSVenugopal Iyer if (i == 0 && hwonly) 25480dc2366fSVenugopal Iyer continue; 2549da14cebeSEric Cheng mac_srs = flent->fe_rx_srs[i]; 2550da14cebeSEric Cheng mac_rx_srs_quiesce(mac_srs, SRS_CONDEMNED); 2551da14cebeSEric Cheng mac_srs_free(mac_srs); 2552da14cebeSEric Cheng flent->fe_rx_srs[i] = NULL; 25530dc2366fSVenugopal Iyer flent->fe_rx_srs_cnt--; 2554da14cebeSEric Cheng } 255584de666eSRyan Zezeski 255684de666eSRyan Zezeski /* 255784de666eSRyan Zezeski * If we are only tearing down the HW SRSes then there must be 255884de666eSRyan Zezeski * one SRS left for SW classification. Otherwise we are tearing 255984de666eSRyan Zezeski * down both HW and SW and there should be no SRSes left. 256084de666eSRyan Zezeski */ 256184de666eSRyan Zezeski if (hwonly) 256284de666eSRyan Zezeski VERIFY3S(flent->fe_rx_srs_cnt, ==, 1); 256384de666eSRyan Zezeski else 256484de666eSRyan Zezeski VERIFY3S(flent->fe_rx_srs_cnt, ==, 0); 25650dc2366fSVenugopal Iyer } 2566da14cebeSEric Cheng 25670dc2366fSVenugopal Iyer /* 25680dc2366fSVenugopal Iyer * Remove the TX SRS. 25690dc2366fSVenugopal Iyer */ 25700dc2366fSVenugopal Iyer void 25710dc2366fSVenugopal Iyer mac_tx_srs_group_teardown(mac_client_impl_t *mcip, flow_entry_t *flent, 25720dc2366fSVenugopal Iyer uint32_t link_type) 25730dc2366fSVenugopal Iyer { 25740dc2366fSVenugopal Iyer mac_soft_ring_set_t *tx_srs; 25750dc2366fSVenugopal Iyer mac_srs_tx_t *tx; 25760dc2366fSVenugopal Iyer 25770dc2366fSVenugopal Iyer if ((tx_srs = flent->fe_tx_srs) == NULL) 25780dc2366fSVenugopal Iyer return; 25790dc2366fSVenugopal Iyer 2580da14cebeSEric Cheng tx = &tx_srs->srs_tx; 2581da14cebeSEric Cheng switch (link_type) { 2582da14cebeSEric Cheng case SRST_FLOW: 2583da14cebeSEric Cheng /* 2584da14cebeSEric Cheng * For flows, we need to work with passed 2585da14cebeSEric Cheng * flent to find the Rx/Tx SRS. 2586da14cebeSEric Cheng */ 2587da14cebeSEric Cheng mac_tx_srs_quiesce(tx_srs, SRS_CONDEMNED); 2588da14cebeSEric Cheng break; 2589da14cebeSEric Cheng case SRST_LINK: 25900dc2366fSVenugopal Iyer mac_tx_client_condemn((mac_client_handle_t)mcip); 2591da14cebeSEric Cheng if (tx->st_arg2 != NULL) { 2592da14cebeSEric Cheng ASSERT(tx_srs->srs_type & SRST_TX); 25930dc2366fSVenugopal Iyer /* 25940dc2366fSVenugopal Iyer * The ring itself will be stopped when 25950dc2366fSVenugopal Iyer * we release the group or in the 25960dc2366fSVenugopal Iyer * mac_datapath_teardown (for the default 25970dc2366fSVenugopal Iyer * group) 25980dc2366fSVenugopal Iyer */ 25990dc2366fSVenugopal Iyer tx->st_arg2 = NULL; 2600da14cebeSEric Cheng } 2601da14cebeSEric Cheng break; 2602da14cebeSEric Cheng default: 2603da14cebeSEric Cheng ASSERT(B_FALSE); 2604da14cebeSEric Cheng break; 2605da14cebeSEric Cheng } 2606da14cebeSEric Cheng mac_srs_free(tx_srs); 2607da14cebeSEric Cheng flent->fe_tx_srs = NULL; 2608da14cebeSEric Cheng } 2609da14cebeSEric Cheng 2610da14cebeSEric Cheng /* 26110dc2366fSVenugopal Iyer * This is the group state machine. 26120dc2366fSVenugopal Iyer * 26130dc2366fSVenugopal Iyer * The state of an Rx group is given by 2614da14cebeSEric Cheng * the following table. The default group and its rings are started in 2615da14cebeSEric Cheng * mac_start itself and the default group stays in SHARED state until 2616da14cebeSEric Cheng * mac_stop at which time the group and rings are stopped and and it 2617da14cebeSEric Cheng * reverts to the Registered state. 2618da14cebeSEric Cheng * 2619da14cebeSEric Cheng * Typically this function is called on a group after adding or removing a 2620da14cebeSEric Cheng * client from it, to find out what should be the new state of the group. 2621da14cebeSEric Cheng * If the new state is RESERVED, then the client that owns this group 2622da14cebeSEric Cheng * exclusively is also returned. Note that adding or removing a client from 2623da14cebeSEric Cheng * a group could also impact the default group and the caller needs to 2624da14cebeSEric Cheng * evaluate the effect on the default group. 2625da14cebeSEric Cheng * 2626da14cebeSEric Cheng * Group type # of clients mi_nactiveclients Group State 2627da14cebeSEric Cheng * in the group 2628da14cebeSEric Cheng * 2629da14cebeSEric Cheng * Non-default 0 N.A. REGISTERED 2630da14cebeSEric Cheng * Non-default 1 N.A. RESERVED 2631da14cebeSEric Cheng * 2632da14cebeSEric Cheng * Default 0 N.A. SHARED 2633da14cebeSEric Cheng * Default 1 1 RESERVED 2634da14cebeSEric Cheng * Default 1 > 1 SHARED 2635da14cebeSEric Cheng * Default > 1 N.A. SHARED 26360dc2366fSVenugopal Iyer * 26370dc2366fSVenugopal Iyer * For a TX group, the following is the state table. 26380dc2366fSVenugopal Iyer * 26390dc2366fSVenugopal Iyer * Group type # of clients Group State 26400dc2366fSVenugopal Iyer * in the group 26410dc2366fSVenugopal Iyer * 26420dc2366fSVenugopal Iyer * Non-default 0 REGISTERED 26430dc2366fSVenugopal Iyer * Non-default 1 RESERVED 26440dc2366fSVenugopal Iyer * 26450dc2366fSVenugopal Iyer * Default 0 REGISTERED 26460dc2366fSVenugopal Iyer * Default 1 RESERVED 26470dc2366fSVenugopal Iyer * Default > 1 SHARED 2648da14cebeSEric Cheng */ 2649da14cebeSEric Cheng mac_group_state_t 26500dc2366fSVenugopal Iyer mac_group_next_state(mac_group_t *grp, mac_client_impl_t **group_only_mcip, 26510dc2366fSVenugopal Iyer mac_group_t *defgrp, boolean_t rx_group) 2652da14cebeSEric Cheng { 2653da14cebeSEric Cheng mac_impl_t *mip = (mac_impl_t *)grp->mrg_mh; 2654da14cebeSEric Cheng 2655da14cebeSEric Cheng *group_only_mcip = NULL; 2656da14cebeSEric Cheng 2657da14cebeSEric Cheng /* Non-default group */ 2658da14cebeSEric Cheng 26590dc2366fSVenugopal Iyer if (grp != defgrp) { 26600dc2366fSVenugopal Iyer if (MAC_GROUP_NO_CLIENT(grp)) 2661da14cebeSEric Cheng return (MAC_GROUP_STATE_REGISTERED); 2662da14cebeSEric Cheng 26630dc2366fSVenugopal Iyer *group_only_mcip = MAC_GROUP_ONLY_CLIENT(grp); 2664da14cebeSEric Cheng if (*group_only_mcip != NULL) 2665da14cebeSEric Cheng return (MAC_GROUP_STATE_RESERVED); 2666da14cebeSEric Cheng 2667da14cebeSEric Cheng return (MAC_GROUP_STATE_SHARED); 2668da14cebeSEric Cheng } 2669da14cebeSEric Cheng 2670da14cebeSEric Cheng /* Default group */ 2671da14cebeSEric Cheng 26720dc2366fSVenugopal Iyer if (MAC_GROUP_NO_CLIENT(grp)) { 26730dc2366fSVenugopal Iyer if (rx_group) 26740dc2366fSVenugopal Iyer return (MAC_GROUP_STATE_SHARED); 26750dc2366fSVenugopal Iyer else 26760dc2366fSVenugopal Iyer return (MAC_GROUP_STATE_REGISTERED); 26770dc2366fSVenugopal Iyer } 26780dc2366fSVenugopal Iyer *group_only_mcip = MAC_GROUP_ONLY_CLIENT(grp); 26790dc2366fSVenugopal Iyer if (*group_only_mcip == NULL) 2680da14cebeSEric Cheng return (MAC_GROUP_STATE_SHARED); 2681da14cebeSEric Cheng 26820dc2366fSVenugopal Iyer if (rx_group && mip->mi_nactiveclients != 1) 26830dc2366fSVenugopal Iyer return (MAC_GROUP_STATE_SHARED); 26840dc2366fSVenugopal Iyer 2685da14cebeSEric Cheng ASSERT(*group_only_mcip != NULL); 2686da14cebeSEric Cheng return (MAC_GROUP_STATE_RESERVED); 2687da14cebeSEric Cheng } 2688da14cebeSEric Cheng 2689da14cebeSEric Cheng /* 2690da14cebeSEric Cheng * OVERVIEW NOTES FOR DATAPATH 2691da14cebeSEric Cheng * =========================== 2692da14cebeSEric Cheng * 2693da14cebeSEric Cheng * Create an SRS and setup the corresponding flow function and args. 2694da14cebeSEric Cheng * Add a classification rule for the flow specified by 'flent' and program 2695da14cebeSEric Cheng * the hardware classifier when applicable. 2696da14cebeSEric Cheng * 2697da14cebeSEric Cheng * Rx ring assignment, SRS, polling and B/W enforcement 2698da14cebeSEric Cheng * ---------------------------------------------------- 2699da14cebeSEric Cheng * 2700da14cebeSEric Cheng * We try to use H/W classification on NIC and assign traffic to a 2701da14cebeSEric Cheng * MAC address to a particular Rx ring. There is a 1-1 mapping 2702da14cebeSEric Cheng * between a SRS and a Rx ring. The SRS (short for soft ring set) 2703da14cebeSEric Cheng * dynamically switches the underlying Rx ring between interrupt 2704da14cebeSEric Cheng * and polling mode and enforces any specified B/W control. 2705da14cebeSEric Cheng * 2706da14cebeSEric Cheng * There is always a SRS created and tied to each H/W and S/W rule. 2707da14cebeSEric Cheng * Whenever we create a H/W rule, we always add the the same rule to 2708da14cebeSEric Cheng * S/W classifier and tie a SRS to it. 2709da14cebeSEric Cheng * 2710da14cebeSEric Cheng * In case a B/W control is specified, its broken into bytes 2711da14cebeSEric Cheng * per ticks and as soon as the quota for a tick is exhausted, 2712da14cebeSEric Cheng * the underlying Rx ring is forced into poll mode for remianing 2713da14cebeSEric Cheng * tick. The SRS poll thread only polls for bytes that are 2714da14cebeSEric Cheng * allowed to come in the SRS. We typically let 4x the configured 2715da14cebeSEric Cheng * B/W worth of packets to come in the SRS (to prevent unnecessary 2716da14cebeSEric Cheng * drops due to bursts) but only process the specified amount. 2717da14cebeSEric Cheng * 2718da14cebeSEric Cheng * A Link (primary NIC, VNIC, VLAN or aggr) can have 1 or more 2719da14cebeSEric Cheng * Rx rings (and corresponding SRSs) assigned to it. The SRS 2720da14cebeSEric Cheng * in turn can have softrings to do protocol level fanout or 2721da14cebeSEric Cheng * softrings to do S/W based fanout or both. In case the NIC 2722da14cebeSEric Cheng * has no Rx rings, we do S/W classification to respective SRS. 2723da14cebeSEric Cheng * The S/W classification rule is always setup and ready. This 2724da14cebeSEric Cheng * allows the MAC layer to reassign Rx rings whenever needed 2725da14cebeSEric Cheng * but packets still continue to flow via the default path and 2726da14cebeSEric Cheng * getting S/W classified to correct SRS. 2727da14cebeSEric Cheng * 2728da14cebeSEric Cheng * In other cases where a NIC or VNIC is plumbed, our goal is use 2729da14cebeSEric Cheng * H/W classifier and get two Rx ring assigned for the Link. One 2730da14cebeSEric Cheng * for TCP and one for UDP|SCTP. The respective SRS still do the 2731da14cebeSEric Cheng * polling on the Rx ring. For Link that is plumbed for IP, there 2732da14cebeSEric Cheng * is a TCP squeue which also does polling and can control the 2733da14cebeSEric Cheng * the Rx ring directly (where SRS is just pass through). For 2734da14cebeSEric Cheng * the following cases, the SRS does the polling underneath. 2735da14cebeSEric Cheng * 1) non IP based Links (Links which are not plumbed via ifconfig) 2736da14cebeSEric Cheng * and paths which have no IP squeues (UDP & SCTP) 2737da14cebeSEric Cheng * 2) If B/W control is specified on the Link 2738da14cebeSEric Cheng * 3) If S/W fanout is secified 2739da14cebeSEric Cheng * 2740da14cebeSEric Cheng * Note1: As of current implementation, we try to assign only 1 Rx 2741da14cebeSEric Cheng * ring per Link and more than 1 Rx ring for primary Link for 2742da14cebeSEric Cheng * H/W based fanout. We always create following softrings per SRS: 2743da14cebeSEric Cheng * 1) TCP softring which is polled by TCP squeue where possible 2744da14cebeSEric Cheng * (and also bypasses DLS) 2745da14cebeSEric Cheng * 2) UDP/SCTP based which bypasses DLS 2746da14cebeSEric Cheng * 3) OTH softring which goes via DLS (currently deal with IPv6 2747da14cebeSEric Cheng * and non TCP/UDP/SCTP for IPv4 packets). 2748da14cebeSEric Cheng * 2749da14cebeSEric Cheng * It is necessary to create 3 softrings since SRS has to poll 2750da14cebeSEric Cheng * the single Rx ring underneath and enforce any link level B/W 2751da14cebeSEric Cheng * control (we can't switch the Rx ring in poll mode just based 2752da14cebeSEric Cheng * on TCP squeue if the same Rx ring is sharing UDP and other 2753da14cebeSEric Cheng * traffic as well). Once polling is done and any Link level B/W 2754da14cebeSEric Cheng * control is specified, the packets are assigned to respective 2755da14cebeSEric Cheng * softring based on protocol. Since TCP has IP based squeue 2756da14cebeSEric Cheng * which benefits by polling, we separate TCP packets into 2757da14cebeSEric Cheng * its own softring which can be polled by IP squeue. We need 2758da14cebeSEric Cheng * to separate out UDP/SCTP to UDP softring since it can bypass 2759da14cebeSEric Cheng * the DLS layer which has heavy performance advanatges and we 2760da14cebeSEric Cheng * need a softring (OTH) for rest. 2761da14cebeSEric Cheng * 2762da14cebeSEric Cheng * ToDo: The 3 softrings for protocol are needed only till we can 2763da14cebeSEric Cheng * get rid of DLS from datapath, make IPv4 and IPv6 paths 2764da14cebeSEric Cheng * symmetric (deal with mac_header_info for v6 and polling for 2765da14cebeSEric Cheng * IPv4 TCP - ip_accept_tcp is IPv4 specific although squeues 2766da14cebeSEric Cheng * are generic), and bring SAP based classification to MAC layer 2767da14cebeSEric Cheng * 2768da14cebeSEric Cheng * H/W and S/W based fanout and multiple Rx rings per Link 2769da14cebeSEric Cheng * ------------------------------------------------------- 2770da14cebeSEric Cheng * 2771da14cebeSEric Cheng * In case, fanout is requested (or determined automatically based 2772da14cebeSEric Cheng * on Link speed and processor speed), we try to assign multiple 2773da14cebeSEric Cheng * Rx rings per Link with their respective SRS. In this case 2774da14cebeSEric Cheng * the NIC should be capable of fanning out incoming packets between 2775da14cebeSEric Cheng * the assigned Rx rings (H/W based fanout). All the SRS 2776da14cebeSEric Cheng * individually switch their Rx ring between interrupt and polling 2777da14cebeSEric Cheng * mode but share a common B/W control counter in case of Link 2778da14cebeSEric Cheng * level B/W is specified. 2779da14cebeSEric Cheng * 2780da14cebeSEric Cheng * If S/W based fanout is specified in lieu of H/W based fanout, 2781da14cebeSEric Cheng * the Link SRS creates the specified number of softrings for 2782da14cebeSEric Cheng * each protocol (TCP, UDP, OTH). Incoming packets are fanned 2783da14cebeSEric Cheng * out to the correct softring based on their protocol and 2784da14cebeSEric Cheng * protocol specific hash function. 2785da14cebeSEric Cheng * 2786da14cebeSEric Cheng * Primary and non primary MAC clients 2787da14cebeSEric Cheng * ----------------------------------- 2788da14cebeSEric Cheng * 2789da14cebeSEric Cheng * The NICs, VNICs, Vlans, and Aggrs are typically termed as Links 2790da14cebeSEric Cheng * and are a Layer 2 construct. 2791da14cebeSEric Cheng * 2792da14cebeSEric Cheng * Primary NIC: 2793da14cebeSEric Cheng * The Link that owns the primary MAC address and typically 2794da14cebeSEric Cheng * is used as the data NIC in non virtualized cases. As such 2795da14cebeSEric Cheng * H/W resources are preferntially given to primary NIC. As 2796da14cebeSEric Cheng * far as code is concerned, there is no difference in the 2797da14cebeSEric Cheng * primary NIC vs VNICs. They are all treated as Links. 2798da14cebeSEric Cheng * At the very first call to mac_unicast_add() we program the S/W 2799da14cebeSEric Cheng * classifier for the primary MAC address, get a soft ring set 2800da14cebeSEric Cheng * (and soft rings based on 'ip_soft_ring_cnt') 2801da14cebeSEric Cheng * and a Rx ring assigned for polling to get enabled. 2802da14cebeSEric Cheng * When IP get plumbed and negotiates polling, we can 2803da14cebeSEric Cheng * let squeue do the polling on TCP softring. 2804da14cebeSEric Cheng * 2805da14cebeSEric Cheng * VNICs: 2806da14cebeSEric Cheng * Same as any other Link. As long as the H/W resource assignments 2807da14cebeSEric Cheng * are equal, the data path and setup for all Links is same. 2808da14cebeSEric Cheng * 2809da14cebeSEric Cheng * Flows: 2810da14cebeSEric Cheng * Can be configured on Links. They have their own SRS and the 2811da14cebeSEric Cheng * S/W classifier is programmed appropriately based on the flow. 2812da14cebeSEric Cheng * The flows typically deal with layer 3 and above and 2813da14cebeSEric Cheng * creates a soft ring set specific to the flow. The receive 2814da14cebeSEric Cheng * side function is switched from mac_rx_srs_process to 2815da14cebeSEric Cheng * mac_rx_srs_subflow_process which first tries to assign the 2816da14cebeSEric Cheng * packet to appropriate flow SRS and failing which assigns it 2817da14cebeSEric Cheng * to link SRS. This allows us to avoid the layered approach 2818da14cebeSEric Cheng * which gets complex. 2819da14cebeSEric Cheng * 2820da14cebeSEric Cheng * By the time mac_datapath_setup() completes, we already have the 2821da14cebeSEric Cheng * soft rings set, Rx rings, soft rings, etc figured out and both H/W 2822da14cebeSEric Cheng * and S/W classifiers programmed. IP is not plumbed yet (and might 2823da14cebeSEric Cheng * never be for Virtual Machines guest OS path). When IP is plumbed 2824da14cebeSEric Cheng * (for both NIC and VNIC), we do a capability negotiation for polling 2825da14cebeSEric Cheng * and upcall functions etc. 2826da14cebeSEric Cheng * 2827da14cebeSEric Cheng * Rx ring Assignement NOTES 2828da14cebeSEric Cheng * ------------------------- 2829da14cebeSEric Cheng * 2830da14cebeSEric Cheng * For NICs which have only 1 Rx ring (we treat NICs with no Rx rings 2831da14cebeSEric Cheng * as NIC with a single default ring), we assign the only ring to 28320dc2366fSVenugopal Iyer * primary Link. The primary Link SRS can do polling on it as long as 28330dc2366fSVenugopal Iyer * it is the only link in use and we compare the MAC address for unicast 28340dc2366fSVenugopal Iyer * packets before accepting an incoming packet (there is no need for S/W 28350dc2366fSVenugopal Iyer * classification in this case). We disable polling on the only ring the 28360dc2366fSVenugopal Iyer * moment 2nd link gets created (the polling remains enabled even though 28370dc2366fSVenugopal Iyer * there are broadcast and * multicast flows created). 2838da14cebeSEric Cheng * 2839da14cebeSEric Cheng * If the NIC has more than 1 Rx ring, we assign the default ring (the 2840da14cebeSEric Cheng * 1st ring) to deal with broadcast, multicast and traffic for other 2841da14cebeSEric Cheng * NICs which needs S/W classification. We assign the primary mac 2842da14cebeSEric Cheng * addresses to another ring by specifiying a classification rule for 2843da14cebeSEric Cheng * primary unicast MAC address to the selected ring. The primary Link 2844da14cebeSEric Cheng * (and its SRS) can continue to poll the assigned Rx ring at all times 2845da14cebeSEric Cheng * independantly. 2846da14cebeSEric Cheng * 2847da14cebeSEric Cheng * Note: In future, if no fanout is specified, we try to assign 2 Rx 2848da14cebeSEric Cheng * rings for the primary Link with the primary MAC address + TCP going 2849da14cebeSEric Cheng * to one ring and primary MAC address + UDP|SCTP going to other ring. 2850da14cebeSEric Cheng * Any remaining traffic for primary MAC address can go to the default 2851da14cebeSEric Cheng * Rx ring and get S/W classified. This way the respective SRSs don't 2852da14cebeSEric Cheng * need to do proto fanout and don't need to have softrings at all and 2853da14cebeSEric Cheng * can poll their respective Rx rings. 2854da14cebeSEric Cheng * 2855da14cebeSEric Cheng * As an optimization, when a new NIC or VNIC is created, we can get 2856da14cebeSEric Cheng * only one Rx ring and make it a TCP specific Rx ring and use the 2857da14cebeSEric Cheng * H/W default Rx ring for the rest (this Rx ring is never polled). 28580dc2366fSVenugopal Iyer * 28590dc2366fSVenugopal Iyer * For clients that don't have MAC address, but want to receive and 28600dc2366fSVenugopal Iyer * transmit packets (e.g, bpf, gvrp etc.), we need to setup the datapath. 28610dc2366fSVenugopal Iyer * For such clients (identified by the MCIS_NO_UNICAST_ADDR flag) we 28620dc2366fSVenugopal Iyer * always give the default group and use software classification (i.e. 28630dc2366fSVenugopal Iyer * even if this is the only client in the default group, we will 28640dc2366fSVenugopal Iyer * leave group as shared). 2865da14cebeSEric Cheng */ 286684de666eSRyan Zezeski 2867da14cebeSEric Cheng int 2868da14cebeSEric Cheng mac_datapath_setup(mac_client_impl_t *mcip, flow_entry_t *flent, 2869da14cebeSEric Cheng uint32_t link_type) 2870da14cebeSEric Cheng { 2871da14cebeSEric Cheng mac_impl_t *mip = mcip->mci_mip; 28720dc2366fSVenugopal Iyer mac_group_t *rgroup = NULL; 28730dc2366fSVenugopal Iyer mac_group_t *tgroup = NULL; 28740dc2366fSVenugopal Iyer mac_group_t *default_rgroup; 28750dc2366fSVenugopal Iyer mac_group_t *default_tgroup; 2876da14cebeSEric Cheng int err; 287784de666eSRyan Zezeski uint16_t vid; 2878da14cebeSEric Cheng uint8_t *mac_addr; 2879da14cebeSEric Cheng mac_group_state_t next_state; 2880da14cebeSEric Cheng mac_client_impl_t *group_only_mcip; 28810dc2366fSVenugopal Iyer mac_resource_props_t *mrp = MCIP_RESOURCE_PROPS(mcip); 28820dc2366fSVenugopal Iyer mac_resource_props_t *emrp = MCIP_EFFECTIVE_PROPS(mcip); 28830dc2366fSVenugopal Iyer boolean_t rxhw; 28840dc2366fSVenugopal Iyer boolean_t txhw; 28850dc2366fSVenugopal Iyer boolean_t use_default = B_FALSE; 28860dc2366fSVenugopal Iyer cpupart_t *cpupart; 28870dc2366fSVenugopal Iyer boolean_t no_unicast; 28880dc2366fSVenugopal Iyer boolean_t isprimary = flent->fe_type & FLOW_PRIMARY_MAC; 28890dc2366fSVenugopal Iyer mac_client_impl_t *reloc_pmcip = NULL; 289084de666eSRyan Zezeski boolean_t use_hw; 2891da14cebeSEric Cheng 2892da14cebeSEric Cheng ASSERT(MAC_PERIM_HELD((mac_handle_t)mip)); 2893da14cebeSEric Cheng 2894da14cebeSEric Cheng switch (link_type) { 2895da14cebeSEric Cheng case SRST_FLOW: 28960dc2366fSVenugopal Iyer mac_srs_group_setup(mcip, flent, link_type); 2897da14cebeSEric Cheng return (0); 2898da14cebeSEric Cheng 2899da14cebeSEric Cheng case SRST_LINK: 29000dc2366fSVenugopal Iyer no_unicast = mcip->mci_state_flags & MCIS_NO_UNICAST_ADDR; 2901da14cebeSEric Cheng mac_addr = flent->fe_flow_desc.fd_dst_mac; 2902da14cebeSEric Cheng 29030dc2366fSVenugopal Iyer /* Default RX group */ 29040dc2366fSVenugopal Iyer default_rgroup = MAC_DEFAULT_RX_GROUP(mip); 2905da14cebeSEric Cheng 29060dc2366fSVenugopal Iyer /* Default TX group */ 29070dc2366fSVenugopal Iyer default_tgroup = MAC_DEFAULT_TX_GROUP(mip); 29080dc2366fSVenugopal Iyer 29090dc2366fSVenugopal Iyer if (no_unicast) { 29100dc2366fSVenugopal Iyer rgroup = default_rgroup; 29110dc2366fSVenugopal Iyer tgroup = default_tgroup; 29120dc2366fSVenugopal Iyer goto grp_found; 29130dc2366fSVenugopal Iyer } 29140dc2366fSVenugopal Iyer rxhw = (mrp->mrp_mask & MRP_RX_RINGS) && 29150dc2366fSVenugopal Iyer (mrp->mrp_nrxrings > 0 || 29160dc2366fSVenugopal Iyer (mrp->mrp_mask & MRP_RXRINGS_UNSPEC)); 29170dc2366fSVenugopal Iyer txhw = (mrp->mrp_mask & MRP_TX_RINGS) && 29180dc2366fSVenugopal Iyer (mrp->mrp_ntxrings > 0 || 29190dc2366fSVenugopal Iyer (mrp->mrp_mask & MRP_TXRINGS_UNSPEC)); 29200dc2366fSVenugopal Iyer 29210dc2366fSVenugopal Iyer /* 292284de666eSRyan Zezeski * All the rings initially belong to the default group 292384de666eSRyan Zezeski * under dynamic grouping. The primary client uses the 292484de666eSRyan Zezeski * default group when it is the only client. The 292584de666eSRyan Zezeski * default group is also used as the destination for 292684de666eSRyan Zezeski * all multicast and broadcast traffic of all clients. 292784de666eSRyan Zezeski * Therefore, the primary client loses its ability to 292884de666eSRyan Zezeski * poll the softrings on addition of a second client. 292984de666eSRyan Zezeski * To avoid a performance penalty, MAC will move the 293084de666eSRyan Zezeski * primary client to a dedicated group when it can. 293184de666eSRyan Zezeski * 293284de666eSRyan Zezeski * When using static grouping, the primary client 293384de666eSRyan Zezeski * begins life on a non-default group. There is 293484de666eSRyan Zezeski * no moving needed upon addition of a second client. 29350dc2366fSVenugopal Iyer */ 29360dc2366fSVenugopal Iyer if (!isprimary && mip->mi_nactiveclients == 2 && 29370dc2366fSVenugopal Iyer (group_only_mcip = mac_primary_client_handle(mip)) != 29380dc2366fSVenugopal Iyer NULL && mip->mi_rx_group_type == MAC_GROUP_TYPE_DYNAMIC) { 29390dc2366fSVenugopal Iyer reloc_pmcip = mac_check_primary_relocation( 29400dc2366fSVenugopal Iyer group_only_mcip, rxhw); 29410dc2366fSVenugopal Iyer } 294284de666eSRyan Zezeski 2943da14cebeSEric Cheng /* 2944da14cebeSEric Cheng * Check to see if we can get an exclusive group for 2945da14cebeSEric Cheng * this mac address or if there already exists a 2946da14cebeSEric Cheng * group that has this mac address (case of VLANs). 2947da14cebeSEric Cheng * If no groups are available, use the default group. 2948da14cebeSEric Cheng */ 29490dc2366fSVenugopal Iyer rgroup = mac_reserve_rx_group(mcip, mac_addr, B_FALSE); 29500dc2366fSVenugopal Iyer if (rgroup == NULL && rxhw) { 29510dc2366fSVenugopal Iyer err = ENOSPC; 29520dc2366fSVenugopal Iyer goto setup_failed; 29530dc2366fSVenugopal Iyer } else if (rgroup == NULL) { 29540dc2366fSVenugopal Iyer rgroup = default_rgroup; 2955da14cebeSEric Cheng } 295684de666eSRyan Zezeski 295784de666eSRyan Zezeski /* 295884de666eSRyan Zezeski * If we are adding a second client to a 295984de666eSRyan Zezeski * non-default group then we need to move the 296084de666eSRyan Zezeski * existing client to the default group and 296184de666eSRyan Zezeski * add the new client to the default group as 296284de666eSRyan Zezeski * well. 296384de666eSRyan Zezeski */ 296484de666eSRyan Zezeski if (rgroup != default_rgroup && 296584de666eSRyan Zezeski rgroup->mrg_state == MAC_GROUP_STATE_RESERVED) { 296684de666eSRyan Zezeski group_only_mcip = MAC_GROUP_ONLY_CLIENT(rgroup); 296784de666eSRyan Zezeski err = mac_rx_switch_group(group_only_mcip, rgroup, 296884de666eSRyan Zezeski default_rgroup); 296984de666eSRyan Zezeski 297084de666eSRyan Zezeski if (err != 0) 297184de666eSRyan Zezeski goto setup_failed; 297284de666eSRyan Zezeski 297384de666eSRyan Zezeski rgroup = default_rgroup; 297484de666eSRyan Zezeski } 297584de666eSRyan Zezeski 29760dc2366fSVenugopal Iyer /* 29770dc2366fSVenugopal Iyer * Check to see if we can get an exclusive group for 29780dc2366fSVenugopal Iyer * this mac client. If no groups are available, use 29790dc2366fSVenugopal Iyer * the default group. 29800dc2366fSVenugopal Iyer */ 29810dc2366fSVenugopal Iyer tgroup = mac_reserve_tx_group(mcip, B_FALSE); 29820dc2366fSVenugopal Iyer if (tgroup == NULL && txhw) { 29830dc2366fSVenugopal Iyer if (rgroup != NULL && rgroup != default_rgroup) 29840dc2366fSVenugopal Iyer mac_release_rx_group(mcip, rgroup); 29850dc2366fSVenugopal Iyer err = ENOSPC; 29860dc2366fSVenugopal Iyer goto setup_failed; 29870dc2366fSVenugopal Iyer } else if (tgroup == NULL) { 29880dc2366fSVenugopal Iyer tgroup = default_tgroup; 2989da14cebeSEric Cheng } 2990da14cebeSEric Cheng 2991da14cebeSEric Cheng /* 2992da14cebeSEric Cheng * Some NICs don't support any Rx rings, so there may not 2993da14cebeSEric Cheng * even be a default group. 2994da14cebeSEric Cheng */ 29950dc2366fSVenugopal Iyer grp_found: 29960dc2366fSVenugopal Iyer if (rgroup != NULL) { 29970dc2366fSVenugopal Iyer if (rgroup != default_rgroup && 29980dc2366fSVenugopal Iyer MAC_GROUP_NO_CLIENT(rgroup) && 299936f99a58SToomas Soome (rxhw || mcip->mci_share != 0)) { 30000dc2366fSVenugopal Iyer MAC_RX_GRP_RESERVED(mip); 30010dc2366fSVenugopal Iyer if (mip->mi_rx_group_type == 30020dc2366fSVenugopal Iyer MAC_GROUP_TYPE_DYNAMIC) { 30030dc2366fSVenugopal Iyer MAC_RX_RING_RESERVED(mip, 30040dc2366fSVenugopal Iyer rgroup->mrg_cur_count); 30050dc2366fSVenugopal Iyer } 30060dc2366fSVenugopal Iyer } 300784de666eSRyan Zezeski 30080dc2366fSVenugopal Iyer flent->fe_rx_ring_group = rgroup; 3009da14cebeSEric Cheng /* 301084de666eSRyan Zezeski * Add the client to the group and update the 301184de666eSRyan Zezeski * group's state. If rgroup != default_group 301284de666eSRyan Zezeski * then the rgroup should only ever have one 301384de666eSRyan Zezeski * client and be in the RESERVED state. But no 301484de666eSRyan Zezeski * matter what, the default_rgroup will enter 301584de666eSRyan Zezeski * the SHARED state since it has to receive 301684de666eSRyan Zezeski * all broadcast and multicast traffic. This 301784de666eSRyan Zezeski * case is handled later in the function. 3018da14cebeSEric Cheng */ 30190dc2366fSVenugopal Iyer mac_group_add_client(rgroup, mcip); 30200dc2366fSVenugopal Iyer next_state = mac_group_next_state(rgroup, 30210dc2366fSVenugopal Iyer &group_only_mcip, default_rgroup, B_TRUE); 30220dc2366fSVenugopal Iyer mac_set_group_state(rgroup, next_state); 3023da14cebeSEric Cheng } 3024da14cebeSEric Cheng 30250dc2366fSVenugopal Iyer if (tgroup != NULL) { 30260dc2366fSVenugopal Iyer if (tgroup != default_tgroup && 30270dc2366fSVenugopal Iyer MAC_GROUP_NO_CLIENT(tgroup) && 302836f99a58SToomas Soome (txhw || mcip->mci_share != 0)) { 30290dc2366fSVenugopal Iyer MAC_TX_GRP_RESERVED(mip); 30300dc2366fSVenugopal Iyer if (mip->mi_tx_group_type == 30310dc2366fSVenugopal Iyer MAC_GROUP_TYPE_DYNAMIC) { 30320dc2366fSVenugopal Iyer MAC_TX_RING_RESERVED(mip, 30330dc2366fSVenugopal Iyer tgroup->mrg_cur_count); 30340dc2366fSVenugopal Iyer } 30350dc2366fSVenugopal Iyer } 30360dc2366fSVenugopal Iyer flent->fe_tx_ring_group = tgroup; 30370dc2366fSVenugopal Iyer mac_group_add_client(tgroup, mcip); 30380dc2366fSVenugopal Iyer next_state = mac_group_next_state(tgroup, 30390dc2366fSVenugopal Iyer &group_only_mcip, default_tgroup, B_FALSE); 30400dc2366fSVenugopal Iyer tgroup->mrg_state = next_state; 30410dc2366fSVenugopal Iyer } 3042da14cebeSEric Cheng 30430dc2366fSVenugopal Iyer /* We are setting up minimal datapath only */ 304484de666eSRyan Zezeski if (no_unicast) { 304584de666eSRyan Zezeski mac_srs_group_setup(mcip, flent, link_type); 30460dc2366fSVenugopal Iyer break; 304784de666eSRyan Zezeski } 304884de666eSRyan Zezeski 304984de666eSRyan Zezeski /* Program software classification. */ 3050da14cebeSEric Cheng if ((err = mac_flow_add(mip->mi_flow_tab, flent)) != 0) 3051da14cebeSEric Cheng goto setup_failed; 3052da14cebeSEric Cheng 305384de666eSRyan Zezeski /* Program hardware classification. */ 305484de666eSRyan Zezeski vid = i_mac_flow_vid(flent); 305584de666eSRyan Zezeski use_hw = (mcip->mci_state_flags & MCIS_UNICAST_HW) != 0; 305684de666eSRyan Zezeski err = mac_add_macaddr_vlan(mip, rgroup, mac_addr, vid, use_hw); 305784de666eSRyan Zezeski 305884de666eSRyan Zezeski if (err != 0) 3059da14cebeSEric Cheng goto setup_failed; 306084de666eSRyan Zezeski 3061da14cebeSEric Cheng mcip->mci_unicast = mac_find_macaddr(mip, mac_addr); 306284de666eSRyan Zezeski VERIFY3P(mcip->mci_unicast, !=, NULL); 306384de666eSRyan Zezeski 306484de666eSRyan Zezeski /* 306584de666eSRyan Zezeski * Setup the Rx and Tx SRSes. If the client has a 306684de666eSRyan Zezeski * reserved group, then mac_srs_group_setup() creates 306784de666eSRyan Zezeski * the required SRSes for the HW rings. If we have a 306884de666eSRyan Zezeski * shared group, mac_srs_group_setup() dismantles the 306984de666eSRyan Zezeski * HW SRSes of the previously exclusive group. 307084de666eSRyan Zezeski */ 307184de666eSRyan Zezeski mac_srs_group_setup(mcip, flent, link_type); 307284de666eSRyan Zezeski 3073797f979dSCody Peter Mello /* (Re)init the v6 token & local addr used by link protection */ 3074797f979dSCody Peter Mello mac_protect_update_mac_token(mcip); 3075da14cebeSEric Cheng break; 3076da14cebeSEric Cheng 3077da14cebeSEric Cheng default: 3078da14cebeSEric Cheng ASSERT(B_FALSE); 3079da14cebeSEric Cheng break; 3080da14cebeSEric Cheng } 3081da14cebeSEric Cheng 3082da14cebeSEric Cheng /* 3083da14cebeSEric Cheng * All broadcast and multicast traffic is received only on the default 3084da14cebeSEric Cheng * group. If we have setup the datapath for a non-default group above 3085da14cebeSEric Cheng * then move the default group to shared state to allow distribution of 3086da14cebeSEric Cheng * incoming broadcast traffic to the other groups and dismantle the 3087da14cebeSEric Cheng * SRSes over the default group. 3088da14cebeSEric Cheng */ 30890dc2366fSVenugopal Iyer if (rgroup != NULL) { 30900dc2366fSVenugopal Iyer if (rgroup != default_rgroup) { 30910dc2366fSVenugopal Iyer if (default_rgroup->mrg_state == 3092da14cebeSEric Cheng MAC_GROUP_STATE_RESERVED) { 30930dc2366fSVenugopal Iyer group_only_mcip = MAC_GROUP_ONLY_CLIENT( 30940dc2366fSVenugopal Iyer default_rgroup); 3095da14cebeSEric Cheng ASSERT(group_only_mcip != NULL && 3096da14cebeSEric Cheng mip->mi_nactiveclients > 1); 3097da14cebeSEric Cheng 30980dc2366fSVenugopal Iyer mac_set_group_state(default_rgroup, 3099da14cebeSEric Cheng MAC_GROUP_STATE_SHARED); 31000dc2366fSVenugopal Iyer mac_rx_srs_group_setup(group_only_mcip, 31010dc2366fSVenugopal Iyer group_only_mcip->mci_flent, SRST_LINK); 31020dc2366fSVenugopal Iyer pool_lock(); 31030dc2366fSVenugopal Iyer cpupart = mac_pset_find(mrp, &use_default); 31040dc2366fSVenugopal Iyer mac_fanout_setup(group_only_mcip, 3105da14cebeSEric Cheng group_only_mcip->mci_flent, 31060dc2366fSVenugopal Iyer MCIP_RESOURCE_PROPS(group_only_mcip), 31070dc2366fSVenugopal Iyer mac_rx_deliver, group_only_mcip, NULL, 31080dc2366fSVenugopal Iyer cpupart); 31090dc2366fSVenugopal Iyer mac_set_pool_effective(use_default, cpupart, 31100dc2366fSVenugopal Iyer mrp, emrp); 31110dc2366fSVenugopal Iyer pool_unlock(); 3112da14cebeSEric Cheng } 31130dc2366fSVenugopal Iyer ASSERT(default_rgroup->mrg_state == 3114da14cebeSEric Cheng MAC_GROUP_STATE_SHARED); 3115da14cebeSEric Cheng } 311684de666eSRyan Zezeski 3117da14cebeSEric Cheng /* 311884de666eSRyan Zezeski * A VLAN MAC client on a reserved group still 311984de666eSRyan Zezeski * requires SW classification if the MAC doesn't 312084de666eSRyan Zezeski * provide VLAN HW filtering. 312184de666eSRyan Zezeski * 312284de666eSRyan Zezeski * Clients with no unicast address also require SW 312384de666eSRyan Zezeski * classification. 3124da14cebeSEric Cheng */ 31250dc2366fSVenugopal Iyer if (rgroup->mrg_state == MAC_GROUP_STATE_RESERVED && 312684de666eSRyan Zezeski ((!MAC_GROUP_HW_VLAN(rgroup) && vid != VLAN_ID_NONE) || 312784de666eSRyan Zezeski no_unicast)) { 31280dc2366fSVenugopal Iyer mac_rx_switch_grp_to_sw(rgroup); 3129da14cebeSEric Cheng } 313084de666eSRyan Zezeski 3131da14cebeSEric Cheng } 313284de666eSRyan Zezeski 31330dc2366fSVenugopal Iyer mac_set_rings_effective(mcip); 3134da14cebeSEric Cheng return (0); 3135da14cebeSEric Cheng 3136da14cebeSEric Cheng setup_failed: 31370dc2366fSVenugopal Iyer /* Switch the primary back to default group */ 31380dc2366fSVenugopal Iyer if (reloc_pmcip != NULL) { 31390dc2366fSVenugopal Iyer (void) mac_rx_switch_group(reloc_pmcip, 31400dc2366fSVenugopal Iyer reloc_pmcip->mci_flent->fe_rx_ring_group, default_rgroup); 31410dc2366fSVenugopal Iyer } 3142da14cebeSEric Cheng mac_datapath_teardown(mcip, flent, link_type); 3143da14cebeSEric Cheng return (err); 3144da14cebeSEric Cheng } 3145da14cebeSEric Cheng 3146da14cebeSEric Cheng void 3147da14cebeSEric Cheng mac_datapath_teardown(mac_client_impl_t *mcip, flow_entry_t *flent, 3148da14cebeSEric Cheng uint32_t link_type) 3149da14cebeSEric Cheng { 3150da14cebeSEric Cheng mac_impl_t *mip = mcip->mci_mip; 3151da14cebeSEric Cheng mac_group_t *group = NULL; 3152da14cebeSEric Cheng mac_client_impl_t *grp_only_mcip; 3153da14cebeSEric Cheng flow_entry_t *group_only_flent; 3154da14cebeSEric Cheng mac_group_t *default_group; 3155da14cebeSEric Cheng boolean_t check_default_group = B_FALSE; 3156da14cebeSEric Cheng mac_group_state_t next_state; 31570dc2366fSVenugopal Iyer mac_resource_props_t *mrp = MCIP_RESOURCE_PROPS(mcip); 315884de666eSRyan Zezeski uint16_t vid; 3159da14cebeSEric Cheng 3160da14cebeSEric Cheng ASSERT(MAC_PERIM_HELD((mac_handle_t)mip)); 3161da14cebeSEric Cheng 3162da14cebeSEric Cheng switch (link_type) { 3163da14cebeSEric Cheng case SRST_FLOW: 31640dc2366fSVenugopal Iyer mac_rx_srs_group_teardown(flent, B_FALSE); 31650dc2366fSVenugopal Iyer mac_tx_srs_group_teardown(mcip, flent, SRST_FLOW); 3166da14cebeSEric Cheng return; 3167da14cebeSEric Cheng 3168da14cebeSEric Cheng case SRST_LINK: 3169da14cebeSEric Cheng /* Stop sending packets */ 3170da14cebeSEric Cheng mac_tx_client_block(mcip); 317184de666eSRyan Zezeski group = flent->fe_rx_ring_group; 317284de666eSRyan Zezeski vid = i_mac_flow_vid(flent); 3173da14cebeSEric Cheng 317484de666eSRyan Zezeski /* 317584de666eSRyan Zezeski * Stop the packet flow from the hardware by disabling 317684de666eSRyan Zezeski * any hardware filters assigned to this client. 317784de666eSRyan Zezeski */ 3178da14cebeSEric Cheng if (mcip->mci_unicast != NULL) { 3179da14cebeSEric Cheng int err; 318084de666eSRyan Zezeski 318184de666eSRyan Zezeski err = mac_remove_macaddr_vlan(mcip->mci_unicast, vid); 318284de666eSRyan Zezeski 3183da14cebeSEric Cheng if (err != 0) { 318484de666eSRyan Zezeski cmn_err(CE_WARN, "%s: failed to remove a MAC HW" 318584de666eSRyan Zezeski " filters because of error 0x%x", 3186da14cebeSEric Cheng mip->mi_name, err); 3187da14cebeSEric Cheng } 318884de666eSRyan Zezeski 3189da14cebeSEric Cheng mcip->mci_unicast = NULL; 3190da14cebeSEric Cheng } 3191da14cebeSEric Cheng 3192da14cebeSEric Cheng /* Stop the packets coming from the S/W classifier */ 3193da14cebeSEric Cheng mac_flow_remove(mip->mi_flow_tab, flent, B_FALSE); 3194da14cebeSEric Cheng mac_flow_wait(flent, FLOW_DRIVER_UPCALL); 3195da14cebeSEric Cheng 319645948e49SRyan Zezeski /* Quiesce and destroy all the SRSes. */ 31970dc2366fSVenugopal Iyer mac_rx_srs_group_teardown(flent, B_FALSE); 31980dc2366fSVenugopal Iyer mac_tx_srs_group_teardown(mcip, flent, SRST_LINK); 31990dc2366fSVenugopal Iyer 320045948e49SRyan Zezeski ASSERT3P(mcip->mci_flent, ==, flent); 320145948e49SRyan Zezeski ASSERT3P(flent->fe_next, ==, NULL); 3202da14cebeSEric Cheng 3203da14cebeSEric Cheng /* 3204da14cebeSEric Cheng * Release our hold on the group as well. We need 3205da14cebeSEric Cheng * to check if the shared group has only one client 3206da14cebeSEric Cheng * left who can use it exclusively. Also, if we 3207da14cebeSEric Cheng * were the last client, release the group. 3208da14cebeSEric Cheng */ 32090dc2366fSVenugopal Iyer default_group = MAC_DEFAULT_RX_GROUP(mip); 3210da14cebeSEric Cheng if (group != NULL) { 32110dc2366fSVenugopal Iyer mac_group_remove_client(group, mcip); 32120dc2366fSVenugopal Iyer next_state = mac_group_next_state(group, 32130dc2366fSVenugopal Iyer &grp_only_mcip, default_group, B_TRUE); 321484de666eSRyan Zezeski 3215da14cebeSEric Cheng if (next_state == MAC_GROUP_STATE_RESERVED) { 3216da14cebeSEric Cheng /* 3217da14cebeSEric Cheng * Only one client left on this RX group. 3218da14cebeSEric Cheng */ 321984de666eSRyan Zezeski VERIFY3P(grp_only_mcip, !=, NULL); 32200dc2366fSVenugopal Iyer mac_set_group_state(group, 3221da14cebeSEric Cheng MAC_GROUP_STATE_RESERVED); 3222da14cebeSEric Cheng group_only_flent = grp_only_mcip->mci_flent; 3223da14cebeSEric Cheng 3224da14cebeSEric Cheng /* 3225da14cebeSEric Cheng * The only remaining client has exclusive 3226da14cebeSEric Cheng * access on the group. Allow it to 3227da14cebeSEric Cheng * dynamically poll the H/W rings etc. 3228da14cebeSEric Cheng */ 32290dc2366fSVenugopal Iyer mac_rx_srs_group_setup(grp_only_mcip, 32300dc2366fSVenugopal Iyer group_only_flent, SRST_LINK); 32310dc2366fSVenugopal Iyer mac_fanout_setup(grp_only_mcip, 32320dc2366fSVenugopal Iyer group_only_flent, 32330dc2366fSVenugopal Iyer MCIP_RESOURCE_PROPS(grp_only_mcip), 32340dc2366fSVenugopal Iyer mac_rx_deliver, grp_only_mcip, NULL, NULL); 3235da14cebeSEric Cheng mac_rx_group_unmark(group, MR_INCIPIENT); 32360dc2366fSVenugopal Iyer mac_set_rings_effective(grp_only_mcip); 3237da14cebeSEric Cheng } else if (next_state == MAC_GROUP_STATE_REGISTERED) { 3238da14cebeSEric Cheng /* 3239da14cebeSEric Cheng * This is a non-default group being freed up. 3240da14cebeSEric Cheng * We need to reevaluate the default group 3241da14cebeSEric Cheng * to see if the primary client can get 3242da14cebeSEric Cheng * exclusive access to the default group. 3243da14cebeSEric Cheng */ 324484de666eSRyan Zezeski VERIFY3P(group, !=, MAC_DEFAULT_RX_GROUP(mip)); 32450dc2366fSVenugopal Iyer if (mrp->mrp_mask & MRP_RX_RINGS) { 32460dc2366fSVenugopal Iyer MAC_RX_GRP_RELEASED(mip); 32470dc2366fSVenugopal Iyer if (mip->mi_rx_group_type == 32480dc2366fSVenugopal Iyer MAC_GROUP_TYPE_DYNAMIC) { 32490dc2366fSVenugopal Iyer MAC_RX_RING_RELEASED(mip, 32500dc2366fSVenugopal Iyer group->mrg_cur_count); 32510dc2366fSVenugopal Iyer } 32520dc2366fSVenugopal Iyer } 3253da14cebeSEric Cheng mac_release_rx_group(mcip, group); 32540dc2366fSVenugopal Iyer mac_set_group_state(group, 3255da14cebeSEric Cheng MAC_GROUP_STATE_REGISTERED); 3256da14cebeSEric Cheng check_default_group = B_TRUE; 3257da14cebeSEric Cheng } else { 325884de666eSRyan Zezeski VERIFY3S(next_state, ==, 325984de666eSRyan Zezeski MAC_GROUP_STATE_SHARED); 32600dc2366fSVenugopal Iyer mac_set_group_state(group, 3261da14cebeSEric Cheng MAC_GROUP_STATE_SHARED); 3262da14cebeSEric Cheng mac_rx_group_unmark(group, MR_CONDEMNED); 3263da14cebeSEric Cheng } 3264da14cebeSEric Cheng flent->fe_rx_ring_group = NULL; 3265da14cebeSEric Cheng } 32660dc2366fSVenugopal Iyer /* 32670dc2366fSVenugopal Iyer * Remove the client from the TX group. Additionally, if 32680dc2366fSVenugopal Iyer * this a non-default group, then we also need to release 32690dc2366fSVenugopal Iyer * the group. 32700dc2366fSVenugopal Iyer */ 32710dc2366fSVenugopal Iyer group = flent->fe_tx_ring_group; 32720dc2366fSVenugopal Iyer default_group = MAC_DEFAULT_TX_GROUP(mip); 32730dc2366fSVenugopal Iyer if (group != NULL) { 32740dc2366fSVenugopal Iyer mac_group_remove_client(group, mcip); 32750dc2366fSVenugopal Iyer next_state = mac_group_next_state(group, 32760dc2366fSVenugopal Iyer &grp_only_mcip, default_group, B_FALSE); 32770dc2366fSVenugopal Iyer if (next_state == MAC_GROUP_STATE_REGISTERED) { 32780dc2366fSVenugopal Iyer if (group != default_group) { 32790dc2366fSVenugopal Iyer if (mrp->mrp_mask & MRP_TX_RINGS) { 32800dc2366fSVenugopal Iyer MAC_TX_GRP_RELEASED(mip); 32810dc2366fSVenugopal Iyer if (mip->mi_tx_group_type == 32820dc2366fSVenugopal Iyer MAC_GROUP_TYPE_DYNAMIC) { 32830dc2366fSVenugopal Iyer MAC_TX_RING_RELEASED( 32840dc2366fSVenugopal Iyer mip, group-> 32850dc2366fSVenugopal Iyer mrg_cur_count); 32860dc2366fSVenugopal Iyer } 32870dc2366fSVenugopal Iyer } 32880dc2366fSVenugopal Iyer mac_release_tx_group(mcip, group); 32890dc2366fSVenugopal Iyer /* 32900dc2366fSVenugopal Iyer * If the default group is reserved, 32910dc2366fSVenugopal Iyer * then we need to set the effective 32920dc2366fSVenugopal Iyer * rings as we would have given 32930dc2366fSVenugopal Iyer * back some rings when the group 32940dc2366fSVenugopal Iyer * was released 32950dc2366fSVenugopal Iyer */ 32960dc2366fSVenugopal Iyer if (mip->mi_tx_group_type == 32970dc2366fSVenugopal Iyer MAC_GROUP_TYPE_DYNAMIC && 32980dc2366fSVenugopal Iyer default_group->mrg_state == 32990dc2366fSVenugopal Iyer MAC_GROUP_STATE_RESERVED) { 33000dc2366fSVenugopal Iyer grp_only_mcip = 33010dc2366fSVenugopal Iyer MAC_GROUP_ONLY_CLIENT 33020dc2366fSVenugopal Iyer (default_group); 33030dc2366fSVenugopal Iyer mac_set_rings_effective( 33040dc2366fSVenugopal Iyer grp_only_mcip); 33050dc2366fSVenugopal Iyer } 33060dc2366fSVenugopal Iyer } else { 33070dc2366fSVenugopal Iyer mac_ring_t *ring; 33080dc2366fSVenugopal Iyer int cnt; 33090dc2366fSVenugopal Iyer int ringcnt; 33100dc2366fSVenugopal Iyer 33110dc2366fSVenugopal Iyer /* 33120dc2366fSVenugopal Iyer * Stop all the rings except the 33130dc2366fSVenugopal Iyer * default ring. 33140dc2366fSVenugopal Iyer */ 33150dc2366fSVenugopal Iyer ringcnt = group->mrg_cur_count; 33160dc2366fSVenugopal Iyer ring = group->mrg_rings; 33170dc2366fSVenugopal Iyer for (cnt = 0; cnt < ringcnt; cnt++) { 33180dc2366fSVenugopal Iyer if (ring->mr_state == 33190dc2366fSVenugopal Iyer MR_INUSE && ring != 33200dc2366fSVenugopal Iyer (mac_ring_t *) 33210dc2366fSVenugopal Iyer mip->mi_default_tx_ring) { 33220dc2366fSVenugopal Iyer mac_stop_ring(ring); 33230dc2366fSVenugopal Iyer ring->mr_flag = 0; 33240dc2366fSVenugopal Iyer } 33250dc2366fSVenugopal Iyer ring = ring->mr_next; 33260dc2366fSVenugopal Iyer } 33270dc2366fSVenugopal Iyer } 33280dc2366fSVenugopal Iyer } else if (next_state == MAC_GROUP_STATE_RESERVED) { 33290dc2366fSVenugopal Iyer mac_set_rings_effective(grp_only_mcip); 33300dc2366fSVenugopal Iyer } 33310dc2366fSVenugopal Iyer flent->fe_tx_ring_group = NULL; 33320dc2366fSVenugopal Iyer group->mrg_state = next_state; 33330dc2366fSVenugopal Iyer } 3334da14cebeSEric Cheng break; 3335da14cebeSEric Cheng default: 3336da14cebeSEric Cheng ASSERT(B_FALSE); 3337da14cebeSEric Cheng break; 3338da14cebeSEric Cheng } 3339da14cebeSEric Cheng 3340da14cebeSEric Cheng /* 3341da14cebeSEric Cheng * The mac client using the default group gets exclusive access to the 3342da14cebeSEric Cheng * default group if and only if it is the sole client on the entire 3343da14cebeSEric Cheng * mip. If so set the group state to reserved, and set up the SRSes 3344da14cebeSEric Cheng * over the default group. 3345da14cebeSEric Cheng */ 3346da14cebeSEric Cheng if (check_default_group) { 33470dc2366fSVenugopal Iyer default_group = MAC_DEFAULT_RX_GROUP(mip); 334884de666eSRyan Zezeski VERIFY3S(default_group->mrg_state, ==, MAC_GROUP_STATE_SHARED); 33490dc2366fSVenugopal Iyer next_state = mac_group_next_state(default_group, 33500dc2366fSVenugopal Iyer &grp_only_mcip, default_group, B_TRUE); 3351da14cebeSEric Cheng if (next_state == MAC_GROUP_STATE_RESERVED) { 335284de666eSRyan Zezeski VERIFY3P(grp_only_mcip, !=, NULL); 335384de666eSRyan Zezeski VERIFY3U(mip->mi_nactiveclients, ==, 1); 33540dc2366fSVenugopal Iyer mac_set_group_state(default_group, 3355da14cebeSEric Cheng MAC_GROUP_STATE_RESERVED); 33560dc2366fSVenugopal Iyer mac_rx_srs_group_setup(grp_only_mcip, 33570dc2366fSVenugopal Iyer grp_only_mcip->mci_flent, SRST_LINK); 33580dc2366fSVenugopal Iyer mac_fanout_setup(grp_only_mcip, 3359da14cebeSEric Cheng grp_only_mcip->mci_flent, 33600dc2366fSVenugopal Iyer MCIP_RESOURCE_PROPS(grp_only_mcip), mac_rx_deliver, 33610dc2366fSVenugopal Iyer grp_only_mcip, NULL, NULL); 3362ae6aa22aSVenugopal Iyer mac_rx_group_unmark(default_group, MR_INCIPIENT); 33630dc2366fSVenugopal Iyer mac_set_rings_effective(grp_only_mcip); 3364da14cebeSEric Cheng } 3365da14cebeSEric Cheng } 33660dc2366fSVenugopal Iyer 33670dc2366fSVenugopal Iyer /* 33680dc2366fSVenugopal Iyer * If the primary is the only one left and the MAC supports 33690dc2366fSVenugopal Iyer * dynamic grouping, we need to see if the primary needs to 33700dc2366fSVenugopal Iyer * be moved to the default group so that it can use all the 33710dc2366fSVenugopal Iyer * H/W rings. 33720dc2366fSVenugopal Iyer */ 33730dc2366fSVenugopal Iyer if (!(flent->fe_type & FLOW_PRIMARY_MAC) && 33740dc2366fSVenugopal Iyer mip->mi_nactiveclients == 1 && 33750dc2366fSVenugopal Iyer mip->mi_rx_group_type == MAC_GROUP_TYPE_DYNAMIC) { 33760dc2366fSVenugopal Iyer default_group = MAC_DEFAULT_RX_GROUP(mip); 33770dc2366fSVenugopal Iyer grp_only_mcip = mac_primary_client_handle(mip); 33780dc2366fSVenugopal Iyer if (grp_only_mcip == NULL) 33790dc2366fSVenugopal Iyer return; 33800dc2366fSVenugopal Iyer group_only_flent = grp_only_mcip->mci_flent; 33810dc2366fSVenugopal Iyer mrp = MCIP_RESOURCE_PROPS(grp_only_mcip); 33820dc2366fSVenugopal Iyer /* 33830dc2366fSVenugopal Iyer * If the primary has an explicit property set, leave it 33840dc2366fSVenugopal Iyer * alone. 33850dc2366fSVenugopal Iyer */ 33860dc2366fSVenugopal Iyer if (mrp->mrp_mask & MRP_RX_RINGS) 33870dc2366fSVenugopal Iyer return; 33880dc2366fSVenugopal Iyer /* 33890dc2366fSVenugopal Iyer * Switch the primary to the default group. 33900dc2366fSVenugopal Iyer */ 33910dc2366fSVenugopal Iyer (void) mac_rx_switch_group(grp_only_mcip, 33920dc2366fSVenugopal Iyer group_only_flent->fe_rx_ring_group, default_group); 33930dc2366fSVenugopal Iyer } 3394da14cebeSEric Cheng } 3395da14cebeSEric Cheng 3396da14cebeSEric Cheng /* DATAPATH TEAR DOWN ROUTINES (SRS and FANOUT teardown) */ 3397da14cebeSEric Cheng 3398da14cebeSEric Cheng static void 3399da14cebeSEric Cheng mac_srs_fanout_list_free(mac_soft_ring_set_t *mac_srs) 3400da14cebeSEric Cheng { 34010dc2366fSVenugopal Iyer if (mac_srs->srs_type & SRST_TX) { 34020dc2366fSVenugopal Iyer mac_srs_tx_t *tx; 34030dc2366fSVenugopal Iyer 34040dc2366fSVenugopal Iyer ASSERT(mac_srs->srs_tcp_soft_rings == NULL); 34050dc2366fSVenugopal Iyer ASSERT(mac_srs->srs_udp_soft_rings == NULL); 34060dc2366fSVenugopal Iyer ASSERT(mac_srs->srs_oth_soft_rings == NULL); 34070dc2366fSVenugopal Iyer ASSERT(mac_srs->srs_tx_soft_rings != NULL); 34080dc2366fSVenugopal Iyer kmem_free(mac_srs->srs_tx_soft_rings, 34090dc2366fSVenugopal Iyer sizeof (mac_soft_ring_t *) * MAX_RINGS_PER_GROUP); 34100dc2366fSVenugopal Iyer mac_srs->srs_tx_soft_rings = NULL; 34110dc2366fSVenugopal Iyer tx = &mac_srs->srs_tx; 34120dc2366fSVenugopal Iyer if (tx->st_soft_rings != NULL) { 34130dc2366fSVenugopal Iyer kmem_free(tx->st_soft_rings, 34140dc2366fSVenugopal Iyer sizeof (mac_soft_ring_t *) * MAX_RINGS_PER_GROUP); 34150dc2366fSVenugopal Iyer } 34160dc2366fSVenugopal Iyer } else { 34170dc2366fSVenugopal Iyer ASSERT(mac_srs->srs_tx_soft_rings == NULL); 3418da14cebeSEric Cheng ASSERT(mac_srs->srs_tcp_soft_rings != NULL); 3419da14cebeSEric Cheng kmem_free(mac_srs->srs_tcp_soft_rings, 3420da14cebeSEric Cheng sizeof (mac_soft_ring_t *) * MAX_SR_FANOUT); 3421da14cebeSEric Cheng mac_srs->srs_tcp_soft_rings = NULL; 3422da14cebeSEric Cheng ASSERT(mac_srs->srs_udp_soft_rings != NULL); 3423da14cebeSEric Cheng kmem_free(mac_srs->srs_udp_soft_rings, 3424da14cebeSEric Cheng sizeof (mac_soft_ring_t *) * MAX_SR_FANOUT); 3425da14cebeSEric Cheng mac_srs->srs_udp_soft_rings = NULL; 3426da14cebeSEric Cheng ASSERT(mac_srs->srs_oth_soft_rings != NULL); 3427da14cebeSEric Cheng kmem_free(mac_srs->srs_oth_soft_rings, 3428da14cebeSEric Cheng sizeof (mac_soft_ring_t *) * MAX_SR_FANOUT); 3429da14cebeSEric Cheng mac_srs->srs_oth_soft_rings = NULL; 3430da14cebeSEric Cheng } 34310dc2366fSVenugopal Iyer } 3432da14cebeSEric Cheng 3433da14cebeSEric Cheng /* 3434da14cebeSEric Cheng * An RX SRS is attached to at most one mac_ring. 3435da14cebeSEric Cheng * A TX SRS has no rings. 3436da14cebeSEric Cheng */ 3437da14cebeSEric Cheng static void 3438da14cebeSEric Cheng mac_srs_ring_free(mac_soft_ring_set_t *mac_srs) 3439da14cebeSEric Cheng { 3440da14cebeSEric Cheng mac_client_impl_t *mcip; 3441da14cebeSEric Cheng mac_ring_t *ring; 3442da14cebeSEric Cheng flow_entry_t *flent; 3443da14cebeSEric Cheng 3444da14cebeSEric Cheng ring = mac_srs->srs_ring; 3445da14cebeSEric Cheng if (mac_srs->srs_type & SRST_TX) { 3446da14cebeSEric Cheng ASSERT(ring == NULL); 3447da14cebeSEric Cheng return; 3448da14cebeSEric Cheng } 3449da14cebeSEric Cheng 3450da14cebeSEric Cheng if (ring == NULL) 3451da14cebeSEric Cheng return; 3452da14cebeSEric Cheng 3453da14cebeSEric Cheng /* 3454da14cebeSEric Cheng * Broadcast flows don't have a client impl association, but they 3455da14cebeSEric Cheng * use only soft rings. 3456da14cebeSEric Cheng */ 3457da14cebeSEric Cheng flent = mac_srs->srs_flent; 3458da14cebeSEric Cheng mcip = flent->fe_mcip; 3459da14cebeSEric Cheng ASSERT(mcip != NULL); 3460da14cebeSEric Cheng 3461da14cebeSEric Cheng ring->mr_classify_type = MAC_NO_CLASSIFIER; 3462da14cebeSEric Cheng ring->mr_srs = NULL; 3463da14cebeSEric Cheng } 3464da14cebeSEric Cheng 3465da14cebeSEric Cheng /* 3466da14cebeSEric Cheng * Physical unlink and free of the data structures happen below. This is 3467da14cebeSEric Cheng * driven from mac_flow_destroy(), on the last refrele of a flow. 3468da14cebeSEric Cheng * 3469da14cebeSEric Cheng * Assumes Rx srs is 1-1 mapped with an ring. 3470da14cebeSEric Cheng */ 3471da14cebeSEric Cheng void 3472da14cebeSEric Cheng mac_srs_free(mac_soft_ring_set_t *mac_srs) 3473da14cebeSEric Cheng { 3474da14cebeSEric Cheng ASSERT(mac_srs->srs_mcip == NULL || 3475da14cebeSEric Cheng MAC_PERIM_HELD((mac_handle_t)mac_srs->srs_mcip->mci_mip)); 3476da14cebeSEric Cheng ASSERT((mac_srs->srs_state & (SRS_CONDEMNED | SRS_CONDEMNED_DONE | 3477da14cebeSEric Cheng SRS_PROC | SRS_PROC_FAST)) == (SRS_CONDEMNED | SRS_CONDEMNED_DONE)); 3478da14cebeSEric Cheng 3479*c61a1653SRyan Zezeski mac_drop_chain(mac_srs->srs_first, "SRS free"); 3480da14cebeSEric Cheng mac_srs_ring_free(mac_srs); 34810dc2366fSVenugopal Iyer mac_srs_soft_rings_free(mac_srs); 3482da14cebeSEric Cheng mac_srs_fanout_list_free(mac_srs); 3483da14cebeSEric Cheng 3484da14cebeSEric Cheng mac_srs->srs_bw = NULL; 34850dc2366fSVenugopal Iyer mac_srs_stat_delete(mac_srs); 3486da14cebeSEric Cheng kmem_cache_free(mac_srs_cache, mac_srs); 3487da14cebeSEric Cheng } 3488da14cebeSEric Cheng 3489da14cebeSEric Cheng static void 3490da14cebeSEric Cheng mac_srs_soft_rings_quiesce(mac_soft_ring_set_t *mac_srs, uint_t s_ring_flag) 3491da14cebeSEric Cheng { 3492da14cebeSEric Cheng mac_soft_ring_t *softring; 3493da14cebeSEric Cheng 3494da14cebeSEric Cheng ASSERT(MUTEX_HELD(&mac_srs->srs_lock)); 3495da14cebeSEric Cheng 3496da14cebeSEric Cheng mac_srs_soft_rings_signal(mac_srs, s_ring_flag); 3497da14cebeSEric Cheng if (s_ring_flag == S_RING_CONDEMNED) { 3498da14cebeSEric Cheng while (mac_srs->srs_soft_ring_condemned_count != 3499da14cebeSEric Cheng mac_srs->srs_soft_ring_count) 3500da14cebeSEric Cheng cv_wait(&mac_srs->srs_async, &mac_srs->srs_lock); 3501da14cebeSEric Cheng } else { 3502da14cebeSEric Cheng while (mac_srs->srs_soft_ring_quiesced_count != 3503da14cebeSEric Cheng mac_srs->srs_soft_ring_count) 3504da14cebeSEric Cheng cv_wait(&mac_srs->srs_async, &mac_srs->srs_lock); 3505da14cebeSEric Cheng } 3506da14cebeSEric Cheng mutex_exit(&mac_srs->srs_lock); 3507da14cebeSEric Cheng 3508da14cebeSEric Cheng for (softring = mac_srs->srs_soft_ring_head; softring != NULL; 3509da906b4eSRyan Zezeski softring = softring->s_ring_next) { 3510da14cebeSEric Cheng (void) untimeout(softring->s_ring_tid); 3511da906b4eSRyan Zezeski softring->s_ring_tid = NULL; 3512da906b4eSRyan Zezeski } 3513da14cebeSEric Cheng 3514da14cebeSEric Cheng (void) untimeout(mac_srs->srs_tid); 3515da906b4eSRyan Zezeski mac_srs->srs_tid = NULL; 3516da14cebeSEric Cheng 3517da14cebeSEric Cheng mutex_enter(&mac_srs->srs_lock); 3518da14cebeSEric Cheng } 3519da14cebeSEric Cheng 3520da14cebeSEric Cheng /* 3521da14cebeSEric Cheng * The block comment above mac_rx_classify_flow_state_change explains the 3522da14cebeSEric Cheng * background. At this point upcalls from the driver (both hardware classified 3523da14cebeSEric Cheng * and software classified) have been cut off. We now need to quiesce the 3524da14cebeSEric Cheng * SRS worker, poll, and softring threads. The SRS worker thread serves as 3525da14cebeSEric Cheng * the master controller. The steps involved are described below in the function 3526da14cebeSEric Cheng */ 3527da14cebeSEric Cheng void 3528da14cebeSEric Cheng mac_srs_worker_quiesce(mac_soft_ring_set_t *mac_srs) 3529da14cebeSEric Cheng { 3530da14cebeSEric Cheng uint_t s_ring_flag; 3531da14cebeSEric Cheng uint_t srs_poll_wait_flag; 3532da14cebeSEric Cheng 3533da14cebeSEric Cheng ASSERT(MUTEX_HELD(&mac_srs->srs_lock)); 3534da14cebeSEric Cheng ASSERT(mac_srs->srs_state & (SRS_CONDEMNED | SRS_QUIESCE)); 3535da14cebeSEric Cheng 3536da14cebeSEric Cheng if (mac_srs->srs_state & SRS_CONDEMNED) { 3537da14cebeSEric Cheng s_ring_flag = S_RING_CONDEMNED; 3538da14cebeSEric Cheng srs_poll_wait_flag = SRS_POLL_THR_EXITED; 3539da14cebeSEric Cheng } else { 3540da14cebeSEric Cheng s_ring_flag = S_RING_QUIESCE; 3541da14cebeSEric Cheng srs_poll_wait_flag = SRS_POLL_THR_QUIESCED; 3542da14cebeSEric Cheng } 3543da14cebeSEric Cheng 3544da14cebeSEric Cheng /* 3545da14cebeSEric Cheng * In the case of Rx SRS wait till the poll thread is done. 3546da14cebeSEric Cheng */ 3547da14cebeSEric Cheng if ((mac_srs->srs_type & SRST_TX) == 0 && 3548da14cebeSEric Cheng mac_srs->srs_poll_thr != NULL) { 3549da14cebeSEric Cheng while (!(mac_srs->srs_state & srs_poll_wait_flag)) 3550da14cebeSEric Cheng cv_wait(&mac_srs->srs_async, &mac_srs->srs_lock); 3551da14cebeSEric Cheng 3552da14cebeSEric Cheng /* 3553da14cebeSEric Cheng * Turn off polling as part of the quiesce operation. 3554da14cebeSEric Cheng */ 3555da14cebeSEric Cheng MAC_SRS_POLLING_OFF(mac_srs); 3556da14cebeSEric Cheng mac_srs->srs_state &= ~(SRS_POLLING | SRS_GET_PKTS); 3557da14cebeSEric Cheng } 3558da14cebeSEric Cheng 3559da14cebeSEric Cheng /* 3560da14cebeSEric Cheng * Then signal the soft ring worker threads to quiesce or quit 3561da14cebeSEric Cheng * as needed and then wait till that happens. 3562da14cebeSEric Cheng */ 3563da14cebeSEric Cheng mac_srs_soft_rings_quiesce(mac_srs, s_ring_flag); 3564da14cebeSEric Cheng 3565da14cebeSEric Cheng if (mac_srs->srs_state & SRS_CONDEMNED) 3566da14cebeSEric Cheng mac_srs->srs_state |= (SRS_QUIESCE_DONE | SRS_CONDEMNED_DONE); 3567da14cebeSEric Cheng else 3568da14cebeSEric Cheng mac_srs->srs_state |= SRS_QUIESCE_DONE; 3569da14cebeSEric Cheng cv_signal(&mac_srs->srs_quiesce_done_cv); 3570da14cebeSEric Cheng } 3571da14cebeSEric Cheng 3572da14cebeSEric Cheng /* 3573da14cebeSEric Cheng * Signal an SRS to start a temporary quiesce, or permanent removal, or restart 3574da14cebeSEric Cheng * a quiesced SRS by setting the appropriate flags and signaling the SRS worker 3575da14cebeSEric Cheng * or poll thread. This function is internal to the quiescing logic and is 3576da14cebeSEric Cheng * called internally from the SRS quiesce or flow quiesce or client quiesce 3577da14cebeSEric Cheng * higher level functions. 3578da14cebeSEric Cheng */ 3579da14cebeSEric Cheng void 3580da14cebeSEric Cheng mac_srs_signal(mac_soft_ring_set_t *mac_srs, uint_t srs_flag) 3581da14cebeSEric Cheng { 3582da14cebeSEric Cheng mac_ring_t *ring; 3583da14cebeSEric Cheng 3584da14cebeSEric Cheng ring = mac_srs->srs_ring; 3585da14cebeSEric Cheng ASSERT(ring == NULL || ring->mr_refcnt == 0); 3586da14cebeSEric Cheng 3587da14cebeSEric Cheng if (srs_flag == SRS_CONDEMNED) { 3588da14cebeSEric Cheng /* 3589da14cebeSEric Cheng * The SRS is going away. We need to unbind the SRS and SR 3590da14cebeSEric Cheng * threads before removing from the global SRS list. Otherwise 3591da14cebeSEric Cheng * there is a small window where the cpu reconfig callbacks 3592da14cebeSEric Cheng * may miss the SRS in the list walk and DR could fail since 3593da14cebeSEric Cheng * there are still bound threads. 3594da14cebeSEric Cheng */ 3595da14cebeSEric Cheng mac_srs_threads_unbind(mac_srs); 3596da14cebeSEric Cheng mac_srs_remove_glist(mac_srs); 3597da14cebeSEric Cheng } 3598da14cebeSEric Cheng /* 3599da14cebeSEric Cheng * Wakeup the SRS worker and poll threads. 3600da14cebeSEric Cheng */ 3601da14cebeSEric Cheng mutex_enter(&mac_srs->srs_lock); 3602da14cebeSEric Cheng mac_srs->srs_state |= srs_flag; 3603da14cebeSEric Cheng cv_signal(&mac_srs->srs_async); 3604da14cebeSEric Cheng cv_signal(&mac_srs->srs_cv); 3605da14cebeSEric Cheng mutex_exit(&mac_srs->srs_lock); 3606da14cebeSEric Cheng } 3607da14cebeSEric Cheng 3608da14cebeSEric Cheng /* 3609da14cebeSEric Cheng * In the Rx side, the quiescing is done bottom up. After the Rx upcalls 3610da14cebeSEric Cheng * from the driver are done, then the Rx SRS is quiesced and only then can 3611da14cebeSEric Cheng * we signal the soft rings. Thus this function can't be called arbitrarily 3612da14cebeSEric Cheng * without satisfying the prerequisites. On the Tx side, the threads from 3613da14cebeSEric Cheng * top need to quiesced, then the Tx SRS and only then can we signal the 3614da14cebeSEric Cheng * Tx soft rings. 3615da14cebeSEric Cheng */ 3616da14cebeSEric Cheng static void 3617da14cebeSEric Cheng mac_srs_soft_rings_signal(mac_soft_ring_set_t *mac_srs, uint_t sr_flag) 3618da14cebeSEric Cheng { 3619da14cebeSEric Cheng mac_soft_ring_t *softring; 3620da14cebeSEric Cheng 3621da14cebeSEric Cheng for (softring = mac_srs->srs_soft_ring_head; softring != NULL; 3622da14cebeSEric Cheng softring = softring->s_ring_next) 3623da14cebeSEric Cheng mac_soft_ring_signal(softring, sr_flag); 3624da14cebeSEric Cheng } 3625da14cebeSEric Cheng 3626da14cebeSEric Cheng /* 3627da14cebeSEric Cheng * The block comment above mac_rx_classify_flow_state_change explains the 3628da14cebeSEric Cheng * background. At this point the SRS is quiesced and we need to restart the 3629da14cebeSEric Cheng * SRS worker, poll, and softring threads. The SRS worker thread serves as 3630da14cebeSEric Cheng * the master controller. The steps involved are described below in the function 3631da14cebeSEric Cheng */ 3632da14cebeSEric Cheng void 3633da14cebeSEric Cheng mac_srs_worker_restart(mac_soft_ring_set_t *mac_srs) 3634da14cebeSEric Cheng { 3635da14cebeSEric Cheng boolean_t iam_rx_srs; 3636da14cebeSEric Cheng mac_soft_ring_t *softring; 3637da14cebeSEric Cheng 3638da14cebeSEric Cheng ASSERT(MUTEX_HELD(&mac_srs->srs_lock)); 3639da14cebeSEric Cheng if ((mac_srs->srs_type & SRST_TX) != 0) { 3640da14cebeSEric Cheng iam_rx_srs = B_FALSE; 3641da14cebeSEric Cheng ASSERT((mac_srs->srs_state & 3642da14cebeSEric Cheng (SRS_POLL_THR_QUIESCED | SRS_QUIESCE_DONE | SRS_QUIESCE)) == 3643da14cebeSEric Cheng (SRS_QUIESCE_DONE | SRS_QUIESCE)); 3644da14cebeSEric Cheng } else { 3645da14cebeSEric Cheng iam_rx_srs = B_TRUE; 3646da14cebeSEric Cheng ASSERT((mac_srs->srs_state & 3647da14cebeSEric Cheng (SRS_QUIESCE_DONE | SRS_QUIESCE)) == 3648da14cebeSEric Cheng (SRS_QUIESCE_DONE | SRS_QUIESCE)); 3649da14cebeSEric Cheng if (mac_srs->srs_poll_thr != NULL) { 3650da14cebeSEric Cheng ASSERT((mac_srs->srs_state & SRS_POLL_THR_QUIESCED) == 3651da14cebeSEric Cheng SRS_POLL_THR_QUIESCED); 3652da14cebeSEric Cheng } 3653da14cebeSEric Cheng } 3654da14cebeSEric Cheng 3655da14cebeSEric Cheng /* 3656da14cebeSEric Cheng * Signal any quiesced soft ring workers to restart and wait for the 3657da14cebeSEric Cheng * soft ring down count to come down to zero. 3658da14cebeSEric Cheng */ 3659da14cebeSEric Cheng if (mac_srs->srs_soft_ring_quiesced_count != 0) { 3660da14cebeSEric Cheng for (softring = mac_srs->srs_soft_ring_head; softring != NULL; 3661da14cebeSEric Cheng softring = softring->s_ring_next) { 3662da14cebeSEric Cheng if (!(softring->s_ring_state & S_RING_QUIESCE)) 3663da14cebeSEric Cheng continue; 3664da14cebeSEric Cheng mac_soft_ring_signal(softring, S_RING_RESTART); 3665da14cebeSEric Cheng } 3666da14cebeSEric Cheng while (mac_srs->srs_soft_ring_quiesced_count != 0) 3667da14cebeSEric Cheng cv_wait(&mac_srs->srs_async, &mac_srs->srs_lock); 3668da14cebeSEric Cheng } 3669da14cebeSEric Cheng 3670da14cebeSEric Cheng mac_srs->srs_state &= ~(SRS_QUIESCE_DONE | SRS_QUIESCE | SRS_RESTART); 3671da14cebeSEric Cheng if (iam_rx_srs && mac_srs->srs_poll_thr != NULL) { 3672da14cebeSEric Cheng /* 3673da14cebeSEric Cheng * Signal the poll thread and ask it to restart. Wait till it 3674da14cebeSEric Cheng * actually restarts and the SRS_POLL_THR_QUIESCED flag gets 3675da14cebeSEric Cheng * cleared. 3676da14cebeSEric Cheng */ 3677da14cebeSEric Cheng mac_srs->srs_state |= SRS_POLL_THR_RESTART; 3678da14cebeSEric Cheng cv_signal(&mac_srs->srs_cv); 3679da14cebeSEric Cheng while (mac_srs->srs_state & SRS_POLL_THR_QUIESCED) 3680da14cebeSEric Cheng cv_wait(&mac_srs->srs_async, &mac_srs->srs_lock); 3681da14cebeSEric Cheng ASSERT(!(mac_srs->srs_state & SRS_POLL_THR_RESTART)); 3682da14cebeSEric Cheng } 3683da14cebeSEric Cheng /* Wake up any waiter waiting for the restart to complete */ 3684da14cebeSEric Cheng mac_srs->srs_state |= SRS_RESTART_DONE; 3685da14cebeSEric Cheng cv_signal(&mac_srs->srs_quiesce_done_cv); 3686da14cebeSEric Cheng } 3687da14cebeSEric Cheng 3688da14cebeSEric Cheng static void 3689da14cebeSEric Cheng mac_srs_worker_unbind(mac_soft_ring_set_t *mac_srs) 3690da14cebeSEric Cheng { 3691da14cebeSEric Cheng mutex_enter(&mac_srs->srs_lock); 3692da14cebeSEric Cheng if (!(mac_srs->srs_state & SRS_WORKER_BOUND)) { 3693da14cebeSEric Cheng ASSERT(mac_srs->srs_worker_cpuid == -1); 3694da14cebeSEric Cheng mutex_exit(&mac_srs->srs_lock); 3695da14cebeSEric Cheng return; 3696da14cebeSEric Cheng } 3697da14cebeSEric Cheng 3698da14cebeSEric Cheng mac_srs->srs_worker_cpuid = -1; 3699da14cebeSEric Cheng mac_srs->srs_state &= ~SRS_WORKER_BOUND; 3700da14cebeSEric Cheng thread_affinity_clear(mac_srs->srs_worker); 3701da14cebeSEric Cheng mutex_exit(&mac_srs->srs_lock); 3702da14cebeSEric Cheng } 3703da14cebeSEric Cheng 3704da14cebeSEric Cheng static void 3705da14cebeSEric Cheng mac_srs_poll_unbind(mac_soft_ring_set_t *mac_srs) 3706da14cebeSEric Cheng { 3707da14cebeSEric Cheng mutex_enter(&mac_srs->srs_lock); 3708da14cebeSEric Cheng if (mac_srs->srs_poll_thr == NULL || 3709da14cebeSEric Cheng (mac_srs->srs_state & SRS_POLL_BOUND) == 0) { 3710da14cebeSEric Cheng ASSERT(mac_srs->srs_poll_cpuid == -1); 3711da14cebeSEric Cheng mutex_exit(&mac_srs->srs_lock); 3712da14cebeSEric Cheng return; 3713da14cebeSEric Cheng } 3714da14cebeSEric Cheng 3715da14cebeSEric Cheng mac_srs->srs_poll_cpuid = -1; 3716da14cebeSEric Cheng mac_srs->srs_state &= ~SRS_POLL_BOUND; 3717da14cebeSEric Cheng thread_affinity_clear(mac_srs->srs_poll_thr); 3718da14cebeSEric Cheng mutex_exit(&mac_srs->srs_lock); 3719da14cebeSEric Cheng } 3720da14cebeSEric Cheng 3721da14cebeSEric Cheng static void 3722da14cebeSEric Cheng mac_srs_threads_unbind(mac_soft_ring_set_t *mac_srs) 3723da14cebeSEric Cheng { 3724da14cebeSEric Cheng mac_soft_ring_t *soft_ring; 3725da14cebeSEric Cheng 3726da14cebeSEric Cheng ASSERT(MAC_PERIM_HELD((mac_handle_t)mac_srs->srs_mcip->mci_mip)); 3727da14cebeSEric Cheng 3728da14cebeSEric Cheng mutex_enter(&cpu_lock); 3729da14cebeSEric Cheng mac_srs_worker_unbind(mac_srs); 3730da14cebeSEric Cheng if (!(mac_srs->srs_type & SRST_TX)) 3731da14cebeSEric Cheng mac_srs_poll_unbind(mac_srs); 3732da14cebeSEric Cheng 3733da14cebeSEric Cheng for (soft_ring = mac_srs->srs_soft_ring_head; soft_ring != NULL; 3734da14cebeSEric Cheng soft_ring = soft_ring->s_ring_next) { 3735da14cebeSEric Cheng mac_soft_ring_unbind(soft_ring); 3736da14cebeSEric Cheng } 3737da14cebeSEric Cheng mutex_exit(&cpu_lock); 3738da14cebeSEric Cheng } 3739da14cebeSEric Cheng 3740da14cebeSEric Cheng /* 3741da14cebeSEric Cheng * When a CPU is going away, unbind all MAC threads which are bound 3742da14cebeSEric Cheng * to that CPU. The affinity of the thread to the CPU is saved to allow 3743da14cebeSEric Cheng * the thread to be rebound to the CPU if it comes back online. 3744da14cebeSEric Cheng */ 3745da14cebeSEric Cheng static void 3746da14cebeSEric Cheng mac_walk_srs_and_unbind(int cpuid) 3747da14cebeSEric Cheng { 3748da14cebeSEric Cheng mac_soft_ring_set_t *mac_srs; 3749da14cebeSEric Cheng mac_soft_ring_t *soft_ring; 3750da14cebeSEric Cheng 3751da14cebeSEric Cheng rw_enter(&mac_srs_g_lock, RW_READER); 3752da14cebeSEric Cheng 3753da14cebeSEric Cheng if ((mac_srs = mac_srs_g_list) == NULL) 3754da14cebeSEric Cheng goto done; 3755da14cebeSEric Cheng 3756da14cebeSEric Cheng for (; mac_srs != NULL; mac_srs = mac_srs->srs_next) { 3757da14cebeSEric Cheng if (mac_srs->srs_worker_cpuid == cpuid) { 3758da14cebeSEric Cheng mac_srs->srs_worker_cpuid_save = cpuid; 3759da14cebeSEric Cheng mac_srs_worker_unbind(mac_srs); 3760da14cebeSEric Cheng } 3761da14cebeSEric Cheng 3762da14cebeSEric Cheng if (!(mac_srs->srs_type & SRST_TX)) { 3763da14cebeSEric Cheng if (mac_srs->srs_poll_cpuid == cpuid) { 3764da14cebeSEric Cheng mac_srs->srs_poll_cpuid_save = cpuid; 3765da14cebeSEric Cheng mac_srs_poll_unbind(mac_srs); 3766da14cebeSEric Cheng } 3767da14cebeSEric Cheng } 3768da14cebeSEric Cheng 3769da14cebeSEric Cheng /* Next tackle the soft rings associated with the srs */ 3770da14cebeSEric Cheng mutex_enter(&mac_srs->srs_lock); 3771da14cebeSEric Cheng for (soft_ring = mac_srs->srs_soft_ring_head; soft_ring != NULL; 3772da14cebeSEric Cheng soft_ring = soft_ring->s_ring_next) { 3773da14cebeSEric Cheng if (soft_ring->s_ring_cpuid == cpuid) { 3774da14cebeSEric Cheng soft_ring->s_ring_cpuid_save = cpuid; 3775da14cebeSEric Cheng mac_soft_ring_unbind(soft_ring); 3776da14cebeSEric Cheng } 3777da14cebeSEric Cheng } 3778da14cebeSEric Cheng mutex_exit(&mac_srs->srs_lock); 3779da14cebeSEric Cheng } 3780da14cebeSEric Cheng done: 3781da14cebeSEric Cheng rw_exit(&mac_srs_g_lock); 3782da14cebeSEric Cheng } 3783da14cebeSEric Cheng 3784da14cebeSEric Cheng /* TX SETUP and TEARDOWN ROUTINES */ 3785da14cebeSEric Cheng 3786da14cebeSEric Cheng /* 3787da14cebeSEric Cheng * XXXHIO need to make sure the two mac_tx_srs_{add,del}_ring() 3788da14cebeSEric Cheng * handle the case where the number of rings is one. I.e. there is 3789da14cebeSEric Cheng * a ring pointed to by mac_srs->srs_tx_arg2. 3790da14cebeSEric Cheng */ 3791da14cebeSEric Cheng void 3792da14cebeSEric Cheng mac_tx_srs_add_ring(mac_soft_ring_set_t *mac_srs, mac_ring_t *tx_ring) 3793da14cebeSEric Cheng { 3794da14cebeSEric Cheng mac_client_impl_t *mcip = mac_srs->srs_mcip; 3795da14cebeSEric Cheng mac_soft_ring_t *soft_ring; 37960dc2366fSVenugopal Iyer int count = mac_srs->srs_tx_ring_count; 37970dc2366fSVenugopal Iyer uint32_t soft_ring_type = ST_RING_TX; 37980dc2366fSVenugopal Iyer uint_t ring_info; 3799da14cebeSEric Cheng 3800da14cebeSEric Cheng ASSERT(mac_srs->srs_state & SRS_QUIESCE); 38010dc2366fSVenugopal Iyer ring_info = mac_hwring_getinfo((mac_ring_handle_t)tx_ring); 38020dc2366fSVenugopal Iyer if (mac_tx_serialize || (ring_info & MAC_RING_TX_SERIALIZE)) 38030dc2366fSVenugopal Iyer soft_ring_type |= ST_RING_WORKER_ONLY; 38040dc2366fSVenugopal Iyer soft_ring = mac_soft_ring_create(count, 0, 38050dc2366fSVenugopal Iyer soft_ring_type, maxclsyspri, mcip, mac_srs, -1, 3806da14cebeSEric Cheng NULL, mcip, (mac_resource_handle_t)tx_ring); 38070dc2366fSVenugopal Iyer mac_srs->srs_tx_ring_count++; 38080dc2366fSVenugopal Iyer mac_srs_update_fanout_list(mac_srs); 3809da14cebeSEric Cheng /* 3810da14cebeSEric Cheng * put this soft ring in quiesce mode too so when we restart 3811da14cebeSEric Cheng * all soft rings in the srs are in the same state. 3812da14cebeSEric Cheng */ 3813da14cebeSEric Cheng mac_soft_ring_signal(soft_ring, S_RING_QUIESCE); 3814da14cebeSEric Cheng } 3815da14cebeSEric Cheng 3816da14cebeSEric Cheng static void 3817da14cebeSEric Cheng mac_soft_ring_remove(mac_soft_ring_set_t *mac_srs, mac_soft_ring_t *softring) 3818da14cebeSEric Cheng { 3819da14cebeSEric Cheng int sringcnt; 3820da14cebeSEric Cheng 3821da14cebeSEric Cheng mutex_enter(&mac_srs->srs_lock); 3822da14cebeSEric Cheng sringcnt = mac_srs->srs_soft_ring_count; 3823da14cebeSEric Cheng ASSERT(sringcnt > 0); 3824da14cebeSEric Cheng mac_soft_ring_signal(softring, S_RING_CONDEMNED); 3825da14cebeSEric Cheng 3826da14cebeSEric Cheng ASSERT(mac_srs->srs_soft_ring_condemned_count == 0); 3827da14cebeSEric Cheng while (mac_srs->srs_soft_ring_condemned_count != 1) 3828da14cebeSEric Cheng cv_wait(&mac_srs->srs_async, &mac_srs->srs_lock); 3829da14cebeSEric Cheng 3830da14cebeSEric Cheng if (softring == mac_srs->srs_soft_ring_head) { 3831da14cebeSEric Cheng mac_srs->srs_soft_ring_head = softring->s_ring_next; 3832da14cebeSEric Cheng if (mac_srs->srs_soft_ring_head != NULL) { 3833da14cebeSEric Cheng mac_srs->srs_soft_ring_head->s_ring_prev = NULL; 3834da14cebeSEric Cheng } else { 3835da14cebeSEric Cheng mac_srs->srs_soft_ring_tail = NULL; 3836da14cebeSEric Cheng } 3837da14cebeSEric Cheng } else { 3838da14cebeSEric Cheng softring->s_ring_prev->s_ring_next = 3839da14cebeSEric Cheng softring->s_ring_next; 3840da14cebeSEric Cheng if (softring->s_ring_next != NULL) { 3841da14cebeSEric Cheng softring->s_ring_next->s_ring_prev = 3842da14cebeSEric Cheng softring->s_ring_prev; 3843da14cebeSEric Cheng } else { 3844da14cebeSEric Cheng mac_srs->srs_soft_ring_tail = 3845da14cebeSEric Cheng softring->s_ring_prev; 3846da14cebeSEric Cheng } 3847da14cebeSEric Cheng } 3848da14cebeSEric Cheng mac_srs->srs_soft_ring_count--; 3849da14cebeSEric Cheng 3850da14cebeSEric Cheng mac_srs->srs_soft_ring_condemned_count--; 3851da14cebeSEric Cheng mutex_exit(&mac_srs->srs_lock); 3852da14cebeSEric Cheng 38530dc2366fSVenugopal Iyer mac_soft_ring_free(softring); 3854da14cebeSEric Cheng } 3855da14cebeSEric Cheng 3856da14cebeSEric Cheng void 3857da14cebeSEric Cheng mac_tx_srs_del_ring(mac_soft_ring_set_t *mac_srs, mac_ring_t *tx_ring) 3858da14cebeSEric Cheng { 3859da14cebeSEric Cheng int i; 3860da14cebeSEric Cheng mac_soft_ring_t *soft_ring, *remove_sring; 38610dc2366fSVenugopal Iyer mac_client_impl_t *mcip = mac_srs->srs_mcip; 3862da14cebeSEric Cheng 3863da14cebeSEric Cheng mutex_enter(&mac_srs->srs_lock); 38640dc2366fSVenugopal Iyer for (i = 0; i < mac_srs->srs_tx_ring_count; i++) { 38650dc2366fSVenugopal Iyer soft_ring = mac_srs->srs_tx_soft_rings[i]; 3866da14cebeSEric Cheng if (soft_ring->s_ring_tx_arg2 == tx_ring) 3867da14cebeSEric Cheng break; 3868da14cebeSEric Cheng } 3869da14cebeSEric Cheng mutex_exit(&mac_srs->srs_lock); 38700dc2366fSVenugopal Iyer ASSERT(i < mac_srs->srs_tx_ring_count); 3871da14cebeSEric Cheng remove_sring = soft_ring; 38720dc2366fSVenugopal Iyer /* 38730dc2366fSVenugopal Iyer * In the case of aggr, the soft ring associated with a Tx ring 38740dc2366fSVenugopal Iyer * is also stored in st_soft_rings[] array. That entry should 38750dc2366fSVenugopal Iyer * be removed. 38760dc2366fSVenugopal Iyer */ 387784de666eSRyan Zezeski if (mcip->mci_state_flags & MCIS_IS_AGGR_CLIENT) { 38780dc2366fSVenugopal Iyer mac_srs_tx_t *tx = &mac_srs->srs_tx; 38790dc2366fSVenugopal Iyer 38800dc2366fSVenugopal Iyer ASSERT(tx->st_soft_rings[tx_ring->mr_index] == remove_sring); 38810dc2366fSVenugopal Iyer tx->st_soft_rings[tx_ring->mr_index] = NULL; 38820dc2366fSVenugopal Iyer } 3883da14cebeSEric Cheng mac_soft_ring_remove(mac_srs, remove_sring); 3884da14cebeSEric Cheng mac_srs_update_fanout_list(mac_srs); 3885da14cebeSEric Cheng } 3886da14cebeSEric Cheng 3887da14cebeSEric Cheng /* 3888da14cebeSEric Cheng * mac_tx_srs_setup(): 3889da14cebeSEric Cheng * Used to setup Tx rings. If no free Tx ring is available, then default 3890da14cebeSEric Cheng * Tx ring is used. 3891da14cebeSEric Cheng */ 3892da14cebeSEric Cheng void 38930dc2366fSVenugopal Iyer mac_tx_srs_setup(mac_client_impl_t *mcip, flow_entry_t *flent) 3894da14cebeSEric Cheng { 3895da14cebeSEric Cheng mac_impl_t *mip = mcip->mci_mip; 38960dc2366fSVenugopal Iyer mac_soft_ring_set_t *tx_srs = flent->fe_tx_srs; 38970dc2366fSVenugopal Iyer int i; 38980dc2366fSVenugopal Iyer int tx_ring_count = 0; 3899da14cebeSEric Cheng uint32_t soft_ring_type; 3900da14cebeSEric Cheng mac_group_t *grp = NULL; 3901da14cebeSEric Cheng mac_ring_t *ring; 39020dc2366fSVenugopal Iyer mac_srs_tx_t *tx = &tx_srs->srs_tx; 39030dc2366fSVenugopal Iyer boolean_t is_aggr; 39040dc2366fSVenugopal Iyer uint_t ring_info = 0; 3905da14cebeSEric Cheng 390684de666eSRyan Zezeski is_aggr = (mcip->mci_state_flags & MCIS_IS_AGGR_CLIENT) != 0; 39070dc2366fSVenugopal Iyer grp = flent->fe_tx_ring_group; 39080dc2366fSVenugopal Iyer if (grp == NULL) { 39090dc2366fSVenugopal Iyer ring = (mac_ring_t *)mip->mi_default_tx_ring; 39100dc2366fSVenugopal Iyer goto no_group; 39110dc2366fSVenugopal Iyer } 3912da14cebeSEric Cheng tx_ring_count = grp->mrg_cur_count; 3913da14cebeSEric Cheng ring = grp->mrg_rings; 3914da14cebeSEric Cheng /* 3915da14cebeSEric Cheng * An attempt is made to reserve 'tx_ring_count' number 3916da14cebeSEric Cheng * of Tx rings. If tx_ring_count is 0, default Tx ring 3917da14cebeSEric Cheng * is used. If it is 1, an attempt is made to reserve one 3918da14cebeSEric Cheng * Tx ring. In both the cases, the ring information is 3919da14cebeSEric Cheng * stored in Tx SRS. If multiple Tx rings are specified, 3920da14cebeSEric Cheng * then each Tx ring will have a Tx-side soft ring. All 3921da14cebeSEric Cheng * these soft rings will be hang off Tx SRS. 3922da14cebeSEric Cheng */ 39230dc2366fSVenugopal Iyer switch (grp->mrg_state) { 39240dc2366fSVenugopal Iyer case MAC_GROUP_STATE_SHARED: 39250dc2366fSVenugopal Iyer case MAC_GROUP_STATE_RESERVED: 39260dc2366fSVenugopal Iyer if (tx_ring_count <= 1 && !is_aggr) { 39270dc2366fSVenugopal Iyer no_group: 39280dc2366fSVenugopal Iyer if (ring != NULL && 39290dc2366fSVenugopal Iyer ring->mr_state != MR_INUSE) { 39300dc2366fSVenugopal Iyer (void) mac_start_ring(ring); 39310dc2366fSVenugopal Iyer ring_info = mac_hwring_getinfo( 39320dc2366fSVenugopal Iyer (mac_ring_handle_t)ring); 39330dc2366fSVenugopal Iyer } 39340dc2366fSVenugopal Iyer tx->st_arg2 = (void *)ring; 39350dc2366fSVenugopal Iyer mac_tx_srs_stat_recreate(tx_srs, B_FALSE); 39360dc2366fSVenugopal Iyer if (tx_srs->srs_type & SRST_BW_CONTROL) { 39370dc2366fSVenugopal Iyer tx->st_mode = SRS_TX_BW; 39380dc2366fSVenugopal Iyer } else if (mac_tx_serialize || 39390dc2366fSVenugopal Iyer (ring_info & MAC_RING_TX_SERIALIZE)) { 39400dc2366fSVenugopal Iyer tx->st_mode = SRS_TX_SERIALIZE; 3941da14cebeSEric Cheng } else { 39420dc2366fSVenugopal Iyer tx->st_mode = SRS_TX_DEFAULT; 39430dc2366fSVenugopal Iyer } 3944da14cebeSEric Cheng break; 3945da14cebeSEric Cheng } 39460dc2366fSVenugopal Iyer soft_ring_type = ST_RING_TX; 3947da14cebeSEric Cheng if (tx_srs->srs_type & SRST_BW_CONTROL) { 39480dc2366fSVenugopal Iyer tx->st_mode = is_aggr ? 39490dc2366fSVenugopal Iyer SRS_TX_BW_AGGR : SRS_TX_BW_FANOUT; 3950da14cebeSEric Cheng } else { 39510dc2366fSVenugopal Iyer tx->st_mode = is_aggr ? SRS_TX_AGGR : 39520dc2366fSVenugopal Iyer SRS_TX_FANOUT; 3953da14cebeSEric Cheng } 39540dc2366fSVenugopal Iyer for (i = 0; i < tx_ring_count; i++) { 39550dc2366fSVenugopal Iyer ASSERT(ring != NULL); 39560dc2366fSVenugopal Iyer switch (ring->mr_state) { 39570dc2366fSVenugopal Iyer case MR_INUSE: 39580dc2366fSVenugopal Iyer case MR_FREE: 39590dc2366fSVenugopal Iyer ASSERT(ring->mr_srs == NULL); 39600dc2366fSVenugopal Iyer 39610dc2366fSVenugopal Iyer if (ring->mr_state != MR_INUSE) 39620dc2366fSVenugopal Iyer (void) mac_start_ring(ring); 39630dc2366fSVenugopal Iyer ring_info = mac_hwring_getinfo( 39640dc2366fSVenugopal Iyer (mac_ring_handle_t)ring); 39650dc2366fSVenugopal Iyer if (mac_tx_serialize || (ring_info & 39660dc2366fSVenugopal Iyer MAC_RING_TX_SERIALIZE)) { 39670dc2366fSVenugopal Iyer soft_ring_type |= 39680dc2366fSVenugopal Iyer ST_RING_WORKER_ONLY; 39690dc2366fSVenugopal Iyer } 39700dc2366fSVenugopal Iyer (void) mac_soft_ring_create(i, 0, 39710dc2366fSVenugopal Iyer soft_ring_type, maxclsyspri, 39720dc2366fSVenugopal Iyer mcip, tx_srs, -1, NULL, mcip, 39730dc2366fSVenugopal Iyer (mac_resource_handle_t)ring); 39740dc2366fSVenugopal Iyer break; 39750dc2366fSVenugopal Iyer default: 39760dc2366fSVenugopal Iyer cmn_err(CE_PANIC, 39770dc2366fSVenugopal Iyer "srs_setup: mcip = %p " 39780dc2366fSVenugopal Iyer "trying to add UNKNOWN ring = %p\n", 39790dc2366fSVenugopal Iyer (void *)mcip, (void *)ring); 39800dc2366fSVenugopal Iyer break; 39810dc2366fSVenugopal Iyer } 39820dc2366fSVenugopal Iyer ring = ring->mr_next; 3983da14cebeSEric Cheng } 3984da14cebeSEric Cheng mac_srs_update_fanout_list(tx_srs); 39850dc2366fSVenugopal Iyer break; 39860dc2366fSVenugopal Iyer default: 39870dc2366fSVenugopal Iyer ASSERT(B_FALSE); 39880dc2366fSVenugopal Iyer break; 3989da14cebeSEric Cheng } 3990da14cebeSEric Cheng tx->st_func = mac_tx_get_func(tx->st_mode); 39910dc2366fSVenugopal Iyer if (is_aggr) { 39920dc2366fSVenugopal Iyer VERIFY(i_mac_capab_get((mac_handle_t)mip, 39930dc2366fSVenugopal Iyer MAC_CAPAB_AGGR, &tx->st_capab_aggr)); 3994da14cebeSEric Cheng } 39950dc2366fSVenugopal Iyer DTRACE_PROBE3(tx__srs___setup__return, mac_soft_ring_set_t *, tx_srs, 39960dc2366fSVenugopal Iyer int, tx->st_mode, int, tx_srs->srs_tx_ring_count); 3997da14cebeSEric Cheng } 3998da14cebeSEric Cheng 3999da14cebeSEric Cheng /* 400008ac1c49SNicolas Droux * Update the fanout of a client if its recorded link speed doesn't match 400108ac1c49SNicolas Droux * its current link speed. 400208ac1c49SNicolas Droux */ 400308ac1c49SNicolas Droux void 40040dc2366fSVenugopal Iyer mac_fanout_recompute_client(mac_client_impl_t *mcip, cpupart_t *cpupart) 400508ac1c49SNicolas Droux { 400608ac1c49SNicolas Droux uint64_t link_speed; 400708ac1c49SNicolas Droux mac_resource_props_t *mcip_mrp; 40080dc2366fSVenugopal Iyer flow_entry_t *flent = mcip->mci_flent; 40090dc2366fSVenugopal Iyer mac_soft_ring_set_t *rx_srs; 40100dc2366fSVenugopal Iyer mac_cpus_t *srs_cpu; 40110dc2366fSVenugopal Iyer int soft_ring_count, maxcpus; 401208ac1c49SNicolas Droux 401308ac1c49SNicolas Droux ASSERT(MAC_PERIM_HELD((mac_handle_t)mcip->mci_mip)); 401408ac1c49SNicolas Droux 401508ac1c49SNicolas Droux link_speed = mac_client_stat_get(mcip->mci_flent->fe_mcip, 401608ac1c49SNicolas Droux MAC_STAT_IFSPEED); 401708ac1c49SNicolas Droux 401808ac1c49SNicolas Droux if ((link_speed != 0) && 401908ac1c49SNicolas Droux (link_speed != mcip->mci_flent->fe_nic_speed)) { 402008ac1c49SNicolas Droux mcip_mrp = MCIP_RESOURCE_PROPS(mcip); 40210dc2366fSVenugopal Iyer /* 40220dc2366fSVenugopal Iyer * Before calling mac_fanout_setup(), check to see if 40230dc2366fSVenugopal Iyer * the SRSes already have the right number of soft 40240dc2366fSVenugopal Iyer * rings. mac_fanout_setup() is a heavy duty operation 40250dc2366fSVenugopal Iyer * where new cpu bindings are done for SRS and soft 40260dc2366fSVenugopal Iyer * ring threads and interrupts re-targeted. 40270dc2366fSVenugopal Iyer */ 40280dc2366fSVenugopal Iyer maxcpus = (cpupart != NULL) ? cpupart->cp_ncpus : ncpus; 40290dc2366fSVenugopal Iyer soft_ring_count = mac_compute_soft_ring_count(flent, 40300dc2366fSVenugopal Iyer flent->fe_rx_srs_cnt - 1, maxcpus); 40310dc2366fSVenugopal Iyer /* 40320dc2366fSVenugopal Iyer * If soft_ring_count returned by 40330dc2366fSVenugopal Iyer * mac_compute_soft_ring_count() is 0, bump it 40340dc2366fSVenugopal Iyer * up by 1 because we always have atleast one 40350dc2366fSVenugopal Iyer * TCP, UDP, and OTH soft ring associated with 40360dc2366fSVenugopal Iyer * an SRS. 40370dc2366fSVenugopal Iyer */ 40380dc2366fSVenugopal Iyer soft_ring_count = (soft_ring_count == 0) ? 40390dc2366fSVenugopal Iyer 1 : soft_ring_count; 40400dc2366fSVenugopal Iyer rx_srs = flent->fe_rx_srs[0]; 40410dc2366fSVenugopal Iyer srs_cpu = &rx_srs->srs_cpu; 40420dc2366fSVenugopal Iyer if (soft_ring_count != srs_cpu->mc_rx_fanout_cnt) { 40430dc2366fSVenugopal Iyer mac_fanout_setup(mcip, flent, mcip_mrp, 40440dc2366fSVenugopal Iyer mac_rx_deliver, mcip, NULL, cpupart); 40450dc2366fSVenugopal Iyer } 404608ac1c49SNicolas Droux } 404708ac1c49SNicolas Droux } 404808ac1c49SNicolas Droux 404908ac1c49SNicolas Droux /* 405045948e49SRyan Zezeski * Walk through the list of MAC clients for the MAC. 405145948e49SRyan Zezeski * For each active MAC client, recompute the number of soft rings 4052da14cebeSEric Cheng * associated with every client, only if current speed is different 4053da14cebeSEric Cheng * from the speed that was previously used for soft ring computation. 4054da14cebeSEric Cheng * If the cable is disconnected whlie the NIC is started, we would get 4055da14cebeSEric Cheng * notification with speed set to 0. We do not recompute in that case. 4056da14cebeSEric Cheng */ 4057da14cebeSEric Cheng void 4058da14cebeSEric Cheng mac_fanout_recompute(mac_impl_t *mip) 4059da14cebeSEric Cheng { 4060da14cebeSEric Cheng mac_client_impl_t *mcip; 40610dc2366fSVenugopal Iyer cpupart_t *cpupart; 40620dc2366fSVenugopal Iyer boolean_t use_default; 40630dc2366fSVenugopal Iyer mac_resource_props_t *mrp, *emrp; 406408ac1c49SNicolas Droux 4065da14cebeSEric Cheng i_mac_perim_enter(mip); 40662c4ec682SEric Cheng if ((mip->mi_state_flags & MIS_IS_VNIC) != 0 || 40672c4ec682SEric Cheng mip->mi_linkstate != LINK_STATE_UP) { 4068da14cebeSEric Cheng i_mac_perim_exit(mip); 4069da14cebeSEric Cheng return; 4070da14cebeSEric Cheng } 4071da14cebeSEric Cheng 4072da14cebeSEric Cheng for (mcip = mip->mi_clients_list; mcip != NULL; 4073da14cebeSEric Cheng mcip = mcip->mci_client_next) { 407445948e49SRyan Zezeski /* Aggr port clients don't have SRSes. */ 407545948e49SRyan Zezeski if ((mcip->mci_state_flags & MCIS_IS_AGGR_PORT) != 0) 407645948e49SRyan Zezeski continue; 407745948e49SRyan Zezeski 407808ac1c49SNicolas Droux if ((mcip->mci_state_flags & MCIS_SHARE_BOUND) != 0 || 407908ac1c49SNicolas Droux !MCIP_DATAPATH_SETUP(mcip)) 4080da14cebeSEric Cheng continue; 40810dc2366fSVenugopal Iyer mrp = MCIP_RESOURCE_PROPS(mcip); 40820dc2366fSVenugopal Iyer emrp = MCIP_EFFECTIVE_PROPS(mcip); 40830dc2366fSVenugopal Iyer use_default = B_FALSE; 40840dc2366fSVenugopal Iyer pool_lock(); 40850dc2366fSVenugopal Iyer cpupart = mac_pset_find(mrp, &use_default); 40860dc2366fSVenugopal Iyer mac_fanout_recompute_client(mcip, cpupart); 40870dc2366fSVenugopal Iyer mac_set_pool_effective(use_default, cpupart, mrp, emrp); 40880dc2366fSVenugopal Iyer pool_unlock(); 4089da14cebeSEric Cheng } 409045948e49SRyan Zezeski 4091da14cebeSEric Cheng i_mac_perim_exit(mip); 4092da14cebeSEric Cheng } 40934eaa4710SRishi Srivatsavai 40944eaa4710SRishi Srivatsavai /* 40954eaa4710SRishi Srivatsavai * Given a MAC, change the polling state for all its MAC clients. 'enable' is 40964eaa4710SRishi Srivatsavai * B_TRUE to enable polling or B_FALSE to disable. Polling is enabled by 40974eaa4710SRishi Srivatsavai * default. 40984eaa4710SRishi Srivatsavai */ 40994eaa4710SRishi Srivatsavai void 41004eaa4710SRishi Srivatsavai mac_poll_state_change(mac_handle_t mh, boolean_t enable) 41014eaa4710SRishi Srivatsavai { 41024eaa4710SRishi Srivatsavai mac_impl_t *mip = (mac_impl_t *)mh; 41034eaa4710SRishi Srivatsavai mac_client_impl_t *mcip; 41044eaa4710SRishi Srivatsavai 41054eaa4710SRishi Srivatsavai i_mac_perim_enter(mip); 41064eaa4710SRishi Srivatsavai if (enable) 41074eaa4710SRishi Srivatsavai mip->mi_state_flags &= ~MIS_POLL_DISABLE; 41084eaa4710SRishi Srivatsavai else 41094eaa4710SRishi Srivatsavai mip->mi_state_flags |= MIS_POLL_DISABLE; 41104eaa4710SRishi Srivatsavai for (mcip = mip->mi_clients_list; mcip != NULL; 41114eaa4710SRishi Srivatsavai mcip = mcip->mci_client_next) 41124eaa4710SRishi Srivatsavai mac_client_update_classifier(mcip, B_TRUE); 41134eaa4710SRishi Srivatsavai i_mac_perim_exit(mip); 41144eaa4710SRishi Srivatsavai } 4115