1da14cebeSEric Cheng /* 2da14cebeSEric Cheng * CDDL HEADER START 3da14cebeSEric Cheng * 4da14cebeSEric Cheng * The contents of this file are subject to the terms of the 5da14cebeSEric Cheng * Common Development and Distribution License (the "License"). 6da14cebeSEric Cheng * You may not use this file except in compliance with the License. 7da14cebeSEric Cheng * 8da14cebeSEric Cheng * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE 9da14cebeSEric Cheng * or http://www.opensolaris.org/os/licensing. 10da14cebeSEric Cheng * See the License for the specific language governing permissions 11da14cebeSEric Cheng * and limitations under the License. 12da14cebeSEric Cheng * 13da14cebeSEric Cheng * When distributing Covered Code, include this CDDL HEADER in each 14da14cebeSEric Cheng * file and include the License file at usr/src/OPENSOLARIS.LICENSE. 15da14cebeSEric Cheng * If applicable, add the following below this CDDL HEADER, with the 16da14cebeSEric Cheng * fields enclosed by brackets "[]" replaced with your own identifying 17da14cebeSEric Cheng * information: Portions Copyright [yyyy] [name of copyright owner] 18da14cebeSEric Cheng * 19da14cebeSEric Cheng * CDDL HEADER END 20da14cebeSEric Cheng */ 21da14cebeSEric Cheng /* 225adf34bdSRajagopal Kunhappan * Copyright (c) 2008, 2010, Oracle and/or its affiliates. All rights reserved. 23da906b4eSRyan Zezeski * Copyright 2017, Joyent, Inc. 24da14cebeSEric Cheng */ 25da14cebeSEric Cheng 26da14cebeSEric Cheng #include <sys/types.h> 27da14cebeSEric Cheng #include <sys/callb.h> 280dc2366fSVenugopal Iyer #include <sys/cpupart.h> 290dc2366fSVenugopal Iyer #include <sys/pool.h> 300dc2366fSVenugopal Iyer #include <sys/pool_pset.h> 31da14cebeSEric Cheng #include <sys/sdt.h> 32da14cebeSEric Cheng #include <sys/strsubr.h> 33da14cebeSEric Cheng #include <sys/strsun.h> 34da14cebeSEric Cheng #include <sys/vlan.h> 35da14cebeSEric Cheng #include <inet/ipsec_impl.h> 36da14cebeSEric Cheng #include <inet/ip_impl.h> 37da14cebeSEric Cheng #include <inet/sadb.h> 38da14cebeSEric Cheng #include <inet/ipsecesp.h> 39da14cebeSEric Cheng #include <inet/ipsecah.h> 40da14cebeSEric Cheng 41da14cebeSEric Cheng #include <sys/mac_impl.h> 42da14cebeSEric Cheng #include <sys/mac_client_impl.h> 43da14cebeSEric Cheng #include <sys/mac_client_priv.h> 44da14cebeSEric Cheng #include <sys/mac_soft_ring.h> 45da14cebeSEric Cheng #include <sys/mac_flow_impl.h> 460dc2366fSVenugopal Iyer #include <sys/mac_stat.h> 47da14cebeSEric Cheng 48da14cebeSEric Cheng static void mac_srs_soft_rings_signal(mac_soft_ring_set_t *, uint_t); 49da14cebeSEric Cheng static void mac_srs_update_fanout_list(mac_soft_ring_set_t *); 50da14cebeSEric Cheng static void mac_srs_poll_unbind(mac_soft_ring_set_t *); 51da14cebeSEric Cheng static void mac_srs_worker_unbind(mac_soft_ring_set_t *); 52da14cebeSEric Cheng static void mac_srs_soft_rings_quiesce(mac_soft_ring_set_t *, uint_t); 53da14cebeSEric Cheng 54da14cebeSEric Cheng static int mac_srs_cpu_setup(cpu_setup_t, int, void *); 55da14cebeSEric Cheng static void mac_srs_worker_bind(mac_soft_ring_set_t *, processorid_t); 56da14cebeSEric Cheng static void mac_srs_poll_bind(mac_soft_ring_set_t *, processorid_t); 57da14cebeSEric Cheng static void mac_srs_threads_unbind(mac_soft_ring_set_t *); 58da14cebeSEric Cheng static void mac_srs_add_glist(mac_soft_ring_set_t *); 59da14cebeSEric Cheng static void mac_srs_remove_glist(mac_soft_ring_set_t *); 60da14cebeSEric Cheng static void mac_srs_fanout_list_free(mac_soft_ring_set_t *); 61da14cebeSEric Cheng static void mac_soft_ring_remove(mac_soft_ring_set_t *, mac_soft_ring_t *); 62da14cebeSEric Cheng 630dc2366fSVenugopal Iyer static int mac_compute_soft_ring_count(flow_entry_t *, int, int); 64da14cebeSEric Cheng static void mac_walk_srs_and_bind(int); 65da14cebeSEric Cheng static void mac_walk_srs_and_unbind(int); 66da14cebeSEric Cheng 67da14cebeSEric Cheng extern boolean_t mac_latency_optimize; 68da14cebeSEric Cheng 69da14cebeSEric Cheng static kmem_cache_t *mac_srs_cache; 70da14cebeSEric Cheng kmem_cache_t *mac_soft_ring_cache; 71da14cebeSEric Cheng 72da14cebeSEric Cheng /* 73da14cebeSEric Cheng * The duration in msec we wait before signalling the soft ring 74da14cebeSEric Cheng * worker thread in case packets get queued. 75da14cebeSEric Cheng */ 76ae6aa22aSVenugopal Iyer uint32_t mac_soft_ring_worker_wait = 0; 77ae6aa22aSVenugopal Iyer 78ae6aa22aSVenugopal Iyer /* 79ae6aa22aSVenugopal Iyer * A global tunable for turning polling on/off. By default, dynamic 80ae6aa22aSVenugopal Iyer * polling is always on and is always very beneficial. It should be 81ae6aa22aSVenugopal Iyer * turned off with absolute care and for the rare workload (very 82ae6aa22aSVenugopal Iyer * low latency sensitive traffic). 83ae6aa22aSVenugopal Iyer */ 84ae6aa22aSVenugopal Iyer int mac_poll_enable = B_TRUE; 85da14cebeSEric Cheng 86da14cebeSEric Cheng /* 87da14cebeSEric Cheng * Need to set mac_soft_ring_max_q_cnt based on bandwidth and perhaps latency. 88da14cebeSEric Cheng * Large values could end up in consuming lot of system memory and cause 89da14cebeSEric Cheng * system hang. 90da14cebeSEric Cheng */ 91ae6aa22aSVenugopal Iyer int mac_soft_ring_max_q_cnt = 1024; 92ae6aa22aSVenugopal Iyer int mac_soft_ring_min_q_cnt = 256; 93ae6aa22aSVenugopal Iyer int mac_soft_ring_poll_thres = 16; 94da14cebeSEric Cheng 95ae6aa22aSVenugopal Iyer boolean_t mac_tx_serialize = B_FALSE; 96da14cebeSEric Cheng 97da14cebeSEric Cheng /* 98da14cebeSEric Cheng * mac_tx_srs_hiwat is the queue depth threshold at which callers of 99da14cebeSEric Cheng * mac_tx() will be notified of flow control condition. 100da14cebeSEric Cheng * 101da14cebeSEric Cheng * TCP does not honour flow control condition sent up by mac_tx(). 102da14cebeSEric Cheng * Thus provision is made for TCP to allow more packets to be queued 103da14cebeSEric Cheng * in SRS upto a maximum of mac_tx_srs_max_q_cnt. 104da14cebeSEric Cheng * 105da14cebeSEric Cheng * Note that mac_tx_srs_hiwat is always be lesser than 106da14cebeSEric Cheng * mac_tx_srs_max_q_cnt. 107da14cebeSEric Cheng */ 108ae6aa22aSVenugopal Iyer uint32_t mac_tx_srs_max_q_cnt = 100000; 109ae6aa22aSVenugopal Iyer uint32_t mac_tx_srs_hiwat = 1000; 110da14cebeSEric Cheng 111da14cebeSEric Cheng /* 112da14cebeSEric Cheng * mac_rx_soft_ring_count, mac_soft_ring_10gig_count: 113da14cebeSEric Cheng * 114da14cebeSEric Cheng * Global tunables that determines the number of soft rings to be used for 115da14cebeSEric Cheng * fanning out incoming traffic on a link. These count will be used only 116da14cebeSEric Cheng * when no explicit set of CPUs was assigned to the data-links. 117da14cebeSEric Cheng * 118da14cebeSEric Cheng * mac_rx_soft_ring_count tunable will come into effect only if 119da14cebeSEric Cheng * mac_soft_ring_enable is set. mac_soft_ring_enable is turned on by 120da14cebeSEric Cheng * default only for sun4v platforms. 121da14cebeSEric Cheng * 122da14cebeSEric Cheng * mac_rx_soft_ring_10gig_count will come into effect if you are running on a 123da14cebeSEric Cheng * 10Gbps link and is not dependent upon mac_soft_ring_enable. 124da14cebeSEric Cheng * 125da14cebeSEric Cheng * The number of soft rings for fanout for a link or a flow is determined 126da14cebeSEric Cheng * by mac_compute_soft_ring_count() routine. This routine will take into 127da14cebeSEric Cheng * account mac_soft_ring_enable, mac_rx_soft_ring_count and 128da14cebeSEric Cheng * mac_rx_soft_ring_10gig_count to determine the soft ring count for a link. 129da14cebeSEric Cheng * 130da14cebeSEric Cheng * If a bandwidth is specified, the determination of the number of soft 131da14cebeSEric Cheng * rings is based on specified bandwidth, CPU speed and number of CPUs in 132da14cebeSEric Cheng * the system. 133da14cebeSEric Cheng */ 134ae6aa22aSVenugopal Iyer uint_t mac_rx_soft_ring_count = 8; 135ae6aa22aSVenugopal Iyer uint_t mac_rx_soft_ring_10gig_count = 8; 136da14cebeSEric Cheng 137da14cebeSEric Cheng /* 138da14cebeSEric Cheng * Every Tx and Rx mac_soft_ring_set_t (mac_srs) created gets added 139da14cebeSEric Cheng * to mac_srs_g_list and mac_srs_g_lock protects mac_srs_g_list. The 140da14cebeSEric Cheng * list is used to walk the list of all MAC threads when a CPU is 141da14cebeSEric Cheng * coming online or going offline. 142da14cebeSEric Cheng */ 143da14cebeSEric Cheng static mac_soft_ring_set_t *mac_srs_g_list = NULL; 144da14cebeSEric Cheng static krwlock_t mac_srs_g_lock; 145da14cebeSEric Cheng 146da14cebeSEric Cheng /* 147da14cebeSEric Cheng * Whether the SRS threads should be bound, or not. 148da14cebeSEric Cheng */ 149ae6aa22aSVenugopal Iyer boolean_t mac_srs_thread_bind = B_TRUE; 150da14cebeSEric Cheng 151da14cebeSEric Cheng /* 1520dc2366fSVenugopal Iyer * Whether Rx/Tx interrupts should be re-targeted. Disabled by default. 1530dc2366fSVenugopal Iyer * dladm command would override this. 154da14cebeSEric Cheng */ 1550dc2366fSVenugopal Iyer boolean_t mac_tx_intr_retarget = B_FALSE; 1560dc2366fSVenugopal Iyer boolean_t mac_rx_intr_retarget = B_FALSE; 157da14cebeSEric Cheng 158da14cebeSEric Cheng /* 159da14cebeSEric Cheng * If cpu bindings are specified by user, then Tx SRS and its soft 160da14cebeSEric Cheng * rings should also be bound to the CPUs specified by user. The 161da14cebeSEric Cheng * CPUs for Tx bindings are at the end of the cpu list provided by 162da14cebeSEric Cheng * the user. If enough CPUs are not available (for Tx and Rx 163da14cebeSEric Cheng * SRSes), then the CPUs are shared by both Tx and Rx SRSes. 164da14cebeSEric Cheng */ 165da14cebeSEric Cheng #define BIND_TX_SRS_AND_SOFT_RINGS(mac_tx_srs, mrp) { \ 166da14cebeSEric Cheng processorid_t cpuid; \ 1670dc2366fSVenugopal Iyer int i; \ 168da14cebeSEric Cheng mac_soft_ring_t *softring; \ 1690dc2366fSVenugopal Iyer mac_cpus_t *srs_cpu; \ 170da14cebeSEric Cheng \ 1710dc2366fSVenugopal Iyer srs_cpu = &mac_tx_srs->srs_cpu; \ 1720dc2366fSVenugopal Iyer cpuid = srs_cpu->mc_tx_fanout_cpus[0]; \ 173da14cebeSEric Cheng mac_srs_worker_bind(mac_tx_srs, cpuid); \ 1740dc2366fSVenugopal Iyer if (MAC_TX_SOFT_RINGS(mac_tx_srs)) { \ 1750dc2366fSVenugopal Iyer for (i = 0; i < mac_tx_srs->srs_tx_ring_count; i++) { \ 1760dc2366fSVenugopal Iyer cpuid = srs_cpu->mc_tx_fanout_cpus[i]; \ 1770dc2366fSVenugopal Iyer softring = mac_tx_srs->srs_tx_soft_rings[i]; \ 1780dc2366fSVenugopal Iyer if (cpuid != -1) { \ 1790dc2366fSVenugopal Iyer (void) mac_soft_ring_bind(softring, \ 1800dc2366fSVenugopal Iyer cpuid); \ 1810dc2366fSVenugopal Iyer } \ 182da14cebeSEric Cheng } \ 183da14cebeSEric Cheng } \ 184da14cebeSEric Cheng } 185da14cebeSEric Cheng 1860dc2366fSVenugopal Iyer /* 1870dc2366fSVenugopal Iyer * Re-targeting is allowed only for exclusive group or for primary. 1880dc2366fSVenugopal Iyer */ 1890dc2366fSVenugopal Iyer #define RETARGETABLE_CLIENT(group, mcip) \ 1900dc2366fSVenugopal Iyer ((((group) != NULL) && \ 1910dc2366fSVenugopal Iyer ((group)->mrg_state == MAC_GROUP_STATE_RESERVED)) || \ 1920dc2366fSVenugopal Iyer mac_is_primary_client(mcip)) 1930dc2366fSVenugopal Iyer 1940dc2366fSVenugopal Iyer #define MAC_RING_RETARGETABLE(ring) \ 1950dc2366fSVenugopal Iyer (((ring) != NULL) && \ 1960dc2366fSVenugopal Iyer ((ring)->mr_info.mri_intr.mi_ddi_handle != NULL) && \ 1970dc2366fSVenugopal Iyer !((ring)->mr_info.mri_intr.mi_ddi_shared)) 1980dc2366fSVenugopal Iyer 1990dc2366fSVenugopal Iyer 200da14cebeSEric Cheng /* INIT and FINI ROUTINES */ 201da14cebeSEric Cheng 202da14cebeSEric Cheng void 203da14cebeSEric Cheng mac_soft_ring_init(void) 204da14cebeSEric Cheng { 205da14cebeSEric Cheng mac_soft_ring_cache = kmem_cache_create("mac_soft_ring_cache", 206da14cebeSEric Cheng sizeof (mac_soft_ring_t), 64, NULL, NULL, NULL, NULL, NULL, 0); 207da14cebeSEric Cheng 208da14cebeSEric Cheng mac_srs_cache = kmem_cache_create("mac_srs_cache", 209da14cebeSEric Cheng sizeof (mac_soft_ring_set_t), 210da14cebeSEric Cheng 64, NULL, NULL, NULL, NULL, NULL, 0); 211da14cebeSEric Cheng 212da14cebeSEric Cheng rw_init(&mac_srs_g_lock, NULL, RW_DEFAULT, NULL); 213da14cebeSEric Cheng mutex_enter(&cpu_lock); 214da14cebeSEric Cheng register_cpu_setup_func(mac_srs_cpu_setup, NULL); 215da14cebeSEric Cheng mutex_exit(&cpu_lock); 216da14cebeSEric Cheng } 217da14cebeSEric Cheng 218da14cebeSEric Cheng void 219da14cebeSEric Cheng mac_soft_ring_finish(void) 220da14cebeSEric Cheng { 221da14cebeSEric Cheng mutex_enter(&cpu_lock); 222da14cebeSEric Cheng unregister_cpu_setup_func(mac_srs_cpu_setup, NULL); 223da14cebeSEric Cheng mutex_exit(&cpu_lock); 224da14cebeSEric Cheng rw_destroy(&mac_srs_g_lock); 225da14cebeSEric Cheng kmem_cache_destroy(mac_soft_ring_cache); 226da14cebeSEric Cheng kmem_cache_destroy(mac_srs_cache); 227da14cebeSEric Cheng } 228da14cebeSEric Cheng 229da14cebeSEric Cheng static void 2300dc2366fSVenugopal Iyer mac_srs_soft_rings_free(mac_soft_ring_set_t *mac_srs) 231da14cebeSEric Cheng { 232da14cebeSEric Cheng mac_soft_ring_t *softring, *next, *head; 233da14cebeSEric Cheng 234da14cebeSEric Cheng /* 235da14cebeSEric Cheng * Synchronize with mac_walk_srs_bind/unbind which are callbacks from 236da14cebeSEric Cheng * DR. The callbacks from DR are called with cpu_lock held, and hence 237da14cebeSEric Cheng * can't wait to grab the mac perimeter. The soft ring list is hence 238da14cebeSEric Cheng * protected for read access by srs_lock. Changing the soft ring list 239da14cebeSEric Cheng * needs the mac perimeter and the srs_lock. 240da14cebeSEric Cheng */ 241da14cebeSEric Cheng mutex_enter(&mac_srs->srs_lock); 242da14cebeSEric Cheng 243da14cebeSEric Cheng head = mac_srs->srs_soft_ring_head; 244da14cebeSEric Cheng mac_srs->srs_soft_ring_head = NULL; 245da14cebeSEric Cheng mac_srs->srs_soft_ring_tail = NULL; 246da14cebeSEric Cheng mac_srs->srs_soft_ring_count = 0; 247da14cebeSEric Cheng 248da14cebeSEric Cheng mutex_exit(&mac_srs->srs_lock); 249da14cebeSEric Cheng 250da14cebeSEric Cheng for (softring = head; softring != NULL; softring = next) { 251da14cebeSEric Cheng next = softring->s_ring_next; 2520dc2366fSVenugopal Iyer mac_soft_ring_free(softring); 253da14cebeSEric Cheng } 254da14cebeSEric Cheng } 255da14cebeSEric Cheng 256da14cebeSEric Cheng static void 257da14cebeSEric Cheng mac_srs_add_glist(mac_soft_ring_set_t *mac_srs) 258da14cebeSEric Cheng { 259da14cebeSEric Cheng ASSERT(mac_srs->srs_next == NULL && mac_srs->srs_prev == NULL); 260da14cebeSEric Cheng ASSERT(MAC_PERIM_HELD((mac_handle_t)mac_srs->srs_mcip->mci_mip)); 261da14cebeSEric Cheng 262da14cebeSEric Cheng rw_enter(&mac_srs_g_lock, RW_WRITER); 263da14cebeSEric Cheng mutex_enter(&mac_srs->srs_lock); 264da14cebeSEric Cheng 265da14cebeSEric Cheng ASSERT((mac_srs->srs_state & SRS_IN_GLIST) == 0); 266da14cebeSEric Cheng 267da14cebeSEric Cheng if (mac_srs_g_list == NULL) { 268da14cebeSEric Cheng mac_srs_g_list = mac_srs; 269da14cebeSEric Cheng } else { 270da14cebeSEric Cheng mac_srs->srs_next = mac_srs_g_list; 271da14cebeSEric Cheng mac_srs_g_list->srs_prev = mac_srs; 272da14cebeSEric Cheng mac_srs->srs_prev = NULL; 273da14cebeSEric Cheng mac_srs_g_list = mac_srs; 274da14cebeSEric Cheng } 275da14cebeSEric Cheng mac_srs->srs_state |= SRS_IN_GLIST; 276da14cebeSEric Cheng 277da14cebeSEric Cheng mutex_exit(&mac_srs->srs_lock); 278da14cebeSEric Cheng rw_exit(&mac_srs_g_lock); 279da14cebeSEric Cheng } 280da14cebeSEric Cheng 281da14cebeSEric Cheng static void 282da14cebeSEric Cheng mac_srs_remove_glist(mac_soft_ring_set_t *mac_srs) 283da14cebeSEric Cheng { 284da14cebeSEric Cheng ASSERT(MAC_PERIM_HELD((mac_handle_t)mac_srs->srs_mcip->mci_mip)); 285da14cebeSEric Cheng 286da14cebeSEric Cheng rw_enter(&mac_srs_g_lock, RW_WRITER); 287da14cebeSEric Cheng mutex_enter(&mac_srs->srs_lock); 288da14cebeSEric Cheng 289da14cebeSEric Cheng ASSERT((mac_srs->srs_state & SRS_IN_GLIST) != 0); 290da14cebeSEric Cheng 291da14cebeSEric Cheng if (mac_srs == mac_srs_g_list) { 292da14cebeSEric Cheng mac_srs_g_list = mac_srs->srs_next; 293da14cebeSEric Cheng if (mac_srs_g_list != NULL) 294da14cebeSEric Cheng mac_srs_g_list->srs_prev = NULL; 295da14cebeSEric Cheng } else { 296da14cebeSEric Cheng mac_srs->srs_prev->srs_next = mac_srs->srs_next; 297da14cebeSEric Cheng if (mac_srs->srs_next != NULL) 298da14cebeSEric Cheng mac_srs->srs_next->srs_prev = mac_srs->srs_prev; 299da14cebeSEric Cheng } 300da14cebeSEric Cheng mac_srs->srs_state &= ~SRS_IN_GLIST; 301da14cebeSEric Cheng 302da14cebeSEric Cheng mutex_exit(&mac_srs->srs_lock); 303da14cebeSEric Cheng rw_exit(&mac_srs_g_lock); 304da14cebeSEric Cheng } 305da14cebeSEric Cheng 306da14cebeSEric Cheng /* POLLING SETUP AND TEAR DOWN ROUTINES */ 307da14cebeSEric Cheng 308da14cebeSEric Cheng /* 309da14cebeSEric Cheng * mac_srs_client_poll_quiesce and mac_srs_client_poll_restart 310da14cebeSEric Cheng * 311da14cebeSEric Cheng * These routines are used to call back into the upper layer 312da14cebeSEric Cheng * (primarily TCP squeue) to stop polling the soft rings or 313da14cebeSEric Cheng * restart polling. 314da14cebeSEric Cheng */ 315da14cebeSEric Cheng void 316da14cebeSEric Cheng mac_srs_client_poll_quiesce(mac_client_impl_t *mcip, 317da14cebeSEric Cheng mac_soft_ring_set_t *mac_srs) 318da14cebeSEric Cheng { 319da14cebeSEric Cheng mac_soft_ring_t *softring; 320da14cebeSEric Cheng 321da14cebeSEric Cheng ASSERT(MAC_PERIM_HELD((mac_handle_t)mcip->mci_mip)); 322da14cebeSEric Cheng 323da14cebeSEric Cheng if (!(mac_srs->srs_type & SRST_CLIENT_POLL_ENABLED)) { 324da14cebeSEric Cheng ASSERT(!(mac_srs->srs_type & SRST_DLS_BYPASS)); 325da14cebeSEric Cheng return; 326da14cebeSEric Cheng } 327da14cebeSEric Cheng 328da14cebeSEric Cheng for (softring = mac_srs->srs_soft_ring_head; 329da14cebeSEric Cheng softring != NULL; softring = softring->s_ring_next) { 330da14cebeSEric Cheng if ((softring->s_ring_type & ST_RING_TCP) && 331da14cebeSEric Cheng (softring->s_ring_rx_arg2 != NULL)) { 332da14cebeSEric Cheng mcip->mci_resource_quiesce(mcip->mci_resource_arg, 333da14cebeSEric Cheng softring->s_ring_rx_arg2); 334da14cebeSEric Cheng } 335da14cebeSEric Cheng } 336da14cebeSEric Cheng } 337da14cebeSEric Cheng 338da14cebeSEric Cheng void 339da14cebeSEric Cheng mac_srs_client_poll_restart(mac_client_impl_t *mcip, 340da14cebeSEric Cheng mac_soft_ring_set_t *mac_srs) 341da14cebeSEric Cheng { 342da14cebeSEric Cheng mac_soft_ring_t *softring; 343da14cebeSEric Cheng 344da14cebeSEric Cheng ASSERT(MAC_PERIM_HELD((mac_handle_t)mcip->mci_mip)); 345da14cebeSEric Cheng 346da14cebeSEric Cheng if (!(mac_srs->srs_type & SRST_CLIENT_POLL_ENABLED)) { 347da14cebeSEric Cheng ASSERT(!(mac_srs->srs_type & SRST_DLS_BYPASS)); 348da14cebeSEric Cheng return; 349da14cebeSEric Cheng } 350da14cebeSEric Cheng 351da14cebeSEric Cheng for (softring = mac_srs->srs_soft_ring_head; 352da14cebeSEric Cheng softring != NULL; softring = softring->s_ring_next) { 353da14cebeSEric Cheng if ((softring->s_ring_type & ST_RING_TCP) && 354da14cebeSEric Cheng (softring->s_ring_rx_arg2 != NULL)) { 355da14cebeSEric Cheng mcip->mci_resource_restart(mcip->mci_resource_arg, 356da14cebeSEric Cheng softring->s_ring_rx_arg2); 357da14cebeSEric Cheng } 358da14cebeSEric Cheng } 359da14cebeSEric Cheng } 360da14cebeSEric Cheng 361da14cebeSEric Cheng /* 362da14cebeSEric Cheng * Register the given SRS and associated soft rings with the consumer and 363da14cebeSEric Cheng * enable the polling interface used by the consumer.(i.e IP) over this 364da14cebeSEric Cheng * SRS and associated soft rings. 365da14cebeSEric Cheng */ 366da14cebeSEric Cheng void 367da14cebeSEric Cheng mac_srs_client_poll_enable(mac_client_impl_t *mcip, 368da14cebeSEric Cheng mac_soft_ring_set_t *mac_srs) 369da14cebeSEric Cheng { 370da14cebeSEric Cheng mac_rx_fifo_t mrf; 371da14cebeSEric Cheng mac_soft_ring_t *softring; 372da14cebeSEric Cheng 373da14cebeSEric Cheng ASSERT(mac_srs->srs_mcip == mcip); 374da14cebeSEric Cheng ASSERT(MAC_PERIM_HELD((mac_handle_t)mcip->mci_mip)); 375da14cebeSEric Cheng 376da14cebeSEric Cheng if (!(mcip->mci_state_flags & MCIS_CLIENT_POLL_CAPABLE)) 377da14cebeSEric Cheng return; 378da14cebeSEric Cheng 379da14cebeSEric Cheng bzero(&mrf, sizeof (mac_rx_fifo_t)); 380da14cebeSEric Cheng mrf.mrf_type = MAC_RX_FIFO; 381da14cebeSEric Cheng 382da14cebeSEric Cheng /* 383da14cebeSEric Cheng * A SRS is capable of acting as a soft ring for cases 384da14cebeSEric Cheng * where no fanout is needed. This is the case for userland 385da14cebeSEric Cheng * flows. 386da14cebeSEric Cheng */ 387da14cebeSEric Cheng if (mac_srs->srs_type & SRST_NO_SOFT_RINGS) 388da14cebeSEric Cheng return; 389da14cebeSEric Cheng 390da14cebeSEric Cheng mrf.mrf_receive = (mac_receive_t)mac_soft_ring_poll; 391da14cebeSEric Cheng mrf.mrf_intr_enable = (mac_intr_enable_t)mac_soft_ring_intr_enable; 392da14cebeSEric Cheng mrf.mrf_intr_disable = (mac_intr_disable_t)mac_soft_ring_intr_disable; 393da14cebeSEric Cheng mac_srs->srs_type |= SRST_CLIENT_POLL_ENABLED; 394da14cebeSEric Cheng 395da14cebeSEric Cheng softring = mac_srs->srs_soft_ring_head; 396da14cebeSEric Cheng while (softring != NULL) { 397da14cebeSEric Cheng if (softring->s_ring_type & (ST_RING_TCP | ST_RING_UDP)) { 398da14cebeSEric Cheng /* 399da14cebeSEric Cheng * TCP and UDP support DLS bypass. Squeue polling 400da14cebeSEric Cheng * support implies DLS bypass since the squeue poll 401da14cebeSEric Cheng * path does not have DLS processing. 402da14cebeSEric Cheng */ 403da14cebeSEric Cheng mac_soft_ring_dls_bypass(softring, 404da14cebeSEric Cheng mcip->mci_direct_rx_fn, mcip->mci_direct_rx_arg); 405da14cebeSEric Cheng } 406da14cebeSEric Cheng /* 407da14cebeSEric Cheng * Non-TCP protocols don't support squeues. Hence we don't 408da14cebeSEric Cheng * make any ring addition callbacks for non-TCP rings 409da14cebeSEric Cheng */ 410da14cebeSEric Cheng if (!(softring->s_ring_type & ST_RING_TCP)) { 411da14cebeSEric Cheng softring->s_ring_rx_arg2 = NULL; 412da14cebeSEric Cheng softring = softring->s_ring_next; 413da14cebeSEric Cheng continue; 414da14cebeSEric Cheng } 415da14cebeSEric Cheng mrf.mrf_rx_arg = softring; 416da14cebeSEric Cheng mrf.mrf_intr_handle = (mac_intr_handle_t)softring; 417da14cebeSEric Cheng mrf.mrf_cpu_id = softring->s_ring_cpuid; 418da14cebeSEric Cheng mrf.mrf_flow_priority = mac_srs->srs_pri; 419da14cebeSEric Cheng 420da14cebeSEric Cheng softring->s_ring_rx_arg2 = mcip->mci_resource_add( 421da14cebeSEric Cheng mcip->mci_resource_arg, (mac_resource_t *)&mrf); 422da14cebeSEric Cheng 423da14cebeSEric Cheng softring = softring->s_ring_next; 424da14cebeSEric Cheng } 425da14cebeSEric Cheng } 426da14cebeSEric Cheng 427da14cebeSEric Cheng /* 428da14cebeSEric Cheng * Unregister the given SRS and associated soft rings with the consumer and 429da14cebeSEric Cheng * disable the polling interface used by the consumer.(i.e IP) over this 430da14cebeSEric Cheng * SRS and associated soft rings. 431da14cebeSEric Cheng */ 432da14cebeSEric Cheng void 433da14cebeSEric Cheng mac_srs_client_poll_disable(mac_client_impl_t *mcip, 434da14cebeSEric Cheng mac_soft_ring_set_t *mac_srs) 435da14cebeSEric Cheng { 436da14cebeSEric Cheng mac_soft_ring_t *softring; 437da14cebeSEric Cheng 438da14cebeSEric Cheng ASSERT(MAC_PERIM_HELD((mac_handle_t)mcip->mci_mip)); 439da14cebeSEric Cheng 440da14cebeSEric Cheng /* 441da14cebeSEric Cheng * A SRS is capable of acting as a soft ring for cases 442da14cebeSEric Cheng * where no protocol fanout is needed. This is the case 443da14cebeSEric Cheng * for userland flows. Nothing to do here. 444da14cebeSEric Cheng */ 445da14cebeSEric Cheng if (mac_srs->srs_type & SRST_NO_SOFT_RINGS) 446da14cebeSEric Cheng return; 447da14cebeSEric Cheng 448da14cebeSEric Cheng mutex_enter(&mac_srs->srs_lock); 449da14cebeSEric Cheng if (!(mac_srs->srs_type & SRST_CLIENT_POLL_ENABLED)) { 450da14cebeSEric Cheng ASSERT(!(mac_srs->srs_type & SRST_DLS_BYPASS)); 451da14cebeSEric Cheng mutex_exit(&mac_srs->srs_lock); 452da14cebeSEric Cheng return; 453da14cebeSEric Cheng } 454da14cebeSEric Cheng mac_srs->srs_type &= ~(SRST_CLIENT_POLL_ENABLED | SRST_DLS_BYPASS); 455da14cebeSEric Cheng mutex_exit(&mac_srs->srs_lock); 456da14cebeSEric Cheng 457da14cebeSEric Cheng /* 458da14cebeSEric Cheng * DLS bypass is now disabled in the case of both TCP and UDP. 459da14cebeSEric Cheng * Reset the soft ring callbacks to the standard 'mac_rx_deliver' 460da14cebeSEric Cheng * callback. In addition, in the case of TCP, invoke IP's callback 461da14cebeSEric Cheng * for ring removal. 462da14cebeSEric Cheng */ 463da14cebeSEric Cheng for (softring = mac_srs->srs_soft_ring_head; 464da14cebeSEric Cheng softring != NULL; softring = softring->s_ring_next) { 465da14cebeSEric Cheng if (!(softring->s_ring_type & (ST_RING_UDP | ST_RING_TCP))) 466da14cebeSEric Cheng continue; 467da14cebeSEric Cheng 468da14cebeSEric Cheng if ((softring->s_ring_type & ST_RING_TCP) && 469da14cebeSEric Cheng softring->s_ring_rx_arg2 != NULL) { 470da14cebeSEric Cheng mcip->mci_resource_remove(mcip->mci_resource_arg, 471da14cebeSEric Cheng softring->s_ring_rx_arg2); 472da14cebeSEric Cheng } 473da14cebeSEric Cheng 474da14cebeSEric Cheng mutex_enter(&softring->s_ring_lock); 475da14cebeSEric Cheng while (softring->s_ring_state & S_RING_PROC) { 476da14cebeSEric Cheng softring->s_ring_state |= S_RING_CLIENT_WAIT; 477da14cebeSEric Cheng cv_wait(&softring->s_ring_client_cv, 478da14cebeSEric Cheng &softring->s_ring_lock); 479da14cebeSEric Cheng } 480da14cebeSEric Cheng softring->s_ring_state &= ~S_RING_CLIENT_WAIT; 481da14cebeSEric Cheng softring->s_ring_rx_arg2 = NULL; 482da14cebeSEric Cheng softring->s_ring_rx_func = mac_rx_deliver; 483da14cebeSEric Cheng softring->s_ring_rx_arg1 = mcip; 484da14cebeSEric Cheng mutex_exit(&softring->s_ring_lock); 485da14cebeSEric Cheng } 486da14cebeSEric Cheng } 487da14cebeSEric Cheng 488da14cebeSEric Cheng /* 489da14cebeSEric Cheng * Enable or disable poll capability of the SRS on the underlying Rx ring. 490da14cebeSEric Cheng * 491da14cebeSEric Cheng * There is a need to enable or disable the poll capability of an SRS over an 492da14cebeSEric Cheng * Rx ring depending on the number of mac clients sharing the ring and also 493da14cebeSEric Cheng * whether user flows are configured on it. However the poll state is actively 494da14cebeSEric Cheng * manipulated by the SRS worker and poll threads and uncoordinated changes by 495da14cebeSEric Cheng * yet another thread to the underlying capability can surprise them leading 496da14cebeSEric Cheng * to assert failures. Instead we quiesce the SRS, make the changes and then 497da14cebeSEric Cheng * restart the SRS. 498da14cebeSEric Cheng */ 499da14cebeSEric Cheng static void 500da14cebeSEric Cheng mac_srs_poll_state_change(mac_soft_ring_set_t *mac_srs, 501da14cebeSEric Cheng boolean_t turn_off_poll_capab, mac_rx_func_t rx_func) 502da14cebeSEric Cheng { 503da14cebeSEric Cheng boolean_t need_restart = B_FALSE; 504da14cebeSEric Cheng mac_srs_rx_t *srs_rx = &mac_srs->srs_rx; 505da14cebeSEric Cheng mac_ring_t *ring; 506da14cebeSEric Cheng 507da14cebeSEric Cheng if (!SRS_QUIESCED(mac_srs)) { 508da14cebeSEric Cheng mac_rx_srs_quiesce(mac_srs, SRS_QUIESCE); 509da14cebeSEric Cheng need_restart = B_TRUE; 510da14cebeSEric Cheng } 511da14cebeSEric Cheng 512da14cebeSEric Cheng ring = mac_srs->srs_ring; 513da14cebeSEric Cheng if ((ring != NULL) && 514da14cebeSEric Cheng (ring->mr_classify_type == MAC_HW_CLASSIFIER)) { 515da14cebeSEric Cheng if (turn_off_poll_capab) 516da14cebeSEric Cheng mac_srs->srs_state &= ~SRS_POLLING_CAPAB; 517ae6aa22aSVenugopal Iyer else if (mac_poll_enable) 518da14cebeSEric Cheng mac_srs->srs_state |= SRS_POLLING_CAPAB; 519da14cebeSEric Cheng } 520da14cebeSEric Cheng srs_rx->sr_lower_proc = rx_func; 521da14cebeSEric Cheng 522da14cebeSEric Cheng if (need_restart) 523da14cebeSEric Cheng mac_rx_srs_restart(mac_srs); 524da14cebeSEric Cheng } 525da14cebeSEric Cheng 526da14cebeSEric Cheng /* CPU RECONFIGURATION AND FANOUT COMPUTATION ROUTINES */ 527da14cebeSEric Cheng 528da14cebeSEric Cheng /* 529da14cebeSEric Cheng * Return the next CPU to be used to bind a MAC kernel thread. 5300dc2366fSVenugopal Iyer * If a cpupart is specified, the cpu chosen must be from that 5310dc2366fSVenugopal Iyer * cpu partition. 532da14cebeSEric Cheng */ 533da14cebeSEric Cheng static processorid_t 5340dc2366fSVenugopal Iyer mac_next_bind_cpu(cpupart_t *cpupart) 535da14cebeSEric Cheng { 5360dc2366fSVenugopal Iyer static cpu_t *cp = NULL; 5370dc2366fSVenugopal Iyer cpu_t *cp_start; 538da14cebeSEric Cheng 539da14cebeSEric Cheng ASSERT(MUTEX_HELD(&cpu_lock)); 540da14cebeSEric Cheng 5410dc2366fSVenugopal Iyer if (cp == NULL) 5420dc2366fSVenugopal Iyer cp = cpu_list; 543da14cebeSEric Cheng 5440dc2366fSVenugopal Iyer cp = cp->cpu_next_onln; 5450dc2366fSVenugopal Iyer cp_start = cp; 5460dc2366fSVenugopal Iyer 5470dc2366fSVenugopal Iyer do { 5480dc2366fSVenugopal Iyer if ((cpupart == NULL) || (cp->cpu_part == cpupart)) 5490dc2366fSVenugopal Iyer return (cp->cpu_id); 5500dc2366fSVenugopal Iyer 5510dc2366fSVenugopal Iyer } while ((cp = cp->cpu_next_onln) != cp_start); 5520dc2366fSVenugopal Iyer 55336f99a58SToomas Soome return (-1); /* No matching CPU found online */ 554da14cebeSEric Cheng } 555da14cebeSEric Cheng 556da14cebeSEric Cheng /* ARGSUSED */ 557da14cebeSEric Cheng static int 558da14cebeSEric Cheng mac_srs_cpu_setup(cpu_setup_t what, int id, void *arg) 559da14cebeSEric Cheng { 560da14cebeSEric Cheng ASSERT(MUTEX_HELD(&cpu_lock)); 561da14cebeSEric Cheng switch (what) { 562da14cebeSEric Cheng case CPU_CONFIG: 563da14cebeSEric Cheng case CPU_ON: 564da14cebeSEric Cheng case CPU_CPUPART_IN: 565da14cebeSEric Cheng mac_walk_srs_and_bind(id); 566da14cebeSEric Cheng break; 567da14cebeSEric Cheng 568da14cebeSEric Cheng case CPU_UNCONFIG: 569da14cebeSEric Cheng case CPU_OFF: 570da14cebeSEric Cheng case CPU_CPUPART_OUT: 571da14cebeSEric Cheng mac_walk_srs_and_unbind(id); 572da14cebeSEric Cheng break; 573da14cebeSEric Cheng 574da14cebeSEric Cheng default: 575da14cebeSEric Cheng break; 576da14cebeSEric Cheng } 577da14cebeSEric Cheng return (0); 578da14cebeSEric Cheng } 579da14cebeSEric Cheng 580da14cebeSEric Cheng /* 581da14cebeSEric Cheng * mac_compute_soft_ring_count(): 582da14cebeSEric Cheng * 583da14cebeSEric Cheng * This routine computes the number of soft rings needed to handle incoming 584da14cebeSEric Cheng * load given a flow_entry. 585da14cebeSEric Cheng * 586da14cebeSEric Cheng * The routine does the following: 587da14cebeSEric Cheng * 1) soft rings will be created if mac_soft_ring_enable is set. 588da14cebeSEric Cheng * 2) If the underlying link is a 10Gbps link, then soft rings will be 589da14cebeSEric Cheng * created even if mac_soft_ring_enable is not set. The number of soft 590da14cebeSEric Cheng * rings, so created, will equal mac_rx_soft_ring_10gig_count. 591da14cebeSEric Cheng * 3) On a sun4v platform (i.e., mac_soft_ring_enable is set), 2 times the 592da14cebeSEric Cheng * mac_rx_soft_ring_10gig_count number of soft rings will be created for a 593da14cebeSEric Cheng * 10Gbps link. 594da14cebeSEric Cheng * 595da14cebeSEric Cheng * If a bandwidth limit is specified, the number that gets computed is 596da14cebeSEric Cheng * dependent upon CPU speed, the number of Rx rings configured, and 597da14cebeSEric Cheng * the bandwidth limit. 598da14cebeSEric Cheng * If more Rx rings are available, less number of soft rings is needed. 599da14cebeSEric Cheng * 600da14cebeSEric Cheng * mac_use_bw_heuristic is another "hidden" variable that can be used to 601da14cebeSEric Cheng * override the default use of soft ring count computation. Depending upon 602da14cebeSEric Cheng * the usefulness of it, mac_use_bw_heuristic can later be made into a 603da14cebeSEric Cheng * data-link property or removed altogether. 604da14cebeSEric Cheng * 605da14cebeSEric Cheng * TODO: Cleanup and tighten some of the assumptions. 606da14cebeSEric Cheng */ 607da14cebeSEric Cheng boolean_t mac_use_bw_heuristic = B_TRUE; 608da14cebeSEric Cheng static int 6090dc2366fSVenugopal Iyer mac_compute_soft_ring_count(flow_entry_t *flent, int rx_srs_cnt, int maxcpus) 610da14cebeSEric Cheng { 611da14cebeSEric Cheng uint64_t cpu_speed, bw = 0; 612da14cebeSEric Cheng int srings = 0; 613da14cebeSEric Cheng boolean_t bw_enabled = B_FALSE; 614da14cebeSEric Cheng 615da14cebeSEric Cheng ASSERT(!(flent->fe_type & FLOW_USER)); 616da14cebeSEric Cheng if (flent->fe_resource_props.mrp_mask & MRP_MAXBW && 617da14cebeSEric Cheng mac_use_bw_heuristic) { 618da14cebeSEric Cheng /* bandwidth enabled */ 619da14cebeSEric Cheng bw_enabled = B_TRUE; 620da14cebeSEric Cheng bw = flent->fe_resource_props.mrp_maxbw; 621da14cebeSEric Cheng } 622da14cebeSEric Cheng if (!bw_enabled) { 623da14cebeSEric Cheng /* No bandwidth enabled */ 624da14cebeSEric Cheng if (mac_soft_ring_enable) 625da14cebeSEric Cheng srings = mac_rx_soft_ring_count; 626da14cebeSEric Cheng 627da14cebeSEric Cheng /* Is this a 10Gig link? */ 628da14cebeSEric Cheng flent->fe_nic_speed = mac_client_stat_get(flent->fe_mcip, 629da14cebeSEric Cheng MAC_STAT_IFSPEED); 630da14cebeSEric Cheng /* convert to Mbps */ 631da14cebeSEric Cheng if (((flent->fe_nic_speed)/1000000) > 1000 && 632da14cebeSEric Cheng mac_rx_soft_ring_10gig_count > 0) { 633da14cebeSEric Cheng /* This is a 10Gig link */ 634da14cebeSEric Cheng srings = mac_rx_soft_ring_10gig_count; 635da14cebeSEric Cheng /* 636da14cebeSEric Cheng * Use 2 times mac_rx_soft_ring_10gig_count for 637da14cebeSEric Cheng * sun4v systems. 638da14cebeSEric Cheng */ 639da14cebeSEric Cheng if (mac_soft_ring_enable) 640da14cebeSEric Cheng srings = srings * 2; 641da14cebeSEric Cheng } 642da14cebeSEric Cheng } else { 643da14cebeSEric Cheng /* 644da14cebeSEric Cheng * Soft ring computation using CPU speed and specified 645da14cebeSEric Cheng * bandwidth limit. 646da14cebeSEric Cheng */ 647da14cebeSEric Cheng /* Assumption: all CPUs have the same frequency */ 648da14cebeSEric Cheng cpu_speed = (uint64_t)CPU->cpu_type_info.pi_clock; 649da14cebeSEric Cheng 650da14cebeSEric Cheng /* cpu_speed is in MHz; make bw in units of Mbps. */ 651da14cebeSEric Cheng bw = bw/1000000; 652da14cebeSEric Cheng 653da14cebeSEric Cheng if (bw >= 1000) { 654da14cebeSEric Cheng /* 655da14cebeSEric Cheng * bw is greater than or equal to 1Gbps. 656da14cebeSEric Cheng * The number of soft rings required is a function 657da14cebeSEric Cheng * of bandwidth and CPU speed. To keep this simple, 658da14cebeSEric Cheng * let's use this rule: 1GHz CPU can handle 1Gbps. 659da14cebeSEric Cheng * If bw is less than 1 Gbps, then there is no need 660da14cebeSEric Cheng * for soft rings. Assumption is that CPU speeds 661da14cebeSEric Cheng * (on modern systems) are at least 1GHz. 662da14cebeSEric Cheng */ 663da14cebeSEric Cheng srings = bw/cpu_speed; 664da14cebeSEric Cheng if (srings <= 1 && mac_soft_ring_enable) { 665da14cebeSEric Cheng /* 666da14cebeSEric Cheng * Give at least 2 soft rings 667da14cebeSEric Cheng * for sun4v systems 668da14cebeSEric Cheng */ 669da14cebeSEric Cheng srings = 2; 670da14cebeSEric Cheng } 671da14cebeSEric Cheng } 672da14cebeSEric Cheng } 673da14cebeSEric Cheng /* 674da14cebeSEric Cheng * If the flent has multiple Rx SRSs, then each SRS need not 675da14cebeSEric Cheng * have that many soft rings on top of it. The number of 676da14cebeSEric Cheng * soft rings for each Rx SRS is found by dividing srings by 677da14cebeSEric Cheng * rx_srs_cnt. 678da14cebeSEric Cheng */ 679da14cebeSEric Cheng if (rx_srs_cnt > 1) { 680da14cebeSEric Cheng int remainder; 681da14cebeSEric Cheng 682da14cebeSEric Cheng remainder = srings%rx_srs_cnt; 683da14cebeSEric Cheng srings = srings/rx_srs_cnt; 684da14cebeSEric Cheng if (remainder != 0) 685da14cebeSEric Cheng srings++; 686da14cebeSEric Cheng /* 687da14cebeSEric Cheng * Fanning out to 1 soft ring is not very useful. 688da14cebeSEric Cheng * Set it as well to 0 and mac_srs_fanout_init() 689da14cebeSEric Cheng * will take care of creating a single soft ring 690da14cebeSEric Cheng * for proto fanout. 691da14cebeSEric Cheng */ 692da14cebeSEric Cheng if (srings == 1) 693da14cebeSEric Cheng srings = 0; 694da14cebeSEric Cheng } 695da14cebeSEric Cheng /* Do some more massaging */ 6960dc2366fSVenugopal Iyer srings = min(srings, maxcpus); 697da14cebeSEric Cheng srings = min(srings, MAX_SR_FANOUT); 698da14cebeSEric Cheng return (srings); 699da14cebeSEric Cheng } 700da14cebeSEric Cheng 701da14cebeSEric Cheng /* 7020dc2366fSVenugopal Iyer * mac_tx_cpu_init: 7030dc2366fSVenugopal Iyer * set up CPUs for Tx interrupt re-targeting and Tx worker 7040dc2366fSVenugopal Iyer * thread binding 7050dc2366fSVenugopal Iyer */ 7060dc2366fSVenugopal Iyer static void 7070dc2366fSVenugopal Iyer mac_tx_cpu_init(flow_entry_t *flent, mac_resource_props_t *mrp, 7080dc2366fSVenugopal Iyer cpupart_t *cpupart) 7090dc2366fSVenugopal Iyer { 7100dc2366fSVenugopal Iyer mac_soft_ring_set_t *tx_srs = flent->fe_tx_srs; 7110dc2366fSVenugopal Iyer mac_srs_tx_t *srs_tx = &tx_srs->srs_tx; 7120dc2366fSVenugopal Iyer mac_cpus_t *srs_cpu = &tx_srs->srs_cpu; 7130dc2366fSVenugopal Iyer mac_soft_ring_t *sringp; 7140dc2366fSVenugopal Iyer mac_ring_t *ring; 7150dc2366fSVenugopal Iyer processorid_t worker_cpuid; 7160dc2366fSVenugopal Iyer boolean_t retargetable_client = B_FALSE; 7170dc2366fSVenugopal Iyer int i, j; 7180dc2366fSVenugopal Iyer 7190dc2366fSVenugopal Iyer if (RETARGETABLE_CLIENT((mac_group_t *)flent->fe_tx_ring_group, 7200dc2366fSVenugopal Iyer flent->fe_mcip)) { 7210dc2366fSVenugopal Iyer retargetable_client = B_TRUE; 7220dc2366fSVenugopal Iyer } 7230dc2366fSVenugopal Iyer 7240dc2366fSVenugopal Iyer if (MAC_TX_SOFT_RINGS(tx_srs)) { 7250dc2366fSVenugopal Iyer if (mrp != NULL) 7260dc2366fSVenugopal Iyer j = mrp->mrp_ncpus - 1; 7270dc2366fSVenugopal Iyer for (i = 0; i < tx_srs->srs_tx_ring_count; i++) { 7280dc2366fSVenugopal Iyer if (mrp != NULL) { 7290dc2366fSVenugopal Iyer if (j < 0) 7300dc2366fSVenugopal Iyer j = mrp->mrp_ncpus - 1; 7310dc2366fSVenugopal Iyer worker_cpuid = mrp->mrp_cpu[j]; 7320dc2366fSVenugopal Iyer } else { 7330dc2366fSVenugopal Iyer /* 7340dc2366fSVenugopal Iyer * Bind interrupt to the next CPU available 7350dc2366fSVenugopal Iyer * and leave the worker unbound. 7360dc2366fSVenugopal Iyer */ 7370dc2366fSVenugopal Iyer worker_cpuid = -1; 7380dc2366fSVenugopal Iyer } 7390dc2366fSVenugopal Iyer sringp = tx_srs->srs_tx_soft_rings[i]; 7400dc2366fSVenugopal Iyer ring = (mac_ring_t *)sringp->s_ring_tx_arg2; 7410dc2366fSVenugopal Iyer srs_cpu->mc_tx_fanout_cpus[i] = worker_cpuid; 7420dc2366fSVenugopal Iyer if (MAC_RING_RETARGETABLE(ring) && 7430dc2366fSVenugopal Iyer retargetable_client) { 7440dc2366fSVenugopal Iyer mutex_enter(&cpu_lock); 7450dc2366fSVenugopal Iyer srs_cpu->mc_tx_intr_cpu[i] = 7460dc2366fSVenugopal Iyer (mrp != NULL) ? mrp->mrp_cpu[j] : 7470dc2366fSVenugopal Iyer (mac_tx_intr_retarget ? 7480dc2366fSVenugopal Iyer mac_next_bind_cpu(cpupart) : -1); 7490dc2366fSVenugopal Iyer mutex_exit(&cpu_lock); 7500dc2366fSVenugopal Iyer } else { 7510dc2366fSVenugopal Iyer srs_cpu->mc_tx_intr_cpu[i] = -1; 7520dc2366fSVenugopal Iyer } 7530dc2366fSVenugopal Iyer if (mrp != NULL) 7540dc2366fSVenugopal Iyer j--; 7550dc2366fSVenugopal Iyer } 7560dc2366fSVenugopal Iyer } else { 7570dc2366fSVenugopal Iyer /* Tx mac_ring_handle_t is stored in st_arg2 */ 7580dc2366fSVenugopal Iyer srs_cpu->mc_tx_fanout_cpus[0] = 7590dc2366fSVenugopal Iyer (mrp != NULL) ? mrp->mrp_cpu[mrp->mrp_ncpus - 1] : -1; 7600dc2366fSVenugopal Iyer ring = (mac_ring_t *)srs_tx->st_arg2; 7610dc2366fSVenugopal Iyer if (MAC_RING_RETARGETABLE(ring) && retargetable_client) { 7620dc2366fSVenugopal Iyer mutex_enter(&cpu_lock); 7630dc2366fSVenugopal Iyer srs_cpu->mc_tx_intr_cpu[0] = (mrp != NULL) ? 7640dc2366fSVenugopal Iyer mrp->mrp_cpu[mrp->mrp_ncpus - 1] : 7650dc2366fSVenugopal Iyer (mac_tx_intr_retarget ? 7660dc2366fSVenugopal Iyer mac_next_bind_cpu(cpupart) : -1); 7670dc2366fSVenugopal Iyer mutex_exit(&cpu_lock); 7680dc2366fSVenugopal Iyer } else { 7690dc2366fSVenugopal Iyer srs_cpu->mc_tx_intr_cpu[0] = -1; 7700dc2366fSVenugopal Iyer } 7710dc2366fSVenugopal Iyer } 7720dc2366fSVenugopal Iyer } 7730dc2366fSVenugopal Iyer 7740dc2366fSVenugopal Iyer /* 775da14cebeSEric Cheng * Assignment of user specified CPUs to a link. 776da14cebeSEric Cheng * 777da14cebeSEric Cheng * Minimum CPUs required to get an optimal assignmet: 778da14cebeSEric Cheng * For each Rx SRS, atleast two CPUs are needed if mac_latency_optimize 779da14cebeSEric Cheng * flag is set -- one for polling, one for fanout soft ring. 780da14cebeSEric Cheng * If mac_latency_optimize is not set, then 3 CPUs are needed -- one 781da14cebeSEric Cheng * for polling, one for SRS worker thread and one for fanout soft ring. 782da14cebeSEric Cheng * 783da14cebeSEric Cheng * The CPUs needed for Tx side is equal to the number of Tx rings 784da14cebeSEric Cheng * the link is using. 785da14cebeSEric Cheng * 786da14cebeSEric Cheng * mac_flow_user_cpu_init() categorizes the CPU assignment depending 787da14cebeSEric Cheng * upon the number of CPUs in 3 different buckets. 788da14cebeSEric Cheng * 789da14cebeSEric Cheng * In the first bucket, the most optimal case is handled. The user has 790da14cebeSEric Cheng * passed enough number of CPUs and every thread gets its own CPU. 791da14cebeSEric Cheng * 792da14cebeSEric Cheng * The second and third are the sub-optimal cases. Enough CPUs are not 793da14cebeSEric Cheng * available. 794da14cebeSEric Cheng * 795da14cebeSEric Cheng * The second bucket handles the case where atleast one distinct CPU is 796da14cebeSEric Cheng * is available for each of the Rx rings (Rx SRSes) and Tx rings (Tx 797da14cebeSEric Cheng * SRS or soft rings). 798da14cebeSEric Cheng * 799da14cebeSEric Cheng * In the third case (worst case scenario), specified CPU count is less 800da14cebeSEric Cheng * than the Rx rings configured for the link. In this case, we round 801da14cebeSEric Cheng * robin the CPUs among the Rx SRSes and Tx SRS/soft rings. 802da14cebeSEric Cheng */ 803da14cebeSEric Cheng static void 804da14cebeSEric Cheng mac_flow_user_cpu_init(flow_entry_t *flent, mac_resource_props_t *mrp) 805da14cebeSEric Cheng { 806da14cebeSEric Cheng mac_soft_ring_set_t *rx_srs, *tx_srs; 807da14cebeSEric Cheng int i, srs_cnt; 808da14cebeSEric Cheng mac_cpus_t *srs_cpu; 809da14cebeSEric Cheng int no_of_cpus, cpu_cnt; 810da14cebeSEric Cheng int rx_srs_cnt, reqd_rx_cpu_cnt; 811da14cebeSEric Cheng int fanout_cpu_cnt, reqd_tx_cpu_cnt; 812da14cebeSEric Cheng int reqd_poll_worker_cnt, fanout_cnt_per_srs; 8130dc2366fSVenugopal Iyer mac_resource_props_t *emrp = &flent->fe_effective_props; 814da14cebeSEric Cheng 815da14cebeSEric Cheng ASSERT(mrp->mrp_fanout_mode == MCM_CPUS); 816da14cebeSEric Cheng /* 817da14cebeSEric Cheng * The check for nbc_ncpus to be within limits for 818da14cebeSEric Cheng * the user specified case was done earlier and if 819da14cebeSEric Cheng * not within limits, an error would have been 820da14cebeSEric Cheng * returned to the user. 821da14cebeSEric Cheng */ 8225adf34bdSRajagopal Kunhappan ASSERT(mrp->mrp_ncpus > 0); 823da14cebeSEric Cheng 824da14cebeSEric Cheng no_of_cpus = mrp->mrp_ncpus; 825da14cebeSEric Cheng 8260dc2366fSVenugopal Iyer if (mrp->mrp_rx_intr_cpu != -1) { 827da14cebeSEric Cheng /* 828da14cebeSEric Cheng * interrupt has been re-targetted. Poll 829da14cebeSEric Cheng * thread needs to be bound to interrupt 8300dc2366fSVenugopal Iyer * CPU. 831da14cebeSEric Cheng * 832da14cebeSEric Cheng * Find where in the list is the intr 833da14cebeSEric Cheng * CPU and swap it with the first one. 834da14cebeSEric Cheng * We will be using the first CPU in the 835da14cebeSEric Cheng * list for poll. 836da14cebeSEric Cheng */ 837da14cebeSEric Cheng for (i = 0; i < no_of_cpus; i++) { 8380dc2366fSVenugopal Iyer if (mrp->mrp_cpu[i] == mrp->mrp_rx_intr_cpu) 839da14cebeSEric Cheng break; 840da14cebeSEric Cheng } 841da14cebeSEric Cheng mrp->mrp_cpu[i] = mrp->mrp_cpu[0]; 8420dc2366fSVenugopal Iyer mrp->mrp_cpu[0] = mrp->mrp_rx_intr_cpu; 843da14cebeSEric Cheng } 844da14cebeSEric Cheng 845da14cebeSEric Cheng /* 846da14cebeSEric Cheng * Requirements: 847da14cebeSEric Cheng * The number of CPUs that each Rx ring needs is dependent 848da14cebeSEric Cheng * upon mac_latency_optimize flag. 849da14cebeSEric Cheng * 1) If set, atleast 2 CPUs are needed -- one for 850da14cebeSEric Cheng * polling, one for fanout soft ring. 851da14cebeSEric Cheng * 2) If not set, then atleast 3 CPUs are needed -- one 852da14cebeSEric Cheng * for polling, one for srs worker thread, and one for 853da14cebeSEric Cheng * fanout soft ring. 854da14cebeSEric Cheng */ 855da14cebeSEric Cheng rx_srs_cnt = (flent->fe_rx_srs_cnt > 1) ? 856da14cebeSEric Cheng (flent->fe_rx_srs_cnt - 1) : flent->fe_rx_srs_cnt; 857da14cebeSEric Cheng reqd_rx_cpu_cnt = mac_latency_optimize ? 858da14cebeSEric Cheng (rx_srs_cnt * 2) : (rx_srs_cnt * 3); 859da14cebeSEric Cheng 860da14cebeSEric Cheng /* How many CPUs are needed for Tx side? */ 861da14cebeSEric Cheng tx_srs = flent->fe_tx_srs; 8620dc2366fSVenugopal Iyer reqd_tx_cpu_cnt = MAC_TX_SOFT_RINGS(tx_srs) ? 8630dc2366fSVenugopal Iyer tx_srs->srs_tx_ring_count : 1; 864da14cebeSEric Cheng 865da14cebeSEric Cheng /* CPUs needed for Rx SRSes poll and worker threads */ 866da14cebeSEric Cheng reqd_poll_worker_cnt = mac_latency_optimize ? 867da14cebeSEric Cheng rx_srs_cnt : rx_srs_cnt * 2; 868da14cebeSEric Cheng 869da14cebeSEric Cheng /* Has the user provided enough CPUs? */ 870da14cebeSEric Cheng if (no_of_cpus >= (reqd_rx_cpu_cnt + reqd_tx_cpu_cnt)) { 871da14cebeSEric Cheng /* 872da14cebeSEric Cheng * Best case scenario. There is enough CPUs. All 873da14cebeSEric Cheng * Rx rings will get their own set of CPUs plus 874da14cebeSEric Cheng * Tx soft rings will get their own. 875da14cebeSEric Cheng */ 876da14cebeSEric Cheng /* 877da14cebeSEric Cheng * fanout_cpu_cnt is the number of CPUs available 878da14cebeSEric Cheng * for Rx side fanout soft rings. 879da14cebeSEric Cheng */ 880da14cebeSEric Cheng fanout_cpu_cnt = no_of_cpus - 881da14cebeSEric Cheng reqd_poll_worker_cnt - reqd_tx_cpu_cnt; 882da14cebeSEric Cheng 883da14cebeSEric Cheng /* 884da14cebeSEric Cheng * Divide fanout_cpu_cnt by rx_srs_cnt to find 885da14cebeSEric Cheng * out how many fanout soft rings each Rx SRS 886da14cebeSEric Cheng * can have. 887da14cebeSEric Cheng */ 888da14cebeSEric Cheng fanout_cnt_per_srs = fanout_cpu_cnt/rx_srs_cnt; 889da14cebeSEric Cheng 8905adf34bdSRajagopal Kunhappan /* fanout_cnt_per_srs should not be > MAX_SR_FANOUT */ 8915adf34bdSRajagopal Kunhappan fanout_cnt_per_srs = min(fanout_cnt_per_srs, MAX_SR_FANOUT); 8925adf34bdSRajagopal Kunhappan 893da14cebeSEric Cheng /* Do the assignment for the default Rx ring */ 894da14cebeSEric Cheng cpu_cnt = 0; 895da14cebeSEric Cheng rx_srs = flent->fe_rx_srs[0]; 896da14cebeSEric Cheng ASSERT(rx_srs->srs_ring == NULL); 897da14cebeSEric Cheng if (rx_srs->srs_fanout_state == SRS_FANOUT_INIT) 898da14cebeSEric Cheng rx_srs->srs_fanout_state = SRS_FANOUT_REINIT; 899da14cebeSEric Cheng srs_cpu = &rx_srs->srs_cpu; 900da14cebeSEric Cheng srs_cpu->mc_ncpus = no_of_cpus; 901da14cebeSEric Cheng bcopy(mrp->mrp_cpu, 902da14cebeSEric Cheng srs_cpu->mc_cpus, sizeof (srs_cpu->mc_cpus)); 9030dc2366fSVenugopal Iyer srs_cpu->mc_rx_fanout_cnt = fanout_cnt_per_srs; 9040dc2366fSVenugopal Iyer srs_cpu->mc_rx_pollid = mrp->mrp_cpu[cpu_cnt++]; 9050dc2366fSVenugopal Iyer /* Retarget the interrupt to the same CPU as the poll */ 9060dc2366fSVenugopal Iyer srs_cpu->mc_rx_intr_cpu = srs_cpu->mc_rx_pollid; 9070dc2366fSVenugopal Iyer srs_cpu->mc_rx_workerid = (mac_latency_optimize ? 9080dc2366fSVenugopal Iyer srs_cpu->mc_rx_pollid : mrp->mrp_cpu[cpu_cnt++]); 909da14cebeSEric Cheng for (i = 0; i < fanout_cnt_per_srs; i++) 9100dc2366fSVenugopal Iyer srs_cpu->mc_rx_fanout_cpus[i] = mrp->mrp_cpu[cpu_cnt++]; 911da14cebeSEric Cheng 912da14cebeSEric Cheng /* Do the assignment for h/w Rx SRSes */ 913da14cebeSEric Cheng if (flent->fe_rx_srs_cnt > 1) { 914da14cebeSEric Cheng cpu_cnt = 0; 915da14cebeSEric Cheng for (srs_cnt = 1; 916da14cebeSEric Cheng srs_cnt < flent->fe_rx_srs_cnt; srs_cnt++) { 917da14cebeSEric Cheng rx_srs = flent->fe_rx_srs[srs_cnt]; 918da14cebeSEric Cheng ASSERT(rx_srs->srs_ring != NULL); 919da14cebeSEric Cheng if (rx_srs->srs_fanout_state == 920da14cebeSEric Cheng SRS_FANOUT_INIT) { 921da14cebeSEric Cheng rx_srs->srs_fanout_state = 922da14cebeSEric Cheng SRS_FANOUT_REINIT; 923da14cebeSEric Cheng } 924da14cebeSEric Cheng srs_cpu = &rx_srs->srs_cpu; 925da14cebeSEric Cheng srs_cpu->mc_ncpus = no_of_cpus; 926da14cebeSEric Cheng bcopy(mrp->mrp_cpu, srs_cpu->mc_cpus, 927da14cebeSEric Cheng sizeof (srs_cpu->mc_cpus)); 9280dc2366fSVenugopal Iyer srs_cpu->mc_rx_fanout_cnt = fanout_cnt_per_srs; 929da14cebeSEric Cheng /* The first CPU in the list is the intr CPU */ 9300dc2366fSVenugopal Iyer srs_cpu->mc_rx_pollid = mrp->mrp_cpu[cpu_cnt++]; 9310dc2366fSVenugopal Iyer srs_cpu->mc_rx_intr_cpu = srs_cpu->mc_rx_pollid; 9320dc2366fSVenugopal Iyer srs_cpu->mc_rx_workerid = 9330dc2366fSVenugopal Iyer (mac_latency_optimize ? 9340dc2366fSVenugopal Iyer srs_cpu->mc_rx_pollid : 9350dc2366fSVenugopal Iyer mrp->mrp_cpu[cpu_cnt++]); 936da14cebeSEric Cheng for (i = 0; i < fanout_cnt_per_srs; i++) { 9370dc2366fSVenugopal Iyer srs_cpu->mc_rx_fanout_cpus[i] = 938da14cebeSEric Cheng mrp->mrp_cpu[cpu_cnt++]; 939da14cebeSEric Cheng } 940da14cebeSEric Cheng ASSERT(cpu_cnt <= no_of_cpus); 941da14cebeSEric Cheng } 942da14cebeSEric Cheng } 9430dc2366fSVenugopal Iyer goto tx_cpu_init; 944da14cebeSEric Cheng } 945da14cebeSEric Cheng 946da14cebeSEric Cheng /* 947da14cebeSEric Cheng * Sub-optimal case. 948da14cebeSEric Cheng * We have the following information: 949da14cebeSEric Cheng * no_of_cpus - no. of cpus that user passed. 950da14cebeSEric Cheng * rx_srs_cnt - no. of rx rings. 951da14cebeSEric Cheng * reqd_rx_cpu_cnt = mac_latency_optimize?rx_srs_cnt*2:rx_srs_cnt*3 952da14cebeSEric Cheng * reqd_tx_cpu_cnt - no. of cpus reqd. for Tx side. 953da14cebeSEric Cheng * reqd_poll_worker_cnt = mac_latency_optimize?rx_srs_cnt:rx_srs_cnt*2 954da14cebeSEric Cheng */ 955da14cebeSEric Cheng /* 956da14cebeSEric Cheng * If we bind the Rx fanout soft rings to the same CPUs 957da14cebeSEric Cheng * as poll/worker, would that be enough? 958da14cebeSEric Cheng */ 959da14cebeSEric Cheng if (no_of_cpus >= (rx_srs_cnt + reqd_tx_cpu_cnt)) { 960da14cebeSEric Cheng boolean_t worker_assign = B_FALSE; 961da14cebeSEric Cheng 962da14cebeSEric Cheng /* 963da14cebeSEric Cheng * If mac_latency_optimize is not set, are there 964da14cebeSEric Cheng * enough CPUs to assign a CPU for worker also? 965da14cebeSEric Cheng */ 966da14cebeSEric Cheng if (no_of_cpus >= (reqd_poll_worker_cnt + reqd_tx_cpu_cnt)) 967da14cebeSEric Cheng worker_assign = B_TRUE; 968da14cebeSEric Cheng /* 969da14cebeSEric Cheng * Zero'th Rx SRS is the default Rx ring. It is not 970da14cebeSEric Cheng * associated with h/w Rx ring. 971da14cebeSEric Cheng */ 972da14cebeSEric Cheng rx_srs = flent->fe_rx_srs[0]; 973da14cebeSEric Cheng ASSERT(rx_srs->srs_ring == NULL); 974da14cebeSEric Cheng if (rx_srs->srs_fanout_state == SRS_FANOUT_INIT) 975da14cebeSEric Cheng rx_srs->srs_fanout_state = SRS_FANOUT_REINIT; 976da14cebeSEric Cheng cpu_cnt = 0; 977da14cebeSEric Cheng srs_cpu = &rx_srs->srs_cpu; 978da14cebeSEric Cheng srs_cpu->mc_ncpus = no_of_cpus; 979da14cebeSEric Cheng bcopy(mrp->mrp_cpu, 980da14cebeSEric Cheng srs_cpu->mc_cpus, sizeof (srs_cpu->mc_cpus)); 9810dc2366fSVenugopal Iyer srs_cpu->mc_rx_fanout_cnt = 1; 9820dc2366fSVenugopal Iyer srs_cpu->mc_rx_pollid = mrp->mrp_cpu[cpu_cnt++]; 9830dc2366fSVenugopal Iyer /* Retarget the interrupt to the same CPU as the poll */ 9840dc2366fSVenugopal Iyer srs_cpu->mc_rx_intr_cpu = srs_cpu->mc_rx_pollid; 9850dc2366fSVenugopal Iyer srs_cpu->mc_rx_workerid = 9860dc2366fSVenugopal Iyer ((!mac_latency_optimize && worker_assign) ? 9870dc2366fSVenugopal Iyer mrp->mrp_cpu[cpu_cnt++] : srs_cpu->mc_rx_pollid); 9880dc2366fSVenugopal Iyer 9890dc2366fSVenugopal Iyer srs_cpu->mc_rx_fanout_cpus[0] = mrp->mrp_cpu[cpu_cnt]; 990da14cebeSEric Cheng 991da14cebeSEric Cheng /* Do CPU bindings for SRSes having h/w Rx rings */ 992da14cebeSEric Cheng if (flent->fe_rx_srs_cnt > 1) { 993da14cebeSEric Cheng cpu_cnt = 0; 994da14cebeSEric Cheng for (srs_cnt = 1; 995da14cebeSEric Cheng srs_cnt < flent->fe_rx_srs_cnt; srs_cnt++) { 996da14cebeSEric Cheng rx_srs = flent->fe_rx_srs[srs_cnt]; 997da14cebeSEric Cheng ASSERT(rx_srs->srs_ring != NULL); 998da14cebeSEric Cheng if (rx_srs->srs_fanout_state == 999da14cebeSEric Cheng SRS_FANOUT_INIT) { 1000da14cebeSEric Cheng rx_srs->srs_fanout_state = 1001da14cebeSEric Cheng SRS_FANOUT_REINIT; 1002da14cebeSEric Cheng } 1003da14cebeSEric Cheng srs_cpu = &rx_srs->srs_cpu; 1004da14cebeSEric Cheng srs_cpu->mc_ncpus = no_of_cpus; 1005da14cebeSEric Cheng bcopy(mrp->mrp_cpu, srs_cpu->mc_cpus, 1006da14cebeSEric Cheng sizeof (srs_cpu->mc_cpus)); 10070dc2366fSVenugopal Iyer srs_cpu->mc_rx_pollid = 1008da14cebeSEric Cheng mrp->mrp_cpu[cpu_cnt]; 10090dc2366fSVenugopal Iyer srs_cpu->mc_rx_intr_cpu = srs_cpu->mc_rx_pollid; 10100dc2366fSVenugopal Iyer srs_cpu->mc_rx_workerid = 10110dc2366fSVenugopal Iyer ((!mac_latency_optimize && worker_assign) ? 10120dc2366fSVenugopal Iyer mrp->mrp_cpu[++cpu_cnt] : 10130dc2366fSVenugopal Iyer srs_cpu->mc_rx_pollid); 10140dc2366fSVenugopal Iyer srs_cpu->mc_rx_fanout_cnt = 1; 10150dc2366fSVenugopal Iyer srs_cpu->mc_rx_fanout_cpus[0] = 1016da14cebeSEric Cheng mrp->mrp_cpu[cpu_cnt]; 1017da14cebeSEric Cheng cpu_cnt++; 1018da14cebeSEric Cheng ASSERT(cpu_cnt <= no_of_cpus); 1019da14cebeSEric Cheng } 1020da14cebeSEric Cheng } 10210dc2366fSVenugopal Iyer goto tx_cpu_init; 1022da14cebeSEric Cheng } 1023da14cebeSEric Cheng 1024da14cebeSEric Cheng /* 1025da14cebeSEric Cheng * Real sub-optimal case. Not enough CPUs for poll and 1026da14cebeSEric Cheng * Tx soft rings. Do a round robin assignment where 1027da14cebeSEric Cheng * each Rx SRS will get the same CPU for poll, worker 1028da14cebeSEric Cheng * and fanout soft ring. 1029da14cebeSEric Cheng */ 1030da14cebeSEric Cheng cpu_cnt = 0; 1031da14cebeSEric Cheng for (srs_cnt = 0; srs_cnt < flent->fe_rx_srs_cnt; srs_cnt++) { 1032da14cebeSEric Cheng rx_srs = flent->fe_rx_srs[srs_cnt]; 1033da14cebeSEric Cheng srs_cpu = &rx_srs->srs_cpu; 1034da14cebeSEric Cheng if (rx_srs->srs_fanout_state == SRS_FANOUT_INIT) 1035da14cebeSEric Cheng rx_srs->srs_fanout_state = SRS_FANOUT_REINIT; 1036da14cebeSEric Cheng srs_cpu->mc_ncpus = no_of_cpus; 1037da14cebeSEric Cheng bcopy(mrp->mrp_cpu, 1038da14cebeSEric Cheng srs_cpu->mc_cpus, sizeof (srs_cpu->mc_cpus)); 10390dc2366fSVenugopal Iyer srs_cpu->mc_rx_fanout_cnt = 1; 10400dc2366fSVenugopal Iyer srs_cpu->mc_rx_pollid = mrp->mrp_cpu[cpu_cnt]; 10410dc2366fSVenugopal Iyer /* Retarget the interrupt to the same CPU as the poll */ 10420dc2366fSVenugopal Iyer srs_cpu->mc_rx_intr_cpu = srs_cpu->mc_rx_pollid; 10430dc2366fSVenugopal Iyer srs_cpu->mc_rx_workerid = mrp->mrp_cpu[cpu_cnt]; 10440dc2366fSVenugopal Iyer srs_cpu->mc_rx_fanout_cpus[0] = mrp->mrp_cpu[cpu_cnt]; 1045da14cebeSEric Cheng if (++cpu_cnt >= no_of_cpus) 1046da14cebeSEric Cheng cpu_cnt = 0; 1047da14cebeSEric Cheng } 10480dc2366fSVenugopal Iyer 10490dc2366fSVenugopal Iyer tx_cpu_init: 10500dc2366fSVenugopal Iyer mac_tx_cpu_init(flent, mrp, NULL); 10510dc2366fSVenugopal Iyer 10520dc2366fSVenugopal Iyer /* 10530dc2366fSVenugopal Iyer * Copy the user specified CPUs to the effective CPUs 10540dc2366fSVenugopal Iyer */ 10550dc2366fSVenugopal Iyer for (i = 0; i < mrp->mrp_ncpus; i++) { 10560dc2366fSVenugopal Iyer emrp->mrp_cpu[i] = mrp->mrp_cpu[i]; 10570dc2366fSVenugopal Iyer } 10580dc2366fSVenugopal Iyer emrp->mrp_ncpus = mrp->mrp_ncpus; 10590dc2366fSVenugopal Iyer emrp->mrp_mask = mrp->mrp_mask; 10600dc2366fSVenugopal Iyer bzero(emrp->mrp_pool, MAXPATHLEN); 1061da14cebeSEric Cheng } 1062da14cebeSEric Cheng 1063da14cebeSEric Cheng /* 1064da14cebeSEric Cheng * mac_flow_cpu_init(): 1065da14cebeSEric Cheng * 1066da14cebeSEric Cheng * Each SRS has a mac_cpu_t structure, srs_cpu. This routine fills in 1067da14cebeSEric Cheng * the CPU binding information in srs_cpu for all Rx SRSes associated 1068da14cebeSEric Cheng * with a flent. 1069da14cebeSEric Cheng */ 1070da14cebeSEric Cheng static void 10710dc2366fSVenugopal Iyer mac_flow_cpu_init(flow_entry_t *flent, cpupart_t *cpupart) 1072da14cebeSEric Cheng { 1073da14cebeSEric Cheng mac_soft_ring_set_t *rx_srs; 1074da14cebeSEric Cheng processorid_t cpuid; 10750dc2366fSVenugopal Iyer int i, j, k, srs_cnt, nscpus, maxcpus, soft_ring_cnt = 0; 1076da14cebeSEric Cheng mac_cpus_t *srs_cpu; 10770dc2366fSVenugopal Iyer mac_resource_props_t *emrp = &flent->fe_effective_props; 10780dc2366fSVenugopal Iyer uint32_t cpus[MRP_NCPUS]; 1079da14cebeSEric Cheng 10800dc2366fSVenugopal Iyer /* 10810dc2366fSVenugopal Iyer * The maximum number of CPUs available can either be 10820dc2366fSVenugopal Iyer * the number of CPUs in the pool or the number of CPUs 10830dc2366fSVenugopal Iyer * in the system. 10840dc2366fSVenugopal Iyer */ 10850dc2366fSVenugopal Iyer maxcpus = (cpupart != NULL) ? cpupart->cp_ncpus : ncpus; 10860dc2366fSVenugopal Iyer 1087da14cebeSEric Cheng /* 1088da14cebeSEric Cheng * Compute the number of soft rings needed on top for each Rx 1089da14cebeSEric Cheng * SRS. "rx_srs_cnt-1" indicates the number of Rx SRS 1090da14cebeSEric Cheng * associated with h/w Rx rings. Soft ring count needed for 1091da14cebeSEric Cheng * each h/w Rx SRS is computed and the same is applied to 1092da14cebeSEric Cheng * software classified Rx SRS. The first Rx SRS in fe_rx_srs[] 1093da14cebeSEric Cheng * is the software classified Rx SRS. 1094da14cebeSEric Cheng */ 1095da14cebeSEric Cheng soft_ring_cnt = mac_compute_soft_ring_count(flent, 10960dc2366fSVenugopal Iyer flent->fe_rx_srs_cnt - 1, maxcpus); 1097da14cebeSEric Cheng if (soft_ring_cnt == 0) { 1098da14cebeSEric Cheng /* 1099da14cebeSEric Cheng * Even when soft_ring_cnt is 0, we still need 1100da14cebeSEric Cheng * to create a soft ring for TCP, UDP and 1101da14cebeSEric Cheng * OTHER. So set it to 1. 1102da14cebeSEric Cheng */ 1103da14cebeSEric Cheng soft_ring_cnt = 1; 1104da14cebeSEric Cheng } 1105da14cebeSEric Cheng for (srs_cnt = 0; srs_cnt < flent->fe_rx_srs_cnt; srs_cnt++) { 1106da14cebeSEric Cheng rx_srs = flent->fe_rx_srs[srs_cnt]; 1107da14cebeSEric Cheng srs_cpu = &rx_srs->srs_cpu; 11080dc2366fSVenugopal Iyer if (rx_srs->srs_fanout_state == SRS_FANOUT_INIT) 1109da14cebeSEric Cheng rx_srs->srs_fanout_state = SRS_FANOUT_REINIT; 1110da14cebeSEric Cheng srs_cpu->mc_ncpus = soft_ring_cnt; 11110dc2366fSVenugopal Iyer srs_cpu->mc_rx_fanout_cnt = soft_ring_cnt; 1112da14cebeSEric Cheng mutex_enter(&cpu_lock); 1113da14cebeSEric Cheng for (j = 0; j < soft_ring_cnt; j++) { 11140dc2366fSVenugopal Iyer cpuid = mac_next_bind_cpu(cpupart); 1115da14cebeSEric Cheng srs_cpu->mc_cpus[j] = cpuid; 11160dc2366fSVenugopal Iyer srs_cpu->mc_rx_fanout_cpus[j] = cpuid; 1117da14cebeSEric Cheng } 11180dc2366fSVenugopal Iyer cpuid = mac_next_bind_cpu(cpupart); 11190dc2366fSVenugopal Iyer srs_cpu->mc_rx_pollid = cpuid; 11200dc2366fSVenugopal Iyer srs_cpu->mc_rx_intr_cpu = (mac_rx_intr_retarget ? 11210dc2366fSVenugopal Iyer srs_cpu->mc_rx_pollid : -1); 1122da14cebeSEric Cheng /* increment ncpus to account for polling cpu */ 1123da14cebeSEric Cheng srs_cpu->mc_ncpus++; 1124da14cebeSEric Cheng srs_cpu->mc_cpus[j++] = cpuid; 1125da14cebeSEric Cheng if (!mac_latency_optimize) { 11260dc2366fSVenugopal Iyer cpuid = mac_next_bind_cpu(cpupart); 1127da14cebeSEric Cheng srs_cpu->mc_ncpus++; 1128da14cebeSEric Cheng srs_cpu->mc_cpus[j++] = cpuid; 1129da14cebeSEric Cheng } 11300dc2366fSVenugopal Iyer srs_cpu->mc_rx_workerid = cpuid; 1131da14cebeSEric Cheng mutex_exit(&cpu_lock); 1132da14cebeSEric Cheng } 11330dc2366fSVenugopal Iyer 11340dc2366fSVenugopal Iyer nscpus = 0; 11350dc2366fSVenugopal Iyer for (srs_cnt = 0; srs_cnt < flent->fe_rx_srs_cnt; srs_cnt++) { 11360dc2366fSVenugopal Iyer rx_srs = flent->fe_rx_srs[srs_cnt]; 11370dc2366fSVenugopal Iyer srs_cpu = &rx_srs->srs_cpu; 11380dc2366fSVenugopal Iyer for (j = 0; j < srs_cpu->mc_ncpus; j++) { 11390dc2366fSVenugopal Iyer cpus[nscpus++] = srs_cpu->mc_cpus[j]; 1140da14cebeSEric Cheng } 1141da14cebeSEric Cheng } 1142da14cebeSEric Cheng 11430dc2366fSVenugopal Iyer 11440dc2366fSVenugopal Iyer /* 11450dc2366fSVenugopal Iyer * Copy cpu list to fe_effective_props 11460dc2366fSVenugopal Iyer * without duplicates. 11470dc2366fSVenugopal Iyer */ 11480dc2366fSVenugopal Iyer k = 0; 11490dc2366fSVenugopal Iyer for (i = 0; i < nscpus; i++) { 11500dc2366fSVenugopal Iyer for (j = 0; j < k; j++) { 11510dc2366fSVenugopal Iyer if (emrp->mrp_cpu[j] == cpus[i]) 11520dc2366fSVenugopal Iyer break; 11530dc2366fSVenugopal Iyer } 11540dc2366fSVenugopal Iyer if (j == k) 11550dc2366fSVenugopal Iyer emrp->mrp_cpu[k++] = cpus[i]; 11560dc2366fSVenugopal Iyer } 11570dc2366fSVenugopal Iyer emrp->mrp_ncpus = k; 11580dc2366fSVenugopal Iyer 11590dc2366fSVenugopal Iyer mac_tx_cpu_init(flent, NULL, cpupart); 11600dc2366fSVenugopal Iyer } 11610dc2366fSVenugopal Iyer 1162da14cebeSEric Cheng /* 1163da14cebeSEric Cheng * DATAPATH SETUP ROUTINES 1164da14cebeSEric Cheng * (setup SRS and set/update FANOUT, B/W and PRIORITY) 1165da14cebeSEric Cheng */ 1166da14cebeSEric Cheng 11670dc2366fSVenugopal Iyer /* 11680dc2366fSVenugopal Iyer * mac_srs_fanout_list_alloc: 11690dc2366fSVenugopal Iyer * 11700dc2366fSVenugopal Iyer * The underlying device can expose upto MAX_RINGS_PER_GROUP worth of 11710dc2366fSVenugopal Iyer * rings to a client. In such a case, MAX_RINGS_PER_GROUP worth of 11720dc2366fSVenugopal Iyer * array space is needed to store Tx soft rings. Thus we allocate so 11730dc2366fSVenugopal Iyer * much array space for srs_tx_soft_rings. 11740dc2366fSVenugopal Iyer * 11750dc2366fSVenugopal Iyer * And when it is an aggr, again we allocate MAX_RINGS_PER_GROUP worth 11760dc2366fSVenugopal Iyer * of space to st_soft_rings. This array is used for quick access to 11770dc2366fSVenugopal Iyer * soft ring associated with a pseudo Tx ring based on the pseudo 11780dc2366fSVenugopal Iyer * ring's index (mr_index). 11790dc2366fSVenugopal Iyer */ 1180da14cebeSEric Cheng static void 1181da14cebeSEric Cheng mac_srs_fanout_list_alloc(mac_soft_ring_set_t *mac_srs) 1182da14cebeSEric Cheng { 11830dc2366fSVenugopal Iyer mac_client_impl_t *mcip = mac_srs->srs_mcip; 11840dc2366fSVenugopal Iyer 11850dc2366fSVenugopal Iyer if (mac_srs->srs_type & SRST_TX) { 11860dc2366fSVenugopal Iyer mac_srs->srs_tx_soft_rings = (mac_soft_ring_t **) 11870dc2366fSVenugopal Iyer kmem_zalloc(sizeof (mac_soft_ring_t *) * 11880dc2366fSVenugopal Iyer MAX_RINGS_PER_GROUP, KM_SLEEP); 11890dc2366fSVenugopal Iyer if (mcip->mci_state_flags & MCIS_IS_AGGR) { 11900dc2366fSVenugopal Iyer mac_srs_tx_t *tx = &mac_srs->srs_tx; 11910dc2366fSVenugopal Iyer 11920dc2366fSVenugopal Iyer tx->st_soft_rings = (mac_soft_ring_t **) 11930dc2366fSVenugopal Iyer kmem_zalloc(sizeof (mac_soft_ring_t *) * 11940dc2366fSVenugopal Iyer MAX_RINGS_PER_GROUP, KM_SLEEP); 11950dc2366fSVenugopal Iyer } 11960dc2366fSVenugopal Iyer } else { 1197da14cebeSEric Cheng mac_srs->srs_tcp_soft_rings = (mac_soft_ring_t **) 11980dc2366fSVenugopal Iyer kmem_zalloc(sizeof (mac_soft_ring_t *) * MAX_SR_FANOUT, 11990dc2366fSVenugopal Iyer KM_SLEEP); 1200da14cebeSEric Cheng mac_srs->srs_udp_soft_rings = (mac_soft_ring_t **) 12010dc2366fSVenugopal Iyer kmem_zalloc(sizeof (mac_soft_ring_t *) * MAX_SR_FANOUT, 12020dc2366fSVenugopal Iyer KM_SLEEP); 1203da14cebeSEric Cheng mac_srs->srs_oth_soft_rings = (mac_soft_ring_t **) 12040dc2366fSVenugopal Iyer kmem_zalloc(sizeof (mac_soft_ring_t *) * MAX_SR_FANOUT, 12050dc2366fSVenugopal Iyer KM_SLEEP); 12060dc2366fSVenugopal Iyer } 1207da14cebeSEric Cheng } 1208da14cebeSEric Cheng 1209da14cebeSEric Cheng static void 1210da14cebeSEric Cheng mac_srs_worker_bind(mac_soft_ring_set_t *mac_srs, processorid_t cpuid) 1211da14cebeSEric Cheng { 1212da14cebeSEric Cheng cpu_t *cp; 1213da14cebeSEric Cheng boolean_t clear = B_FALSE; 1214da14cebeSEric Cheng 1215da14cebeSEric Cheng ASSERT(MUTEX_HELD(&cpu_lock)); 1216da14cebeSEric Cheng 1217da14cebeSEric Cheng if (!mac_srs_thread_bind) 1218da14cebeSEric Cheng return; 1219da14cebeSEric Cheng 1220da14cebeSEric Cheng cp = cpu_get(cpuid); 1221da14cebeSEric Cheng if (cp == NULL || !cpu_is_online(cp)) 1222da14cebeSEric Cheng return; 1223da14cebeSEric Cheng 1224da14cebeSEric Cheng mutex_enter(&mac_srs->srs_lock); 1225da14cebeSEric Cheng mac_srs->srs_state |= SRS_WORKER_BOUND; 1226da14cebeSEric Cheng if (mac_srs->srs_worker_cpuid != -1) 1227da14cebeSEric Cheng clear = B_TRUE; 1228da14cebeSEric Cheng mac_srs->srs_worker_cpuid = cpuid; 1229da14cebeSEric Cheng mutex_exit(&mac_srs->srs_lock); 1230da14cebeSEric Cheng 1231da14cebeSEric Cheng if (clear) 1232da14cebeSEric Cheng thread_affinity_clear(mac_srs->srs_worker); 1233da14cebeSEric Cheng 1234da14cebeSEric Cheng thread_affinity_set(mac_srs->srs_worker, cpuid); 1235da14cebeSEric Cheng DTRACE_PROBE1(worker__CPU, processorid_t, cpuid); 1236da14cebeSEric Cheng } 1237da14cebeSEric Cheng 1238da14cebeSEric Cheng static void 1239da14cebeSEric Cheng mac_srs_poll_bind(mac_soft_ring_set_t *mac_srs, processorid_t cpuid) 1240da14cebeSEric Cheng { 1241da14cebeSEric Cheng cpu_t *cp; 1242da14cebeSEric Cheng boolean_t clear = B_FALSE; 1243da14cebeSEric Cheng 1244da14cebeSEric Cheng ASSERT(MUTEX_HELD(&cpu_lock)); 1245da14cebeSEric Cheng 1246da14cebeSEric Cheng if (!mac_srs_thread_bind || mac_srs->srs_poll_thr == NULL) 1247da14cebeSEric Cheng return; 1248da14cebeSEric Cheng 1249da14cebeSEric Cheng cp = cpu_get(cpuid); 1250da14cebeSEric Cheng if (cp == NULL || !cpu_is_online(cp)) 1251da14cebeSEric Cheng return; 1252da14cebeSEric Cheng 1253da14cebeSEric Cheng mutex_enter(&mac_srs->srs_lock); 1254da14cebeSEric Cheng mac_srs->srs_state |= SRS_POLL_BOUND; 1255da14cebeSEric Cheng if (mac_srs->srs_poll_cpuid != -1) 1256da14cebeSEric Cheng clear = B_TRUE; 1257da14cebeSEric Cheng mac_srs->srs_poll_cpuid = cpuid; 1258da14cebeSEric Cheng mutex_exit(&mac_srs->srs_lock); 1259da14cebeSEric Cheng 1260da14cebeSEric Cheng if (clear) 1261da14cebeSEric Cheng thread_affinity_clear(mac_srs->srs_poll_thr); 1262da14cebeSEric Cheng 1263da14cebeSEric Cheng thread_affinity_set(mac_srs->srs_poll_thr, cpuid); 1264da14cebeSEric Cheng DTRACE_PROBE1(poll__CPU, processorid_t, cpuid); 1265da14cebeSEric Cheng } 1266da14cebeSEric Cheng 1267da14cebeSEric Cheng /* 12680dc2366fSVenugopal Iyer * Re-target interrupt to the passed CPU. If re-target is successful, 12690dc2366fSVenugopal Iyer * set mc_rx_intr_cpu to the re-targeted CPU. Otherwise set it to -1. 12700dc2366fSVenugopal Iyer */ 12710dc2366fSVenugopal Iyer void 12720dc2366fSVenugopal Iyer mac_rx_srs_retarget_intr(mac_soft_ring_set_t *mac_srs, processorid_t cpuid) 12730dc2366fSVenugopal Iyer { 12740dc2366fSVenugopal Iyer cpu_t *cp; 12750dc2366fSVenugopal Iyer mac_ring_t *ring = mac_srs->srs_ring; 12760dc2366fSVenugopal Iyer mac_intr_t *mintr = &ring->mr_info.mri_intr; 12770dc2366fSVenugopal Iyer flow_entry_t *flent = mac_srs->srs_flent; 12780dc2366fSVenugopal Iyer boolean_t primary = mac_is_primary_client(mac_srs->srs_mcip); 12790dc2366fSVenugopal Iyer 12800dc2366fSVenugopal Iyer ASSERT(MUTEX_HELD(&cpu_lock)); 12810dc2366fSVenugopal Iyer 12820dc2366fSVenugopal Iyer /* 12830dc2366fSVenugopal Iyer * Don't re-target the interrupt for these cases: 12840dc2366fSVenugopal Iyer * 1) ring is NULL 12850dc2366fSVenugopal Iyer * 2) the interrupt is shared (mi_ddi_shared) 12860dc2366fSVenugopal Iyer * 3) ddi_handle is NULL and !primary 12870dc2366fSVenugopal Iyer * 4) primary, ddi_handle is NULL but fe_rx_srs_cnt > 2 12880dc2366fSVenugopal Iyer * Case 3 & 4 are because of mac_client_intr_cpu() routine. 12890dc2366fSVenugopal Iyer * This routine will re-target fixed interrupt for primary 12900dc2366fSVenugopal Iyer * mac client if the client has only one ring. In that 12910dc2366fSVenugopal Iyer * case, mc_rx_intr_cpu will already have the correct value. 12920dc2366fSVenugopal Iyer */ 12930dc2366fSVenugopal Iyer if (ring == NULL || mintr->mi_ddi_shared || cpuid == -1 || 12940dc2366fSVenugopal Iyer (mintr->mi_ddi_handle == NULL && !primary) || (primary && 12950dc2366fSVenugopal Iyer mintr->mi_ddi_handle == NULL && flent->fe_rx_srs_cnt > 2)) { 12960dc2366fSVenugopal Iyer mac_srs->srs_cpu.mc_rx_intr_cpu = -1; 12970dc2366fSVenugopal Iyer return; 12980dc2366fSVenugopal Iyer } 12990dc2366fSVenugopal Iyer 13000dc2366fSVenugopal Iyer if (mintr->mi_ddi_handle == NULL) 13010dc2366fSVenugopal Iyer return; 13020dc2366fSVenugopal Iyer 13030dc2366fSVenugopal Iyer cp = cpu_get(cpuid); 13040dc2366fSVenugopal Iyer if (cp == NULL || !cpu_is_online(cp)) 13050dc2366fSVenugopal Iyer return; 13060dc2366fSVenugopal Iyer 1307657f87deSgongtian zhao - Sun Microsystems - Beijing China /* Drop the cpu_lock as set_intr_affinity() holds it */ 13080dc2366fSVenugopal Iyer mutex_exit(&cpu_lock); 1309657f87deSgongtian zhao - Sun Microsystems - Beijing China if (set_intr_affinity(mintr->mi_ddi_handle, cpuid) == DDI_SUCCESS) 13100dc2366fSVenugopal Iyer mac_srs->srs_cpu.mc_rx_intr_cpu = cpuid; 13110dc2366fSVenugopal Iyer else 13120dc2366fSVenugopal Iyer mac_srs->srs_cpu.mc_rx_intr_cpu = -1; 13130dc2366fSVenugopal Iyer mutex_enter(&cpu_lock); 13140dc2366fSVenugopal Iyer } 13150dc2366fSVenugopal Iyer 13160dc2366fSVenugopal Iyer /* 13170dc2366fSVenugopal Iyer * Re-target Tx interrupts 13180dc2366fSVenugopal Iyer */ 13190dc2366fSVenugopal Iyer void 13200dc2366fSVenugopal Iyer mac_tx_srs_retarget_intr(mac_soft_ring_set_t *mac_srs) 13210dc2366fSVenugopal Iyer { 13220dc2366fSVenugopal Iyer cpu_t *cp; 13230dc2366fSVenugopal Iyer mac_ring_t *ring; 13240dc2366fSVenugopal Iyer mac_intr_t *mintr; 13250dc2366fSVenugopal Iyer mac_soft_ring_t *sringp; 13260dc2366fSVenugopal Iyer mac_srs_tx_t *srs_tx; 13270dc2366fSVenugopal Iyer mac_cpus_t *srs_cpu; 13280dc2366fSVenugopal Iyer processorid_t cpuid; 13290dc2366fSVenugopal Iyer int i; 13300dc2366fSVenugopal Iyer 13310dc2366fSVenugopal Iyer ASSERT(MUTEX_HELD(&cpu_lock)); 13320dc2366fSVenugopal Iyer 13330dc2366fSVenugopal Iyer srs_cpu = &mac_srs->srs_cpu; 13340dc2366fSVenugopal Iyer if (MAC_TX_SOFT_RINGS(mac_srs)) { 13350dc2366fSVenugopal Iyer for (i = 0; i < mac_srs->srs_tx_ring_count; i++) { 13360dc2366fSVenugopal Iyer sringp = mac_srs->srs_tx_soft_rings[i]; 13370dc2366fSVenugopal Iyer ring = (mac_ring_t *)sringp->s_ring_tx_arg2; 13380dc2366fSVenugopal Iyer cpuid = srs_cpu->mc_tx_intr_cpu[i]; 13390dc2366fSVenugopal Iyer cp = cpu_get(cpuid); 13400dc2366fSVenugopal Iyer if (cp == NULL || !cpu_is_online(cp) || 13410dc2366fSVenugopal Iyer !MAC_RING_RETARGETABLE(ring)) { 13420dc2366fSVenugopal Iyer srs_cpu->mc_tx_retargeted_cpu[i] = -1; 13430dc2366fSVenugopal Iyer continue; 13440dc2366fSVenugopal Iyer } 13450dc2366fSVenugopal Iyer mintr = &ring->mr_info.mri_intr; 13460dc2366fSVenugopal Iyer /* 1347657f87deSgongtian zhao - Sun Microsystems - Beijing China * Drop the cpu_lock as set_intr_affinity() 13480dc2366fSVenugopal Iyer * holds it 13490dc2366fSVenugopal Iyer */ 13500dc2366fSVenugopal Iyer mutex_exit(&cpu_lock); 1351657f87deSgongtian zhao - Sun Microsystems - Beijing China if (set_intr_affinity(mintr->mi_ddi_handle, 13520dc2366fSVenugopal Iyer cpuid) == DDI_SUCCESS) { 13530dc2366fSVenugopal Iyer srs_cpu->mc_tx_retargeted_cpu[i] = cpuid; 13540dc2366fSVenugopal Iyer } else { 13550dc2366fSVenugopal Iyer srs_cpu->mc_tx_retargeted_cpu[i] = -1; 13560dc2366fSVenugopal Iyer } 13570dc2366fSVenugopal Iyer mutex_enter(&cpu_lock); 13580dc2366fSVenugopal Iyer } 13590dc2366fSVenugopal Iyer } else { 13600dc2366fSVenugopal Iyer cpuid = srs_cpu->mc_tx_intr_cpu[0]; 13610dc2366fSVenugopal Iyer cp = cpu_get(cpuid); 13620dc2366fSVenugopal Iyer if (cp == NULL || !cpu_is_online(cp)) { 13630dc2366fSVenugopal Iyer srs_cpu->mc_tx_retargeted_cpu[0] = -1; 13640dc2366fSVenugopal Iyer return; 13650dc2366fSVenugopal Iyer } 13660dc2366fSVenugopal Iyer srs_tx = &mac_srs->srs_tx; 13670dc2366fSVenugopal Iyer ring = (mac_ring_t *)srs_tx->st_arg2; 13680dc2366fSVenugopal Iyer if (MAC_RING_RETARGETABLE(ring)) { 13690dc2366fSVenugopal Iyer mintr = &ring->mr_info.mri_intr; 13700dc2366fSVenugopal Iyer mutex_exit(&cpu_lock); 1371657f87deSgongtian zhao - Sun Microsystems - Beijing China if ((set_intr_affinity(mintr->mi_ddi_handle, 13720dc2366fSVenugopal Iyer cpuid) == DDI_SUCCESS)) { 13730dc2366fSVenugopal Iyer srs_cpu->mc_tx_retargeted_cpu[0] = cpuid; 13740dc2366fSVenugopal Iyer } else { 13750dc2366fSVenugopal Iyer srs_cpu->mc_tx_retargeted_cpu[0] = -1; 13760dc2366fSVenugopal Iyer } 13770dc2366fSVenugopal Iyer mutex_enter(&cpu_lock); 13780dc2366fSVenugopal Iyer } 13790dc2366fSVenugopal Iyer } 13800dc2366fSVenugopal Iyer } 13810dc2366fSVenugopal Iyer 13820dc2366fSVenugopal Iyer /* 1383da14cebeSEric Cheng * When a CPU comes back online, bind the MAC kernel threads which 1384da14cebeSEric Cheng * were previously bound to that CPU, and had to be unbound because 1385da14cebeSEric Cheng * the CPU was going away. 1386da14cebeSEric Cheng * 1387da14cebeSEric Cheng * These functions are called with cpu_lock held and hence we can't 1388da14cebeSEric Cheng * cv_wait to grab the mac perimeter. Since these functions walk the soft 1389da14cebeSEric Cheng * ring list of an SRS without being in the perimeter, the list itself 1390da14cebeSEric Cheng * is protected by the SRS lock. 1391da14cebeSEric Cheng */ 1392da14cebeSEric Cheng static void 1393da14cebeSEric Cheng mac_walk_srs_and_bind(int cpuid) 1394da14cebeSEric Cheng { 1395da14cebeSEric Cheng mac_soft_ring_set_t *mac_srs; 1396da14cebeSEric Cheng mac_soft_ring_t *soft_ring; 1397da14cebeSEric Cheng 1398da14cebeSEric Cheng rw_enter(&mac_srs_g_lock, RW_READER); 1399da14cebeSEric Cheng 1400da14cebeSEric Cheng if ((mac_srs = mac_srs_g_list) == NULL) 1401da14cebeSEric Cheng goto done; 1402da14cebeSEric Cheng 1403da14cebeSEric Cheng for (; mac_srs != NULL; mac_srs = mac_srs->srs_next) { 1404da14cebeSEric Cheng if (mac_srs->srs_worker_cpuid == -1 && 1405da14cebeSEric Cheng mac_srs->srs_worker_cpuid_save == cpuid) { 1406da14cebeSEric Cheng mac_srs->srs_worker_cpuid_save = -1; 1407da14cebeSEric Cheng mac_srs_worker_bind(mac_srs, cpuid); 1408da14cebeSEric Cheng } 1409da14cebeSEric Cheng 1410da14cebeSEric Cheng if (!(mac_srs->srs_type & SRST_TX)) { 1411da14cebeSEric Cheng if (mac_srs->srs_poll_cpuid == -1 && 1412da14cebeSEric Cheng mac_srs->srs_poll_cpuid_save == cpuid) { 1413da14cebeSEric Cheng mac_srs->srs_poll_cpuid_save = -1; 1414da14cebeSEric Cheng mac_srs_poll_bind(mac_srs, cpuid); 1415da14cebeSEric Cheng } 1416da14cebeSEric Cheng } 1417da14cebeSEric Cheng 1418da14cebeSEric Cheng /* Next tackle the soft rings associated with the srs */ 1419da14cebeSEric Cheng mutex_enter(&mac_srs->srs_lock); 1420da14cebeSEric Cheng for (soft_ring = mac_srs->srs_soft_ring_head; soft_ring != NULL; 1421da14cebeSEric Cheng soft_ring = soft_ring->s_ring_next) { 1422da14cebeSEric Cheng if (soft_ring->s_ring_cpuid == -1 && 1423da14cebeSEric Cheng soft_ring->s_ring_cpuid_save == cpuid) { 1424da14cebeSEric Cheng soft_ring->s_ring_cpuid_save = -1; 1425da14cebeSEric Cheng (void) mac_soft_ring_bind(soft_ring, cpuid); 1426da14cebeSEric Cheng } 1427da14cebeSEric Cheng } 1428da14cebeSEric Cheng mutex_exit(&mac_srs->srs_lock); 1429da14cebeSEric Cheng } 1430da14cebeSEric Cheng done: 1431da14cebeSEric Cheng rw_exit(&mac_srs_g_lock); 1432da14cebeSEric Cheng } 1433da14cebeSEric Cheng 1434da14cebeSEric Cheng /* 1435da14cebeSEric Cheng * Change the priority of the SRS's poll and worker thread. Additionally, 1436da14cebeSEric Cheng * update the priority of the worker threads for the SRS's soft rings. 1437da14cebeSEric Cheng * Need to modify any associated squeue threads. 1438da14cebeSEric Cheng */ 1439da14cebeSEric Cheng void 1440da14cebeSEric Cheng mac_update_srs_priority(mac_soft_ring_set_t *mac_srs, pri_t prival) 1441da14cebeSEric Cheng { 1442da14cebeSEric Cheng mac_soft_ring_t *ringp; 1443da14cebeSEric Cheng 1444da14cebeSEric Cheng mac_srs->srs_pri = prival; 1445da14cebeSEric Cheng thread_lock(mac_srs->srs_worker); 1446da14cebeSEric Cheng (void) thread_change_pri(mac_srs->srs_worker, mac_srs->srs_pri, 0); 1447da14cebeSEric Cheng thread_unlock(mac_srs->srs_worker); 1448da14cebeSEric Cheng if (mac_srs->srs_poll_thr != NULL) { 1449da14cebeSEric Cheng thread_lock(mac_srs->srs_poll_thr); 1450da14cebeSEric Cheng (void) thread_change_pri(mac_srs->srs_poll_thr, 1451da14cebeSEric Cheng mac_srs->srs_pri, 0); 1452da14cebeSEric Cheng thread_unlock(mac_srs->srs_poll_thr); 1453da14cebeSEric Cheng } 1454da14cebeSEric Cheng if ((ringp = mac_srs->srs_soft_ring_head) == NULL) 1455da14cebeSEric Cheng return; 1456da14cebeSEric Cheng while (ringp != mac_srs->srs_soft_ring_tail) { 1457da14cebeSEric Cheng thread_lock(ringp->s_ring_worker); 1458da14cebeSEric Cheng (void) thread_change_pri(ringp->s_ring_worker, 1459da14cebeSEric Cheng mac_srs->srs_pri, 0); 1460da14cebeSEric Cheng thread_unlock(ringp->s_ring_worker); 1461da14cebeSEric Cheng ringp = ringp->s_ring_next; 1462da14cebeSEric Cheng } 1463da14cebeSEric Cheng ASSERT(ringp == mac_srs->srs_soft_ring_tail); 1464da14cebeSEric Cheng thread_lock(ringp->s_ring_worker); 1465da14cebeSEric Cheng (void) thread_change_pri(ringp->s_ring_worker, mac_srs->srs_pri, 0); 1466da14cebeSEric Cheng thread_unlock(ringp->s_ring_worker); 1467da14cebeSEric Cheng } 1468da14cebeSEric Cheng 1469da14cebeSEric Cheng /* 1470da14cebeSEric Cheng * Change the receive bandwidth limit. 1471da14cebeSEric Cheng */ 1472da14cebeSEric Cheng static void 1473da14cebeSEric Cheng mac_rx_srs_update_bwlimit(mac_soft_ring_set_t *srs, mac_resource_props_t *mrp) 1474da14cebeSEric Cheng { 1475da14cebeSEric Cheng mac_soft_ring_t *softring; 1476da14cebeSEric Cheng 1477da14cebeSEric Cheng mutex_enter(&srs->srs_lock); 1478da14cebeSEric Cheng mutex_enter(&srs->srs_bw->mac_bw_lock); 1479da14cebeSEric Cheng 1480da14cebeSEric Cheng if (mrp->mrp_maxbw == MRP_MAXBW_RESETVAL) { 1481da14cebeSEric Cheng /* Reset bandwidth limit */ 1482da14cebeSEric Cheng if (srs->srs_type & SRST_BW_CONTROL) { 1483da14cebeSEric Cheng softring = srs->srs_soft_ring_head; 1484da14cebeSEric Cheng while (softring != NULL) { 1485da14cebeSEric Cheng softring->s_ring_type &= ~ST_RING_BW_CTL; 1486da14cebeSEric Cheng softring = softring->s_ring_next; 1487da14cebeSEric Cheng } 1488da14cebeSEric Cheng srs->srs_type &= ~SRST_BW_CONTROL; 1489da14cebeSEric Cheng srs->srs_drain_func = mac_rx_srs_drain; 1490da14cebeSEric Cheng } 1491da14cebeSEric Cheng } else { 1492da14cebeSEric Cheng /* Set/Modify bandwidth limit */ 1493da14cebeSEric Cheng srs->srs_bw->mac_bw_limit = FLOW_BYTES_PER_TICK(mrp->mrp_maxbw); 1494da14cebeSEric Cheng /* 1495da14cebeSEric Cheng * Give twice the queuing capability before 1496da14cebeSEric Cheng * dropping packets. The unit is bytes/tick. 1497da14cebeSEric Cheng */ 1498da14cebeSEric Cheng srs->srs_bw->mac_bw_drop_threshold = 1499da14cebeSEric Cheng srs->srs_bw->mac_bw_limit << 1; 1500da14cebeSEric Cheng if (!(srs->srs_type & SRST_BW_CONTROL)) { 1501da14cebeSEric Cheng softring = srs->srs_soft_ring_head; 1502da14cebeSEric Cheng while (softring != NULL) { 1503da14cebeSEric Cheng softring->s_ring_type |= ST_RING_BW_CTL; 1504da14cebeSEric Cheng softring = softring->s_ring_next; 1505da14cebeSEric Cheng } 1506da14cebeSEric Cheng srs->srs_type |= SRST_BW_CONTROL; 1507da14cebeSEric Cheng srs->srs_drain_func = mac_rx_srs_drain_bw; 1508da14cebeSEric Cheng } 1509da14cebeSEric Cheng } 1510da14cebeSEric Cheng done: 1511da14cebeSEric Cheng mutex_exit(&srs->srs_bw->mac_bw_lock); 1512da14cebeSEric Cheng mutex_exit(&srs->srs_lock); 1513da14cebeSEric Cheng } 1514da14cebeSEric Cheng 1515da14cebeSEric Cheng /* Change the transmit bandwidth limit */ 1516da14cebeSEric Cheng static void 1517da14cebeSEric Cheng mac_tx_srs_update_bwlimit(mac_soft_ring_set_t *srs, mac_resource_props_t *mrp) 1518da14cebeSEric Cheng { 15190dc2366fSVenugopal Iyer uint32_t tx_mode, ring_info = 0; 1520f94ede51SEric Cheng mac_srs_tx_t *srs_tx = &srs->srs_tx; 1521f94ede51SEric Cheng mac_client_impl_t *mcip = srs->srs_mcip; 1522f94ede51SEric Cheng 1523f94ede51SEric Cheng /* 1524f94ede51SEric Cheng * We need to quiesce/restart the client here because mac_tx() and 1525f94ede51SEric Cheng * srs->srs_tx->st_func do not hold srs->srs_lock while accessing 1526f94ede51SEric Cheng * st_mode and related fields, which are modified by the code below. 1527f94ede51SEric Cheng */ 15280dc2366fSVenugopal Iyer mac_tx_client_quiesce((mac_client_handle_t)mcip); 1529da14cebeSEric Cheng 1530da14cebeSEric Cheng mutex_enter(&srs->srs_lock); 1531da14cebeSEric Cheng mutex_enter(&srs->srs_bw->mac_bw_lock); 1532da14cebeSEric Cheng 1533da14cebeSEric Cheng tx_mode = srs_tx->st_mode; 1534da14cebeSEric Cheng if (mrp->mrp_maxbw == MRP_MAXBW_RESETVAL) { 1535da14cebeSEric Cheng /* Reset bandwidth limit */ 1536da14cebeSEric Cheng if (tx_mode == SRS_TX_BW) { 15370dc2366fSVenugopal Iyer if (srs_tx->st_arg2 != NULL) 15380dc2366fSVenugopal Iyer ring_info = mac_hwring_getinfo(srs_tx->st_arg2); 1539da14cebeSEric Cheng if (mac_tx_serialize || 15400dc2366fSVenugopal Iyer (ring_info & MAC_RING_TX_SERIALIZE)) { 1541da14cebeSEric Cheng srs_tx->st_mode = SRS_TX_SERIALIZE; 1542da14cebeSEric Cheng } else { 1543da14cebeSEric Cheng srs_tx->st_mode = SRS_TX_DEFAULT; 1544da14cebeSEric Cheng } 1545da14cebeSEric Cheng } else if (tx_mode == SRS_TX_BW_FANOUT) { 1546da14cebeSEric Cheng srs_tx->st_mode = SRS_TX_FANOUT; 15470dc2366fSVenugopal Iyer } else if (tx_mode == SRS_TX_BW_AGGR) { 15480dc2366fSVenugopal Iyer srs_tx->st_mode = SRS_TX_AGGR; 1549da14cebeSEric Cheng } 1550da14cebeSEric Cheng srs->srs_type &= ~SRST_BW_CONTROL; 1551da14cebeSEric Cheng } else { 1552da14cebeSEric Cheng /* Set/Modify bandwidth limit */ 1553da14cebeSEric Cheng srs->srs_bw->mac_bw_limit = FLOW_BYTES_PER_TICK(mrp->mrp_maxbw); 1554da14cebeSEric Cheng /* 1555da14cebeSEric Cheng * Give twice the queuing capability before 1556da14cebeSEric Cheng * dropping packets. The unit is bytes/tick. 1557da14cebeSEric Cheng */ 1558da14cebeSEric Cheng srs->srs_bw->mac_bw_drop_threshold = 1559da14cebeSEric Cheng srs->srs_bw->mac_bw_limit << 1; 1560da14cebeSEric Cheng srs->srs_type |= SRST_BW_CONTROL; 15610dc2366fSVenugopal Iyer if (tx_mode != SRS_TX_BW && tx_mode != SRS_TX_BW_FANOUT && 15620dc2366fSVenugopal Iyer tx_mode != SRS_TX_BW_AGGR) { 1563da14cebeSEric Cheng if (tx_mode == SRS_TX_SERIALIZE || 1564da14cebeSEric Cheng tx_mode == SRS_TX_DEFAULT) { 1565da14cebeSEric Cheng srs_tx->st_mode = SRS_TX_BW; 1566da14cebeSEric Cheng } else if (tx_mode == SRS_TX_FANOUT) { 1567da14cebeSEric Cheng srs_tx->st_mode = SRS_TX_BW_FANOUT; 15680dc2366fSVenugopal Iyer } else if (tx_mode == SRS_TX_AGGR) { 15690dc2366fSVenugopal Iyer srs_tx->st_mode = SRS_TX_BW_AGGR; 1570da14cebeSEric Cheng } else { 1571da14cebeSEric Cheng ASSERT(0); 1572da14cebeSEric Cheng } 1573da14cebeSEric Cheng } 1574da14cebeSEric Cheng } 1575da14cebeSEric Cheng done: 1576da14cebeSEric Cheng srs_tx->st_func = mac_tx_get_func(srs_tx->st_mode); 1577da14cebeSEric Cheng mutex_exit(&srs->srs_bw->mac_bw_lock); 1578da14cebeSEric Cheng mutex_exit(&srs->srs_lock); 1579f94ede51SEric Cheng 15800dc2366fSVenugopal Iyer mac_tx_client_restart((mac_client_handle_t)mcip); 1581da14cebeSEric Cheng } 1582da14cebeSEric Cheng 1583da14cebeSEric Cheng /* 1584da14cebeSEric Cheng * The uber function that deals with any update to bandwidth limits. 1585da14cebeSEric Cheng */ 1586da14cebeSEric Cheng void 1587da14cebeSEric Cheng mac_srs_update_bwlimit(flow_entry_t *flent, mac_resource_props_t *mrp) 1588da14cebeSEric Cheng { 1589da14cebeSEric Cheng int count; 1590da14cebeSEric Cheng 1591da14cebeSEric Cheng for (count = 0; count < flent->fe_rx_srs_cnt; count++) 1592da14cebeSEric Cheng mac_rx_srs_update_bwlimit(flent->fe_rx_srs[count], mrp); 1593da14cebeSEric Cheng mac_tx_srs_update_bwlimit(flent->fe_tx_srs, mrp); 1594da14cebeSEric Cheng } 1595da14cebeSEric Cheng 1596da14cebeSEric Cheng /* 1597da14cebeSEric Cheng * When the first sub-flow is added to a link, we disable polling on the 1598da14cebeSEric Cheng * link and also modify the entry point to mac_rx_srs_subflow_process. 1599da14cebeSEric Cheng * (polling is disabled because with the subflow added, accounting 1600da14cebeSEric Cheng * for polling needs additional logic, it is assumed that when a subflow is 1601da14cebeSEric Cheng * added, we can take some hit as a result of disabling polling rather than 1602da14cebeSEric Cheng * adding more complexity - if this becomes a perf. issue we need to 1603da14cebeSEric Cheng * re-rvaluate this logic). When the last subflow is removed, we turn back 1604da14cebeSEric Cheng * polling and also reset the entry point to mac_rx_srs_process. 1605da14cebeSEric Cheng * 1606da14cebeSEric Cheng * In the future if there are multiple SRS, we can simply 1607da14cebeSEric Cheng * take one and give it to the flow rather than disabling polling and 1608da14cebeSEric Cheng * resetting the entry point. 1609da14cebeSEric Cheng */ 1610da14cebeSEric Cheng void 1611da14cebeSEric Cheng mac_client_update_classifier(mac_client_impl_t *mcip, boolean_t enable) 1612da14cebeSEric Cheng { 1613da14cebeSEric Cheng flow_entry_t *flent = mcip->mci_flent; 1614da14cebeSEric Cheng int i; 1615da14cebeSEric Cheng mac_impl_t *mip = mcip->mci_mip; 1616da14cebeSEric Cheng mac_rx_func_t rx_func; 1617da14cebeSEric Cheng uint_t rx_srs_cnt; 1618da14cebeSEric Cheng boolean_t enable_classifier; 1619da14cebeSEric Cheng 1620da14cebeSEric Cheng ASSERT(MAC_PERIM_HELD((mac_handle_t)mip)); 1621da14cebeSEric Cheng 1622da14cebeSEric Cheng enable_classifier = !FLOW_TAB_EMPTY(mcip->mci_subflow_tab) && enable; 1623da14cebeSEric Cheng 1624da14cebeSEric Cheng rx_func = enable_classifier ? mac_rx_srs_subflow_process : 1625da14cebeSEric Cheng mac_rx_srs_process; 1626da14cebeSEric Cheng 16274eaa4710SRishi Srivatsavai /* Tell mac_srs_poll_state_change to disable polling if necessary */ 16284eaa4710SRishi Srivatsavai if (mip->mi_state_flags & MIS_POLL_DISABLE) 16294eaa4710SRishi Srivatsavai enable_classifier = B_TRUE; 16304eaa4710SRishi Srivatsavai 1631da14cebeSEric Cheng /* 1632da14cebeSEric Cheng * If receive function has already been configured correctly for 1633da14cebeSEric Cheng * current subflow configuration, do nothing. 1634da14cebeSEric Cheng */ 1635da14cebeSEric Cheng if (flent->fe_cb_fn == (flow_fn_t)rx_func) 1636da14cebeSEric Cheng return; 1637da14cebeSEric Cheng 1638da14cebeSEric Cheng rx_srs_cnt = flent->fe_rx_srs_cnt; 1639da14cebeSEric Cheng for (i = 0; i < rx_srs_cnt; i++) { 1640da14cebeSEric Cheng ASSERT(flent->fe_rx_srs[i] != NULL); 1641da14cebeSEric Cheng mac_srs_poll_state_change(flent->fe_rx_srs[i], 1642da14cebeSEric Cheng enable_classifier, rx_func); 1643da14cebeSEric Cheng } 1644da14cebeSEric Cheng 1645da14cebeSEric Cheng /* 1646da14cebeSEric Cheng * Change the S/W classifier so that we can land in the 1647da14cebeSEric Cheng * correct processing function with correct argument. 1648da14cebeSEric Cheng * If all subflows have been removed we can revert to 1649da14cebeSEric Cheng * mac_rx_srsprocess, else we need mac_rx_srs_subflow_process. 1650da14cebeSEric Cheng */ 1651da14cebeSEric Cheng mutex_enter(&flent->fe_lock); 1652da14cebeSEric Cheng flent->fe_cb_fn = (flow_fn_t)rx_func; 1653da14cebeSEric Cheng flent->fe_cb_arg1 = (void *)mip; 1654da14cebeSEric Cheng flent->fe_cb_arg2 = flent->fe_rx_srs[0]; 1655da14cebeSEric Cheng mutex_exit(&flent->fe_lock); 1656da14cebeSEric Cheng } 1657da14cebeSEric Cheng 1658da14cebeSEric Cheng static void 1659da14cebeSEric Cheng mac_srs_update_fanout_list(mac_soft_ring_set_t *mac_srs) 1660da14cebeSEric Cheng { 16610dc2366fSVenugopal Iyer int tcp_count = 0, udp_count = 0, oth_count = 0, tx_count = 0; 1662da14cebeSEric Cheng mac_soft_ring_t *softring; 1663da14cebeSEric Cheng 1664da14cebeSEric Cheng softring = mac_srs->srs_soft_ring_head; 1665da14cebeSEric Cheng if (softring == NULL) { 1666da14cebeSEric Cheng ASSERT(mac_srs->srs_soft_ring_count == 0); 1667da14cebeSEric Cheng mac_srs->srs_tcp_ring_count = 0; 1668da14cebeSEric Cheng mac_srs->srs_udp_ring_count = 0; 1669da14cebeSEric Cheng mac_srs->srs_oth_ring_count = 0; 16700dc2366fSVenugopal Iyer mac_srs->srs_tx_ring_count = 0; 1671da14cebeSEric Cheng return; 1672da14cebeSEric Cheng } 1673da14cebeSEric Cheng 1674da14cebeSEric Cheng while (softring != NULL) { 16750dc2366fSVenugopal Iyer if (softring->s_ring_type & ST_RING_TCP) { 1676da14cebeSEric Cheng mac_srs->srs_tcp_soft_rings[tcp_count++] = softring; 16770dc2366fSVenugopal Iyer } else if (softring->s_ring_type & ST_RING_UDP) { 1678da14cebeSEric Cheng mac_srs->srs_udp_soft_rings[udp_count++] = softring; 16790dc2366fSVenugopal Iyer } else if (softring->s_ring_type & ST_RING_OTH) { 1680da14cebeSEric Cheng mac_srs->srs_oth_soft_rings[oth_count++] = softring; 16810dc2366fSVenugopal Iyer } else { 16820dc2366fSVenugopal Iyer ASSERT(softring->s_ring_type & ST_RING_TX); 16830dc2366fSVenugopal Iyer mac_srs->srs_tx_soft_rings[tx_count++] = softring; 16840dc2366fSVenugopal Iyer } 1685da14cebeSEric Cheng softring = softring->s_ring_next; 1686da14cebeSEric Cheng } 1687da14cebeSEric Cheng 1688da14cebeSEric Cheng ASSERT(mac_srs->srs_soft_ring_count == 16890dc2366fSVenugopal Iyer (tcp_count + udp_count + oth_count + tx_count)); 1690da14cebeSEric Cheng mac_srs->srs_tcp_ring_count = tcp_count; 1691da14cebeSEric Cheng mac_srs->srs_udp_ring_count = udp_count; 1692da14cebeSEric Cheng mac_srs->srs_oth_ring_count = oth_count; 16930dc2366fSVenugopal Iyer mac_srs->srs_tx_ring_count = tx_count; 1694da14cebeSEric Cheng } 1695da14cebeSEric Cheng 1696da14cebeSEric Cheng void 16970dc2366fSVenugopal Iyer mac_srs_create_proto_softrings(int id, uint16_t type, pri_t pri, 16980dc2366fSVenugopal Iyer mac_client_impl_t *mcip, mac_soft_ring_set_t *mac_srs, 1699da14cebeSEric Cheng processorid_t cpuid, mac_direct_rx_t rx_func, void *x_arg1, 1700da14cebeSEric Cheng mac_resource_handle_t x_arg2, boolean_t set_bypass) 1701da14cebeSEric Cheng { 1702da14cebeSEric Cheng mac_soft_ring_t *softring; 1703da14cebeSEric Cheng mac_rx_fifo_t mrf; 1704da14cebeSEric Cheng 1705da14cebeSEric Cheng bzero(&mrf, sizeof (mac_rx_fifo_t)); 1706da14cebeSEric Cheng mrf.mrf_type = MAC_RX_FIFO; 1707da14cebeSEric Cheng mrf.mrf_receive = (mac_receive_t)mac_soft_ring_poll; 1708*2c465844SToomas Soome mrf.mrf_intr_enable = (mac_intr_enable_t)mac_soft_ring_intr_enable; 1709*2c465844SToomas Soome mrf.mrf_intr_disable = (mac_intr_disable_t)mac_soft_ring_intr_disable; 1710da14cebeSEric Cheng mrf.mrf_flow_priority = pri; 1711da14cebeSEric Cheng 1712da14cebeSEric Cheng softring = mac_soft_ring_create(id, mac_soft_ring_worker_wait, 17130dc2366fSVenugopal Iyer (type|ST_RING_TCP), pri, mcip, mac_srs, 1714da14cebeSEric Cheng cpuid, rx_func, x_arg1, x_arg2); 1715da14cebeSEric Cheng softring->s_ring_rx_arg2 = NULL; 1716da14cebeSEric Cheng 1717da14cebeSEric Cheng /* 1718da14cebeSEric Cheng * TCP and UDP support DLS bypass. In addition TCP 1719da14cebeSEric Cheng * squeue can also poll their corresponding soft rings. 1720da14cebeSEric Cheng */ 1721da14cebeSEric Cheng if (set_bypass && (mcip->mci_resource_arg != NULL)) { 1722da14cebeSEric Cheng mac_soft_ring_dls_bypass(softring, 1723da14cebeSEric Cheng mcip->mci_direct_rx_fn, 1724da14cebeSEric Cheng mcip->mci_direct_rx_arg); 1725da14cebeSEric Cheng 1726da14cebeSEric Cheng mrf.mrf_rx_arg = softring; 1727da14cebeSEric Cheng mrf.mrf_intr_handle = (mac_intr_handle_t)softring; 1728da14cebeSEric Cheng 1729da14cebeSEric Cheng /* 1730da14cebeSEric Cheng * Make a call in IP to get a TCP squeue assigned to 1731da14cebeSEric Cheng * this softring to maintain full CPU locality through 1732da14cebeSEric Cheng * the stack and allow the squeue to be able to poll 1733da14cebeSEric Cheng * the softring so the flow control can be pushed 1734da14cebeSEric Cheng * all the way to H/W. 1735da14cebeSEric Cheng */ 1736da14cebeSEric Cheng softring->s_ring_rx_arg2 = 1737da14cebeSEric Cheng mcip->mci_resource_add((void *)mcip->mci_resource_arg, 1738da14cebeSEric Cheng (mac_resource_t *)&mrf); 1739da14cebeSEric Cheng } 1740da14cebeSEric Cheng 1741da14cebeSEric Cheng /* 1742da14cebeSEric Cheng * Non-TCP protocols don't support squeues. Hence we 1743da14cebeSEric Cheng * don't make any ring addition callbacks for non-TCP 1744da14cebeSEric Cheng * rings. Now create the UDP softring and allow it to 1745da14cebeSEric Cheng * bypass the DLS layer. 1746da14cebeSEric Cheng */ 1747da14cebeSEric Cheng softring = mac_soft_ring_create(id, mac_soft_ring_worker_wait, 17480dc2366fSVenugopal Iyer (type|ST_RING_UDP), pri, mcip, mac_srs, 1749da14cebeSEric Cheng cpuid, rx_func, x_arg1, x_arg2); 1750da14cebeSEric Cheng softring->s_ring_rx_arg2 = NULL; 1751da14cebeSEric Cheng 1752da14cebeSEric Cheng if (set_bypass && (mcip->mci_resource_arg != NULL)) { 1753da14cebeSEric Cheng mac_soft_ring_dls_bypass(softring, 1754da14cebeSEric Cheng mcip->mci_direct_rx_fn, 1755da14cebeSEric Cheng mcip->mci_direct_rx_arg); 1756da14cebeSEric Cheng } 1757da14cebeSEric Cheng 1758da14cebeSEric Cheng /* Create the Oth softrings which has to go through the DLS */ 1759da14cebeSEric Cheng softring = mac_soft_ring_create(id, mac_soft_ring_worker_wait, 17600dc2366fSVenugopal Iyer (type|ST_RING_OTH), pri, mcip, mac_srs, 1761da14cebeSEric Cheng cpuid, rx_func, x_arg1, x_arg2); 1762da14cebeSEric Cheng softring->s_ring_rx_arg2 = NULL; 1763da14cebeSEric Cheng } 1764da14cebeSEric Cheng 1765da14cebeSEric Cheng /* 1766da14cebeSEric Cheng * This routine associates a CPU or a set of CPU to process incoming 1767da14cebeSEric Cheng * traffic from a mac client. If multiple CPUs are specified, then 1768da14cebeSEric Cheng * so many soft rings are created with each soft ring worker thread 1769da14cebeSEric Cheng * bound to a CPU in the set. Each soft ring in turn will be 1770da14cebeSEric Cheng * associated with an squeue and the squeue will be moved to the 1771da14cebeSEric Cheng * same CPU as that of the soft ring's. 1772da14cebeSEric Cheng */ 1773da14cebeSEric Cheng static void 17740dc2366fSVenugopal Iyer mac_srs_fanout_modify(mac_client_impl_t *mcip, mac_direct_rx_t rx_func, 17750dc2366fSVenugopal Iyer void *x_arg1, mac_resource_handle_t x_arg2, 17760dc2366fSVenugopal Iyer mac_soft_ring_set_t *mac_rx_srs, mac_soft_ring_set_t *mac_tx_srs) 1777da14cebeSEric Cheng { 1778da14cebeSEric Cheng mac_soft_ring_t *softring; 1779ae6aa22aSVenugopal Iyer uint32_t soft_ring_flag = 0; 1780da14cebeSEric Cheng processorid_t cpuid = -1; 1781da14cebeSEric Cheng int i, srings_present, new_fanout_cnt; 1782da14cebeSEric Cheng mac_cpus_t *srs_cpu; 1783da14cebeSEric Cheng 1784da14cebeSEric Cheng /* fanout state is REINIT. Set it back to INIT */ 1785da14cebeSEric Cheng ASSERT(mac_rx_srs->srs_fanout_state == SRS_FANOUT_REINIT); 1786da14cebeSEric Cheng mac_rx_srs->srs_fanout_state = SRS_FANOUT_INIT; 1787da14cebeSEric Cheng 1788da14cebeSEric Cheng /* how many are present right now */ 1789da14cebeSEric Cheng srings_present = mac_rx_srs->srs_tcp_ring_count; 1790da14cebeSEric Cheng /* new request */ 1791da14cebeSEric Cheng srs_cpu = &mac_rx_srs->srs_cpu; 17920dc2366fSVenugopal Iyer new_fanout_cnt = srs_cpu->mc_rx_fanout_cnt; 1793da14cebeSEric Cheng 1794da14cebeSEric Cheng mutex_enter(&mac_rx_srs->srs_lock); 1795da14cebeSEric Cheng if (mac_rx_srs->srs_type & SRST_BW_CONTROL) 1796da14cebeSEric Cheng soft_ring_flag |= ST_RING_BW_CTL; 1797da14cebeSEric Cheng mutex_exit(&mac_rx_srs->srs_lock); 1798da14cebeSEric Cheng 1799da14cebeSEric Cheng if (new_fanout_cnt > srings_present) { 1800da14cebeSEric Cheng /* soft rings increased */ 1801da14cebeSEric Cheng mutex_enter(&mac_rx_srs->srs_lock); 1802da14cebeSEric Cheng mac_rx_srs->srs_type |= SRST_FANOUT_SRC_IP; 1803da14cebeSEric Cheng mutex_exit(&mac_rx_srs->srs_lock); 1804da14cebeSEric Cheng 1805da14cebeSEric Cheng for (i = mac_rx_srs->srs_tcp_ring_count; 1806da14cebeSEric Cheng i < new_fanout_cnt; i++) { 1807da14cebeSEric Cheng /* 1808da14cebeSEric Cheng * Create the protocol softrings and set the 1809da14cebeSEric Cheng * DLS bypass where possible. 1810da14cebeSEric Cheng */ 18110dc2366fSVenugopal Iyer mac_srs_create_proto_softrings(i, soft_ring_flag, 1812da14cebeSEric Cheng mac_rx_srs->srs_pri, mcip, mac_rx_srs, cpuid, 1813da14cebeSEric Cheng rx_func, x_arg1, x_arg2, B_TRUE); 1814da14cebeSEric Cheng } 1815da14cebeSEric Cheng mac_srs_update_fanout_list(mac_rx_srs); 1816da14cebeSEric Cheng } else if (new_fanout_cnt < srings_present) { 1817da14cebeSEric Cheng /* soft rings decreased */ 1818da14cebeSEric Cheng if (new_fanout_cnt == 1) { 1819da14cebeSEric Cheng mutex_enter(&mac_rx_srs->srs_lock); 1820da14cebeSEric Cheng mac_rx_srs->srs_type &= ~SRST_FANOUT_SRC_IP; 1821da14cebeSEric Cheng ASSERT(mac_rx_srs->srs_type & SRST_FANOUT_PROTO); 1822da14cebeSEric Cheng mutex_exit(&mac_rx_srs->srs_lock); 1823da14cebeSEric Cheng } 1824da14cebeSEric Cheng /* Get rid of extra soft rings */ 1825da14cebeSEric Cheng for (i = new_fanout_cnt; 1826da14cebeSEric Cheng i < mac_rx_srs->srs_tcp_ring_count; i++) { 1827da14cebeSEric Cheng softring = mac_rx_srs->srs_tcp_soft_rings[i]; 1828da14cebeSEric Cheng if (softring->s_ring_rx_arg2 != NULL) { 1829da14cebeSEric Cheng mcip->mci_resource_remove( 1830da14cebeSEric Cheng (void *)mcip->mci_resource_arg, 1831da14cebeSEric Cheng softring->s_ring_rx_arg2); 1832da14cebeSEric Cheng } 1833da14cebeSEric Cheng mac_soft_ring_remove(mac_rx_srs, 1834da14cebeSEric Cheng mac_rx_srs->srs_tcp_soft_rings[i]); 1835da14cebeSEric Cheng mac_soft_ring_remove(mac_rx_srs, 1836da14cebeSEric Cheng mac_rx_srs->srs_udp_soft_rings[i]); 1837da14cebeSEric Cheng mac_soft_ring_remove(mac_rx_srs, 1838da14cebeSEric Cheng mac_rx_srs->srs_oth_soft_rings[i]); 1839da14cebeSEric Cheng } 1840da14cebeSEric Cheng mac_srs_update_fanout_list(mac_rx_srs); 1841da14cebeSEric Cheng } 1842da14cebeSEric Cheng 1843da14cebeSEric Cheng ASSERT(new_fanout_cnt == mac_rx_srs->srs_tcp_ring_count); 1844da14cebeSEric Cheng mutex_enter(&cpu_lock); 1845da14cebeSEric Cheng for (i = 0; i < mac_rx_srs->srs_tcp_ring_count; i++) { 18460dc2366fSVenugopal Iyer cpuid = srs_cpu->mc_rx_fanout_cpus[i]; 1847da14cebeSEric Cheng (void) mac_soft_ring_bind(mac_rx_srs->srs_udp_soft_rings[i], 1848da14cebeSEric Cheng cpuid); 1849da14cebeSEric Cheng (void) mac_soft_ring_bind(mac_rx_srs->srs_oth_soft_rings[i], 1850da14cebeSEric Cheng cpuid); 1851da14cebeSEric Cheng (void) mac_soft_ring_bind(mac_rx_srs->srs_tcp_soft_rings[i], 1852da14cebeSEric Cheng cpuid); 1853da14cebeSEric Cheng softring = mac_rx_srs->srs_tcp_soft_rings[i]; 1854da14cebeSEric Cheng if (softring->s_ring_rx_arg2 != NULL) { 1855da14cebeSEric Cheng mcip->mci_resource_bind((void *)mcip->mci_resource_arg, 1856da14cebeSEric Cheng softring->s_ring_rx_arg2, cpuid); 1857da14cebeSEric Cheng } 1858da14cebeSEric Cheng } 1859da14cebeSEric Cheng 18600dc2366fSVenugopal Iyer mac_srs_worker_bind(mac_rx_srs, srs_cpu->mc_rx_workerid); 18610dc2366fSVenugopal Iyer mac_srs_poll_bind(mac_rx_srs, srs_cpu->mc_rx_pollid); 18620dc2366fSVenugopal Iyer mac_rx_srs_retarget_intr(mac_rx_srs, srs_cpu->mc_rx_intr_cpu); 1863da14cebeSEric Cheng /* 1864da14cebeSEric Cheng * Bind Tx srs and soft ring threads too. Let's bind tx 1865da14cebeSEric Cheng * srs to the last cpu in mrp list. 1866da14cebeSEric Cheng */ 18670dc2366fSVenugopal Iyer if (mac_tx_srs != NULL) { 1868da14cebeSEric Cheng BIND_TX_SRS_AND_SOFT_RINGS(mac_tx_srs, mrp); 18690dc2366fSVenugopal Iyer mac_tx_srs_retarget_intr(mac_tx_srs); 1870da14cebeSEric Cheng } 1871da14cebeSEric Cheng mutex_exit(&cpu_lock); 1872da14cebeSEric Cheng } 1873da14cebeSEric Cheng 1874da14cebeSEric Cheng /* 1875da14cebeSEric Cheng * Bind SRS threads and soft rings to CPUs/create fanout list. 1876da14cebeSEric Cheng */ 1877da14cebeSEric Cheng void 18780dc2366fSVenugopal Iyer mac_srs_fanout_init(mac_client_impl_t *mcip, mac_resource_props_t *mrp, 18790dc2366fSVenugopal Iyer mac_direct_rx_t rx_func, void *x_arg1, mac_resource_handle_t x_arg2, 18800dc2366fSVenugopal Iyer mac_soft_ring_set_t *mac_rx_srs, mac_soft_ring_set_t *mac_tx_srs, 18810dc2366fSVenugopal Iyer cpupart_t *cpupart) 1882da14cebeSEric Cheng { 1883da14cebeSEric Cheng int i; 18840dc2366fSVenugopal Iyer processorid_t cpuid; 1885ae6aa22aSVenugopal Iyer uint32_t soft_ring_flag = 0; 1886da14cebeSEric Cheng int soft_ring_cnt; 1887da14cebeSEric Cheng mac_cpus_t *srs_cpu = &mac_rx_srs->srs_cpu; 1888da14cebeSEric Cheng 1889da14cebeSEric Cheng /* 1890da14cebeSEric Cheng * Remove the no soft ring flag and we will adjust it 1891da14cebeSEric Cheng * appropriately further down. 1892da14cebeSEric Cheng */ 1893da14cebeSEric Cheng mutex_enter(&mac_rx_srs->srs_lock); 1894da14cebeSEric Cheng mac_rx_srs->srs_type &= ~SRST_NO_SOFT_RINGS; 1895da14cebeSEric Cheng mutex_exit(&mac_rx_srs->srs_lock); 1896da14cebeSEric Cheng 1897da14cebeSEric Cheng ASSERT(mac_rx_srs->srs_soft_ring_head == NULL); 1898da14cebeSEric Cheng 1899da14cebeSEric Cheng if (mac_rx_srs->srs_type & SRST_BW_CONTROL) 1900da14cebeSEric Cheng soft_ring_flag |= ST_RING_BW_CTL; 1901da14cebeSEric Cheng 1902da14cebeSEric Cheng ASSERT(mac_rx_srs->srs_fanout_state == SRS_FANOUT_UNINIT); 1903da14cebeSEric Cheng mac_rx_srs->srs_fanout_state = SRS_FANOUT_INIT; 1904da14cebeSEric Cheng /* 1905da14cebeSEric Cheng * Ring count can be 0 if no fanout is required and no cpu 1906da14cebeSEric Cheng * were specified. Leave the SRS worker and poll thread 1907da14cebeSEric Cheng * unbound 1908da14cebeSEric Cheng */ 1909da14cebeSEric Cheng ASSERT(mrp != NULL); 19100dc2366fSVenugopal Iyer soft_ring_cnt = srs_cpu->mc_rx_fanout_cnt; 1911da14cebeSEric Cheng 1912da14cebeSEric Cheng /* Step 1: bind cpu contains cpu list where threads need to bind */ 1913da14cebeSEric Cheng if (soft_ring_cnt > 0) { 1914da14cebeSEric Cheng mutex_enter(&cpu_lock); 1915da14cebeSEric Cheng for (i = 0; i < soft_ring_cnt; i++) { 19160dc2366fSVenugopal Iyer cpuid = srs_cpu->mc_rx_fanout_cpus[i]; 1917da14cebeSEric Cheng /* Create the protocol softrings */ 19180dc2366fSVenugopal Iyer mac_srs_create_proto_softrings(i, soft_ring_flag, 19190dc2366fSVenugopal Iyer mac_rx_srs->srs_pri, mcip, mac_rx_srs, cpuid, 19200dc2366fSVenugopal Iyer rx_func, x_arg1, x_arg2, B_FALSE); 1921da14cebeSEric Cheng } 19220dc2366fSVenugopal Iyer mac_srs_worker_bind(mac_rx_srs, srs_cpu->mc_rx_workerid); 19230dc2366fSVenugopal Iyer mac_srs_poll_bind(mac_rx_srs, srs_cpu->mc_rx_pollid); 19240dc2366fSVenugopal Iyer mac_rx_srs_retarget_intr(mac_rx_srs, srs_cpu->mc_rx_intr_cpu); 1925da14cebeSEric Cheng /* 1926da14cebeSEric Cheng * Bind Tx srs and soft ring threads too. 1927da14cebeSEric Cheng * Let's bind tx srs to the last cpu in 1928da14cebeSEric Cheng * mrp list. 1929da14cebeSEric Cheng */ 1930da14cebeSEric Cheng if (mac_tx_srs == NULL) { 1931da14cebeSEric Cheng mutex_exit(&cpu_lock); 1932da14cebeSEric Cheng goto alldone; 1933da14cebeSEric Cheng } 1934da14cebeSEric Cheng 1935da14cebeSEric Cheng BIND_TX_SRS_AND_SOFT_RINGS(mac_tx_srs, mrp); 19360dc2366fSVenugopal Iyer mac_tx_srs_retarget_intr(mac_tx_srs); 1937da14cebeSEric Cheng mutex_exit(&cpu_lock); 1938da14cebeSEric Cheng } else { 1939da14cebeSEric Cheng mutex_enter(&cpu_lock); 1940da14cebeSEric Cheng /* 1941da14cebeSEric Cheng * For a subflow, mrp_workerid and mrp_pollid 1942da14cebeSEric Cheng * is not set. 1943da14cebeSEric Cheng */ 19440dc2366fSVenugopal Iyer mac_srs_worker_bind(mac_rx_srs, mrp->mrp_rx_workerid); 19450dc2366fSVenugopal Iyer mac_srs_poll_bind(mac_rx_srs, mrp->mrp_rx_pollid); 1946da14cebeSEric Cheng mutex_exit(&cpu_lock); 1947da14cebeSEric Cheng goto no_softrings; 1948da14cebeSEric Cheng } 1949da14cebeSEric Cheng 1950da14cebeSEric Cheng alldone: 1951da14cebeSEric Cheng if (soft_ring_cnt > 1) 1952da14cebeSEric Cheng mac_rx_srs->srs_type |= SRST_FANOUT_SRC_IP; 1953da14cebeSEric Cheng mac_srs_update_fanout_list(mac_rx_srs); 1954da14cebeSEric Cheng mac_srs_client_poll_enable(mcip, mac_rx_srs); 1955da14cebeSEric Cheng return; 1956da14cebeSEric Cheng 1957da14cebeSEric Cheng no_softrings: 1958da14cebeSEric Cheng if (mac_rx_srs->srs_type & SRST_FANOUT_PROTO) { 1959da14cebeSEric Cheng mutex_enter(&cpu_lock); 19600dc2366fSVenugopal Iyer cpuid = mac_next_bind_cpu(cpupart); 1961da14cebeSEric Cheng /* Create the protocol softrings */ 19620dc2366fSVenugopal Iyer mac_srs_create_proto_softrings(0, soft_ring_flag, 19630dc2366fSVenugopal Iyer mac_rx_srs->srs_pri, mcip, mac_rx_srs, cpuid, 19640dc2366fSVenugopal Iyer rx_func, x_arg1, x_arg2, B_FALSE); 1965da14cebeSEric Cheng mutex_exit(&cpu_lock); 1966da14cebeSEric Cheng } else { 1967da14cebeSEric Cheng /* 1968da14cebeSEric Cheng * This is the case when there is no fanout which is 1969da14cebeSEric Cheng * true for subflows. 1970da14cebeSEric Cheng */ 1971da14cebeSEric Cheng mac_rx_srs->srs_type |= SRST_NO_SOFT_RINGS; 1972da14cebeSEric Cheng } 1973da14cebeSEric Cheng mac_srs_update_fanout_list(mac_rx_srs); 1974da14cebeSEric Cheng mac_srs_client_poll_enable(mcip, mac_rx_srs); 1975da14cebeSEric Cheng } 1976da14cebeSEric Cheng 1977da14cebeSEric Cheng /* 1978da14cebeSEric Cheng * mac_fanout_setup: 1979da14cebeSEric Cheng * 1980da14cebeSEric Cheng * Calls mac_srs_fanout_init() or modify() depending upon whether 1981da14cebeSEric Cheng * the SRS is getting initialized or re-initialized. 1982da14cebeSEric Cheng */ 1983da14cebeSEric Cheng void 1984da14cebeSEric Cheng mac_fanout_setup(mac_client_impl_t *mcip, flow_entry_t *flent, 1985da14cebeSEric Cheng mac_resource_props_t *mrp, mac_direct_rx_t rx_func, void *x_arg1, 19860dc2366fSVenugopal Iyer mac_resource_handle_t x_arg2, cpupart_t *cpupart) 1987da14cebeSEric Cheng { 1988da14cebeSEric Cheng mac_soft_ring_set_t *mac_rx_srs, *mac_tx_srs; 1989da14cebeSEric Cheng int i, rx_srs_cnt; 1990da14cebeSEric Cheng 1991da14cebeSEric Cheng ASSERT(MAC_PERIM_HELD((mac_handle_t)mcip->mci_mip)); 1992da14cebeSEric Cheng /* 1993da14cebeSEric Cheng * This is an aggregation port. Fanout will be setup 1994da14cebeSEric Cheng * over the aggregation itself. 1995da14cebeSEric Cheng */ 19960dc2366fSVenugopal Iyer if (mcip->mci_state_flags & MCIS_EXCLUSIVE) 1997da14cebeSEric Cheng return; 1998da14cebeSEric Cheng 1999da14cebeSEric Cheng mac_rx_srs = flent->fe_rx_srs[0]; 2000da14cebeSEric Cheng /* 2001da14cebeSEric Cheng * Set up the fanout on the tx side only once, with the 2002da14cebeSEric Cheng * first rx SRS. The CPU binding, fanout, and bandwidth 2003da14cebeSEric Cheng * criteria are common to both RX and TX, so 2004da14cebeSEric Cheng * initializing them along side avoids redundant code. 2005da14cebeSEric Cheng */ 2006da14cebeSEric Cheng mac_tx_srs = flent->fe_tx_srs; 2007da14cebeSEric Cheng rx_srs_cnt = flent->fe_rx_srs_cnt; 2008da14cebeSEric Cheng 2009da14cebeSEric Cheng /* No fanout for subflows */ 2010da14cebeSEric Cheng if (flent->fe_type & FLOW_USER) { 20110dc2366fSVenugopal Iyer mac_srs_fanout_init(mcip, mrp, rx_func, 20120dc2366fSVenugopal Iyer x_arg1, x_arg2, mac_rx_srs, mac_tx_srs, 20130dc2366fSVenugopal Iyer cpupart); 2014da14cebeSEric Cheng return; 2015da14cebeSEric Cheng } 2016da14cebeSEric Cheng 20170dc2366fSVenugopal Iyer if (mrp->mrp_mask & MRP_CPUS_USERSPEC) 20180dc2366fSVenugopal Iyer mac_flow_user_cpu_init(flent, mrp); 20190dc2366fSVenugopal Iyer else 20200dc2366fSVenugopal Iyer mac_flow_cpu_init(flent, cpupart); 20210dc2366fSVenugopal Iyer 20220dc2366fSVenugopal Iyer mrp->mrp_rx_fanout_cnt = mac_rx_srs->srs_cpu.mc_rx_fanout_cnt; 2023da14cebeSEric Cheng 2024da14cebeSEric Cheng /* 2025da14cebeSEric Cheng * Set up fanout for both SW (0th SRS) and HW classified 2026da14cebeSEric Cheng * SRS (the rest of Rx SRSs in flent). 2027da14cebeSEric Cheng */ 2028da14cebeSEric Cheng for (i = 0; i < rx_srs_cnt; i++) { 2029da14cebeSEric Cheng mac_rx_srs = flent->fe_rx_srs[i]; 2030da14cebeSEric Cheng if (i != 0) 2031da14cebeSEric Cheng mac_tx_srs = NULL; 2032da14cebeSEric Cheng switch (mac_rx_srs->srs_fanout_state) { 2033da14cebeSEric Cheng case SRS_FANOUT_UNINIT: 20340dc2366fSVenugopal Iyer mac_srs_fanout_init(mcip, mrp, rx_func, 20350dc2366fSVenugopal Iyer x_arg1, x_arg2, mac_rx_srs, mac_tx_srs, 20360dc2366fSVenugopal Iyer cpupart); 2037da14cebeSEric Cheng break; 2038da14cebeSEric Cheng case SRS_FANOUT_INIT: 2039da14cebeSEric Cheng break; 2040da14cebeSEric Cheng case SRS_FANOUT_REINIT: 2041da14cebeSEric Cheng mac_rx_srs_quiesce(mac_rx_srs, SRS_QUIESCE); 20420dc2366fSVenugopal Iyer mac_srs_fanout_modify(mcip, rx_func, x_arg1, 20430dc2366fSVenugopal Iyer x_arg2, mac_rx_srs, mac_tx_srs); 2044da14cebeSEric Cheng mac_rx_srs_restart(mac_rx_srs); 2045da14cebeSEric Cheng break; 2046da14cebeSEric Cheng default: 2047da14cebeSEric Cheng VERIFY(mac_rx_srs->srs_fanout_state <= 2048da14cebeSEric Cheng SRS_FANOUT_REINIT); 2049da14cebeSEric Cheng break; 2050da14cebeSEric Cheng } 2051da14cebeSEric Cheng } 2052da14cebeSEric Cheng } 2053da14cebeSEric Cheng 2054da14cebeSEric Cheng /* 20550dc2366fSVenugopal Iyer * mac_srs_create: 2056da14cebeSEric Cheng * 2057da14cebeSEric Cheng * Create a mac_soft_ring_set_t (SRS). If soft_ring_fanout_type is 2058da14cebeSEric Cheng * SRST_TX, an SRS for Tx side is created. Otherwise an SRS for Rx side 2059da14cebeSEric Cheng * processing is created. 2060da14cebeSEric Cheng * 2061da14cebeSEric Cheng * Details on Rx SRS: 2062da14cebeSEric Cheng * Create a SRS and also add the necessary soft rings for TCP and 2063da14cebeSEric Cheng * non-TCP based on fanout type and count specified. 2064da14cebeSEric Cheng * 2065da14cebeSEric Cheng * mac_soft_ring_fanout, mac_srs_fanout_modify (?), 2066da14cebeSEric Cheng * mac_soft_ring_stop_workers, mac_soft_ring_set_destroy, etc need 2067da14cebeSEric Cheng * to be heavily modified. 2068da14cebeSEric Cheng * 2069da14cebeSEric Cheng * mi_soft_ring_list_size, mi_soft_ring_size, etc need to disappear. 2070da14cebeSEric Cheng */ 2071da14cebeSEric Cheng mac_soft_ring_set_t * 2072da14cebeSEric Cheng mac_srs_create(mac_client_impl_t *mcip, flow_entry_t *flent, uint32_t srs_type, 2073da14cebeSEric Cheng mac_direct_rx_t rx_func, void *x_arg1, mac_resource_handle_t x_arg2, 2074da14cebeSEric Cheng mac_ring_t *ring) 2075da14cebeSEric Cheng { 2076da14cebeSEric Cheng mac_soft_ring_set_t *mac_srs; 2077da14cebeSEric Cheng mac_srs_rx_t *srs_rx; 2078da14cebeSEric Cheng mac_srs_tx_t *srs_tx; 2079da14cebeSEric Cheng mac_bw_ctl_t *mac_bw; 2080da14cebeSEric Cheng mac_resource_props_t *mrp; 2081da14cebeSEric Cheng boolean_t is_tx_srs = ((srs_type & SRST_TX) != 0); 2082da14cebeSEric Cheng 2083da14cebeSEric Cheng mac_srs = kmem_cache_alloc(mac_srs_cache, KM_SLEEP); 2084da14cebeSEric Cheng bzero(mac_srs, sizeof (mac_soft_ring_set_t)); 2085da14cebeSEric Cheng srs_rx = &mac_srs->srs_rx; 2086da14cebeSEric Cheng srs_tx = &mac_srs->srs_tx; 2087da14cebeSEric Cheng 2088da14cebeSEric Cheng mutex_enter(&flent->fe_lock); 2089da14cebeSEric Cheng 2090da14cebeSEric Cheng /* 2091da14cebeSEric Cheng * Get the bandwidth control structure from the flent. Get 2092da14cebeSEric Cheng * rid of any residual values in the control structure for 2093da14cebeSEric Cheng * the tx bw struct and also for the rx, if the rx srs is 2094da14cebeSEric Cheng * the 1st one being brought up (the rx bw ctl struct may 2095da14cebeSEric Cheng * be shared by multiple SRSs) 2096da14cebeSEric Cheng */ 2097da14cebeSEric Cheng if (is_tx_srs) { 2098da14cebeSEric Cheng mac_srs->srs_bw = &flent->fe_tx_bw; 2099da14cebeSEric Cheng bzero(mac_srs->srs_bw, sizeof (mac_bw_ctl_t)); 2100da14cebeSEric Cheng flent->fe_tx_srs = mac_srs; 2101da14cebeSEric Cheng } else { 2102da14cebeSEric Cheng /* 2103da14cebeSEric Cheng * The bw counter (stored in the flent) is shared 2104da14cebeSEric Cheng * by SRS's within an rx group. 2105da14cebeSEric Cheng */ 2106da14cebeSEric Cheng mac_srs->srs_bw = &flent->fe_rx_bw; 2107da14cebeSEric Cheng /* First rx SRS, clear the bw structure */ 2108da14cebeSEric Cheng if (flent->fe_rx_srs_cnt == 0) 2109da14cebeSEric Cheng bzero(mac_srs->srs_bw, sizeof (mac_bw_ctl_t)); 211077e3ef6dSVenugopal Iyer 211177e3ef6dSVenugopal Iyer /* 211277e3ef6dSVenugopal Iyer * It is better to panic here rather than just assert because 211377e3ef6dSVenugopal Iyer * on a non-debug kernel we might end up courrupting memory 211477e3ef6dSVenugopal Iyer * and making it difficult to debug. 211577e3ef6dSVenugopal Iyer */ 211677e3ef6dSVenugopal Iyer if (flent->fe_rx_srs_cnt >= MAX_RINGS_PER_GROUP) { 211777e3ef6dSVenugopal Iyer panic("Array Overrun detected due to MAC client %p " 211877e3ef6dSVenugopal Iyer " having more rings than %d", (void *)mcip, 211977e3ef6dSVenugopal Iyer MAX_RINGS_PER_GROUP); 212077e3ef6dSVenugopal Iyer } 2121da14cebeSEric Cheng flent->fe_rx_srs[flent->fe_rx_srs_cnt] = mac_srs; 2122da14cebeSEric Cheng flent->fe_rx_srs_cnt++; 2123da14cebeSEric Cheng } 2124da14cebeSEric Cheng mac_srs->srs_flent = flent; 2125da14cebeSEric Cheng mutex_exit(&flent->fe_lock); 2126da14cebeSEric Cheng 2127da14cebeSEric Cheng mac_srs->srs_state = 0; 2128da14cebeSEric Cheng mac_srs->srs_type = (srs_type | SRST_NO_SOFT_RINGS); 2129da14cebeSEric Cheng mac_srs->srs_worker_cpuid = mac_srs->srs_worker_cpuid_save = -1; 2130da14cebeSEric Cheng mac_srs->srs_poll_cpuid = mac_srs->srs_poll_cpuid_save = -1; 21310dc2366fSVenugopal Iyer mac_srs->srs_mcip = mcip; 2132da14cebeSEric Cheng mac_srs_fanout_list_alloc(mac_srs); 2133da14cebeSEric Cheng 2134da14cebeSEric Cheng /* 2135da14cebeSEric Cheng * For a flow we use the underlying MAC client's priority range with 2136da14cebeSEric Cheng * the priority value to find an absolute priority value. For a MAC 2137da14cebeSEric Cheng * client we use the MAC client's maximum priority as the value. 2138da14cebeSEric Cheng */ 2139da14cebeSEric Cheng mrp = &flent->fe_effective_props; 2140da14cebeSEric Cheng if ((mac_srs->srs_type & SRST_FLOW) != 0) { 2141da14cebeSEric Cheng mac_srs->srs_pri = FLOW_PRIORITY(mcip->mci_min_pri, 2142da14cebeSEric Cheng mcip->mci_max_pri, mrp->mrp_priority); 2143da14cebeSEric Cheng } else { 2144da14cebeSEric Cheng mac_srs->srs_pri = mcip->mci_max_pri; 2145da14cebeSEric Cheng } 2146da14cebeSEric Cheng /* 2147da14cebeSEric Cheng * We need to insert the SRS in the global list before 2148da14cebeSEric Cheng * binding the SRS and SR threads. Otherwise there is a 2149da14cebeSEric Cheng * is a small window where the cpu reconfig callbacks 2150da14cebeSEric Cheng * may miss the SRS in the list walk and DR could fail 2151da14cebeSEric Cheng * as there are bound threads. 2152da14cebeSEric Cheng */ 2153da14cebeSEric Cheng mac_srs_add_glist(mac_srs); 2154da14cebeSEric Cheng 2155da14cebeSEric Cheng /* Initialize bw limit */ 2156da14cebeSEric Cheng if ((mrp->mrp_mask & MRP_MAXBW) != 0) { 2157da14cebeSEric Cheng mac_srs->srs_drain_func = mac_rx_srs_drain_bw; 2158da14cebeSEric Cheng 2159da14cebeSEric Cheng mac_bw = mac_srs->srs_bw; 2160da14cebeSEric Cheng mutex_enter(&mac_bw->mac_bw_lock); 2161da14cebeSEric Cheng mac_bw->mac_bw_limit = FLOW_BYTES_PER_TICK(mrp->mrp_maxbw); 2162da14cebeSEric Cheng 2163da14cebeSEric Cheng /* 2164da14cebeSEric Cheng * Give twice the queuing capability before 2165da14cebeSEric Cheng * dropping packets. The unit is bytes/tick. 2166da14cebeSEric Cheng */ 2167da14cebeSEric Cheng mac_bw->mac_bw_drop_threshold = mac_bw->mac_bw_limit << 1; 2168da14cebeSEric Cheng mutex_exit(&mac_bw->mac_bw_lock); 2169da14cebeSEric Cheng mac_srs->srs_type |= SRST_BW_CONTROL; 2170da14cebeSEric Cheng } else { 2171da14cebeSEric Cheng mac_srs->srs_drain_func = mac_rx_srs_drain; 2172da14cebeSEric Cheng } 2173da14cebeSEric Cheng 2174da14cebeSEric Cheng /* 2175da14cebeSEric Cheng * We use the following policy to control Receive 2176da14cebeSEric Cheng * Side Dynamic Polling: 2177da14cebeSEric Cheng * 1) We switch to poll mode anytime the processing thread causes 2178da14cebeSEric Cheng * a backlog to build up in SRS and its associated Soft Rings 2179da14cebeSEric Cheng * (sr_poll_pkt_cnt > 0). 2180da14cebeSEric Cheng * 2) As long as the backlog stays under the low water mark 2181da14cebeSEric Cheng * (sr_lowat), we poll the H/W for more packets. 2182da14cebeSEric Cheng * 3) If the backlog (sr_poll_pkt_cnt) exceeds low water mark, we 2183da14cebeSEric Cheng * stay in poll mode but don't poll the H/W for more packets. 2184da14cebeSEric Cheng * 4) Anytime in polling mode, if we poll the H/W for packets and 2185da14cebeSEric Cheng * find nothing plus we have an existing backlog 2186da14cebeSEric Cheng * (sr_poll_pkt_cnt > 0), we stay in polling mode but don't poll 2187da14cebeSEric Cheng * the H/W for packets anymore (let the polling thread go to sleep). 2188da14cebeSEric Cheng * 5) Once the backlog is relived (packets are processed) we reenable 2189da14cebeSEric Cheng * polling (by signalling the poll thread) only when the backlog 2190da14cebeSEric Cheng * dips below sr_poll_thres. 2191da14cebeSEric Cheng * 6) sr_hiwat is used exclusively when we are not polling capable 2192da14cebeSEric Cheng * and is used to decide when to drop packets so the SRS queue 2193da14cebeSEric Cheng * length doesn't grow infinitely. 2194da14cebeSEric Cheng */ 2195da14cebeSEric Cheng if (!is_tx_srs) { 2196da14cebeSEric Cheng srs_rx->sr_hiwat = mac_soft_ring_max_q_cnt; 2197da14cebeSEric Cheng /* Low water mark needs to be less than high water mark */ 2198da14cebeSEric Cheng srs_rx->sr_lowat = mac_soft_ring_min_q_cnt <= 2199da14cebeSEric Cheng mac_soft_ring_max_q_cnt ? mac_soft_ring_min_q_cnt : 2200da14cebeSEric Cheng (mac_soft_ring_max_q_cnt >> 2); 2201da14cebeSEric Cheng /* Poll threshold need to be half of low water mark or less */ 2202da14cebeSEric Cheng srs_rx->sr_poll_thres = mac_soft_ring_poll_thres <= 2203da14cebeSEric Cheng (srs_rx->sr_lowat >> 1) ? mac_soft_ring_poll_thres : 2204da14cebeSEric Cheng (srs_rx->sr_lowat >> 1); 2205da14cebeSEric Cheng if (mac_latency_optimize) 22063631b19bSRajagopal Kunhappan mac_srs->srs_state |= SRS_LATENCY_OPT; 22073631b19bSRajagopal Kunhappan else 22083631b19bSRajagopal Kunhappan mac_srs->srs_state |= SRS_SOFTRING_QUEUE; 2209da14cebeSEric Cheng } 2210da14cebeSEric Cheng 2211da14cebeSEric Cheng mac_srs->srs_worker = thread_create(NULL, 0, 2212da14cebeSEric Cheng mac_srs_worker, mac_srs, 0, &p0, TS_RUN, mac_srs->srs_pri); 2213da14cebeSEric Cheng 2214da14cebeSEric Cheng if (is_tx_srs) { 2215da14cebeSEric Cheng /* Handle everything about Tx SRS and return */ 2216da14cebeSEric Cheng mac_srs->srs_drain_func = mac_tx_srs_drain; 2217da14cebeSEric Cheng srs_tx->st_max_q_cnt = mac_tx_srs_max_q_cnt; 2218da14cebeSEric Cheng srs_tx->st_hiwat = 2219da14cebeSEric Cheng (mac_tx_srs_hiwat > mac_tx_srs_max_q_cnt) ? 2220da14cebeSEric Cheng mac_tx_srs_max_q_cnt : mac_tx_srs_hiwat; 2221da14cebeSEric Cheng srs_tx->st_arg1 = x_arg1; 2222da14cebeSEric Cheng srs_tx->st_arg2 = x_arg2; 22230dc2366fSVenugopal Iyer goto done; 2224da14cebeSEric Cheng } 2225da14cebeSEric Cheng 2226da14cebeSEric Cheng if ((srs_type & SRST_FLOW) != 0 || 2227da14cebeSEric Cheng FLOW_TAB_EMPTY(mcip->mci_subflow_tab)) 2228da14cebeSEric Cheng srs_rx->sr_lower_proc = mac_rx_srs_process; 2229da14cebeSEric Cheng else 2230da14cebeSEric Cheng srs_rx->sr_lower_proc = mac_rx_srs_subflow_process; 2231da14cebeSEric Cheng 2232da14cebeSEric Cheng srs_rx->sr_func = rx_func; 2233da14cebeSEric Cheng srs_rx->sr_arg1 = x_arg1; 2234da14cebeSEric Cheng srs_rx->sr_arg2 = x_arg2; 2235da14cebeSEric Cheng 2236da14cebeSEric Cheng if (ring != NULL) { 22370dc2366fSVenugopal Iyer uint_t ring_info; 22380dc2366fSVenugopal Iyer 2239da14cebeSEric Cheng /* Is the mac_srs created over the RX default group? */ 2240da14cebeSEric Cheng if (ring->mr_gh == (mac_group_handle_t) 22410dc2366fSVenugopal Iyer MAC_DEFAULT_RX_GROUP(mcip->mci_mip)) { 2242da14cebeSEric Cheng mac_srs->srs_type |= SRST_DEFAULT_GRP; 22430dc2366fSVenugopal Iyer } 2244da14cebeSEric Cheng mac_srs->srs_ring = ring; 2245da14cebeSEric Cheng ring->mr_srs = mac_srs; 2246da14cebeSEric Cheng ring->mr_classify_type = MAC_HW_CLASSIFIER; 2247da14cebeSEric Cheng ring->mr_flag |= MR_INCIPIENT; 2248da14cebeSEric Cheng 22494eaa4710SRishi Srivatsavai if (!(mcip->mci_mip->mi_state_flags & MIS_POLL_DISABLE) && 22504eaa4710SRishi Srivatsavai FLOW_TAB_EMPTY(mcip->mci_subflow_tab) && mac_poll_enable) 2251da14cebeSEric Cheng mac_srs->srs_state |= SRS_POLLING_CAPAB; 2252da14cebeSEric Cheng 2253da14cebeSEric Cheng mac_srs->srs_poll_thr = thread_create(NULL, 0, 2254da14cebeSEric Cheng mac_rx_srs_poll_ring, mac_srs, 0, &p0, TS_RUN, 2255da14cebeSEric Cheng mac_srs->srs_pri); 2256ae6aa22aSVenugopal Iyer /* 2257ae6aa22aSVenugopal Iyer * Some drivers require serialization and don't send 2258ae6aa22aSVenugopal Iyer * packet chains in interrupt context. For such 2259ae6aa22aSVenugopal Iyer * drivers, we should always queue in soft ring 2260ae6aa22aSVenugopal Iyer * so that we get a chance to switch into a polling 2261ae6aa22aSVenugopal Iyer * mode under backlog. 2262ae6aa22aSVenugopal Iyer */ 22630dc2366fSVenugopal Iyer ring_info = mac_hwring_getinfo((mac_ring_handle_t)ring); 22640dc2366fSVenugopal Iyer if (ring_info & MAC_RING_RX_ENQUEUE) 2265ae6aa22aSVenugopal Iyer mac_srs->srs_state |= SRS_SOFTRING_QUEUE; 2266da14cebeSEric Cheng } 22670dc2366fSVenugopal Iyer done: 22680dc2366fSVenugopal Iyer mac_srs_stat_create(mac_srs); 2269da14cebeSEric Cheng return (mac_srs); 2270da14cebeSEric Cheng } 2271da14cebeSEric Cheng 2272da14cebeSEric Cheng /* 2273da14cebeSEric Cheng * Figure out the number of soft rings required. Its dependant on 2274da14cebeSEric Cheng * if protocol fanout is required (for LINKs), global settings 2275da14cebeSEric Cheng * require us to do fanout for performance (based on mac_soft_ring_enable), 2276da14cebeSEric Cheng * or user has specifically requested fanout. 2277da14cebeSEric Cheng */ 2278da14cebeSEric Cheng static uint32_t 2279da14cebeSEric Cheng mac_find_fanout(flow_entry_t *flent, uint32_t link_type) 2280da14cebeSEric Cheng { 2281da14cebeSEric Cheng uint32_t fanout_type; 2282da14cebeSEric Cheng mac_resource_props_t *mrp = &flent->fe_effective_props; 2283da14cebeSEric Cheng 2284da14cebeSEric Cheng /* no fanout for subflows */ 2285da14cebeSEric Cheng switch (link_type) { 2286da14cebeSEric Cheng case SRST_FLOW: 2287da14cebeSEric Cheng fanout_type = SRST_NO_SOFT_RINGS; 2288da14cebeSEric Cheng break; 2289da14cebeSEric Cheng case SRST_LINK: 2290da14cebeSEric Cheng fanout_type = SRST_FANOUT_PROTO; 2291da14cebeSEric Cheng break; 2292da14cebeSEric Cheng } 2293da14cebeSEric Cheng 2294da14cebeSEric Cheng /* A primary NIC/link is being plumbed */ 2295da14cebeSEric Cheng if (flent->fe_type & FLOW_PRIMARY_MAC) { 2296da14cebeSEric Cheng if (mac_soft_ring_enable && mac_rx_soft_ring_count > 1) { 2297da14cebeSEric Cheng fanout_type |= SRST_FANOUT_SRC_IP; 2298da14cebeSEric Cheng } 2299da14cebeSEric Cheng } else if (flent->fe_type & FLOW_VNIC) { 2300da14cebeSEric Cheng /* A VNIC is being created */ 2301da14cebeSEric Cheng if (mrp != NULL && mrp->mrp_ncpus > 0) { 2302da14cebeSEric Cheng fanout_type |= SRST_FANOUT_SRC_IP; 2303da14cebeSEric Cheng } 2304da14cebeSEric Cheng } 2305da14cebeSEric Cheng 2306da14cebeSEric Cheng return (fanout_type); 2307da14cebeSEric Cheng } 2308da14cebeSEric Cheng 2309da14cebeSEric Cheng /* 2310da14cebeSEric Cheng * Change a group from h/w to s/w classification. 2311da14cebeSEric Cheng */ 23120dc2366fSVenugopal Iyer void 2313da14cebeSEric Cheng mac_rx_switch_grp_to_sw(mac_group_t *group) 2314da14cebeSEric Cheng { 2315da14cebeSEric Cheng mac_ring_t *ring; 2316da14cebeSEric Cheng mac_soft_ring_set_t *mac_srs; 2317da14cebeSEric Cheng 2318da14cebeSEric Cheng for (ring = group->mrg_rings; ring != NULL; ring = ring->mr_next) { 2319da14cebeSEric Cheng if (ring->mr_classify_type == MAC_HW_CLASSIFIER) { 2320da14cebeSEric Cheng /* 2321da14cebeSEric Cheng * Remove the SRS associated with the HW ring. 2322da14cebeSEric Cheng * As a result, polling will be disabled. 2323da14cebeSEric Cheng */ 2324da14cebeSEric Cheng mac_srs = ring->mr_srs; 2325da14cebeSEric Cheng ASSERT(mac_srs != NULL); 2326da14cebeSEric Cheng mac_rx_srs_remove(mac_srs); 2327da14cebeSEric Cheng ring->mr_srs = NULL; 2328da14cebeSEric Cheng } 2329da14cebeSEric Cheng 2330da14cebeSEric Cheng if (ring->mr_state != MR_INUSE) 2331da14cebeSEric Cheng (void) mac_start_ring(ring); 23320dc2366fSVenugopal Iyer 2333da14cebeSEric Cheng /* 2334da14cebeSEric Cheng * We need to perform SW classification 2335da14cebeSEric Cheng * for packets landing in these rings 2336da14cebeSEric Cheng */ 2337da14cebeSEric Cheng ring->mr_flag = 0; 2338da14cebeSEric Cheng ring->mr_classify_type = MAC_SW_CLASSIFIER; 2339da14cebeSEric Cheng } 2340da14cebeSEric Cheng } 2341da14cebeSEric Cheng 2342da14cebeSEric Cheng /* 2343da14cebeSEric Cheng * Create the Rx SRS for S/W classifier and for each ring in the 2344da14cebeSEric Cheng * group (if exclusive group). Also create the Tx SRS. 2345da14cebeSEric Cheng */ 2346da14cebeSEric Cheng void 2347da14cebeSEric Cheng mac_srs_group_setup(mac_client_impl_t *mcip, flow_entry_t *flent, 23480dc2366fSVenugopal Iyer uint32_t link_type) 23490dc2366fSVenugopal Iyer { 23500dc2366fSVenugopal Iyer cpupart_t *cpupart; 23510dc2366fSVenugopal Iyer mac_resource_props_t *mrp = MCIP_RESOURCE_PROPS(mcip); 23520dc2366fSVenugopal Iyer mac_resource_props_t *emrp = MCIP_EFFECTIVE_PROPS(mcip); 23530dc2366fSVenugopal Iyer boolean_t use_default = B_FALSE; 23540dc2366fSVenugopal Iyer 23550dc2366fSVenugopal Iyer mac_rx_srs_group_setup(mcip, flent, link_type); 23560dc2366fSVenugopal Iyer mac_tx_srs_group_setup(mcip, flent, link_type); 23570dc2366fSVenugopal Iyer 23580dc2366fSVenugopal Iyer pool_lock(); 23590dc2366fSVenugopal Iyer cpupart = mac_pset_find(mrp, &use_default); 23600dc2366fSVenugopal Iyer mac_fanout_setup(mcip, flent, MCIP_RESOURCE_PROPS(mcip), 23610dc2366fSVenugopal Iyer mac_rx_deliver, mcip, NULL, cpupart); 23620dc2366fSVenugopal Iyer mac_set_pool_effective(use_default, cpupart, mrp, emrp); 23630dc2366fSVenugopal Iyer pool_unlock(); 23640dc2366fSVenugopal Iyer } 23650dc2366fSVenugopal Iyer 23660dc2366fSVenugopal Iyer /* 23670dc2366fSVenugopal Iyer * Set up the RX SRSs. If the S/W SRS is not set, set it up, if there 23680dc2366fSVenugopal Iyer * is a group associated with this MAC client, set up SRSs for individual 23690dc2366fSVenugopal Iyer * h/w rings. 23700dc2366fSVenugopal Iyer */ 23710dc2366fSVenugopal Iyer void 23720dc2366fSVenugopal Iyer mac_rx_srs_group_setup(mac_client_impl_t *mcip, flow_entry_t *flent, 23730dc2366fSVenugopal Iyer uint32_t link_type) 2374da14cebeSEric Cheng { 2375da14cebeSEric Cheng mac_impl_t *mip = mcip->mci_mip; 2376da14cebeSEric Cheng mac_soft_ring_set_t *mac_srs; 2377da14cebeSEric Cheng mac_ring_t *ring; 2378da14cebeSEric Cheng uint32_t fanout_type; 23790dc2366fSVenugopal Iyer mac_group_t *rx_group = flent->fe_rx_ring_group; 2380da14cebeSEric Cheng 2381da14cebeSEric Cheng fanout_type = mac_find_fanout(flent, link_type); 2382da14cebeSEric Cheng 2383da14cebeSEric Cheng /* Create the SRS for S/W classification if none exists */ 2384da14cebeSEric Cheng if (flent->fe_rx_srs[0] == NULL) { 2385da14cebeSEric Cheng ASSERT(flent->fe_rx_srs_cnt == 0); 2386da14cebeSEric Cheng /* Setup the Rx SRS */ 2387da14cebeSEric Cheng mac_srs = mac_srs_create(mcip, flent, fanout_type | link_type, 2388da14cebeSEric Cheng mac_rx_deliver, mcip, NULL, NULL); 2389da14cebeSEric Cheng mutex_enter(&flent->fe_lock); 2390da14cebeSEric Cheng flent->fe_cb_fn = (flow_fn_t)mac_srs->srs_rx.sr_lower_proc; 2391da14cebeSEric Cheng flent->fe_cb_arg1 = (void *)mip; 2392da14cebeSEric Cheng flent->fe_cb_arg2 = (void *)mac_srs; 2393da14cebeSEric Cheng mutex_exit(&flent->fe_lock); 2394da14cebeSEric Cheng } 2395da14cebeSEric Cheng 23960dc2366fSVenugopal Iyer if (rx_group == NULL) 2397da14cebeSEric Cheng return; 2398da14cebeSEric Cheng /* 2399da14cebeSEric Cheng * fanout for default SRS is done when default SRS are created 2400da14cebeSEric Cheng * above. As each ring is added to the group, we setup the 2401da14cebeSEric Cheng * SRS and fanout to it. 2402da14cebeSEric Cheng */ 24030dc2366fSVenugopal Iyer switch (rx_group->mrg_state) { 2404da14cebeSEric Cheng case MAC_GROUP_STATE_RESERVED: 24050dc2366fSVenugopal Iyer for (ring = rx_group->mrg_rings; ring != NULL; 2406da14cebeSEric Cheng ring = ring->mr_next) { 2407da14cebeSEric Cheng switch (ring->mr_state) { 2408da14cebeSEric Cheng case MR_INUSE: 2409da14cebeSEric Cheng case MR_FREE: 2410da14cebeSEric Cheng if (ring->mr_srs != NULL) 2411da14cebeSEric Cheng break; 2412da14cebeSEric Cheng if (ring->mr_state != MR_INUSE) 2413da14cebeSEric Cheng (void) mac_start_ring(ring); 2414da14cebeSEric Cheng 24150dc2366fSVenugopal Iyer /* 24160dc2366fSVenugopal Iyer * Since the group is exclusively ours create 24170dc2366fSVenugopal Iyer * an SRS for this ring to allow the 24180dc2366fSVenugopal Iyer * individual SRS to dynamically poll the 24190dc2366fSVenugopal Iyer * ring. Do this only if the client is not 24200dc2366fSVenugopal Iyer * a VLAN MAC client, since for VLAN we do 24210dc2366fSVenugopal Iyer * s/w classification for the VID check, and 24220dc2366fSVenugopal Iyer * if it has a unicast address. 24230dc2366fSVenugopal Iyer */ 24240dc2366fSVenugopal Iyer if ((mcip->mci_state_flags & 24250dc2366fSVenugopal Iyer MCIS_NO_UNICAST_ADDR) || 24260dc2366fSVenugopal Iyer i_mac_flow_vid(mcip->mci_flent) != 24270dc2366fSVenugopal Iyer VLAN_ID_NONE) { 24280dc2366fSVenugopal Iyer break; 24290dc2366fSVenugopal Iyer } 2430da14cebeSEric Cheng mac_srs = mac_srs_create(mcip, flent, 2431da14cebeSEric Cheng fanout_type | link_type, 2432da14cebeSEric Cheng mac_rx_deliver, mcip, NULL, ring); 2433da14cebeSEric Cheng break; 2434da14cebeSEric Cheng default: 24350dc2366fSVenugopal Iyer cmn_err(CE_PANIC, 24360dc2366fSVenugopal Iyer "srs_setup: mcip = %p " 2437da14cebeSEric Cheng "trying to add UNKNOWN ring = %p\n", 2438da14cebeSEric Cheng (void *)mcip, (void *)ring); 2439da14cebeSEric Cheng break; 2440da14cebeSEric Cheng } 2441da14cebeSEric Cheng } 2442da14cebeSEric Cheng break; 2443da14cebeSEric Cheng case MAC_GROUP_STATE_SHARED: 2444da14cebeSEric Cheng /* 2445da14cebeSEric Cheng * Set all rings of this group to software classified. 2446da14cebeSEric Cheng * 24470dc2366fSVenugopal Iyer * If the group is current RESERVED, the existing mac 24480dc2366fSVenugopal Iyer * client (the only client on this group) is using 24490dc2366fSVenugopal Iyer * this group exclusively. In that case we need to 24500dc2366fSVenugopal Iyer * disable polling on the rings of the group (if it 24510dc2366fSVenugopal Iyer * was enabled), and free the SRS associated with the 24520dc2366fSVenugopal Iyer * rings. 2453da14cebeSEric Cheng */ 24540dc2366fSVenugopal Iyer mac_rx_switch_grp_to_sw(rx_group); 2455da14cebeSEric Cheng break; 2456da14cebeSEric Cheng default: 2457da14cebeSEric Cheng ASSERT(B_FALSE); 2458da14cebeSEric Cheng break; 2459da14cebeSEric Cheng } 2460da14cebeSEric Cheng } 2461da14cebeSEric Cheng 24620dc2366fSVenugopal Iyer /* 24630dc2366fSVenugopal Iyer * Set up the TX SRS. 24640dc2366fSVenugopal Iyer */ 2465da14cebeSEric Cheng void 24660dc2366fSVenugopal Iyer mac_tx_srs_group_setup(mac_client_impl_t *mcip, flow_entry_t *flent, 2467da14cebeSEric Cheng uint32_t link_type) 2468da14cebeSEric Cheng { 24690dc2366fSVenugopal Iyer int cnt; 24700dc2366fSVenugopal Iyer int ringcnt; 24710dc2366fSVenugopal Iyer mac_ring_t *ring; 24720dc2366fSVenugopal Iyer mac_group_t *grp; 2473da14cebeSEric Cheng 24740dc2366fSVenugopal Iyer /* 24750dc2366fSVenugopal Iyer * If we are opened exclusively (like aggr does for aggr_ports), 24760dc2366fSVenugopal Iyer * don't set up Tx SRS and Tx soft rings as they won't be used. 24770dc2366fSVenugopal Iyer * The same thing has to be done for Rx side also. See bug: 24780dc2366fSVenugopal Iyer * 6880080 24790dc2366fSVenugopal Iyer */ 24800dc2366fSVenugopal Iyer if (mcip->mci_state_flags & MCIS_EXCLUSIVE) { 24810dc2366fSVenugopal Iyer /* 24820dc2366fSVenugopal Iyer * If we have rings, start them here. 24830dc2366fSVenugopal Iyer */ 24840dc2366fSVenugopal Iyer if (flent->fe_tx_ring_group == NULL) 24850dc2366fSVenugopal Iyer return; 24860dc2366fSVenugopal Iyer grp = (mac_group_t *)flent->fe_tx_ring_group; 24870dc2366fSVenugopal Iyer ringcnt = grp->mrg_cur_count; 24880dc2366fSVenugopal Iyer ring = grp->mrg_rings; 24890dc2366fSVenugopal Iyer for (cnt = 0; cnt < ringcnt; cnt++) { 24900dc2366fSVenugopal Iyer if (ring->mr_state != MR_INUSE) { 24910dc2366fSVenugopal Iyer (void) mac_start_ring(ring); 24920dc2366fSVenugopal Iyer } 24930dc2366fSVenugopal Iyer ring = ring->mr_next; 24940dc2366fSVenugopal Iyer } 24950dc2366fSVenugopal Iyer return; 24960dc2366fSVenugopal Iyer } 24970dc2366fSVenugopal Iyer if (flent->fe_tx_srs == NULL) { 24980dc2366fSVenugopal Iyer (void) mac_srs_create(mcip, flent, SRST_TX | link_type, 24990dc2366fSVenugopal Iyer NULL, mcip, NULL, NULL); 25000dc2366fSVenugopal Iyer } 25010dc2366fSVenugopal Iyer mac_tx_srs_setup(mcip, flent); 25020dc2366fSVenugopal Iyer } 25030dc2366fSVenugopal Iyer 25040dc2366fSVenugopal Iyer /* 25050dc2366fSVenugopal Iyer * Remove all the RX SRSs. If we want to remove only the SRSs associated 25060dc2366fSVenugopal Iyer * with h/w rings, leave the S/W SRS alone. This is used when we want to 25070dc2366fSVenugopal Iyer * move the MAC client from one group to another, so we need to teardown 25080dc2366fSVenugopal Iyer * on the h/w SRSs. 25090dc2366fSVenugopal Iyer */ 25100dc2366fSVenugopal Iyer void 25110dc2366fSVenugopal Iyer mac_rx_srs_group_teardown(flow_entry_t *flent, boolean_t hwonly) 25120dc2366fSVenugopal Iyer { 25130dc2366fSVenugopal Iyer mac_soft_ring_set_t *mac_srs; 25140dc2366fSVenugopal Iyer int i; 25150dc2366fSVenugopal Iyer int count = flent->fe_rx_srs_cnt; 25160dc2366fSVenugopal Iyer 25170dc2366fSVenugopal Iyer for (i = 0; i < count; i++) { 25180dc2366fSVenugopal Iyer if (i == 0 && hwonly) 25190dc2366fSVenugopal Iyer continue; 2520da14cebeSEric Cheng mac_srs = flent->fe_rx_srs[i]; 2521da14cebeSEric Cheng mac_rx_srs_quiesce(mac_srs, SRS_CONDEMNED); 2522da14cebeSEric Cheng mac_srs_free(mac_srs); 2523da14cebeSEric Cheng flent->fe_rx_srs[i] = NULL; 25240dc2366fSVenugopal Iyer flent->fe_rx_srs_cnt--; 2525da14cebeSEric Cheng } 25260dc2366fSVenugopal Iyer ASSERT(!hwonly || flent->fe_rx_srs_cnt == 1); 25270dc2366fSVenugopal Iyer ASSERT(hwonly || flent->fe_rx_srs_cnt == 0); 25280dc2366fSVenugopal Iyer } 2529da14cebeSEric Cheng 25300dc2366fSVenugopal Iyer /* 25310dc2366fSVenugopal Iyer * Remove the TX SRS. 25320dc2366fSVenugopal Iyer */ 25330dc2366fSVenugopal Iyer void 25340dc2366fSVenugopal Iyer mac_tx_srs_group_teardown(mac_client_impl_t *mcip, flow_entry_t *flent, 25350dc2366fSVenugopal Iyer uint32_t link_type) 25360dc2366fSVenugopal Iyer { 25370dc2366fSVenugopal Iyer mac_soft_ring_set_t *tx_srs; 25380dc2366fSVenugopal Iyer mac_srs_tx_t *tx; 25390dc2366fSVenugopal Iyer 25400dc2366fSVenugopal Iyer if ((tx_srs = flent->fe_tx_srs) == NULL) 25410dc2366fSVenugopal Iyer return; 25420dc2366fSVenugopal Iyer 2543da14cebeSEric Cheng tx = &tx_srs->srs_tx; 2544da14cebeSEric Cheng switch (link_type) { 2545da14cebeSEric Cheng case SRST_FLOW: 2546da14cebeSEric Cheng /* 2547da14cebeSEric Cheng * For flows, we need to work with passed 2548da14cebeSEric Cheng * flent to find the Rx/Tx SRS. 2549da14cebeSEric Cheng */ 2550da14cebeSEric Cheng mac_tx_srs_quiesce(tx_srs, SRS_CONDEMNED); 2551da14cebeSEric Cheng break; 2552da14cebeSEric Cheng case SRST_LINK: 25530dc2366fSVenugopal Iyer mac_tx_client_condemn((mac_client_handle_t)mcip); 2554da14cebeSEric Cheng if (tx->st_arg2 != NULL) { 2555da14cebeSEric Cheng ASSERT(tx_srs->srs_type & SRST_TX); 25560dc2366fSVenugopal Iyer /* 25570dc2366fSVenugopal Iyer * The ring itself will be stopped when 25580dc2366fSVenugopal Iyer * we release the group or in the 25590dc2366fSVenugopal Iyer * mac_datapath_teardown (for the default 25600dc2366fSVenugopal Iyer * group) 25610dc2366fSVenugopal Iyer */ 25620dc2366fSVenugopal Iyer tx->st_arg2 = NULL; 2563da14cebeSEric Cheng } 2564da14cebeSEric Cheng break; 2565da14cebeSEric Cheng default: 2566da14cebeSEric Cheng ASSERT(B_FALSE); 2567da14cebeSEric Cheng break; 2568da14cebeSEric Cheng } 2569da14cebeSEric Cheng mac_srs_free(tx_srs); 2570da14cebeSEric Cheng flent->fe_tx_srs = NULL; 2571da14cebeSEric Cheng } 2572da14cebeSEric Cheng 2573da14cebeSEric Cheng /* 25740dc2366fSVenugopal Iyer * This is the group state machine. 25750dc2366fSVenugopal Iyer * 25760dc2366fSVenugopal Iyer * The state of an Rx group is given by 2577da14cebeSEric Cheng * the following table. The default group and its rings are started in 2578da14cebeSEric Cheng * mac_start itself and the default group stays in SHARED state until 2579da14cebeSEric Cheng * mac_stop at which time the group and rings are stopped and and it 2580da14cebeSEric Cheng * reverts to the Registered state. 2581da14cebeSEric Cheng * 2582da14cebeSEric Cheng * Typically this function is called on a group after adding or removing a 2583da14cebeSEric Cheng * client from it, to find out what should be the new state of the group. 2584da14cebeSEric Cheng * If the new state is RESERVED, then the client that owns this group 2585da14cebeSEric Cheng * exclusively is also returned. Note that adding or removing a client from 2586da14cebeSEric Cheng * a group could also impact the default group and the caller needs to 2587da14cebeSEric Cheng * evaluate the effect on the default group. 2588da14cebeSEric Cheng * 2589da14cebeSEric Cheng * Group type # of clients mi_nactiveclients Group State 2590da14cebeSEric Cheng * in the group 2591da14cebeSEric Cheng * 2592da14cebeSEric Cheng * Non-default 0 N.A. REGISTERED 2593da14cebeSEric Cheng * Non-default 1 N.A. RESERVED 2594da14cebeSEric Cheng * 2595da14cebeSEric Cheng * Default 0 N.A. SHARED 2596da14cebeSEric Cheng * Default 1 1 RESERVED 2597da14cebeSEric Cheng * Default 1 > 1 SHARED 2598da14cebeSEric Cheng * Default > 1 N.A. SHARED 25990dc2366fSVenugopal Iyer * 26000dc2366fSVenugopal Iyer * For a TX group, the following is the state table. 26010dc2366fSVenugopal Iyer * 26020dc2366fSVenugopal Iyer * Group type # of clients Group State 26030dc2366fSVenugopal Iyer * in the group 26040dc2366fSVenugopal Iyer * 26050dc2366fSVenugopal Iyer * Non-default 0 REGISTERED 26060dc2366fSVenugopal Iyer * Non-default 1 RESERVED 26070dc2366fSVenugopal Iyer * 26080dc2366fSVenugopal Iyer * Default 0 REGISTERED 26090dc2366fSVenugopal Iyer * Default 1 RESERVED 26100dc2366fSVenugopal Iyer * Default > 1 SHARED 2611da14cebeSEric Cheng */ 2612da14cebeSEric Cheng mac_group_state_t 26130dc2366fSVenugopal Iyer mac_group_next_state(mac_group_t *grp, mac_client_impl_t **group_only_mcip, 26140dc2366fSVenugopal Iyer mac_group_t *defgrp, boolean_t rx_group) 2615da14cebeSEric Cheng { 2616da14cebeSEric Cheng mac_impl_t *mip = (mac_impl_t *)grp->mrg_mh; 2617da14cebeSEric Cheng 2618da14cebeSEric Cheng *group_only_mcip = NULL; 2619da14cebeSEric Cheng 2620da14cebeSEric Cheng /* Non-default group */ 2621da14cebeSEric Cheng 26220dc2366fSVenugopal Iyer if (grp != defgrp) { 26230dc2366fSVenugopal Iyer if (MAC_GROUP_NO_CLIENT(grp)) 2624da14cebeSEric Cheng return (MAC_GROUP_STATE_REGISTERED); 2625da14cebeSEric Cheng 26260dc2366fSVenugopal Iyer *group_only_mcip = MAC_GROUP_ONLY_CLIENT(grp); 2627da14cebeSEric Cheng if (*group_only_mcip != NULL) 2628da14cebeSEric Cheng return (MAC_GROUP_STATE_RESERVED); 2629da14cebeSEric Cheng 2630da14cebeSEric Cheng return (MAC_GROUP_STATE_SHARED); 2631da14cebeSEric Cheng } 2632da14cebeSEric Cheng 2633da14cebeSEric Cheng /* Default group */ 2634da14cebeSEric Cheng 26350dc2366fSVenugopal Iyer if (MAC_GROUP_NO_CLIENT(grp)) { 26360dc2366fSVenugopal Iyer if (rx_group) 26370dc2366fSVenugopal Iyer return (MAC_GROUP_STATE_SHARED); 26380dc2366fSVenugopal Iyer else 26390dc2366fSVenugopal Iyer return (MAC_GROUP_STATE_REGISTERED); 26400dc2366fSVenugopal Iyer } 26410dc2366fSVenugopal Iyer *group_only_mcip = MAC_GROUP_ONLY_CLIENT(grp); 26420dc2366fSVenugopal Iyer if (*group_only_mcip == NULL) 2643da14cebeSEric Cheng return (MAC_GROUP_STATE_SHARED); 2644da14cebeSEric Cheng 26450dc2366fSVenugopal Iyer if (rx_group && mip->mi_nactiveclients != 1) 26460dc2366fSVenugopal Iyer return (MAC_GROUP_STATE_SHARED); 26470dc2366fSVenugopal Iyer 2648da14cebeSEric Cheng ASSERT(*group_only_mcip != NULL); 2649da14cebeSEric Cheng return (MAC_GROUP_STATE_RESERVED); 2650da14cebeSEric Cheng } 2651da14cebeSEric Cheng 2652da14cebeSEric Cheng /* 2653da14cebeSEric Cheng * OVERVIEW NOTES FOR DATAPATH 2654da14cebeSEric Cheng * =========================== 2655da14cebeSEric Cheng * 2656da14cebeSEric Cheng * Create an SRS and setup the corresponding flow function and args. 2657da14cebeSEric Cheng * Add a classification rule for the flow specified by 'flent' and program 2658da14cebeSEric Cheng * the hardware classifier when applicable. 2659da14cebeSEric Cheng * 2660da14cebeSEric Cheng * Rx ring assignment, SRS, polling and B/W enforcement 2661da14cebeSEric Cheng * ---------------------------------------------------- 2662da14cebeSEric Cheng * 2663da14cebeSEric Cheng * We try to use H/W classification on NIC and assign traffic to a 2664da14cebeSEric Cheng * MAC address to a particular Rx ring. There is a 1-1 mapping 2665da14cebeSEric Cheng * between a SRS and a Rx ring. The SRS (short for soft ring set) 2666da14cebeSEric Cheng * dynamically switches the underlying Rx ring between interrupt 2667da14cebeSEric Cheng * and polling mode and enforces any specified B/W control. 2668da14cebeSEric Cheng * 2669da14cebeSEric Cheng * There is always a SRS created and tied to each H/W and S/W rule. 2670da14cebeSEric Cheng * Whenever we create a H/W rule, we always add the the same rule to 2671da14cebeSEric Cheng * S/W classifier and tie a SRS to it. 2672da14cebeSEric Cheng * 2673da14cebeSEric Cheng * In case a B/W control is specified, its broken into bytes 2674da14cebeSEric Cheng * per ticks and as soon as the quota for a tick is exhausted, 2675da14cebeSEric Cheng * the underlying Rx ring is forced into poll mode for remianing 2676da14cebeSEric Cheng * tick. The SRS poll thread only polls for bytes that are 2677da14cebeSEric Cheng * allowed to come in the SRS. We typically let 4x the configured 2678da14cebeSEric Cheng * B/W worth of packets to come in the SRS (to prevent unnecessary 2679da14cebeSEric Cheng * drops due to bursts) but only process the specified amount. 2680da14cebeSEric Cheng * 2681da14cebeSEric Cheng * A Link (primary NIC, VNIC, VLAN or aggr) can have 1 or more 2682da14cebeSEric Cheng * Rx rings (and corresponding SRSs) assigned to it. The SRS 2683da14cebeSEric Cheng * in turn can have softrings to do protocol level fanout or 2684da14cebeSEric Cheng * softrings to do S/W based fanout or both. In case the NIC 2685da14cebeSEric Cheng * has no Rx rings, we do S/W classification to respective SRS. 2686da14cebeSEric Cheng * The S/W classification rule is always setup and ready. This 2687da14cebeSEric Cheng * allows the MAC layer to reassign Rx rings whenever needed 2688da14cebeSEric Cheng * but packets still continue to flow via the default path and 2689da14cebeSEric Cheng * getting S/W classified to correct SRS. 2690da14cebeSEric Cheng * 2691da14cebeSEric Cheng * In other cases where a NIC or VNIC is plumbed, our goal is use 2692da14cebeSEric Cheng * H/W classifier and get two Rx ring assigned for the Link. One 2693da14cebeSEric Cheng * for TCP and one for UDP|SCTP. The respective SRS still do the 2694da14cebeSEric Cheng * polling on the Rx ring. For Link that is plumbed for IP, there 2695da14cebeSEric Cheng * is a TCP squeue which also does polling and can control the 2696da14cebeSEric Cheng * the Rx ring directly (where SRS is just pass through). For 2697da14cebeSEric Cheng * the following cases, the SRS does the polling underneath. 2698da14cebeSEric Cheng * 1) non IP based Links (Links which are not plumbed via ifconfig) 2699da14cebeSEric Cheng * and paths which have no IP squeues (UDP & SCTP) 2700da14cebeSEric Cheng * 2) If B/W control is specified on the Link 2701da14cebeSEric Cheng * 3) If S/W fanout is secified 2702da14cebeSEric Cheng * 2703da14cebeSEric Cheng * Note1: As of current implementation, we try to assign only 1 Rx 2704da14cebeSEric Cheng * ring per Link and more than 1 Rx ring for primary Link for 2705da14cebeSEric Cheng * H/W based fanout. We always create following softrings per SRS: 2706da14cebeSEric Cheng * 1) TCP softring which is polled by TCP squeue where possible 2707da14cebeSEric Cheng * (and also bypasses DLS) 2708da14cebeSEric Cheng * 2) UDP/SCTP based which bypasses DLS 2709da14cebeSEric Cheng * 3) OTH softring which goes via DLS (currently deal with IPv6 2710da14cebeSEric Cheng * and non TCP/UDP/SCTP for IPv4 packets). 2711da14cebeSEric Cheng * 2712da14cebeSEric Cheng * It is necessary to create 3 softrings since SRS has to poll 2713da14cebeSEric Cheng * the single Rx ring underneath and enforce any link level B/W 2714da14cebeSEric Cheng * control (we can't switch the Rx ring in poll mode just based 2715da14cebeSEric Cheng * on TCP squeue if the same Rx ring is sharing UDP and other 2716da14cebeSEric Cheng * traffic as well). Once polling is done and any Link level B/W 2717da14cebeSEric Cheng * control is specified, the packets are assigned to respective 2718da14cebeSEric Cheng * softring based on protocol. Since TCP has IP based squeue 2719da14cebeSEric Cheng * which benefits by polling, we separate TCP packets into 2720da14cebeSEric Cheng * its own softring which can be polled by IP squeue. We need 2721da14cebeSEric Cheng * to separate out UDP/SCTP to UDP softring since it can bypass 2722da14cebeSEric Cheng * the DLS layer which has heavy performance advanatges and we 2723da14cebeSEric Cheng * need a softring (OTH) for rest. 2724da14cebeSEric Cheng * 2725da14cebeSEric Cheng * ToDo: The 3 softrings for protocol are needed only till we can 2726da14cebeSEric Cheng * get rid of DLS from datapath, make IPv4 and IPv6 paths 2727da14cebeSEric Cheng * symmetric (deal with mac_header_info for v6 and polling for 2728da14cebeSEric Cheng * IPv4 TCP - ip_accept_tcp is IPv4 specific although squeues 2729da14cebeSEric Cheng * are generic), and bring SAP based classification to MAC layer 2730da14cebeSEric Cheng * 2731da14cebeSEric Cheng * H/W and S/W based fanout and multiple Rx rings per Link 2732da14cebeSEric Cheng * ------------------------------------------------------- 2733da14cebeSEric Cheng * 2734da14cebeSEric Cheng * In case, fanout is requested (or determined automatically based 2735da14cebeSEric Cheng * on Link speed and processor speed), we try to assign multiple 2736da14cebeSEric Cheng * Rx rings per Link with their respective SRS. In this case 2737da14cebeSEric Cheng * the NIC should be capable of fanning out incoming packets between 2738da14cebeSEric Cheng * the assigned Rx rings (H/W based fanout). All the SRS 2739da14cebeSEric Cheng * individually switch their Rx ring between interrupt and polling 2740da14cebeSEric Cheng * mode but share a common B/W control counter in case of Link 2741da14cebeSEric Cheng * level B/W is specified. 2742da14cebeSEric Cheng * 2743da14cebeSEric Cheng * If S/W based fanout is specified in lieu of H/W based fanout, 2744da14cebeSEric Cheng * the Link SRS creates the specified number of softrings for 2745da14cebeSEric Cheng * each protocol (TCP, UDP, OTH). Incoming packets are fanned 2746da14cebeSEric Cheng * out to the correct softring based on their protocol and 2747da14cebeSEric Cheng * protocol specific hash function. 2748da14cebeSEric Cheng * 2749da14cebeSEric Cheng * Primary and non primary MAC clients 2750da14cebeSEric Cheng * ----------------------------------- 2751da14cebeSEric Cheng * 2752da14cebeSEric Cheng * The NICs, VNICs, Vlans, and Aggrs are typically termed as Links 2753da14cebeSEric Cheng * and are a Layer 2 construct. 2754da14cebeSEric Cheng * 2755da14cebeSEric Cheng * Primary NIC: 2756da14cebeSEric Cheng * The Link that owns the primary MAC address and typically 2757da14cebeSEric Cheng * is used as the data NIC in non virtualized cases. As such 2758da14cebeSEric Cheng * H/W resources are preferntially given to primary NIC. As 2759da14cebeSEric Cheng * far as code is concerned, there is no difference in the 2760da14cebeSEric Cheng * primary NIC vs VNICs. They are all treated as Links. 2761da14cebeSEric Cheng * At the very first call to mac_unicast_add() we program the S/W 2762da14cebeSEric Cheng * classifier for the primary MAC address, get a soft ring set 2763da14cebeSEric Cheng * (and soft rings based on 'ip_soft_ring_cnt') 2764da14cebeSEric Cheng * and a Rx ring assigned for polling to get enabled. 2765da14cebeSEric Cheng * When IP get plumbed and negotiates polling, we can 2766da14cebeSEric Cheng * let squeue do the polling on TCP softring. 2767da14cebeSEric Cheng * 2768da14cebeSEric Cheng * VNICs: 2769da14cebeSEric Cheng * Same as any other Link. As long as the H/W resource assignments 2770da14cebeSEric Cheng * are equal, the data path and setup for all Links is same. 2771da14cebeSEric Cheng * 2772da14cebeSEric Cheng * Flows: 2773da14cebeSEric Cheng * Can be configured on Links. They have their own SRS and the 2774da14cebeSEric Cheng * S/W classifier is programmed appropriately based on the flow. 2775da14cebeSEric Cheng * The flows typically deal with layer 3 and above and 2776da14cebeSEric Cheng * creates a soft ring set specific to the flow. The receive 2777da14cebeSEric Cheng * side function is switched from mac_rx_srs_process to 2778da14cebeSEric Cheng * mac_rx_srs_subflow_process which first tries to assign the 2779da14cebeSEric Cheng * packet to appropriate flow SRS and failing which assigns it 2780da14cebeSEric Cheng * to link SRS. This allows us to avoid the layered approach 2781da14cebeSEric Cheng * which gets complex. 2782da14cebeSEric Cheng * 2783da14cebeSEric Cheng * By the time mac_datapath_setup() completes, we already have the 2784da14cebeSEric Cheng * soft rings set, Rx rings, soft rings, etc figured out and both H/W 2785da14cebeSEric Cheng * and S/W classifiers programmed. IP is not plumbed yet (and might 2786da14cebeSEric Cheng * never be for Virtual Machines guest OS path). When IP is plumbed 2787da14cebeSEric Cheng * (for both NIC and VNIC), we do a capability negotiation for polling 2788da14cebeSEric Cheng * and upcall functions etc. 2789da14cebeSEric Cheng * 2790da14cebeSEric Cheng * Rx ring Assignement NOTES 2791da14cebeSEric Cheng * ------------------------- 2792da14cebeSEric Cheng * 2793da14cebeSEric Cheng * For NICs which have only 1 Rx ring (we treat NICs with no Rx rings 2794da14cebeSEric Cheng * as NIC with a single default ring), we assign the only ring to 27950dc2366fSVenugopal Iyer * primary Link. The primary Link SRS can do polling on it as long as 27960dc2366fSVenugopal Iyer * it is the only link in use and we compare the MAC address for unicast 27970dc2366fSVenugopal Iyer * packets before accepting an incoming packet (there is no need for S/W 27980dc2366fSVenugopal Iyer * classification in this case). We disable polling on the only ring the 27990dc2366fSVenugopal Iyer * moment 2nd link gets created (the polling remains enabled even though 28000dc2366fSVenugopal Iyer * there are broadcast and * multicast flows created). 2801da14cebeSEric Cheng * 2802da14cebeSEric Cheng * If the NIC has more than 1 Rx ring, we assign the default ring (the 2803da14cebeSEric Cheng * 1st ring) to deal with broadcast, multicast and traffic for other 2804da14cebeSEric Cheng * NICs which needs S/W classification. We assign the primary mac 2805da14cebeSEric Cheng * addresses to another ring by specifiying a classification rule for 2806da14cebeSEric Cheng * primary unicast MAC address to the selected ring. The primary Link 2807da14cebeSEric Cheng * (and its SRS) can continue to poll the assigned Rx ring at all times 2808da14cebeSEric Cheng * independantly. 2809da14cebeSEric Cheng * 2810da14cebeSEric Cheng * Note: In future, if no fanout is specified, we try to assign 2 Rx 2811da14cebeSEric Cheng * rings for the primary Link with the primary MAC address + TCP going 2812da14cebeSEric Cheng * to one ring and primary MAC address + UDP|SCTP going to other ring. 2813da14cebeSEric Cheng * Any remaining traffic for primary MAC address can go to the default 2814da14cebeSEric Cheng * Rx ring and get S/W classified. This way the respective SRSs don't 2815da14cebeSEric Cheng * need to do proto fanout and don't need to have softrings at all and 2816da14cebeSEric Cheng * can poll their respective Rx rings. 2817da14cebeSEric Cheng * 2818da14cebeSEric Cheng * As an optimization, when a new NIC or VNIC is created, we can get 2819da14cebeSEric Cheng * only one Rx ring and make it a TCP specific Rx ring and use the 2820da14cebeSEric Cheng * H/W default Rx ring for the rest (this Rx ring is never polled). 28210dc2366fSVenugopal Iyer * 28220dc2366fSVenugopal Iyer * For clients that don't have MAC address, but want to receive and 28230dc2366fSVenugopal Iyer * transmit packets (e.g, bpf, gvrp etc.), we need to setup the datapath. 28240dc2366fSVenugopal Iyer * For such clients (identified by the MCIS_NO_UNICAST_ADDR flag) we 28250dc2366fSVenugopal Iyer * always give the default group and use software classification (i.e. 28260dc2366fSVenugopal Iyer * even if this is the only client in the default group, we will 28270dc2366fSVenugopal Iyer * leave group as shared). 2828da14cebeSEric Cheng */ 2829da14cebeSEric Cheng int 2830da14cebeSEric Cheng mac_datapath_setup(mac_client_impl_t *mcip, flow_entry_t *flent, 2831da14cebeSEric Cheng uint32_t link_type) 2832da14cebeSEric Cheng { 2833da14cebeSEric Cheng mac_impl_t *mip = mcip->mci_mip; 28340dc2366fSVenugopal Iyer mac_group_t *rgroup = NULL; 28350dc2366fSVenugopal Iyer mac_group_t *tgroup = NULL; 28360dc2366fSVenugopal Iyer mac_group_t *default_rgroup; 28370dc2366fSVenugopal Iyer mac_group_t *default_tgroup; 2838da14cebeSEric Cheng int err; 2839da14cebeSEric Cheng uint8_t *mac_addr; 2840da14cebeSEric Cheng mac_group_state_t next_state; 2841da14cebeSEric Cheng mac_client_impl_t *group_only_mcip; 28420dc2366fSVenugopal Iyer mac_resource_props_t *mrp = MCIP_RESOURCE_PROPS(mcip); 28430dc2366fSVenugopal Iyer mac_resource_props_t *emrp = MCIP_EFFECTIVE_PROPS(mcip); 28440dc2366fSVenugopal Iyer boolean_t rxhw; 28450dc2366fSVenugopal Iyer boolean_t txhw; 28460dc2366fSVenugopal Iyer boolean_t use_default = B_FALSE; 28470dc2366fSVenugopal Iyer cpupart_t *cpupart; 28480dc2366fSVenugopal Iyer boolean_t no_unicast; 28490dc2366fSVenugopal Iyer boolean_t isprimary = flent->fe_type & FLOW_PRIMARY_MAC; 28500dc2366fSVenugopal Iyer mac_client_impl_t *reloc_pmcip = NULL; 2851da14cebeSEric Cheng 2852da14cebeSEric Cheng ASSERT(MAC_PERIM_HELD((mac_handle_t)mip)); 2853da14cebeSEric Cheng 2854da14cebeSEric Cheng switch (link_type) { 2855da14cebeSEric Cheng case SRST_FLOW: 28560dc2366fSVenugopal Iyer mac_srs_group_setup(mcip, flent, link_type); 2857da14cebeSEric Cheng return (0); 2858da14cebeSEric Cheng 2859da14cebeSEric Cheng case SRST_LINK: 28600dc2366fSVenugopal Iyer no_unicast = mcip->mci_state_flags & MCIS_NO_UNICAST_ADDR; 2861da14cebeSEric Cheng mac_addr = flent->fe_flow_desc.fd_dst_mac; 2862da14cebeSEric Cheng 28630dc2366fSVenugopal Iyer /* Default RX group */ 28640dc2366fSVenugopal Iyer default_rgroup = MAC_DEFAULT_RX_GROUP(mip); 2865da14cebeSEric Cheng 28660dc2366fSVenugopal Iyer /* Default TX group */ 28670dc2366fSVenugopal Iyer default_tgroup = MAC_DEFAULT_TX_GROUP(mip); 28680dc2366fSVenugopal Iyer 28690dc2366fSVenugopal Iyer if (no_unicast) { 28700dc2366fSVenugopal Iyer rgroup = default_rgroup; 28710dc2366fSVenugopal Iyer tgroup = default_tgroup; 28720dc2366fSVenugopal Iyer goto grp_found; 28730dc2366fSVenugopal Iyer } 28740dc2366fSVenugopal Iyer rxhw = (mrp->mrp_mask & MRP_RX_RINGS) && 28750dc2366fSVenugopal Iyer (mrp->mrp_nrxrings > 0 || 28760dc2366fSVenugopal Iyer (mrp->mrp_mask & MRP_RXRINGS_UNSPEC)); 28770dc2366fSVenugopal Iyer txhw = (mrp->mrp_mask & MRP_TX_RINGS) && 28780dc2366fSVenugopal Iyer (mrp->mrp_ntxrings > 0 || 28790dc2366fSVenugopal Iyer (mrp->mrp_mask & MRP_TXRINGS_UNSPEC)); 28800dc2366fSVenugopal Iyer 28810dc2366fSVenugopal Iyer /* 28820dc2366fSVenugopal Iyer * By default we have given the primary all the rings 28830dc2366fSVenugopal Iyer * i.e. the default group. Let's see if the primary 28840dc2366fSVenugopal Iyer * needs to be relocated so that the addition of this 28850dc2366fSVenugopal Iyer * client doesn't impact the primary's performance, 28860dc2366fSVenugopal Iyer * i.e. if the primary is in the default group and 28870dc2366fSVenugopal Iyer * we add this client, the primary will lose polling. 28880dc2366fSVenugopal Iyer * We do this only for NICs supporting dynamic ring 28890dc2366fSVenugopal Iyer * grouping and only when this is the first client 28900dc2366fSVenugopal Iyer * after the primary (i.e. nactiveclients is 2) 28910dc2366fSVenugopal Iyer */ 28920dc2366fSVenugopal Iyer if (!isprimary && mip->mi_nactiveclients == 2 && 28930dc2366fSVenugopal Iyer (group_only_mcip = mac_primary_client_handle(mip)) != 28940dc2366fSVenugopal Iyer NULL && mip->mi_rx_group_type == MAC_GROUP_TYPE_DYNAMIC) { 28950dc2366fSVenugopal Iyer reloc_pmcip = mac_check_primary_relocation( 28960dc2366fSVenugopal Iyer group_only_mcip, rxhw); 28970dc2366fSVenugopal Iyer } 2898da14cebeSEric Cheng /* 2899da14cebeSEric Cheng * Check to see if we can get an exclusive group for 2900da14cebeSEric Cheng * this mac address or if there already exists a 2901da14cebeSEric Cheng * group that has this mac address (case of VLANs). 2902da14cebeSEric Cheng * If no groups are available, use the default group. 2903da14cebeSEric Cheng */ 29040dc2366fSVenugopal Iyer rgroup = mac_reserve_rx_group(mcip, mac_addr, B_FALSE); 29050dc2366fSVenugopal Iyer if (rgroup == NULL && rxhw) { 29060dc2366fSVenugopal Iyer err = ENOSPC; 29070dc2366fSVenugopal Iyer goto setup_failed; 29080dc2366fSVenugopal Iyer } else if (rgroup == NULL) { 29090dc2366fSVenugopal Iyer rgroup = default_rgroup; 2910da14cebeSEric Cheng } 29110dc2366fSVenugopal Iyer /* 29120dc2366fSVenugopal Iyer * Check to see if we can get an exclusive group for 29130dc2366fSVenugopal Iyer * this mac client. If no groups are available, use 29140dc2366fSVenugopal Iyer * the default group. 29150dc2366fSVenugopal Iyer */ 29160dc2366fSVenugopal Iyer tgroup = mac_reserve_tx_group(mcip, B_FALSE); 29170dc2366fSVenugopal Iyer if (tgroup == NULL && txhw) { 29180dc2366fSVenugopal Iyer if (rgroup != NULL && rgroup != default_rgroup) 29190dc2366fSVenugopal Iyer mac_release_rx_group(mcip, rgroup); 29200dc2366fSVenugopal Iyer err = ENOSPC; 29210dc2366fSVenugopal Iyer goto setup_failed; 29220dc2366fSVenugopal Iyer } else if (tgroup == NULL) { 29230dc2366fSVenugopal Iyer tgroup = default_tgroup; 2924da14cebeSEric Cheng } 2925da14cebeSEric Cheng 2926da14cebeSEric Cheng /* 2927da14cebeSEric Cheng * Some NICs don't support any Rx rings, so there may not 2928da14cebeSEric Cheng * even be a default group. 2929da14cebeSEric Cheng */ 29300dc2366fSVenugopal Iyer grp_found: 29310dc2366fSVenugopal Iyer if (rgroup != NULL) { 29320dc2366fSVenugopal Iyer if (rgroup != default_rgroup && 29330dc2366fSVenugopal Iyer MAC_GROUP_NO_CLIENT(rgroup) && 293436f99a58SToomas Soome (rxhw || mcip->mci_share != 0)) { 29350dc2366fSVenugopal Iyer MAC_RX_GRP_RESERVED(mip); 29360dc2366fSVenugopal Iyer if (mip->mi_rx_group_type == 29370dc2366fSVenugopal Iyer MAC_GROUP_TYPE_DYNAMIC) { 29380dc2366fSVenugopal Iyer MAC_RX_RING_RESERVED(mip, 29390dc2366fSVenugopal Iyer rgroup->mrg_cur_count); 29400dc2366fSVenugopal Iyer } 29410dc2366fSVenugopal Iyer } 29420dc2366fSVenugopal Iyer flent->fe_rx_ring_group = rgroup; 2943da14cebeSEric Cheng /* 2944da14cebeSEric Cheng * Add the client to the group. This could cause 2945da14cebeSEric Cheng * either this group to move to the shared state or 2946da14cebeSEric Cheng * cause the default group to move to the shared state. 2947da14cebeSEric Cheng * The actions on this group are done here, while the 2948da14cebeSEric Cheng * actions on the default group are postponed to 2949da14cebeSEric Cheng * the end of this function. 2950da14cebeSEric Cheng */ 29510dc2366fSVenugopal Iyer mac_group_add_client(rgroup, mcip); 29520dc2366fSVenugopal Iyer next_state = mac_group_next_state(rgroup, 29530dc2366fSVenugopal Iyer &group_only_mcip, default_rgroup, B_TRUE); 29540dc2366fSVenugopal Iyer mac_set_group_state(rgroup, next_state); 2955da14cebeSEric Cheng } 2956da14cebeSEric Cheng 29570dc2366fSVenugopal Iyer if (tgroup != NULL) { 29580dc2366fSVenugopal Iyer if (tgroup != default_tgroup && 29590dc2366fSVenugopal Iyer MAC_GROUP_NO_CLIENT(tgroup) && 296036f99a58SToomas Soome (txhw || mcip->mci_share != 0)) { 29610dc2366fSVenugopal Iyer MAC_TX_GRP_RESERVED(mip); 29620dc2366fSVenugopal Iyer if (mip->mi_tx_group_type == 29630dc2366fSVenugopal Iyer MAC_GROUP_TYPE_DYNAMIC) { 29640dc2366fSVenugopal Iyer MAC_TX_RING_RESERVED(mip, 29650dc2366fSVenugopal Iyer tgroup->mrg_cur_count); 29660dc2366fSVenugopal Iyer } 29670dc2366fSVenugopal Iyer } 29680dc2366fSVenugopal Iyer flent->fe_tx_ring_group = tgroup; 29690dc2366fSVenugopal Iyer mac_group_add_client(tgroup, mcip); 29700dc2366fSVenugopal Iyer next_state = mac_group_next_state(tgroup, 29710dc2366fSVenugopal Iyer &group_only_mcip, default_tgroup, B_FALSE); 29720dc2366fSVenugopal Iyer tgroup->mrg_state = next_state; 29730dc2366fSVenugopal Iyer } 2974da14cebeSEric Cheng /* 2975da14cebeSEric Cheng * Setup the Rx and Tx SRSes. If we got a pristine group 2976da14cebeSEric Cheng * exclusively above, mac_srs_group_setup would simply create 2977da14cebeSEric Cheng * the required SRSes. If we ended up sharing a previously 2978da14cebeSEric Cheng * reserved group, mac_srs_group_setup would also dismantle the 2979da14cebeSEric Cheng * SRSes of the previously exclusive group 2980da14cebeSEric Cheng */ 29810dc2366fSVenugopal Iyer mac_srs_group_setup(mcip, flent, link_type); 2982da14cebeSEric Cheng 29830dc2366fSVenugopal Iyer /* We are setting up minimal datapath only */ 29840dc2366fSVenugopal Iyer if (no_unicast) 29850dc2366fSVenugopal Iyer break; 2986da14cebeSEric Cheng /* Program the S/W Classifer */ 2987da14cebeSEric Cheng if ((err = mac_flow_add(mip->mi_flow_tab, flent)) != 0) 2988da14cebeSEric Cheng goto setup_failed; 2989da14cebeSEric Cheng 2990da14cebeSEric Cheng /* Program the H/W Classifier */ 29910dc2366fSVenugopal Iyer if ((err = mac_add_macaddr(mip, rgroup, mac_addr, 299208ac1c49SNicolas Droux (mcip->mci_state_flags & MCIS_UNICAST_HW) != 0)) != 0) 2993da14cebeSEric Cheng goto setup_failed; 2994da14cebeSEric Cheng mcip->mci_unicast = mac_find_macaddr(mip, mac_addr); 2995da14cebeSEric Cheng ASSERT(mcip->mci_unicast != NULL); 2996797f979dSCody Peter Mello /* (Re)init the v6 token & local addr used by link protection */ 2997797f979dSCody Peter Mello mac_protect_update_mac_token(mcip); 2998da14cebeSEric Cheng break; 2999da14cebeSEric Cheng 3000da14cebeSEric Cheng default: 3001da14cebeSEric Cheng ASSERT(B_FALSE); 3002da14cebeSEric Cheng break; 3003da14cebeSEric Cheng } 3004da14cebeSEric Cheng 3005da14cebeSEric Cheng /* 3006da14cebeSEric Cheng * All broadcast and multicast traffic is received only on the default 3007da14cebeSEric Cheng * group. If we have setup the datapath for a non-default group above 3008da14cebeSEric Cheng * then move the default group to shared state to allow distribution of 3009da14cebeSEric Cheng * incoming broadcast traffic to the other groups and dismantle the 3010da14cebeSEric Cheng * SRSes over the default group. 3011da14cebeSEric Cheng */ 30120dc2366fSVenugopal Iyer if (rgroup != NULL) { 30130dc2366fSVenugopal Iyer if (rgroup != default_rgroup) { 30140dc2366fSVenugopal Iyer if (default_rgroup->mrg_state == 3015da14cebeSEric Cheng MAC_GROUP_STATE_RESERVED) { 30160dc2366fSVenugopal Iyer group_only_mcip = MAC_GROUP_ONLY_CLIENT( 30170dc2366fSVenugopal Iyer default_rgroup); 3018da14cebeSEric Cheng ASSERT(group_only_mcip != NULL && 3019da14cebeSEric Cheng mip->mi_nactiveclients > 1); 3020da14cebeSEric Cheng 30210dc2366fSVenugopal Iyer mac_set_group_state(default_rgroup, 3022da14cebeSEric Cheng MAC_GROUP_STATE_SHARED); 30230dc2366fSVenugopal Iyer mac_rx_srs_group_setup(group_only_mcip, 30240dc2366fSVenugopal Iyer group_only_mcip->mci_flent, SRST_LINK); 30250dc2366fSVenugopal Iyer pool_lock(); 30260dc2366fSVenugopal Iyer cpupart = mac_pset_find(mrp, &use_default); 30270dc2366fSVenugopal Iyer mac_fanout_setup(group_only_mcip, 3028da14cebeSEric Cheng group_only_mcip->mci_flent, 30290dc2366fSVenugopal Iyer MCIP_RESOURCE_PROPS(group_only_mcip), 30300dc2366fSVenugopal Iyer mac_rx_deliver, group_only_mcip, NULL, 30310dc2366fSVenugopal Iyer cpupart); 30320dc2366fSVenugopal Iyer mac_set_pool_effective(use_default, cpupart, 30330dc2366fSVenugopal Iyer mrp, emrp); 30340dc2366fSVenugopal Iyer pool_unlock(); 3035da14cebeSEric Cheng } 30360dc2366fSVenugopal Iyer ASSERT(default_rgroup->mrg_state == 3037da14cebeSEric Cheng MAC_GROUP_STATE_SHARED); 3038da14cebeSEric Cheng } 3039da14cebeSEric Cheng /* 3040da14cebeSEric Cheng * If we get an exclusive group for a VLAN MAC client we 3041da14cebeSEric Cheng * need to take the s/w path to make the additional check for 3042da14cebeSEric Cheng * the vid. Disable polling and set it to s/w classification. 30430dc2366fSVenugopal Iyer * Similarly for clients that don't have a unicast address. 3044da14cebeSEric Cheng */ 30450dc2366fSVenugopal Iyer if (rgroup->mrg_state == MAC_GROUP_STATE_RESERVED && 30460dc2366fSVenugopal Iyer (i_mac_flow_vid(flent) != VLAN_ID_NONE || no_unicast)) { 30470dc2366fSVenugopal Iyer mac_rx_switch_grp_to_sw(rgroup); 3048da14cebeSEric Cheng } 3049da14cebeSEric Cheng } 30500dc2366fSVenugopal Iyer mac_set_rings_effective(mcip); 3051da14cebeSEric Cheng return (0); 3052da14cebeSEric Cheng 3053da14cebeSEric Cheng setup_failed: 30540dc2366fSVenugopal Iyer /* Switch the primary back to default group */ 30550dc2366fSVenugopal Iyer if (reloc_pmcip != NULL) { 30560dc2366fSVenugopal Iyer (void) mac_rx_switch_group(reloc_pmcip, 30570dc2366fSVenugopal Iyer reloc_pmcip->mci_flent->fe_rx_ring_group, default_rgroup); 30580dc2366fSVenugopal Iyer } 3059da14cebeSEric Cheng mac_datapath_teardown(mcip, flent, link_type); 3060da14cebeSEric Cheng return (err); 3061da14cebeSEric Cheng } 3062da14cebeSEric Cheng 3063da14cebeSEric Cheng void 3064da14cebeSEric Cheng mac_datapath_teardown(mac_client_impl_t *mcip, flow_entry_t *flent, 3065da14cebeSEric Cheng uint32_t link_type) 3066da14cebeSEric Cheng { 3067da14cebeSEric Cheng mac_impl_t *mip = mcip->mci_mip; 3068da14cebeSEric Cheng mac_group_t *group = NULL; 3069da14cebeSEric Cheng mac_client_impl_t *grp_only_mcip; 3070da14cebeSEric Cheng flow_entry_t *group_only_flent; 3071da14cebeSEric Cheng mac_group_t *default_group; 3072da14cebeSEric Cheng boolean_t check_default_group = B_FALSE; 3073da14cebeSEric Cheng mac_group_state_t next_state; 30740dc2366fSVenugopal Iyer mac_resource_props_t *mrp = MCIP_RESOURCE_PROPS(mcip); 3075da14cebeSEric Cheng 3076da14cebeSEric Cheng ASSERT(MAC_PERIM_HELD((mac_handle_t)mip)); 3077da14cebeSEric Cheng 3078da14cebeSEric Cheng switch (link_type) { 3079da14cebeSEric Cheng case SRST_FLOW: 30800dc2366fSVenugopal Iyer mac_rx_srs_group_teardown(flent, B_FALSE); 30810dc2366fSVenugopal Iyer mac_tx_srs_group_teardown(mcip, flent, SRST_FLOW); 3082da14cebeSEric Cheng return; 3083da14cebeSEric Cheng 3084da14cebeSEric Cheng case SRST_LINK: 3085da14cebeSEric Cheng /* Stop sending packets */ 3086da14cebeSEric Cheng mac_tx_client_block(mcip); 3087da14cebeSEric Cheng 3088da14cebeSEric Cheng /* Stop the packets coming from the H/W */ 3089da14cebeSEric Cheng if (mcip->mci_unicast != NULL) { 3090da14cebeSEric Cheng int err; 3091da14cebeSEric Cheng err = mac_remove_macaddr(mcip->mci_unicast); 3092da14cebeSEric Cheng if (err != 0) { 3093da14cebeSEric Cheng cmn_err(CE_WARN, "%s: failed to remove a MAC" 3094da14cebeSEric Cheng " address because of error 0x%x", 3095da14cebeSEric Cheng mip->mi_name, err); 3096da14cebeSEric Cheng } 3097da14cebeSEric Cheng mcip->mci_unicast = NULL; 3098da14cebeSEric Cheng } 3099da14cebeSEric Cheng 3100da14cebeSEric Cheng /* Stop the packets coming from the S/W classifier */ 3101da14cebeSEric Cheng mac_flow_remove(mip->mi_flow_tab, flent, B_FALSE); 3102da14cebeSEric Cheng mac_flow_wait(flent, FLOW_DRIVER_UPCALL); 3103da14cebeSEric Cheng 3104da14cebeSEric Cheng /* Now quiesce and destroy all SRS and soft rings */ 31050dc2366fSVenugopal Iyer mac_rx_srs_group_teardown(flent, B_FALSE); 31060dc2366fSVenugopal Iyer mac_tx_srs_group_teardown(mcip, flent, SRST_LINK); 31070dc2366fSVenugopal Iyer 3108da14cebeSEric Cheng ASSERT((mcip->mci_flent == flent) && 3109da14cebeSEric Cheng (flent->fe_next == NULL)); 3110da14cebeSEric Cheng 3111da14cebeSEric Cheng /* 3112da14cebeSEric Cheng * Release our hold on the group as well. We need 3113da14cebeSEric Cheng * to check if the shared group has only one client 3114da14cebeSEric Cheng * left who can use it exclusively. Also, if we 3115da14cebeSEric Cheng * were the last client, release the group. 3116da14cebeSEric Cheng */ 3117da14cebeSEric Cheng group = flent->fe_rx_ring_group; 31180dc2366fSVenugopal Iyer default_group = MAC_DEFAULT_RX_GROUP(mip); 3119da14cebeSEric Cheng if (group != NULL) { 31200dc2366fSVenugopal Iyer mac_group_remove_client(group, mcip); 31210dc2366fSVenugopal Iyer next_state = mac_group_next_state(group, 31220dc2366fSVenugopal Iyer &grp_only_mcip, default_group, B_TRUE); 3123da14cebeSEric Cheng if (next_state == MAC_GROUP_STATE_RESERVED) { 3124da14cebeSEric Cheng /* 3125da14cebeSEric Cheng * Only one client left on this RX group. 3126da14cebeSEric Cheng */ 3127da14cebeSEric Cheng ASSERT(grp_only_mcip != NULL); 31280dc2366fSVenugopal Iyer mac_set_group_state(group, 3129da14cebeSEric Cheng MAC_GROUP_STATE_RESERVED); 3130da14cebeSEric Cheng group_only_flent = grp_only_mcip->mci_flent; 3131da14cebeSEric Cheng 3132da14cebeSEric Cheng /* 3133da14cebeSEric Cheng * The only remaining client has exclusive 3134da14cebeSEric Cheng * access on the group. Allow it to 3135da14cebeSEric Cheng * dynamically poll the H/W rings etc. 3136da14cebeSEric Cheng */ 31370dc2366fSVenugopal Iyer mac_rx_srs_group_setup(grp_only_mcip, 31380dc2366fSVenugopal Iyer group_only_flent, SRST_LINK); 31390dc2366fSVenugopal Iyer mac_fanout_setup(grp_only_mcip, 31400dc2366fSVenugopal Iyer group_only_flent, 31410dc2366fSVenugopal Iyer MCIP_RESOURCE_PROPS(grp_only_mcip), 31420dc2366fSVenugopal Iyer mac_rx_deliver, grp_only_mcip, NULL, NULL); 3143da14cebeSEric Cheng mac_rx_group_unmark(group, MR_INCIPIENT); 31440dc2366fSVenugopal Iyer mac_set_rings_effective(grp_only_mcip); 3145da14cebeSEric Cheng } else if (next_state == MAC_GROUP_STATE_REGISTERED) { 3146da14cebeSEric Cheng /* 3147da14cebeSEric Cheng * This is a non-default group being freed up. 3148da14cebeSEric Cheng * We need to reevaluate the default group 3149da14cebeSEric Cheng * to see if the primary client can get 3150da14cebeSEric Cheng * exclusive access to the default group. 3151da14cebeSEric Cheng */ 31520dc2366fSVenugopal Iyer ASSERT(group != MAC_DEFAULT_RX_GROUP(mip)); 31530dc2366fSVenugopal Iyer if (mrp->mrp_mask & MRP_RX_RINGS) { 31540dc2366fSVenugopal Iyer MAC_RX_GRP_RELEASED(mip); 31550dc2366fSVenugopal Iyer if (mip->mi_rx_group_type == 31560dc2366fSVenugopal Iyer MAC_GROUP_TYPE_DYNAMIC) { 31570dc2366fSVenugopal Iyer MAC_RX_RING_RELEASED(mip, 31580dc2366fSVenugopal Iyer group->mrg_cur_count); 31590dc2366fSVenugopal Iyer } 31600dc2366fSVenugopal Iyer } 3161da14cebeSEric Cheng mac_release_rx_group(mcip, group); 31620dc2366fSVenugopal Iyer mac_set_group_state(group, 3163da14cebeSEric Cheng MAC_GROUP_STATE_REGISTERED); 3164da14cebeSEric Cheng check_default_group = B_TRUE; 3165da14cebeSEric Cheng } else { 3166da14cebeSEric Cheng ASSERT(next_state == MAC_GROUP_STATE_SHARED); 31670dc2366fSVenugopal Iyer mac_set_group_state(group, 3168da14cebeSEric Cheng MAC_GROUP_STATE_SHARED); 3169da14cebeSEric Cheng mac_rx_group_unmark(group, MR_CONDEMNED); 3170da14cebeSEric Cheng } 3171da14cebeSEric Cheng flent->fe_rx_ring_group = NULL; 3172da14cebeSEric Cheng } 31730dc2366fSVenugopal Iyer /* 31740dc2366fSVenugopal Iyer * Remove the client from the TX group. Additionally, if 31750dc2366fSVenugopal Iyer * this a non-default group, then we also need to release 31760dc2366fSVenugopal Iyer * the group. 31770dc2366fSVenugopal Iyer */ 31780dc2366fSVenugopal Iyer group = flent->fe_tx_ring_group; 31790dc2366fSVenugopal Iyer default_group = MAC_DEFAULT_TX_GROUP(mip); 31800dc2366fSVenugopal Iyer if (group != NULL) { 31810dc2366fSVenugopal Iyer mac_group_remove_client(group, mcip); 31820dc2366fSVenugopal Iyer next_state = mac_group_next_state(group, 31830dc2366fSVenugopal Iyer &grp_only_mcip, default_group, B_FALSE); 31840dc2366fSVenugopal Iyer if (next_state == MAC_GROUP_STATE_REGISTERED) { 31850dc2366fSVenugopal Iyer if (group != default_group) { 31860dc2366fSVenugopal Iyer if (mrp->mrp_mask & MRP_TX_RINGS) { 31870dc2366fSVenugopal Iyer MAC_TX_GRP_RELEASED(mip); 31880dc2366fSVenugopal Iyer if (mip->mi_tx_group_type == 31890dc2366fSVenugopal Iyer MAC_GROUP_TYPE_DYNAMIC) { 31900dc2366fSVenugopal Iyer MAC_TX_RING_RELEASED( 31910dc2366fSVenugopal Iyer mip, group-> 31920dc2366fSVenugopal Iyer mrg_cur_count); 31930dc2366fSVenugopal Iyer } 31940dc2366fSVenugopal Iyer } 31950dc2366fSVenugopal Iyer mac_release_tx_group(mcip, group); 31960dc2366fSVenugopal Iyer /* 31970dc2366fSVenugopal Iyer * If the default group is reserved, 31980dc2366fSVenugopal Iyer * then we need to set the effective 31990dc2366fSVenugopal Iyer * rings as we would have given 32000dc2366fSVenugopal Iyer * back some rings when the group 32010dc2366fSVenugopal Iyer * was released 32020dc2366fSVenugopal Iyer */ 32030dc2366fSVenugopal Iyer if (mip->mi_tx_group_type == 32040dc2366fSVenugopal Iyer MAC_GROUP_TYPE_DYNAMIC && 32050dc2366fSVenugopal Iyer default_group->mrg_state == 32060dc2366fSVenugopal Iyer MAC_GROUP_STATE_RESERVED) { 32070dc2366fSVenugopal Iyer grp_only_mcip = 32080dc2366fSVenugopal Iyer MAC_GROUP_ONLY_CLIENT 32090dc2366fSVenugopal Iyer (default_group); 32100dc2366fSVenugopal Iyer mac_set_rings_effective( 32110dc2366fSVenugopal Iyer grp_only_mcip); 32120dc2366fSVenugopal Iyer } 32130dc2366fSVenugopal Iyer } else { 32140dc2366fSVenugopal Iyer mac_ring_t *ring; 32150dc2366fSVenugopal Iyer int cnt; 32160dc2366fSVenugopal Iyer int ringcnt; 32170dc2366fSVenugopal Iyer 32180dc2366fSVenugopal Iyer /* 32190dc2366fSVenugopal Iyer * Stop all the rings except the 32200dc2366fSVenugopal Iyer * default ring. 32210dc2366fSVenugopal Iyer */ 32220dc2366fSVenugopal Iyer ringcnt = group->mrg_cur_count; 32230dc2366fSVenugopal Iyer ring = group->mrg_rings; 32240dc2366fSVenugopal Iyer for (cnt = 0; cnt < ringcnt; cnt++) { 32250dc2366fSVenugopal Iyer if (ring->mr_state == 32260dc2366fSVenugopal Iyer MR_INUSE && ring != 32270dc2366fSVenugopal Iyer (mac_ring_t *) 32280dc2366fSVenugopal Iyer mip->mi_default_tx_ring) { 32290dc2366fSVenugopal Iyer mac_stop_ring(ring); 32300dc2366fSVenugopal Iyer ring->mr_flag = 0; 32310dc2366fSVenugopal Iyer } 32320dc2366fSVenugopal Iyer ring = ring->mr_next; 32330dc2366fSVenugopal Iyer } 32340dc2366fSVenugopal Iyer } 32350dc2366fSVenugopal Iyer } else if (next_state == MAC_GROUP_STATE_RESERVED) { 32360dc2366fSVenugopal Iyer mac_set_rings_effective(grp_only_mcip); 32370dc2366fSVenugopal Iyer } 32380dc2366fSVenugopal Iyer flent->fe_tx_ring_group = NULL; 32390dc2366fSVenugopal Iyer group->mrg_state = next_state; 32400dc2366fSVenugopal Iyer } 3241da14cebeSEric Cheng break; 3242da14cebeSEric Cheng default: 3243da14cebeSEric Cheng ASSERT(B_FALSE); 3244da14cebeSEric Cheng break; 3245da14cebeSEric Cheng } 3246da14cebeSEric Cheng 3247da14cebeSEric Cheng /* 3248da14cebeSEric Cheng * The mac client using the default group gets exclusive access to the 3249da14cebeSEric Cheng * default group if and only if it is the sole client on the entire 3250da14cebeSEric Cheng * mip. If so set the group state to reserved, and set up the SRSes 3251da14cebeSEric Cheng * over the default group. 3252da14cebeSEric Cheng */ 3253da14cebeSEric Cheng if (check_default_group) { 32540dc2366fSVenugopal Iyer default_group = MAC_DEFAULT_RX_GROUP(mip); 3255da14cebeSEric Cheng ASSERT(default_group->mrg_state == MAC_GROUP_STATE_SHARED); 32560dc2366fSVenugopal Iyer next_state = mac_group_next_state(default_group, 32570dc2366fSVenugopal Iyer &grp_only_mcip, default_group, B_TRUE); 3258da14cebeSEric Cheng if (next_state == MAC_GROUP_STATE_RESERVED) { 3259da14cebeSEric Cheng ASSERT(grp_only_mcip != NULL && 3260da14cebeSEric Cheng mip->mi_nactiveclients == 1); 32610dc2366fSVenugopal Iyer mac_set_group_state(default_group, 3262da14cebeSEric Cheng MAC_GROUP_STATE_RESERVED); 32630dc2366fSVenugopal Iyer mac_rx_srs_group_setup(grp_only_mcip, 32640dc2366fSVenugopal Iyer grp_only_mcip->mci_flent, SRST_LINK); 32650dc2366fSVenugopal Iyer mac_fanout_setup(grp_only_mcip, 3266da14cebeSEric Cheng grp_only_mcip->mci_flent, 32670dc2366fSVenugopal Iyer MCIP_RESOURCE_PROPS(grp_only_mcip), mac_rx_deliver, 32680dc2366fSVenugopal Iyer grp_only_mcip, NULL, NULL); 3269ae6aa22aSVenugopal Iyer mac_rx_group_unmark(default_group, MR_INCIPIENT); 32700dc2366fSVenugopal Iyer mac_set_rings_effective(grp_only_mcip); 3271da14cebeSEric Cheng } 3272da14cebeSEric Cheng } 32730dc2366fSVenugopal Iyer 32740dc2366fSVenugopal Iyer /* 32750dc2366fSVenugopal Iyer * If the primary is the only one left and the MAC supports 32760dc2366fSVenugopal Iyer * dynamic grouping, we need to see if the primary needs to 32770dc2366fSVenugopal Iyer * be moved to the default group so that it can use all the 32780dc2366fSVenugopal Iyer * H/W rings. 32790dc2366fSVenugopal Iyer */ 32800dc2366fSVenugopal Iyer if (!(flent->fe_type & FLOW_PRIMARY_MAC) && 32810dc2366fSVenugopal Iyer mip->mi_nactiveclients == 1 && 32820dc2366fSVenugopal Iyer mip->mi_rx_group_type == MAC_GROUP_TYPE_DYNAMIC) { 32830dc2366fSVenugopal Iyer default_group = MAC_DEFAULT_RX_GROUP(mip); 32840dc2366fSVenugopal Iyer grp_only_mcip = mac_primary_client_handle(mip); 32850dc2366fSVenugopal Iyer if (grp_only_mcip == NULL) 32860dc2366fSVenugopal Iyer return; 32870dc2366fSVenugopal Iyer group_only_flent = grp_only_mcip->mci_flent; 32880dc2366fSVenugopal Iyer mrp = MCIP_RESOURCE_PROPS(grp_only_mcip); 32890dc2366fSVenugopal Iyer /* 32900dc2366fSVenugopal Iyer * If the primary has an explicit property set, leave it 32910dc2366fSVenugopal Iyer * alone. 32920dc2366fSVenugopal Iyer */ 32930dc2366fSVenugopal Iyer if (mrp->mrp_mask & MRP_RX_RINGS) 32940dc2366fSVenugopal Iyer return; 32950dc2366fSVenugopal Iyer /* 32960dc2366fSVenugopal Iyer * Switch the primary to the default group. 32970dc2366fSVenugopal Iyer */ 32980dc2366fSVenugopal Iyer (void) mac_rx_switch_group(grp_only_mcip, 32990dc2366fSVenugopal Iyer group_only_flent->fe_rx_ring_group, default_group); 33000dc2366fSVenugopal Iyer } 3301da14cebeSEric Cheng } 3302da14cebeSEric Cheng 3303da14cebeSEric Cheng /* DATAPATH TEAR DOWN ROUTINES (SRS and FANOUT teardown) */ 3304da14cebeSEric Cheng 3305da14cebeSEric Cheng static void 3306da14cebeSEric Cheng mac_srs_fanout_list_free(mac_soft_ring_set_t *mac_srs) 3307da14cebeSEric Cheng { 33080dc2366fSVenugopal Iyer if (mac_srs->srs_type & SRST_TX) { 33090dc2366fSVenugopal Iyer mac_srs_tx_t *tx; 33100dc2366fSVenugopal Iyer 33110dc2366fSVenugopal Iyer ASSERT(mac_srs->srs_tcp_soft_rings == NULL); 33120dc2366fSVenugopal Iyer ASSERT(mac_srs->srs_udp_soft_rings == NULL); 33130dc2366fSVenugopal Iyer ASSERT(mac_srs->srs_oth_soft_rings == NULL); 33140dc2366fSVenugopal Iyer ASSERT(mac_srs->srs_tx_soft_rings != NULL); 33150dc2366fSVenugopal Iyer kmem_free(mac_srs->srs_tx_soft_rings, 33160dc2366fSVenugopal Iyer sizeof (mac_soft_ring_t *) * MAX_RINGS_PER_GROUP); 33170dc2366fSVenugopal Iyer mac_srs->srs_tx_soft_rings = NULL; 33180dc2366fSVenugopal Iyer tx = &mac_srs->srs_tx; 33190dc2366fSVenugopal Iyer if (tx->st_soft_rings != NULL) { 33200dc2366fSVenugopal Iyer kmem_free(tx->st_soft_rings, 33210dc2366fSVenugopal Iyer sizeof (mac_soft_ring_t *) * MAX_RINGS_PER_GROUP); 33220dc2366fSVenugopal Iyer } 33230dc2366fSVenugopal Iyer } else { 33240dc2366fSVenugopal Iyer ASSERT(mac_srs->srs_tx_soft_rings == NULL); 3325da14cebeSEric Cheng ASSERT(mac_srs->srs_tcp_soft_rings != NULL); 3326da14cebeSEric Cheng kmem_free(mac_srs->srs_tcp_soft_rings, 3327da14cebeSEric Cheng sizeof (mac_soft_ring_t *) * MAX_SR_FANOUT); 3328da14cebeSEric Cheng mac_srs->srs_tcp_soft_rings = NULL; 3329da14cebeSEric Cheng ASSERT(mac_srs->srs_udp_soft_rings != NULL); 3330da14cebeSEric Cheng kmem_free(mac_srs->srs_udp_soft_rings, 3331da14cebeSEric Cheng sizeof (mac_soft_ring_t *) * MAX_SR_FANOUT); 3332da14cebeSEric Cheng mac_srs->srs_udp_soft_rings = NULL; 3333da14cebeSEric Cheng ASSERT(mac_srs->srs_oth_soft_rings != NULL); 3334da14cebeSEric Cheng kmem_free(mac_srs->srs_oth_soft_rings, 3335da14cebeSEric Cheng sizeof (mac_soft_ring_t *) * MAX_SR_FANOUT); 3336da14cebeSEric Cheng mac_srs->srs_oth_soft_rings = NULL; 3337da14cebeSEric Cheng } 33380dc2366fSVenugopal Iyer } 3339da14cebeSEric Cheng 3340da14cebeSEric Cheng /* 3341da14cebeSEric Cheng * An RX SRS is attached to at most one mac_ring. 3342da14cebeSEric Cheng * A TX SRS has no rings. 3343da14cebeSEric Cheng */ 3344da14cebeSEric Cheng static void 3345da14cebeSEric Cheng mac_srs_ring_free(mac_soft_ring_set_t *mac_srs) 3346da14cebeSEric Cheng { 3347da14cebeSEric Cheng mac_client_impl_t *mcip; 3348da14cebeSEric Cheng mac_ring_t *ring; 3349da14cebeSEric Cheng flow_entry_t *flent; 3350da14cebeSEric Cheng 3351da14cebeSEric Cheng ring = mac_srs->srs_ring; 3352da14cebeSEric Cheng if (mac_srs->srs_type & SRST_TX) { 3353da14cebeSEric Cheng ASSERT(ring == NULL); 3354da14cebeSEric Cheng return; 3355da14cebeSEric Cheng } 3356da14cebeSEric Cheng 3357da14cebeSEric Cheng if (ring == NULL) 3358da14cebeSEric Cheng return; 3359da14cebeSEric Cheng 3360da14cebeSEric Cheng /* 3361da14cebeSEric Cheng * Broadcast flows don't have a client impl association, but they 3362da14cebeSEric Cheng * use only soft rings. 3363da14cebeSEric Cheng */ 3364da14cebeSEric Cheng flent = mac_srs->srs_flent; 3365da14cebeSEric Cheng mcip = flent->fe_mcip; 3366da14cebeSEric Cheng ASSERT(mcip != NULL); 3367da14cebeSEric Cheng 3368da14cebeSEric Cheng ring->mr_classify_type = MAC_NO_CLASSIFIER; 3369da14cebeSEric Cheng ring->mr_srs = NULL; 3370da14cebeSEric Cheng } 3371da14cebeSEric Cheng 3372da14cebeSEric Cheng /* 3373da14cebeSEric Cheng * Physical unlink and free of the data structures happen below. This is 3374da14cebeSEric Cheng * driven from mac_flow_destroy(), on the last refrele of a flow. 3375da14cebeSEric Cheng * 3376da14cebeSEric Cheng * Assumes Rx srs is 1-1 mapped with an ring. 3377da14cebeSEric Cheng */ 3378da14cebeSEric Cheng void 3379da14cebeSEric Cheng mac_srs_free(mac_soft_ring_set_t *mac_srs) 3380da14cebeSEric Cheng { 3381da14cebeSEric Cheng ASSERT(mac_srs->srs_mcip == NULL || 3382da14cebeSEric Cheng MAC_PERIM_HELD((mac_handle_t)mac_srs->srs_mcip->mci_mip)); 3383da14cebeSEric Cheng ASSERT((mac_srs->srs_state & (SRS_CONDEMNED | SRS_CONDEMNED_DONE | 3384da14cebeSEric Cheng SRS_PROC | SRS_PROC_FAST)) == (SRS_CONDEMNED | SRS_CONDEMNED_DONE)); 3385da14cebeSEric Cheng 3386da14cebeSEric Cheng mac_pkt_drop(NULL, NULL, mac_srs->srs_first, B_FALSE); 3387da14cebeSEric Cheng mac_srs_ring_free(mac_srs); 33880dc2366fSVenugopal Iyer mac_srs_soft_rings_free(mac_srs); 3389da14cebeSEric Cheng mac_srs_fanout_list_free(mac_srs); 3390da14cebeSEric Cheng 3391da14cebeSEric Cheng mac_srs->srs_bw = NULL; 33920dc2366fSVenugopal Iyer mac_srs_stat_delete(mac_srs); 3393da14cebeSEric Cheng kmem_cache_free(mac_srs_cache, mac_srs); 3394da14cebeSEric Cheng } 3395da14cebeSEric Cheng 3396da14cebeSEric Cheng static void 3397da14cebeSEric Cheng mac_srs_soft_rings_quiesce(mac_soft_ring_set_t *mac_srs, uint_t s_ring_flag) 3398da14cebeSEric Cheng { 3399da14cebeSEric Cheng mac_soft_ring_t *softring; 3400da14cebeSEric Cheng 3401da14cebeSEric Cheng ASSERT(MUTEX_HELD(&mac_srs->srs_lock)); 3402da14cebeSEric Cheng 3403da14cebeSEric Cheng mac_srs_soft_rings_signal(mac_srs, s_ring_flag); 3404da14cebeSEric Cheng if (s_ring_flag == S_RING_CONDEMNED) { 3405da14cebeSEric Cheng while (mac_srs->srs_soft_ring_condemned_count != 3406da14cebeSEric Cheng mac_srs->srs_soft_ring_count) 3407da14cebeSEric Cheng cv_wait(&mac_srs->srs_async, &mac_srs->srs_lock); 3408da14cebeSEric Cheng } else { 3409da14cebeSEric Cheng while (mac_srs->srs_soft_ring_quiesced_count != 3410da14cebeSEric Cheng mac_srs->srs_soft_ring_count) 3411da14cebeSEric Cheng cv_wait(&mac_srs->srs_async, &mac_srs->srs_lock); 3412da14cebeSEric Cheng } 3413da14cebeSEric Cheng mutex_exit(&mac_srs->srs_lock); 3414da14cebeSEric Cheng 3415da14cebeSEric Cheng for (softring = mac_srs->srs_soft_ring_head; softring != NULL; 3416da906b4eSRyan Zezeski softring = softring->s_ring_next) { 3417da14cebeSEric Cheng (void) untimeout(softring->s_ring_tid); 3418da906b4eSRyan Zezeski softring->s_ring_tid = NULL; 3419da906b4eSRyan Zezeski } 3420da14cebeSEric Cheng 3421da14cebeSEric Cheng (void) untimeout(mac_srs->srs_tid); 3422da906b4eSRyan Zezeski mac_srs->srs_tid = NULL; 3423da14cebeSEric Cheng 3424da14cebeSEric Cheng mutex_enter(&mac_srs->srs_lock); 3425da14cebeSEric Cheng } 3426da14cebeSEric Cheng 3427da14cebeSEric Cheng /* 3428da14cebeSEric Cheng * The block comment above mac_rx_classify_flow_state_change explains the 3429da14cebeSEric Cheng * background. At this point upcalls from the driver (both hardware classified 3430da14cebeSEric Cheng * and software classified) have been cut off. We now need to quiesce the 3431da14cebeSEric Cheng * SRS worker, poll, and softring threads. The SRS worker thread serves as 3432da14cebeSEric Cheng * the master controller. The steps involved are described below in the function 3433da14cebeSEric Cheng */ 3434da14cebeSEric Cheng void 3435da14cebeSEric Cheng mac_srs_worker_quiesce(mac_soft_ring_set_t *mac_srs) 3436da14cebeSEric Cheng { 3437da14cebeSEric Cheng uint_t s_ring_flag; 3438da14cebeSEric Cheng uint_t srs_poll_wait_flag; 3439da14cebeSEric Cheng 3440da14cebeSEric Cheng ASSERT(MUTEX_HELD(&mac_srs->srs_lock)); 3441da14cebeSEric Cheng ASSERT(mac_srs->srs_state & (SRS_CONDEMNED | SRS_QUIESCE)); 3442da14cebeSEric Cheng 3443da14cebeSEric Cheng if (mac_srs->srs_state & SRS_CONDEMNED) { 3444da14cebeSEric Cheng s_ring_flag = S_RING_CONDEMNED; 3445da14cebeSEric Cheng srs_poll_wait_flag = SRS_POLL_THR_EXITED; 3446da14cebeSEric Cheng } else { 3447da14cebeSEric Cheng s_ring_flag = S_RING_QUIESCE; 3448da14cebeSEric Cheng srs_poll_wait_flag = SRS_POLL_THR_QUIESCED; 3449da14cebeSEric Cheng } 3450da14cebeSEric Cheng 3451da14cebeSEric Cheng /* 3452da14cebeSEric Cheng * In the case of Rx SRS wait till the poll thread is done. 3453da14cebeSEric Cheng */ 3454da14cebeSEric Cheng if ((mac_srs->srs_type & SRST_TX) == 0 && 3455da14cebeSEric Cheng mac_srs->srs_poll_thr != NULL) { 3456da14cebeSEric Cheng while (!(mac_srs->srs_state & srs_poll_wait_flag)) 3457da14cebeSEric Cheng cv_wait(&mac_srs->srs_async, &mac_srs->srs_lock); 3458da14cebeSEric Cheng 3459da14cebeSEric Cheng /* 3460da14cebeSEric Cheng * Turn off polling as part of the quiesce operation. 3461da14cebeSEric Cheng */ 3462da14cebeSEric Cheng MAC_SRS_POLLING_OFF(mac_srs); 3463da14cebeSEric Cheng mac_srs->srs_state &= ~(SRS_POLLING | SRS_GET_PKTS); 3464da14cebeSEric Cheng } 3465da14cebeSEric Cheng 3466da14cebeSEric Cheng /* 3467da14cebeSEric Cheng * Then signal the soft ring worker threads to quiesce or quit 3468da14cebeSEric Cheng * as needed and then wait till that happens. 3469da14cebeSEric Cheng */ 3470da14cebeSEric Cheng mac_srs_soft_rings_quiesce(mac_srs, s_ring_flag); 3471da14cebeSEric Cheng 3472da14cebeSEric Cheng if (mac_srs->srs_state & SRS_CONDEMNED) 3473da14cebeSEric Cheng mac_srs->srs_state |= (SRS_QUIESCE_DONE | SRS_CONDEMNED_DONE); 3474da14cebeSEric Cheng else 3475da14cebeSEric Cheng mac_srs->srs_state |= SRS_QUIESCE_DONE; 3476da14cebeSEric Cheng cv_signal(&mac_srs->srs_quiesce_done_cv); 3477da14cebeSEric Cheng } 3478da14cebeSEric Cheng 3479da14cebeSEric Cheng /* 3480da14cebeSEric Cheng * Signal an SRS to start a temporary quiesce, or permanent removal, or restart 3481da14cebeSEric Cheng * a quiesced SRS by setting the appropriate flags and signaling the SRS worker 3482da14cebeSEric Cheng * or poll thread. This function is internal to the quiescing logic and is 3483da14cebeSEric Cheng * called internally from the SRS quiesce or flow quiesce or client quiesce 3484da14cebeSEric Cheng * higher level functions. 3485da14cebeSEric Cheng */ 3486da14cebeSEric Cheng void 3487da14cebeSEric Cheng mac_srs_signal(mac_soft_ring_set_t *mac_srs, uint_t srs_flag) 3488da14cebeSEric Cheng { 3489da14cebeSEric Cheng mac_ring_t *ring; 3490da14cebeSEric Cheng 3491da14cebeSEric Cheng ring = mac_srs->srs_ring; 3492da14cebeSEric Cheng ASSERT(ring == NULL || ring->mr_refcnt == 0); 3493da14cebeSEric Cheng 3494da14cebeSEric Cheng if (srs_flag == SRS_CONDEMNED) { 3495da14cebeSEric Cheng /* 3496da14cebeSEric Cheng * The SRS is going away. We need to unbind the SRS and SR 3497da14cebeSEric Cheng * threads before removing from the global SRS list. Otherwise 3498da14cebeSEric Cheng * there is a small window where the cpu reconfig callbacks 3499da14cebeSEric Cheng * may miss the SRS in the list walk and DR could fail since 3500da14cebeSEric Cheng * there are still bound threads. 3501da14cebeSEric Cheng */ 3502da14cebeSEric Cheng mac_srs_threads_unbind(mac_srs); 3503da14cebeSEric Cheng mac_srs_remove_glist(mac_srs); 3504da14cebeSEric Cheng } 3505da14cebeSEric Cheng /* 3506da14cebeSEric Cheng * Wakeup the SRS worker and poll threads. 3507da14cebeSEric Cheng */ 3508da14cebeSEric Cheng mutex_enter(&mac_srs->srs_lock); 3509da14cebeSEric Cheng mac_srs->srs_state |= srs_flag; 3510da14cebeSEric Cheng cv_signal(&mac_srs->srs_async); 3511da14cebeSEric Cheng cv_signal(&mac_srs->srs_cv); 3512da14cebeSEric Cheng mutex_exit(&mac_srs->srs_lock); 3513da14cebeSEric Cheng } 3514da14cebeSEric Cheng 3515da14cebeSEric Cheng /* 3516da14cebeSEric Cheng * In the Rx side, the quiescing is done bottom up. After the Rx upcalls 3517da14cebeSEric Cheng * from the driver are done, then the Rx SRS is quiesced and only then can 3518da14cebeSEric Cheng * we signal the soft rings. Thus this function can't be called arbitrarily 3519da14cebeSEric Cheng * without satisfying the prerequisites. On the Tx side, the threads from 3520da14cebeSEric Cheng * top need to quiesced, then the Tx SRS and only then can we signal the 3521da14cebeSEric Cheng * Tx soft rings. 3522da14cebeSEric Cheng */ 3523da14cebeSEric Cheng static void 3524da14cebeSEric Cheng mac_srs_soft_rings_signal(mac_soft_ring_set_t *mac_srs, uint_t sr_flag) 3525da14cebeSEric Cheng { 3526da14cebeSEric Cheng mac_soft_ring_t *softring; 3527da14cebeSEric Cheng 3528da14cebeSEric Cheng for (softring = mac_srs->srs_soft_ring_head; softring != NULL; 3529da14cebeSEric Cheng softring = softring->s_ring_next) 3530da14cebeSEric Cheng mac_soft_ring_signal(softring, sr_flag); 3531da14cebeSEric Cheng } 3532da14cebeSEric Cheng 3533da14cebeSEric Cheng /* 3534da14cebeSEric Cheng * The block comment above mac_rx_classify_flow_state_change explains the 3535da14cebeSEric Cheng * background. At this point the SRS is quiesced and we need to restart the 3536da14cebeSEric Cheng * SRS worker, poll, and softring threads. The SRS worker thread serves as 3537da14cebeSEric Cheng * the master controller. The steps involved are described below in the function 3538da14cebeSEric Cheng */ 3539da14cebeSEric Cheng void 3540da14cebeSEric Cheng mac_srs_worker_restart(mac_soft_ring_set_t *mac_srs) 3541da14cebeSEric Cheng { 3542da14cebeSEric Cheng boolean_t iam_rx_srs; 3543da14cebeSEric Cheng mac_soft_ring_t *softring; 3544da14cebeSEric Cheng 3545da14cebeSEric Cheng ASSERT(MUTEX_HELD(&mac_srs->srs_lock)); 3546da14cebeSEric Cheng if ((mac_srs->srs_type & SRST_TX) != 0) { 3547da14cebeSEric Cheng iam_rx_srs = B_FALSE; 3548da14cebeSEric Cheng ASSERT((mac_srs->srs_state & 3549da14cebeSEric Cheng (SRS_POLL_THR_QUIESCED | SRS_QUIESCE_DONE | SRS_QUIESCE)) == 3550da14cebeSEric Cheng (SRS_QUIESCE_DONE | SRS_QUIESCE)); 3551da14cebeSEric Cheng } else { 3552da14cebeSEric Cheng iam_rx_srs = B_TRUE; 3553da14cebeSEric Cheng ASSERT((mac_srs->srs_state & 3554da14cebeSEric Cheng (SRS_QUIESCE_DONE | SRS_QUIESCE)) == 3555da14cebeSEric Cheng (SRS_QUIESCE_DONE | SRS_QUIESCE)); 3556da14cebeSEric Cheng if (mac_srs->srs_poll_thr != NULL) { 3557da14cebeSEric Cheng ASSERT((mac_srs->srs_state & SRS_POLL_THR_QUIESCED) == 3558da14cebeSEric Cheng SRS_POLL_THR_QUIESCED); 3559da14cebeSEric Cheng } 3560da14cebeSEric Cheng } 3561da14cebeSEric Cheng 3562da14cebeSEric Cheng /* 3563da14cebeSEric Cheng * Signal any quiesced soft ring workers to restart and wait for the 3564da14cebeSEric Cheng * soft ring down count to come down to zero. 3565da14cebeSEric Cheng */ 3566da14cebeSEric Cheng if (mac_srs->srs_soft_ring_quiesced_count != 0) { 3567da14cebeSEric Cheng for (softring = mac_srs->srs_soft_ring_head; softring != NULL; 3568da14cebeSEric Cheng softring = softring->s_ring_next) { 3569da14cebeSEric Cheng if (!(softring->s_ring_state & S_RING_QUIESCE)) 3570da14cebeSEric Cheng continue; 3571da14cebeSEric Cheng mac_soft_ring_signal(softring, S_RING_RESTART); 3572da14cebeSEric Cheng } 3573da14cebeSEric Cheng while (mac_srs->srs_soft_ring_quiesced_count != 0) 3574da14cebeSEric Cheng cv_wait(&mac_srs->srs_async, &mac_srs->srs_lock); 3575da14cebeSEric Cheng } 3576da14cebeSEric Cheng 3577da14cebeSEric Cheng mac_srs->srs_state &= ~(SRS_QUIESCE_DONE | SRS_QUIESCE | SRS_RESTART); 3578da14cebeSEric Cheng if (iam_rx_srs && mac_srs->srs_poll_thr != NULL) { 3579da14cebeSEric Cheng /* 3580da14cebeSEric Cheng * Signal the poll thread and ask it to restart. Wait till it 3581da14cebeSEric Cheng * actually restarts and the SRS_POLL_THR_QUIESCED flag gets 3582da14cebeSEric Cheng * cleared. 3583da14cebeSEric Cheng */ 3584da14cebeSEric Cheng mac_srs->srs_state |= SRS_POLL_THR_RESTART; 3585da14cebeSEric Cheng cv_signal(&mac_srs->srs_cv); 3586da14cebeSEric Cheng while (mac_srs->srs_state & SRS_POLL_THR_QUIESCED) 3587da14cebeSEric Cheng cv_wait(&mac_srs->srs_async, &mac_srs->srs_lock); 3588da14cebeSEric Cheng ASSERT(!(mac_srs->srs_state & SRS_POLL_THR_RESTART)); 3589da14cebeSEric Cheng } 3590da14cebeSEric Cheng /* Wake up any waiter waiting for the restart to complete */ 3591da14cebeSEric Cheng mac_srs->srs_state |= SRS_RESTART_DONE; 3592da14cebeSEric Cheng cv_signal(&mac_srs->srs_quiesce_done_cv); 3593da14cebeSEric Cheng } 3594da14cebeSEric Cheng 3595da14cebeSEric Cheng static void 3596da14cebeSEric Cheng mac_srs_worker_unbind(mac_soft_ring_set_t *mac_srs) 3597da14cebeSEric Cheng { 3598da14cebeSEric Cheng mutex_enter(&mac_srs->srs_lock); 3599da14cebeSEric Cheng if (!(mac_srs->srs_state & SRS_WORKER_BOUND)) { 3600da14cebeSEric Cheng ASSERT(mac_srs->srs_worker_cpuid == -1); 3601da14cebeSEric Cheng mutex_exit(&mac_srs->srs_lock); 3602da14cebeSEric Cheng return; 3603da14cebeSEric Cheng } 3604da14cebeSEric Cheng 3605da14cebeSEric Cheng mac_srs->srs_worker_cpuid = -1; 3606da14cebeSEric Cheng mac_srs->srs_state &= ~SRS_WORKER_BOUND; 3607da14cebeSEric Cheng thread_affinity_clear(mac_srs->srs_worker); 3608da14cebeSEric Cheng mutex_exit(&mac_srs->srs_lock); 3609da14cebeSEric Cheng } 3610da14cebeSEric Cheng 3611da14cebeSEric Cheng static void 3612da14cebeSEric Cheng mac_srs_poll_unbind(mac_soft_ring_set_t *mac_srs) 3613da14cebeSEric Cheng { 3614da14cebeSEric Cheng mutex_enter(&mac_srs->srs_lock); 3615da14cebeSEric Cheng if (mac_srs->srs_poll_thr == NULL || 3616da14cebeSEric Cheng (mac_srs->srs_state & SRS_POLL_BOUND) == 0) { 3617da14cebeSEric Cheng ASSERT(mac_srs->srs_poll_cpuid == -1); 3618da14cebeSEric Cheng mutex_exit(&mac_srs->srs_lock); 3619da14cebeSEric Cheng return; 3620da14cebeSEric Cheng } 3621da14cebeSEric Cheng 3622da14cebeSEric Cheng mac_srs->srs_poll_cpuid = -1; 3623da14cebeSEric Cheng mac_srs->srs_state &= ~SRS_POLL_BOUND; 3624da14cebeSEric Cheng thread_affinity_clear(mac_srs->srs_poll_thr); 3625da14cebeSEric Cheng mutex_exit(&mac_srs->srs_lock); 3626da14cebeSEric Cheng } 3627da14cebeSEric Cheng 3628da14cebeSEric Cheng static void 3629da14cebeSEric Cheng mac_srs_threads_unbind(mac_soft_ring_set_t *mac_srs) 3630da14cebeSEric Cheng { 3631da14cebeSEric Cheng mac_soft_ring_t *soft_ring; 3632da14cebeSEric Cheng 3633da14cebeSEric Cheng ASSERT(MAC_PERIM_HELD((mac_handle_t)mac_srs->srs_mcip->mci_mip)); 3634da14cebeSEric Cheng 3635da14cebeSEric Cheng mutex_enter(&cpu_lock); 3636da14cebeSEric Cheng mac_srs_worker_unbind(mac_srs); 3637da14cebeSEric Cheng if (!(mac_srs->srs_type & SRST_TX)) 3638da14cebeSEric Cheng mac_srs_poll_unbind(mac_srs); 3639da14cebeSEric Cheng 3640da14cebeSEric Cheng for (soft_ring = mac_srs->srs_soft_ring_head; soft_ring != NULL; 3641da14cebeSEric Cheng soft_ring = soft_ring->s_ring_next) { 3642da14cebeSEric Cheng mac_soft_ring_unbind(soft_ring); 3643da14cebeSEric Cheng } 3644da14cebeSEric Cheng mutex_exit(&cpu_lock); 3645da14cebeSEric Cheng } 3646da14cebeSEric Cheng 3647da14cebeSEric Cheng /* 3648da14cebeSEric Cheng * When a CPU is going away, unbind all MAC threads which are bound 3649da14cebeSEric Cheng * to that CPU. The affinity of the thread to the CPU is saved to allow 3650da14cebeSEric Cheng * the thread to be rebound to the CPU if it comes back online. 3651da14cebeSEric Cheng */ 3652da14cebeSEric Cheng static void 3653da14cebeSEric Cheng mac_walk_srs_and_unbind(int cpuid) 3654da14cebeSEric Cheng { 3655da14cebeSEric Cheng mac_soft_ring_set_t *mac_srs; 3656da14cebeSEric Cheng mac_soft_ring_t *soft_ring; 3657da14cebeSEric Cheng 3658da14cebeSEric Cheng rw_enter(&mac_srs_g_lock, RW_READER); 3659da14cebeSEric Cheng 3660da14cebeSEric Cheng if ((mac_srs = mac_srs_g_list) == NULL) 3661da14cebeSEric Cheng goto done; 3662da14cebeSEric Cheng 3663da14cebeSEric Cheng for (; mac_srs != NULL; mac_srs = mac_srs->srs_next) { 3664da14cebeSEric Cheng if (mac_srs->srs_worker_cpuid == cpuid) { 3665da14cebeSEric Cheng mac_srs->srs_worker_cpuid_save = cpuid; 3666da14cebeSEric Cheng mac_srs_worker_unbind(mac_srs); 3667da14cebeSEric Cheng } 3668da14cebeSEric Cheng 3669da14cebeSEric Cheng if (!(mac_srs->srs_type & SRST_TX)) { 3670da14cebeSEric Cheng if (mac_srs->srs_poll_cpuid == cpuid) { 3671da14cebeSEric Cheng mac_srs->srs_poll_cpuid_save = cpuid; 3672da14cebeSEric Cheng mac_srs_poll_unbind(mac_srs); 3673da14cebeSEric Cheng } 3674da14cebeSEric Cheng } 3675da14cebeSEric Cheng 3676da14cebeSEric Cheng /* Next tackle the soft rings associated with the srs */ 3677da14cebeSEric Cheng mutex_enter(&mac_srs->srs_lock); 3678da14cebeSEric Cheng for (soft_ring = mac_srs->srs_soft_ring_head; soft_ring != NULL; 3679da14cebeSEric Cheng soft_ring = soft_ring->s_ring_next) { 3680da14cebeSEric Cheng if (soft_ring->s_ring_cpuid == cpuid) { 3681da14cebeSEric Cheng soft_ring->s_ring_cpuid_save = cpuid; 3682da14cebeSEric Cheng mac_soft_ring_unbind(soft_ring); 3683da14cebeSEric Cheng } 3684da14cebeSEric Cheng } 3685da14cebeSEric Cheng mutex_exit(&mac_srs->srs_lock); 3686da14cebeSEric Cheng } 3687da14cebeSEric Cheng done: 3688da14cebeSEric Cheng rw_exit(&mac_srs_g_lock); 3689da14cebeSEric Cheng } 3690da14cebeSEric Cheng 3691da14cebeSEric Cheng /* TX SETUP and TEARDOWN ROUTINES */ 3692da14cebeSEric Cheng 3693da14cebeSEric Cheng /* 3694da14cebeSEric Cheng * XXXHIO need to make sure the two mac_tx_srs_{add,del}_ring() 3695da14cebeSEric Cheng * handle the case where the number of rings is one. I.e. there is 3696da14cebeSEric Cheng * a ring pointed to by mac_srs->srs_tx_arg2. 3697da14cebeSEric Cheng */ 3698da14cebeSEric Cheng void 3699da14cebeSEric Cheng mac_tx_srs_add_ring(mac_soft_ring_set_t *mac_srs, mac_ring_t *tx_ring) 3700da14cebeSEric Cheng { 3701da14cebeSEric Cheng mac_client_impl_t *mcip = mac_srs->srs_mcip; 3702da14cebeSEric Cheng mac_soft_ring_t *soft_ring; 37030dc2366fSVenugopal Iyer int count = mac_srs->srs_tx_ring_count; 37040dc2366fSVenugopal Iyer uint32_t soft_ring_type = ST_RING_TX; 37050dc2366fSVenugopal Iyer uint_t ring_info; 3706da14cebeSEric Cheng 3707da14cebeSEric Cheng ASSERT(mac_srs->srs_state & SRS_QUIESCE); 37080dc2366fSVenugopal Iyer ring_info = mac_hwring_getinfo((mac_ring_handle_t)tx_ring); 37090dc2366fSVenugopal Iyer if (mac_tx_serialize || (ring_info & MAC_RING_TX_SERIALIZE)) 37100dc2366fSVenugopal Iyer soft_ring_type |= ST_RING_WORKER_ONLY; 37110dc2366fSVenugopal Iyer soft_ring = mac_soft_ring_create(count, 0, 37120dc2366fSVenugopal Iyer soft_ring_type, maxclsyspri, mcip, mac_srs, -1, 3713da14cebeSEric Cheng NULL, mcip, (mac_resource_handle_t)tx_ring); 37140dc2366fSVenugopal Iyer mac_srs->srs_tx_ring_count++; 37150dc2366fSVenugopal Iyer mac_srs_update_fanout_list(mac_srs); 3716da14cebeSEric Cheng /* 3717da14cebeSEric Cheng * put this soft ring in quiesce mode too so when we restart 3718da14cebeSEric Cheng * all soft rings in the srs are in the same state. 3719da14cebeSEric Cheng */ 3720da14cebeSEric Cheng mac_soft_ring_signal(soft_ring, S_RING_QUIESCE); 3721da14cebeSEric Cheng } 3722da14cebeSEric Cheng 3723da14cebeSEric Cheng static void 3724da14cebeSEric Cheng mac_soft_ring_remove(mac_soft_ring_set_t *mac_srs, mac_soft_ring_t *softring) 3725da14cebeSEric Cheng { 3726da14cebeSEric Cheng int sringcnt; 3727da14cebeSEric Cheng 3728da14cebeSEric Cheng mutex_enter(&mac_srs->srs_lock); 3729da14cebeSEric Cheng sringcnt = mac_srs->srs_soft_ring_count; 3730da14cebeSEric Cheng ASSERT(sringcnt > 0); 3731da14cebeSEric Cheng mac_soft_ring_signal(softring, S_RING_CONDEMNED); 3732da14cebeSEric Cheng 3733da14cebeSEric Cheng ASSERT(mac_srs->srs_soft_ring_condemned_count == 0); 3734da14cebeSEric Cheng while (mac_srs->srs_soft_ring_condemned_count != 1) 3735da14cebeSEric Cheng cv_wait(&mac_srs->srs_async, &mac_srs->srs_lock); 3736da14cebeSEric Cheng 3737da14cebeSEric Cheng if (softring == mac_srs->srs_soft_ring_head) { 3738da14cebeSEric Cheng mac_srs->srs_soft_ring_head = softring->s_ring_next; 3739da14cebeSEric Cheng if (mac_srs->srs_soft_ring_head != NULL) { 3740da14cebeSEric Cheng mac_srs->srs_soft_ring_head->s_ring_prev = NULL; 3741da14cebeSEric Cheng } else { 3742da14cebeSEric Cheng mac_srs->srs_soft_ring_tail = NULL; 3743da14cebeSEric Cheng } 3744da14cebeSEric Cheng } else { 3745da14cebeSEric Cheng softring->s_ring_prev->s_ring_next = 3746da14cebeSEric Cheng softring->s_ring_next; 3747da14cebeSEric Cheng if (softring->s_ring_next != NULL) { 3748da14cebeSEric Cheng softring->s_ring_next->s_ring_prev = 3749da14cebeSEric Cheng softring->s_ring_prev; 3750da14cebeSEric Cheng } else { 3751da14cebeSEric Cheng mac_srs->srs_soft_ring_tail = 3752da14cebeSEric Cheng softring->s_ring_prev; 3753da14cebeSEric Cheng } 3754da14cebeSEric Cheng } 3755da14cebeSEric Cheng mac_srs->srs_soft_ring_count--; 3756da14cebeSEric Cheng 3757da14cebeSEric Cheng mac_srs->srs_soft_ring_condemned_count--; 3758da14cebeSEric Cheng mutex_exit(&mac_srs->srs_lock); 3759da14cebeSEric Cheng 37600dc2366fSVenugopal Iyer mac_soft_ring_free(softring); 3761da14cebeSEric Cheng } 3762da14cebeSEric Cheng 3763da14cebeSEric Cheng void 3764da14cebeSEric Cheng mac_tx_srs_del_ring(mac_soft_ring_set_t *mac_srs, mac_ring_t *tx_ring) 3765da14cebeSEric Cheng { 3766da14cebeSEric Cheng int i; 3767da14cebeSEric Cheng mac_soft_ring_t *soft_ring, *remove_sring; 37680dc2366fSVenugopal Iyer mac_client_impl_t *mcip = mac_srs->srs_mcip; 3769da14cebeSEric Cheng 3770da14cebeSEric Cheng mutex_enter(&mac_srs->srs_lock); 37710dc2366fSVenugopal Iyer for (i = 0; i < mac_srs->srs_tx_ring_count; i++) { 37720dc2366fSVenugopal Iyer soft_ring = mac_srs->srs_tx_soft_rings[i]; 3773da14cebeSEric Cheng if (soft_ring->s_ring_tx_arg2 == tx_ring) 3774da14cebeSEric Cheng break; 3775da14cebeSEric Cheng } 3776da14cebeSEric Cheng mutex_exit(&mac_srs->srs_lock); 37770dc2366fSVenugopal Iyer ASSERT(i < mac_srs->srs_tx_ring_count); 3778da14cebeSEric Cheng remove_sring = soft_ring; 37790dc2366fSVenugopal Iyer /* 37800dc2366fSVenugopal Iyer * In the case of aggr, the soft ring associated with a Tx ring 37810dc2366fSVenugopal Iyer * is also stored in st_soft_rings[] array. That entry should 37820dc2366fSVenugopal Iyer * be removed. 37830dc2366fSVenugopal Iyer */ 37840dc2366fSVenugopal Iyer if (mcip->mci_state_flags & MCIS_IS_AGGR) { 37850dc2366fSVenugopal Iyer mac_srs_tx_t *tx = &mac_srs->srs_tx; 37860dc2366fSVenugopal Iyer 37870dc2366fSVenugopal Iyer ASSERT(tx->st_soft_rings[tx_ring->mr_index] == remove_sring); 37880dc2366fSVenugopal Iyer tx->st_soft_rings[tx_ring->mr_index] = NULL; 37890dc2366fSVenugopal Iyer } 3790da14cebeSEric Cheng mac_soft_ring_remove(mac_srs, remove_sring); 3791da14cebeSEric Cheng mac_srs_update_fanout_list(mac_srs); 3792da14cebeSEric Cheng } 3793da14cebeSEric Cheng 3794da14cebeSEric Cheng /* 3795da14cebeSEric Cheng * mac_tx_srs_setup(): 3796da14cebeSEric Cheng * Used to setup Tx rings. If no free Tx ring is available, then default 3797da14cebeSEric Cheng * Tx ring is used. 3798da14cebeSEric Cheng */ 3799da14cebeSEric Cheng void 38000dc2366fSVenugopal Iyer mac_tx_srs_setup(mac_client_impl_t *mcip, flow_entry_t *flent) 3801da14cebeSEric Cheng { 3802da14cebeSEric Cheng mac_impl_t *mip = mcip->mci_mip; 38030dc2366fSVenugopal Iyer mac_soft_ring_set_t *tx_srs = flent->fe_tx_srs; 38040dc2366fSVenugopal Iyer int i; 38050dc2366fSVenugopal Iyer int tx_ring_count = 0; 3806da14cebeSEric Cheng uint32_t soft_ring_type; 3807da14cebeSEric Cheng mac_group_t *grp = NULL; 3808da14cebeSEric Cheng mac_ring_t *ring; 38090dc2366fSVenugopal Iyer mac_srs_tx_t *tx = &tx_srs->srs_tx; 38100dc2366fSVenugopal Iyer boolean_t is_aggr; 38110dc2366fSVenugopal Iyer uint_t ring_info = 0; 3812da14cebeSEric Cheng 38130dc2366fSVenugopal Iyer is_aggr = (mcip->mci_state_flags & MCIS_IS_AGGR) != 0; 38140dc2366fSVenugopal Iyer grp = flent->fe_tx_ring_group; 38150dc2366fSVenugopal Iyer if (grp == NULL) { 38160dc2366fSVenugopal Iyer ring = (mac_ring_t *)mip->mi_default_tx_ring; 38170dc2366fSVenugopal Iyer goto no_group; 38180dc2366fSVenugopal Iyer } 3819da14cebeSEric Cheng tx_ring_count = grp->mrg_cur_count; 3820da14cebeSEric Cheng ring = grp->mrg_rings; 3821da14cebeSEric Cheng /* 3822da14cebeSEric Cheng * An attempt is made to reserve 'tx_ring_count' number 3823da14cebeSEric Cheng * of Tx rings. If tx_ring_count is 0, default Tx ring 3824da14cebeSEric Cheng * is used. If it is 1, an attempt is made to reserve one 3825da14cebeSEric Cheng * Tx ring. In both the cases, the ring information is 3826da14cebeSEric Cheng * stored in Tx SRS. If multiple Tx rings are specified, 3827da14cebeSEric Cheng * then each Tx ring will have a Tx-side soft ring. All 3828da14cebeSEric Cheng * these soft rings will be hang off Tx SRS. 3829da14cebeSEric Cheng */ 38300dc2366fSVenugopal Iyer switch (grp->mrg_state) { 38310dc2366fSVenugopal Iyer case MAC_GROUP_STATE_SHARED: 38320dc2366fSVenugopal Iyer case MAC_GROUP_STATE_RESERVED: 38330dc2366fSVenugopal Iyer if (tx_ring_count <= 1 && !is_aggr) { 38340dc2366fSVenugopal Iyer no_group: 38350dc2366fSVenugopal Iyer if (ring != NULL && 38360dc2366fSVenugopal Iyer ring->mr_state != MR_INUSE) { 38370dc2366fSVenugopal Iyer (void) mac_start_ring(ring); 38380dc2366fSVenugopal Iyer ring_info = mac_hwring_getinfo( 38390dc2366fSVenugopal Iyer (mac_ring_handle_t)ring); 38400dc2366fSVenugopal Iyer } 38410dc2366fSVenugopal Iyer tx->st_arg2 = (void *)ring; 38420dc2366fSVenugopal Iyer mac_tx_srs_stat_recreate(tx_srs, B_FALSE); 38430dc2366fSVenugopal Iyer if (tx_srs->srs_type & SRST_BW_CONTROL) { 38440dc2366fSVenugopal Iyer tx->st_mode = SRS_TX_BW; 38450dc2366fSVenugopal Iyer } else if (mac_tx_serialize || 38460dc2366fSVenugopal Iyer (ring_info & MAC_RING_TX_SERIALIZE)) { 38470dc2366fSVenugopal Iyer tx->st_mode = SRS_TX_SERIALIZE; 3848da14cebeSEric Cheng } else { 38490dc2366fSVenugopal Iyer tx->st_mode = SRS_TX_DEFAULT; 38500dc2366fSVenugopal Iyer } 3851da14cebeSEric Cheng break; 3852da14cebeSEric Cheng } 38530dc2366fSVenugopal Iyer soft_ring_type = ST_RING_TX; 3854da14cebeSEric Cheng if (tx_srs->srs_type & SRST_BW_CONTROL) { 38550dc2366fSVenugopal Iyer tx->st_mode = is_aggr ? 38560dc2366fSVenugopal Iyer SRS_TX_BW_AGGR : SRS_TX_BW_FANOUT; 3857da14cebeSEric Cheng } else { 38580dc2366fSVenugopal Iyer tx->st_mode = is_aggr ? SRS_TX_AGGR : 38590dc2366fSVenugopal Iyer SRS_TX_FANOUT; 3860da14cebeSEric Cheng } 38610dc2366fSVenugopal Iyer for (i = 0; i < tx_ring_count; i++) { 38620dc2366fSVenugopal Iyer ASSERT(ring != NULL); 38630dc2366fSVenugopal Iyer switch (ring->mr_state) { 38640dc2366fSVenugopal Iyer case MR_INUSE: 38650dc2366fSVenugopal Iyer case MR_FREE: 38660dc2366fSVenugopal Iyer ASSERT(ring->mr_srs == NULL); 38670dc2366fSVenugopal Iyer 38680dc2366fSVenugopal Iyer if (ring->mr_state != MR_INUSE) 38690dc2366fSVenugopal Iyer (void) mac_start_ring(ring); 38700dc2366fSVenugopal Iyer ring_info = mac_hwring_getinfo( 38710dc2366fSVenugopal Iyer (mac_ring_handle_t)ring); 38720dc2366fSVenugopal Iyer if (mac_tx_serialize || (ring_info & 38730dc2366fSVenugopal Iyer MAC_RING_TX_SERIALIZE)) { 38740dc2366fSVenugopal Iyer soft_ring_type |= 38750dc2366fSVenugopal Iyer ST_RING_WORKER_ONLY; 38760dc2366fSVenugopal Iyer } 38770dc2366fSVenugopal Iyer (void) mac_soft_ring_create(i, 0, 38780dc2366fSVenugopal Iyer soft_ring_type, maxclsyspri, 38790dc2366fSVenugopal Iyer mcip, tx_srs, -1, NULL, mcip, 38800dc2366fSVenugopal Iyer (mac_resource_handle_t)ring); 38810dc2366fSVenugopal Iyer break; 38820dc2366fSVenugopal Iyer default: 38830dc2366fSVenugopal Iyer cmn_err(CE_PANIC, 38840dc2366fSVenugopal Iyer "srs_setup: mcip = %p " 38850dc2366fSVenugopal Iyer "trying to add UNKNOWN ring = %p\n", 38860dc2366fSVenugopal Iyer (void *)mcip, (void *)ring); 38870dc2366fSVenugopal Iyer break; 38880dc2366fSVenugopal Iyer } 38890dc2366fSVenugopal Iyer ring = ring->mr_next; 3890da14cebeSEric Cheng } 3891da14cebeSEric Cheng mac_srs_update_fanout_list(tx_srs); 38920dc2366fSVenugopal Iyer break; 38930dc2366fSVenugopal Iyer default: 38940dc2366fSVenugopal Iyer ASSERT(B_FALSE); 38950dc2366fSVenugopal Iyer break; 3896da14cebeSEric Cheng } 3897da14cebeSEric Cheng tx->st_func = mac_tx_get_func(tx->st_mode); 38980dc2366fSVenugopal Iyer if (is_aggr) { 38990dc2366fSVenugopal Iyer VERIFY(i_mac_capab_get((mac_handle_t)mip, 39000dc2366fSVenugopal Iyer MAC_CAPAB_AGGR, &tx->st_capab_aggr)); 3901da14cebeSEric Cheng } 39020dc2366fSVenugopal Iyer DTRACE_PROBE3(tx__srs___setup__return, mac_soft_ring_set_t *, tx_srs, 39030dc2366fSVenugopal Iyer int, tx->st_mode, int, tx_srs->srs_tx_ring_count); 3904da14cebeSEric Cheng } 3905da14cebeSEric Cheng 3906da14cebeSEric Cheng /* 390708ac1c49SNicolas Droux * Update the fanout of a client if its recorded link speed doesn't match 390808ac1c49SNicolas Droux * its current link speed. 390908ac1c49SNicolas Droux */ 391008ac1c49SNicolas Droux void 39110dc2366fSVenugopal Iyer mac_fanout_recompute_client(mac_client_impl_t *mcip, cpupart_t *cpupart) 391208ac1c49SNicolas Droux { 391308ac1c49SNicolas Droux uint64_t link_speed; 391408ac1c49SNicolas Droux mac_resource_props_t *mcip_mrp; 39150dc2366fSVenugopal Iyer flow_entry_t *flent = mcip->mci_flent; 39160dc2366fSVenugopal Iyer mac_soft_ring_set_t *rx_srs; 39170dc2366fSVenugopal Iyer mac_cpus_t *srs_cpu; 39180dc2366fSVenugopal Iyer int soft_ring_count, maxcpus; 391908ac1c49SNicolas Droux 392008ac1c49SNicolas Droux ASSERT(MAC_PERIM_HELD((mac_handle_t)mcip->mci_mip)); 392108ac1c49SNicolas Droux 392208ac1c49SNicolas Droux link_speed = mac_client_stat_get(mcip->mci_flent->fe_mcip, 392308ac1c49SNicolas Droux MAC_STAT_IFSPEED); 392408ac1c49SNicolas Droux 392508ac1c49SNicolas Droux if ((link_speed != 0) && 392608ac1c49SNicolas Droux (link_speed != mcip->mci_flent->fe_nic_speed)) { 392708ac1c49SNicolas Droux mcip_mrp = MCIP_RESOURCE_PROPS(mcip); 39280dc2366fSVenugopal Iyer /* 39290dc2366fSVenugopal Iyer * Before calling mac_fanout_setup(), check to see if 39300dc2366fSVenugopal Iyer * the SRSes already have the right number of soft 39310dc2366fSVenugopal Iyer * rings. mac_fanout_setup() is a heavy duty operation 39320dc2366fSVenugopal Iyer * where new cpu bindings are done for SRS and soft 39330dc2366fSVenugopal Iyer * ring threads and interrupts re-targeted. 39340dc2366fSVenugopal Iyer */ 39350dc2366fSVenugopal Iyer maxcpus = (cpupart != NULL) ? cpupart->cp_ncpus : ncpus; 39360dc2366fSVenugopal Iyer soft_ring_count = mac_compute_soft_ring_count(flent, 39370dc2366fSVenugopal Iyer flent->fe_rx_srs_cnt - 1, maxcpus); 39380dc2366fSVenugopal Iyer /* 39390dc2366fSVenugopal Iyer * If soft_ring_count returned by 39400dc2366fSVenugopal Iyer * mac_compute_soft_ring_count() is 0, bump it 39410dc2366fSVenugopal Iyer * up by 1 because we always have atleast one 39420dc2366fSVenugopal Iyer * TCP, UDP, and OTH soft ring associated with 39430dc2366fSVenugopal Iyer * an SRS. 39440dc2366fSVenugopal Iyer */ 39450dc2366fSVenugopal Iyer soft_ring_count = (soft_ring_count == 0) ? 39460dc2366fSVenugopal Iyer 1 : soft_ring_count; 39470dc2366fSVenugopal Iyer rx_srs = flent->fe_rx_srs[0]; 39480dc2366fSVenugopal Iyer srs_cpu = &rx_srs->srs_cpu; 39490dc2366fSVenugopal Iyer if (soft_ring_count != srs_cpu->mc_rx_fanout_cnt) { 39500dc2366fSVenugopal Iyer mac_fanout_setup(mcip, flent, mcip_mrp, 39510dc2366fSVenugopal Iyer mac_rx_deliver, mcip, NULL, cpupart); 39520dc2366fSVenugopal Iyer } 395308ac1c49SNicolas Droux } 395408ac1c49SNicolas Droux } 395508ac1c49SNicolas Droux 395608ac1c49SNicolas Droux /* 3957da14cebeSEric Cheng * Walk through the list of mac clients for the MAC. 3958da14cebeSEric Cheng * For each active mac client, recompute the number of soft rings 3959da14cebeSEric Cheng * associated with every client, only if current speed is different 3960da14cebeSEric Cheng * from the speed that was previously used for soft ring computation. 3961da14cebeSEric Cheng * If the cable is disconnected whlie the NIC is started, we would get 3962da14cebeSEric Cheng * notification with speed set to 0. We do not recompute in that case. 3963da14cebeSEric Cheng */ 3964da14cebeSEric Cheng void 3965da14cebeSEric Cheng mac_fanout_recompute(mac_impl_t *mip) 3966da14cebeSEric Cheng { 3967da14cebeSEric Cheng mac_client_impl_t *mcip; 39680dc2366fSVenugopal Iyer cpupart_t *cpupart; 39690dc2366fSVenugopal Iyer boolean_t use_default; 39700dc2366fSVenugopal Iyer mac_resource_props_t *mrp, *emrp; 397108ac1c49SNicolas Droux 3972da14cebeSEric Cheng i_mac_perim_enter(mip); 39732c4ec682SEric Cheng if ((mip->mi_state_flags & MIS_IS_VNIC) != 0 || 39742c4ec682SEric Cheng mip->mi_linkstate != LINK_STATE_UP) { 3975da14cebeSEric Cheng i_mac_perim_exit(mip); 3976da14cebeSEric Cheng return; 3977da14cebeSEric Cheng } 3978da14cebeSEric Cheng 3979da14cebeSEric Cheng for (mcip = mip->mi_clients_list; mcip != NULL; 3980da14cebeSEric Cheng mcip = mcip->mci_client_next) { 398108ac1c49SNicolas Droux if ((mcip->mci_state_flags & MCIS_SHARE_BOUND) != 0 || 398208ac1c49SNicolas Droux !MCIP_DATAPATH_SETUP(mcip)) 3983da14cebeSEric Cheng continue; 39840dc2366fSVenugopal Iyer mrp = MCIP_RESOURCE_PROPS(mcip); 39850dc2366fSVenugopal Iyer emrp = MCIP_EFFECTIVE_PROPS(mcip); 39860dc2366fSVenugopal Iyer use_default = B_FALSE; 39870dc2366fSVenugopal Iyer pool_lock(); 39880dc2366fSVenugopal Iyer cpupart = mac_pset_find(mrp, &use_default); 39890dc2366fSVenugopal Iyer mac_fanout_recompute_client(mcip, cpupart); 39900dc2366fSVenugopal Iyer mac_set_pool_effective(use_default, cpupart, mrp, emrp); 39910dc2366fSVenugopal Iyer pool_unlock(); 3992da14cebeSEric Cheng } 3993da14cebeSEric Cheng i_mac_perim_exit(mip); 3994da14cebeSEric Cheng } 39954eaa4710SRishi Srivatsavai 39964eaa4710SRishi Srivatsavai /* 39974eaa4710SRishi Srivatsavai * Given a MAC, change the polling state for all its MAC clients. 'enable' is 39984eaa4710SRishi Srivatsavai * B_TRUE to enable polling or B_FALSE to disable. Polling is enabled by 39994eaa4710SRishi Srivatsavai * default. 40004eaa4710SRishi Srivatsavai */ 40014eaa4710SRishi Srivatsavai void 40024eaa4710SRishi Srivatsavai mac_poll_state_change(mac_handle_t mh, boolean_t enable) 40034eaa4710SRishi Srivatsavai { 40044eaa4710SRishi Srivatsavai mac_impl_t *mip = (mac_impl_t *)mh; 40054eaa4710SRishi Srivatsavai mac_client_impl_t *mcip; 40064eaa4710SRishi Srivatsavai 40074eaa4710SRishi Srivatsavai i_mac_perim_enter(mip); 40084eaa4710SRishi Srivatsavai if (enable) 40094eaa4710SRishi Srivatsavai mip->mi_state_flags &= ~MIS_POLL_DISABLE; 40104eaa4710SRishi Srivatsavai else 40114eaa4710SRishi Srivatsavai mip->mi_state_flags |= MIS_POLL_DISABLE; 40124eaa4710SRishi Srivatsavai for (mcip = mip->mi_clients_list; mcip != NULL; 40134eaa4710SRishi Srivatsavai mcip = mcip->mci_client_next) 40144eaa4710SRishi Srivatsavai mac_client_update_classifier(mcip, B_TRUE); 40154eaa4710SRishi Srivatsavai i_mac_perim_exit(mip); 40164eaa4710SRishi Srivatsavai } 4017