1 /* 2 * CDDL HEADER START 3 * 4 * The contents of this file are subject to the terms of the 5 * Common Development and Distribution License (the "License"). 6 * You may not use this file except in compliance with the License. 7 * 8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE 9 * or http://www.opensolaris.org/os/licensing. 10 * See the License for the specific language governing permissions 11 * and limitations under the License. 12 * 13 * When distributing Covered Code, include this CDDL HEADER in each 14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE. 15 * If applicable, add the following below this CDDL HEADER, with the 16 * fields enclosed by brackets "[]" replaced with your own identifying 17 * information: Portions Copyright [yyyy] [name of copyright owner] 18 * 19 * CDDL HEADER END 20 */ 21 /* 22 * Copyright 2007 Sun Microsystems, Inc. All rights reserved. 23 * Use is subject to license terms. 24 */ 25 26 #pragma ident "%Z%%M% %I% %E% SMI" 27 28 #include <sys/types.h> 29 #include <sys/param.h> 30 #include <sys/t_lock.h> 31 #include <sys/systm.h> 32 #include <sys/spl.h> 33 #include <sys/cmn_err.h> 34 #include <sys/debug.h> 35 #include <sys/kdi_impl.h> 36 #include <sys/cpuvar.h> 37 #include <sys/cpuvar.h> 38 #include <sys/archsystm.h> 39 40 /* 41 * Handle software interrupts through 'softcall' mechanism 42 * 43 * At present softcall mechanism uses a global list headed by softhead. 44 * Entries are added to tail and removed from head so as to preserve FIFO 45 * nature of entries in the softcall list. softcall() takes care of adding 46 * entries to the softtail. 47 * 48 * softint must take care of executing the entries in the FIFO 49 * order. It could be called simultaneously from multiple cpus, however only 50 * one instance of softint should process the softcall list with the exception 51 * when CPU is stuck due to high interrupt load and can't execute callbacks. 52 * State diagram is as follows :- 53 * 54 * - Upper half which is same as old state machine 55 * (IDLE->PEND->DRAIN->IDLE) 56 * 57 * - Lower half which steals the entries from softcall queue and execute 58 * in the context of softint interrupt handler. The interrupt handler 59 * is fired on a different CPU by sending a cross-call. 60 * 61 * Starting state is IDLE. 62 * 63 * softint() 64 * 65 * 66 * (c) 67 * ____________________________________________________ 68 * | ^ ^ 69 * v (a) | (b) | 70 * IDLE--------------------->PEND--------------------->DRAIN 71 * ^ | | 72 * | | | 73 * | | | 74 * | | | 75 * | | | 76 * | d d 77 * | | | 78 * | v v 79 * | PEND DRAIN 80 * | (e) & & 81 * |<-----------------------STEAL STEAL 82 * ^ | 83 * | | 84 * | (e) v 85 * |_________________________<__________________________| 86 * 87 * 88 * 89 * Edge (a)->(b)->(c) are same as old state machine and these 90 * are mutually exclusive state. 91 * 92 * a - When an entry is being enqueued to softcall queue then the state 93 * moves from IDLE to PEND. 94 * 95 * b - When interrupt handler has started processing softcall queue. 96 * 97 * c - When interrupt handler finished processing softcall queue, the 98 * state of machines goes back to IDLE. 99 * 100 * d - softcall() generates another softlevel1 iff interrupt handler 101 * hasn't run recently. 102 * 103 * e - Either PEND|STEAL or DRAIN|STEAL is set. We let softlevel1 104 * handler exit because we have processed all the entries. 105 * 106 * When CPU is being pinned by higher level interrupts for more than 107 * softcall_delay clock ticks, SOFT_STEAL is OR'ed so that softlevel1 108 * handler on the other CPU can drain the queue. 109 * 110 * These states are needed for softcall mechanism since Solaris has only 111 * one interface (ie. siron ) as of now for : 112 * 113 * - raising a soft interrupt architecture independently (ie not through 114 * setsoftint(..) ) 115 * - to process the softcall queue. 116 */ 117 118 #define NSOFTCALLS 200 119 120 /* 121 * Defined states for softcall processing. 122 */ 123 #define SOFT_IDLE 0x01 /* no processing is needed */ 124 #define SOFT_PEND 0x02 /* softcall list needs processing */ 125 #define SOFT_DRAIN 0x04 /* list is being processed */ 126 #define SOFT_STEAL 0x08 /* list is being stolen for draining */ 127 128 typedef struct softcall { 129 void (*sc_func)(void *); /* function to call */ 130 void *sc_arg; /* arg to pass to func */ 131 struct softcall *sc_next; /* next in list */ 132 } softcall_t; 133 134 /* 135 * softcall list and state variables. 136 */ 137 static softcall_t *softcalls; 138 static softcall_t *softhead, *softtail, *softfree; 139 static uint_t softcall_state; 140 static clock_t softcall_tick; 141 142 /* 143 * This ensures that softcall entries don't get stuck for long. It's expressed 144 * in 10 milliseconds as 1 unit. When hires_tick is set or other clock frequency 145 * is used, softcall_init() ensures that it's still expressed as 1 = 10 milli 146 * seconds. 147 */ 148 static int softcall_delay = 1; 149 150 /* 151 * The last CPU which will drain softcall queue. 152 */ 153 static int softcall_latest_cpuid = -1; 154 155 /* 156 * CPUSET to hold the CPU which is processing softcall queue 157 * currently. There can be more than one CPU having bit set 158 * but it will happen only when they are stuck. 159 */ 160 static cpuset_t *softcall_cpuset = NULL; 161 162 /* 163 * protects softcall lists and control variable softcall_state. 164 */ 165 static kmutex_t softcall_lock; 166 167 static void (*kdi_softcall_func)(void); 168 extern void siron_poke_cpu(cpuset_t); 169 170 extern void siron(void); 171 172 void 173 softcall_init(void) 174 { 175 softcall_t *sc; 176 177 softcalls = kmem_zalloc(sizeof (softcall_t) * NSOFTCALLS, KM_SLEEP); 178 softcall_cpuset = kmem_zalloc(sizeof (cpuset_t), KM_SLEEP); 179 for (sc = softcalls; sc < &softcalls[NSOFTCALLS]; sc++) { 180 sc->sc_next = softfree; 181 softfree = sc; 182 } 183 mutex_init(&softcall_lock, NULL, MUTEX_SPIN, 184 (void *)ipltospl(SPL8)); 185 softcall_state = SOFT_IDLE; 186 softcall_tick = lbolt; 187 188 if (softcall_delay < 0) 189 softcall_delay = 1; 190 191 /* 192 * Since softcall_delay is expressed as 1 = 10 milliseconds. 193 */ 194 softcall_delay = softcall_delay * (hz/100); 195 CPUSET_ZERO(*softcall_cpuset); 196 } 197 198 /* 199 * Gets called when softcall queue is not moving forward. We choose 200 * a CPU and poke except the ones which are already poked. 201 */ 202 static int 203 softcall_choose_cpu() 204 { 205 cpu_t *cplist = CPU; 206 cpu_t *cp; 207 int intr_load = INT_MAX; 208 int cpuid = -1; 209 cpuset_t poke; 210 int s; 211 212 ASSERT(getpil() >= DISP_LEVEL); 213 ASSERT(ncpus > 1); 214 ASSERT(MUTEX_HELD(&softcall_lock)); 215 216 CPUSET_ZERO(poke); 217 218 /* 219 * The hint is to start from current CPU. 220 */ 221 cp = cplist; 222 do { 223 if (CPU_IN_SET(*softcall_cpuset, cp->cpu_id) || 224 (cp->cpu_flags & CPU_ENABLE) == 0) 225 continue; 226 227 /* if CPU is not busy */ 228 if (cp->cpu_intrload == 0) { 229 cpuid = cp->cpu_id; 230 break; 231 } 232 233 if (cp->cpu_intrload < intr_load) { 234 cpuid = cp->cpu_id; 235 intr_load = cp->cpu_intrload; 236 } else if (cp->cpu_intrload == intr_load) { 237 /* 238 * We want to poke CPUs having similar 239 * load because we don't know which CPU is 240 * can acknowledge level1 interrupt. The 241 * list of such CPUs should not be large. 242 */ 243 if (cpuid != -1) { 244 /* 245 * Put the last CPU chosen because 246 * it also has same interrupt load. 247 */ 248 CPUSET_ADD(poke, cpuid); 249 cpuid = -1; 250 } 251 252 CPUSET_ADD(poke, cp->cpu_id); 253 } 254 } while ((cp = cp->cpu_next_onln) != cplist); 255 256 /* if we found a CPU which suits best to poke */ 257 if (cpuid != -1) { 258 CPUSET_ZERO(poke); 259 CPUSET_ADD(poke, cpuid); 260 } 261 262 if (CPUSET_ISNULL(poke)) { 263 mutex_exit(&softcall_lock); 264 return (0); 265 } 266 267 /* 268 * We first set the bit in cpuset and then poke. 269 */ 270 CPUSET_XOR(*softcall_cpuset, poke); 271 mutex_exit(&softcall_lock); 272 273 /* 274 * If softcall() was called at low pil then we may 275 * get preempted before we raise PIL. It should be okay 276 * because we are just going to poke CPUs now or at most 277 * another thread may start choosing CPUs in this routine. 278 */ 279 s = splhigh(); 280 siron_poke_cpu(poke); 281 splx(s); 282 return (1); 283 } 284 285 /* 286 * Call function func with argument arg 287 * at some later time at software interrupt priority 288 */ 289 void 290 softcall(void (*func)(void *), void *arg) 291 { 292 softcall_t *sc; 293 clock_t w; 294 295 /* 296 * protect against cross-calls 297 */ 298 mutex_enter(&softcall_lock); 299 /* coalesce identical softcalls */ 300 for (sc = softhead; sc != 0; sc = sc->sc_next) { 301 if (sc->sc_func == func && sc->sc_arg == arg) { 302 goto intr; 303 } 304 } 305 306 if ((sc = softfree) == 0) 307 panic("too many softcalls"); 308 309 softfree = sc->sc_next; 310 sc->sc_func = func; 311 sc->sc_arg = arg; 312 sc->sc_next = 0; 313 314 if (softhead) { 315 softtail->sc_next = sc; 316 softtail = sc; 317 } else 318 softhead = softtail = sc; 319 320 intr: 321 if (softcall_state & SOFT_IDLE) { 322 softcall_state = SOFT_PEND; 323 softcall_tick = lbolt; 324 mutex_exit(&softcall_lock); 325 siron(); 326 } else if (softcall_state & (SOFT_DRAIN|SOFT_PEND)) { 327 w = lbolt - softcall_tick; 328 if (w <= softcall_delay || ncpus == 1) { 329 mutex_exit(&softcall_lock); 330 return; 331 } 332 333 if (!(softcall_state & SOFT_STEAL)) { 334 softcall_state |= SOFT_STEAL; 335 336 /* 337 * We want to give some more chance before 338 * fishing around again. 339 */ 340 softcall_tick = lbolt; 341 } 342 343 /* softcall_lock will be released by this routine */ 344 (void) softcall_choose_cpu(); 345 } 346 } 347 348 void 349 kdi_softcall(void (*func)(void)) 350 { 351 kdi_softcall_func = func; 352 353 if (softhead == NULL) 354 siron(); 355 } 356 357 /* 358 * Called to process software interrupts take one off queue, call it, 359 * repeat. 360 * 361 * Note queue may change during call; softcall_lock, state variables 362 * softcall_state and softcall_latest_cpuid ensures that - 363 * - we don't have multiple cpus pulling from the list (thus causing 364 * a violation of FIFO order with an exception when we are stuck). 365 * - we don't miss a new entry having been added to the head. 366 * - we don't miss a wakeup. 367 */ 368 369 void 370 softint(void) 371 { 372 softcall_t *sc = NULL; 373 void (*func)(); 374 caddr_t arg; 375 int cpu_id = CPU->cpu_id; 376 377 mutex_enter(&softcall_lock); 378 379 if (softcall_state & (SOFT_STEAL|SOFT_PEND)) { 380 softcall_state = SOFT_DRAIN; 381 } else { 382 /* 383 * The check for softcall_cpuset being 384 * NULL is required because it may get 385 * called very early during boot. 386 */ 387 if (softcall_cpuset != NULL && 388 CPU_IN_SET(*softcall_cpuset, cpu_id)) 389 CPUSET_DEL(*softcall_cpuset, cpu_id); 390 mutex_exit(&softcall_lock); 391 goto out; 392 } 393 394 /* 395 * Setting softcall_latest_cpuid to current CPU ensures 396 * that there is only one active softlevel1 handler to 397 * process softcall queues. 398 * 399 * Since softcall_lock lock is dropped before calling 400 * func (callback), we need softcall_latest_cpuid 401 * to prevent two softlevel1 hanlders working on the 402 * queue when the first softlevel1 handler gets 403 * stuck due to high interrupt load. 404 */ 405 softcall_latest_cpuid = cpu_id; 406 407 /* add ourself to the cpuset */ 408 if (!CPU_IN_SET(*softcall_cpuset, cpu_id)) 409 CPUSET_ADD(*softcall_cpuset, cpu_id); 410 411 for (;;) { 412 softcall_tick = lbolt; 413 if ((sc = softhead) != NULL) { 414 func = sc->sc_func; 415 arg = sc->sc_arg; 416 softhead = sc->sc_next; 417 sc->sc_next = softfree; 418 softfree = sc; 419 } 420 421 if (sc == NULL) { 422 if (CPU_IN_SET(*softcall_cpuset, cpu_id)) 423 CPUSET_DEL(*softcall_cpuset, cpu_id); 424 425 softcall_state = SOFT_IDLE; 426 ASSERT(softcall_latest_cpuid == cpu_id); 427 softcall_latest_cpuid = -1; 428 429 mutex_exit(&softcall_lock); 430 break; 431 } 432 433 mutex_exit(&softcall_lock); 434 func(arg); 435 mutex_enter(&softcall_lock); 436 437 /* 438 * No longer need softcall processing from current 439 * interrupt handler because either 440 * (a) softcall is in SOFT_IDLE state or 441 * (b) There is a CPU already draining softcall 442 * queue and the current softlevel1 is no 443 * longer required. 444 */ 445 if (softcall_latest_cpuid != cpu_id) { 446 if (CPU_IN_SET(*softcall_cpuset, cpu_id)) 447 CPUSET_DEL(*softcall_cpuset, cpu_id); 448 449 mutex_exit(&softcall_lock); 450 break; 451 } 452 } 453 454 out: 455 if ((func = kdi_softcall_func) != NULL) { 456 kdi_softcall_func = NULL; 457 func(); 458 } 459 } 460