1 /* 2 * Copyright (c) 2017 The DragonFly Project. All rights reserved. 3 * 4 * This code is derived from software contributed to The DragonFly Project 5 * by Sepherosa Ziehau <sepherosa@gmail.com> 6 * 7 * Redistribution and use in source and binary forms, with or without 8 * modification, are permitted provided that the following conditions 9 * are met: 10 * 11 * 1. Redistributions of source code must retain the above copyright 12 * notice, this list of conditions and the following disclaimer. 13 * 2. Redistributions in binary form must reproduce the above copyright 14 * notice, this list of conditions and the following disclaimer in 15 * the documentation and/or other materials provided with the 16 * distribution. 17 * 3. Neither the name of The DragonFly Project nor the names of its 18 * contributors may be used to endorse or promote products derived 19 * from this software without specific, prior written permission. 20 * 21 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 22 * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 23 * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS 24 * FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE 25 * COPYRIGHT HOLDERS OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, 26 * INCIDENTAL, SPECIAL, EXEMPLARY OR CONSEQUENTIAL DAMAGES (INCLUDING, 27 * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; 28 * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED 29 * AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, 30 * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT 31 * OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 32 * SUCH DAMAGE. 33 */ 34 35 #include <sys/param.h> 36 #include <sys/systm.h> 37 #include <sys/kernel.h> 38 #include <sys/thread.h> 39 40 #include <sys/cpuhelper.h> 41 42 static struct thread *cpuhelper[MAXCPU]; 43 static int (*cpuhelper_saved_putport)(lwkt_port_t, lwkt_msg_t); 44 45 static __inline lwkt_port_t 46 cpuhelper_port(int cpuid) 47 { 48 49 return (&cpuhelper[cpuid]->td_msgport); 50 } 51 52 /* 53 * To prevent deadlocking, we must execute these self-referential messages 54 * synchronously, effectively turning the message into a direct procedure 55 * call. 56 */ 57 static int 58 cpuhelper_putport(lwkt_port_t port, lwkt_msg_t lmsg) 59 { 60 struct cpuhelper_msg *msg = (struct cpuhelper_msg *)lmsg; 61 62 63 if ((lmsg->ms_flags & MSGF_SYNC) && port == &curthread->td_msgport) { 64 msg->ch_cb(msg); 65 return (EASYNC); 66 } else { 67 return (cpuhelper_saved_putport(port, lmsg)); 68 } 69 } 70 71 static void 72 cpuhelper_mainloop(void *arg __unused) 73 { 74 struct cpuhelper_msg *msg; 75 76 77 while ((msg = lwkt_waitport(&curthread->td_msgport, 0))) { 78 KASSERT(msg->ch_cb != NULL, ("cpuhelper%d: badmsg", mycpuid)); 79 msg->ch_cb(msg); 80 } 81 /* NEVER REACHED */ 82 } 83 84 void 85 cpuhelper_initmsg(struct cpuhelper_msg *msg, lwkt_port_t rport, 86 cpuhelper_cb_t cb, void *cbarg, int flags) 87 { 88 89 lwkt_initmsg(&msg->ch_lmsg, rport, flags); 90 msg->ch_cb = cb; 91 msg->ch_cbarg = cbarg; 92 msg->ch_cbarg1 = 0; 93 } 94 95 void 96 cpuhelper_replymsg(struct cpuhelper_msg *msg, int error) 97 { 98 99 lwkt_replymsg(&msg->ch_lmsg, error); 100 } 101 102 int 103 cpuhelper_domsg(struct cpuhelper_msg *msg, int cpuid) 104 { 105 106 KASSERT(cpuid >= 0 && cpuid < ncpus, ("invalid cpuid")); 107 return (lwkt_domsg(cpuhelper_port(cpuid), &msg->ch_lmsg, 0)); 108 } 109 110 void 111 cpuhelper_assert(int cpuid, bool in) 112 { 113 114 #ifdef INVARIANTS 115 KASSERT(cpuid >= 0 && cpuid < ncpus, ("invalid cpuid")); 116 if (in) { 117 KASSERT(&curthread->td_msgport == cpuhelper_port(cpuid), 118 ("not in cpuhelper%d", cpuid)); 119 } else { 120 KASSERT(&curthread->td_msgport != cpuhelper_port(cpuid), 121 ("in cpuhelper%d", cpuid)); 122 } 123 #endif 124 } 125 126 static void 127 cpuhelper_init(void) 128 { 129 int i; 130 131 /* 132 * Create per-cpu helper threads. 133 */ 134 for (i = 0; i < ncpus; ++i) { 135 lwkt_port_t port; 136 137 lwkt_create(cpuhelper_mainloop, NULL, &cpuhelper[i], NULL, 138 TDF_NOSTART|TDF_FORCE_SPINPORT|TDF_FIXEDCPU, i, 139 "cpuhelper %d", i); 140 141 /* 142 * Override the putport function. Our custom function checks 143 * for self-references. 144 */ 145 port = cpuhelper_port(i); 146 if (cpuhelper_saved_putport == NULL) 147 cpuhelper_saved_putport = port->mp_putport; 148 KKASSERT(cpuhelper_saved_putport == port->mp_putport); 149 port->mp_putport = cpuhelper_putport; 150 151 lwkt_schedule(cpuhelper[i]); 152 } 153 } 154 SYSINIT(cpuhelper, SI_SUB_PRE_DRIVERS, SI_ORDER_FIRST, cpuhelper_init, NULL); 155