1 /* 2 * Copyright (c) 2020 The DragonFly Project. All rights reserved. 3 * 4 * This code is derived from software contributed to The DragonFly Project 5 * by Matthew Dillon <dillon@backplane.com> 6 * 7 * Redistribution and use in source and binary forms, with or without 8 * modification, are permitted provided that the following conditions 9 * are met: 10 * 11 * 1. Redistributions of source code must retain the above copyright 12 * notice, this list of conditions and the following disclaimer. 13 * 2. Redistributions in binary form must reproduce the above copyright 14 * notice, this list of conditions and the following disclaimer in 15 * the documentation and/or other materials provided with the 16 * distribution. 17 * 3. Neither the name of The DragonFly Project nor the names of its 18 * contributors may be used to endorse or promote products derived 19 * from this software without specific, prior written permission. 20 * 21 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 22 * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 23 * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS 24 * FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE 25 * COPYRIGHT HOLDERS OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, 26 * INCIDENTAL, SPECIAL, EXEMPLARY OR CONSEQUENTIAL DAMAGES (INCLUDING, 27 * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; 28 * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED 29 * AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, 30 * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT 31 * OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 32 * SUCH DAMAGE. 33 */ 34 /* 35 * Brute-force implementation for linux RCU functions. Just use a delay 36 * line and per-cpu callouts. 37 */ 38 #include <sys/types.h> 39 #include <sys/errno.h> 40 #include <sys/kernel.h> 41 #include <sys/spinlock.h> 42 #include <sys/spinlock2.h> 43 #include <linux/gfp.h> 44 #include <linux/slab.h> 45 #include <linux/rcupdate.h> 46 47 #include <machine/atomic.h> 48 49 typedef struct rcu_elm { 50 enum { RCU_NULL, RCU_CALL, RCU_FREE } type; 51 int ticks; 52 void (*func)(struct rcu_head *arg); 53 void *ptr; 54 } rcu_elm_t; 55 56 typedef struct rcu_pcpu { 57 rcu_elm_t *elms; 58 int size; 59 int mask; 60 int s; 61 int e; 62 int running; 63 struct callout timer_callout; 64 } rcu_pcpu_t; 65 66 static rcu_pcpu_t *rcupcpu; 67 68 /* 69 * Timer callout (pcpu) 70 */ 71 static void 72 rcu_timer(void *arg) 73 { 74 rcu_pcpu_t *rcu = arg; 75 rcu_elm_t *elm; 76 int delta; 77 78 crit_enter(); 79 while (rcu->s != rcu->e) { 80 elm = &rcu->elms[rcu->s & rcu->mask]; 81 delta = ticks - elm->ticks; /* 2s compl underflow */ 82 if (delta < hz) 83 break; 84 85 switch(elm->type) { 86 case RCU_NULL: 87 break; 88 case RCU_CALL: 89 elm->func(elm->ptr); 90 break; 91 case RCU_FREE: 92 kfree(elm->ptr); 93 break; 94 } 95 elm->type = RCU_NULL; 96 ++rcu->s; 97 } 98 if (rcu->s == rcu->e) { 99 rcu->running = 0; 100 } else { 101 callout_reset_bycpu(&rcu->timer_callout, hz / 10, rcu_timer, 102 rcu, mycpuid); 103 } 104 crit_exit(); 105 } 106 107 108 /* 109 * Ping timer callout (pcpu). 110 * 111 * Must be in critical section. 112 */ 113 static void 114 rcu_ping(rcu_pcpu_t *rcu) 115 { 116 if (rcu->running == 0) { 117 rcu->running = 1; 118 callout_reset_bycpu(&rcu->timer_callout, hz / 10, rcu_timer, 119 rcu, mycpuid); 120 } 121 } 122 123 /* 124 * Expand the rcu array for the current cpu 125 */ 126 static void 127 rcu_expand(rcu_pcpu_t *rcu) 128 { 129 rcu_elm_t *oelms; 130 rcu_elm_t *nelms; 131 int count; 132 int nsize; 133 int nmask; 134 int n; 135 136 count = rcu->e - rcu->s; /* note: 2s complement underflow */ 137 while (unlikely(count == rcu->size)) { 138 nsize = count ? count * 2 : 16; 139 nelms = kzalloc(nsize * sizeof(*nelms), GFP_KERNEL); 140 kprintf("drm: expand RCU cpu %d to %d\n", mycpuid, nsize); 141 if (likely(count == rcu->size)) { 142 nmask = nsize - 1; 143 oelms = rcu->elms; 144 n = rcu->s; 145 while (n != rcu->e) { 146 nelms[n & nmask] = oelms[n & rcu->mask]; 147 ++n; 148 } 149 rcu->elms = nelms; 150 rcu->size = nsize; 151 rcu->mask = nmask; 152 nelms = oelms; 153 } 154 if (likely(nelms != NULL)) 155 kfree(nelms); 156 count = rcu->e - rcu->s; 157 } 158 KKASSERT(count >= 0 && count < rcu->size); 159 } 160 161 void 162 __kfree_rcu(void *ptr) 163 { 164 rcu_pcpu_t *rcu; 165 rcu_elm_t *elm; 166 167 if (unlikely(rcupcpu == NULL)) { 168 kfree(ptr); 169 return; 170 } 171 rcu = &rcupcpu[mycpuid]; 172 173 crit_enter(); 174 rcu_expand(rcu); 175 elm = &rcu->elms[rcu->e & rcu->mask]; 176 ++rcu->e; 177 178 elm->type = RCU_FREE; 179 elm->ticks = ticks; 180 elm->ptr = ptr; 181 182 rcu_ping(rcu); 183 crit_exit(); 184 } 185 186 void 187 call_rcu(struct rcu_head *head, void (*func)(struct rcu_head *)) 188 { 189 rcu_pcpu_t *rcu; 190 rcu_elm_t *elm; 191 192 if (unlikely(rcupcpu == NULL)) { 193 func(head); 194 return; 195 } 196 rcu = &rcupcpu[mycpuid]; 197 198 crit_enter(); 199 rcu_expand(rcu); 200 elm = &rcu->elms[rcu->e & rcu->mask]; 201 ++rcu->e; 202 203 elm->type = RCU_CALL; 204 elm->ticks = ticks; 205 elm->func = func; 206 elm->ptr = head; 207 208 rcu_ping(rcu); 209 crit_exit(); 210 } 211 212 static int 213 init_rcu(void *dummy __unused) 214 { 215 int i; 216 217 rcupcpu = kzalloc(ncpus * sizeof(*rcupcpu), GFP_KERNEL); 218 for (i = 0; i < ncpus; ++i) { 219 callout_init_mp(&rcupcpu[i].timer_callout); 220 } 221 return 0; 222 } 223 224 SYSINIT(linux_rcu_init, SI_SUB_DRIVERS, SI_ORDER_MIDDLE, init_rcu, NULL); 225