1 /*
2 * Copyright (c) 2020 The DragonFly Project. All rights reserved.
3 *
4 * This code is derived from software contributed to The DragonFly Project
5 * by Matthew Dillon <dillon@backplane.com>
6 *
7 * Redistribution and use in source and binary forms, with or without
8 * modification, are permitted provided that the following conditions
9 * are met:
10 *
11 * 1. Redistributions of source code must retain the above copyright
12 * notice, this list of conditions and the following disclaimer.
13 * 2. Redistributions in binary form must reproduce the above copyright
14 * notice, this list of conditions and the following disclaimer in
15 * the documentation and/or other materials provided with the
16 * distribution.
17 * 3. Neither the name of The DragonFly Project nor the names of its
18 * contributors may be used to endorse or promote products derived
19 * from this software without specific, prior written permission.
20 *
21 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
22 * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
23 * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS
24 * FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE
25 * COPYRIGHT HOLDERS OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT,
26 * INCIDENTAL, SPECIAL, EXEMPLARY OR CONSEQUENTIAL DAMAGES (INCLUDING,
27 * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
28 * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED
29 * AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
30 * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT
31 * OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
32 * SUCH DAMAGE.
33 */
34 /*
35 * Brute-force implementation for linux RCU functions. Just use a delay
36 * line and per-cpu callouts.
37 */
38 #include <sys/types.h>
39 #include <sys/errno.h>
40 #include <sys/kernel.h>
41 #include <sys/spinlock.h>
42 #include <sys/spinlock2.h>
43 #include <linux/gfp.h>
44 #include <linux/slab.h>
45 #include <linux/rcupdate.h>
46
47 #include <machine/atomic.h>
48
49 typedef struct rcu_elm {
50 enum { RCU_NULL, RCU_CALL, RCU_FREE } type;
51 int ticks;
52 void (*func)(struct rcu_head *arg);
53 void *ptr;
54 } rcu_elm_t;
55
56 typedef struct rcu_pcpu {
57 rcu_elm_t *elms;
58 int size;
59 int mask;
60 int s;
61 int e;
62 int running;
63 struct callout timer_callout;
64 } rcu_pcpu_t;
65
66 static rcu_pcpu_t *rcupcpu;
67
68 /*
69 * Timer callout (pcpu)
70 */
71 static void
rcu_timer(void * arg)72 rcu_timer(void *arg)
73 {
74 rcu_pcpu_t *rcu = arg;
75 rcu_elm_t *elm;
76 int delta;
77
78 crit_enter();
79 while (rcu->s != rcu->e) {
80 elm = &rcu->elms[rcu->s & rcu->mask];
81 delta = ticks - elm->ticks; /* 2s compl underflow */
82 if (delta < hz)
83 break;
84
85 switch(elm->type) {
86 case RCU_NULL:
87 break;
88 case RCU_CALL:
89 elm->func(elm->ptr);
90 break;
91 case RCU_FREE:
92 kfree(elm->ptr);
93 break;
94 }
95 elm->type = RCU_NULL;
96 ++rcu->s;
97 }
98 if (rcu->s == rcu->e) {
99 rcu->running = 0;
100 } else {
101 callout_reset_bycpu(&rcu->timer_callout, hz / 10, rcu_timer,
102 rcu, mycpuid);
103 }
104 crit_exit();
105 }
106
107
108 /*
109 * Ping timer callout (pcpu).
110 *
111 * Must be in critical section.
112 */
113 static void
rcu_ping(rcu_pcpu_t * rcu)114 rcu_ping(rcu_pcpu_t *rcu)
115 {
116 if (rcu->running == 0) {
117 rcu->running = 1;
118 callout_reset_bycpu(&rcu->timer_callout, hz / 10, rcu_timer,
119 rcu, mycpuid);
120 }
121 }
122
123 /*
124 * Expand the rcu array for the current cpu
125 */
126 static void
rcu_expand(rcu_pcpu_t * rcu)127 rcu_expand(rcu_pcpu_t *rcu)
128 {
129 rcu_elm_t *oelms;
130 rcu_elm_t *nelms;
131 int count;
132 int nsize;
133 int nmask;
134 int n;
135
136 count = rcu->e - rcu->s; /* note: 2s complement underflow */
137 while (unlikely(count == rcu->size)) {
138 nsize = count ? count * 2 : 16;
139 nelms = kzalloc(nsize * sizeof(*nelms), GFP_KERNEL);
140 kprintf("drm: expand RCU cpu %d to %d\n", mycpuid, nsize);
141 if (likely(count == rcu->size)) {
142 nmask = nsize - 1;
143 oelms = rcu->elms;
144 n = rcu->s;
145 while (n != rcu->e) {
146 nelms[n & nmask] = oelms[n & rcu->mask];
147 ++n;
148 }
149 rcu->elms = nelms;
150 rcu->size = nsize;
151 rcu->mask = nmask;
152 nelms = oelms;
153 }
154 if (likely(nelms != NULL))
155 kfree(nelms);
156 count = rcu->e - rcu->s;
157 }
158 KKASSERT(count >= 0 && count < rcu->size);
159 }
160
161 void
__kfree_rcu(void * ptr)162 __kfree_rcu(void *ptr)
163 {
164 rcu_pcpu_t *rcu;
165 rcu_elm_t *elm;
166
167 if (unlikely(rcupcpu == NULL)) {
168 kfree(ptr);
169 return;
170 }
171 rcu = &rcupcpu[mycpuid];
172
173 crit_enter();
174 rcu_expand(rcu);
175 elm = &rcu->elms[rcu->e & rcu->mask];
176 ++rcu->e;
177
178 elm->type = RCU_FREE;
179 elm->ticks = ticks;
180 elm->ptr = ptr;
181
182 rcu_ping(rcu);
183 crit_exit();
184 }
185
186 void
call_rcu(struct rcu_head * head,void (* func)(struct rcu_head *))187 call_rcu(struct rcu_head *head, void (*func)(struct rcu_head *))
188 {
189 rcu_pcpu_t *rcu;
190 rcu_elm_t *elm;
191
192 if (unlikely(rcupcpu == NULL)) {
193 func(head);
194 return;
195 }
196 rcu = &rcupcpu[mycpuid];
197
198 crit_enter();
199 rcu_expand(rcu);
200 elm = &rcu->elms[rcu->e & rcu->mask];
201 ++rcu->e;
202
203 elm->type = RCU_CALL;
204 elm->ticks = ticks;
205 elm->func = func;
206 elm->ptr = head;
207
208 rcu_ping(rcu);
209 crit_exit();
210 }
211
212 static int
init_rcu(void * dummy __unused)213 init_rcu(void *dummy __unused)
214 {
215 int i;
216
217 rcupcpu = kzalloc(ncpus * sizeof(*rcupcpu), GFP_KERNEL);
218 for (i = 0; i < ncpus; ++i) {
219 callout_init_mp(&rcupcpu[i].timer_callout);
220 }
221 return 0;
222 }
223
224 SYSINIT(linux_rcu_init, SI_SUB_DRIVERS, SI_ORDER_MIDDLE, init_rcu, NULL);
225