1 /* $NetBSD: pl310.c,v 1.20 2021/10/02 20:52:09 skrll Exp $ */
2
3 /*-
4 * Copyright (c) 2012 The NetBSD Foundation, Inc.
5 * All rights reserved.
6 *
7 * This code is derived from software contributed to The NetBSD Foundation
8 * by Matt Thomas
9 *
10 * Redistribution and use in source and binary forms, with or without
11 * modification, are permitted provided that the following conditions
12 * are met:
13 * 1. Redistributions of source code must retain the above copyright
14 * notice, this list of conditions and the following disclaimer.
15 * 2. Redistributions in binary form must reproduce the above copyright
16 * notice, this list of conditions and the following disclaimer in the
17 * documentation and/or other materials provided with the distribution.
18 *
19 * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS
20 * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
21 * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
22 * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS
23 * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
24 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
25 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
26 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
27 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
28 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
29 * POSSIBILITY OF SUCH DAMAGE.
30 */
31
32 #include <sys/cdefs.h>
33 __KERNEL_RCSID(0, "$NetBSD: pl310.c,v 1.20 2021/10/02 20:52:09 skrll Exp $");
34
35 #include <sys/param.h>
36 #include <sys/bus.h>
37 #include <sys/cpu.h>
38 #include <sys/device.h>
39 #include <sys/atomic.h>
40
41 #include <arm/locore.h>
42
43 #include <arm/cortex/mpcore_var.h>
44 #include <arm/cortex/pl310_reg.h>
45 #include <arm/cortex/pl310_var.h>
46
47 static int arml2cc_match(device_t, cfdata_t, void *);
48 static void arml2cc_attach(device_t, device_t, void *);
49
50 #define L2CC_SIZE 0x1000
51
52 struct arml2cc_softc {
53 device_t sc_dev;
54 bus_space_tag_t sc_memt;
55 bus_space_handle_t sc_memh;
56 kmutex_t sc_lock;
57 uint32_t sc_waymask;
58 struct evcnt sc_ev_inv __aligned(8);
59 struct evcnt sc_ev_wb;
60 struct evcnt sc_ev_wbinv;
61 bool sc_enabled;
62 };
63
64 __CTASSERT(offsetof(struct arml2cc_softc, sc_ev_inv.ev_count) % 8 == 0);
65 __CTASSERT(offsetof(struct arml2cc_softc, sc_ev_wb.ev_count) % 8 == 0);
66 __CTASSERT(offsetof(struct arml2cc_softc, sc_ev_wbinv.ev_count) % 8 == 0);
67
68 CFATTACH_DECL_NEW(arml2cc, sizeof(struct arml2cc_softc),
69 arml2cc_match, arml2cc_attach, NULL, NULL);
70
71 static inline void arml2cc_disable(struct arml2cc_softc *);
72 static inline void arml2cc_enable(struct arml2cc_softc *);
73 static void arml2cc_sdcache_wb_range(vaddr_t, paddr_t, psize_t);
74 static void arml2cc_sdcache_inv_range(vaddr_t, paddr_t, psize_t);
75 static void arml2cc_sdcache_wbinv_range(vaddr_t, paddr_t, psize_t);
76
77 static struct arml2cc_softc *arml2cc_sc;
78
79 static inline uint32_t
arml2cc_read_4(struct arml2cc_softc * sc,bus_size_t o)80 arml2cc_read_4(struct arml2cc_softc *sc, bus_size_t o)
81 {
82 return bus_space_read_4(sc->sc_memt, sc->sc_memh, o);
83 }
84
85 static inline void
arml2cc_write_4(struct arml2cc_softc * sc,bus_size_t o,uint32_t v)86 arml2cc_write_4(struct arml2cc_softc *sc, bus_size_t o, uint32_t v)
87 {
88 bus_space_write_4(sc->sc_memt, sc->sc_memh, o, v);
89 }
90
91
92 /* ARGSUSED */
93 static int
arml2cc_match(device_t parent,cfdata_t cf,void * aux)94 arml2cc_match(device_t parent, cfdata_t cf, void *aux)
95 {
96 struct mpcore_attach_args * const mpcaa = aux;
97
98 if (arml2cc_sc)
99 return 0;
100
101 if (!CPU_ID_CORTEX_A9_P(curcpu()->ci_arm_cpuid) &&
102 !CPU_ID_CORTEX_A5_P(curcpu()->ci_arm_cpuid))
103 return 0;
104
105 if (strcmp(mpcaa->mpcaa_name, cf->cf_name) != 0)
106 return 0;
107
108 /*
109 * This isn't present on UP A9s (since CBAR isn't present).
110 */
111 uint32_t mpidr = armreg_mpidr_read();
112 if (mpidr == 0 || (mpidr & MPIDR_U))
113 return 0;
114
115 return 1;
116 }
117
118 static const struct {
119 uint8_t rev;
120 uint8_t str[7];
121 } pl310_revs[] = {
122 { 0, " r0p0" },
123 { 2, " r1p0" },
124 { 4, " r2p0" },
125 { 5, " r3p0" },
126 { 6, " r3p1" },
127 { 7, " r3p1a" },
128 { 8, " r3p2" },
129 { 9, " r3p3" },
130 };
131
132 static void
arml2cc_attach(device_t parent,device_t self,void * aux)133 arml2cc_attach(device_t parent, device_t self, void *aux)
134 {
135 struct arml2cc_softc * const sc = device_private(self);
136 struct mpcore_attach_args * const mpcaa = aux;
137 const char * const xname = device_xname(self);
138 prop_dictionary_t dict = device_properties(self);
139 uint32_t off;
140
141 aprint_naive("\n");
142
143 if (!prop_dictionary_get_uint32(dict, "offset", &off))
144 off = mpcaa->mpcaa_off1;
145
146 arml2cc_sc = sc;
147 sc->sc_dev = self;
148 sc->sc_memt = mpcaa->mpcaa_memt;
149 sc->sc_waymask = __BIT(arm_scache.dcache_ways) - 1;
150
151 evcnt_attach_dynamic(&sc->sc_ev_inv, EVCNT_TYPE_MISC, NULL,
152 xname, "L2 inv requests");
153 evcnt_attach_dynamic(&sc->sc_ev_wb, EVCNT_TYPE_MISC, NULL,
154 xname, "L2 wb requests");
155 evcnt_attach_dynamic(&sc->sc_ev_wbinv, EVCNT_TYPE_MISC, NULL,
156 xname, "L2 wbinv requests");
157
158 mutex_init(&sc->sc_lock, MUTEX_DEFAULT, IPL_HIGH);
159
160 bus_space_subregion(sc->sc_memt, mpcaa->mpcaa_memh,
161 off, L2CC_SIZE, &sc->sc_memh);
162
163 uint32_t id = arml2cc_read_4(sc, L2C_CACHE_ID);
164 u_int rev = __SHIFTOUT(id, CACHE_ID_REV);
165
166 const char *revstr = "";
167 for (size_t i = 0; i < __arraycount(pl310_revs); i++) {
168 if (rev == pl310_revs[i].rev) {
169 revstr = pl310_revs[i].str;
170 break;
171 }
172 }
173
174 const bool enabled_p = arml2cc_read_4(sc, L2C_CTL) != 0;
175
176 aprint_normal(": ARM PL310%s L2 Cache Controller%s\n",
177 revstr, enabled_p ? "" : " (disabled)");
178
179 if (enabled_p) {
180 if (device_cfdata(self)->cf_flags & 1) {
181 arml2cc_disable(sc);
182 aprint_normal_dev(self, "cache %s\n",
183 arml2cc_read_4(sc, L2C_CTL) ? "enabled" : "disabled");
184 sc->sc_enabled = false;
185 } else {
186 cpufuncs.cf_sdcache_wb_range = arml2cc_sdcache_wb_range;
187 cpufuncs.cf_sdcache_inv_range = arml2cc_sdcache_inv_range;
188 cpufuncs.cf_sdcache_wbinv_range = arml2cc_sdcache_wbinv_range;
189 sc->sc_enabled = true;
190 }
191 } else if ((device_cfdata(self)->cf_flags & 1) == 0) {
192 if (!enabled_p) {
193 arml2cc_enable(sc);
194 aprint_normal_dev(self, "cache %s\n",
195 arml2cc_read_4(sc, L2C_CTL) ? "enabled" : "disabled");
196 }
197 cpufuncs.cf_sdcache_wb_range = arml2cc_sdcache_wb_range;
198 cpufuncs.cf_sdcache_inv_range = arml2cc_sdcache_inv_range;
199 cpufuncs.cf_sdcache_wbinv_range = arml2cc_sdcache_wbinv_range;
200 sc->sc_enabled = true;
201 }
202
203 KASSERTMSG(arm_pcache.dcache_line_size == arm_scache.dcache_line_size,
204 "pcache %u scache %u",
205 arm_pcache.dcache_line_size, arm_scache.dcache_line_size);
206 }
207
208 static inline void
arml2cc_cache_op(struct arml2cc_softc * sc,bus_size_t off,uint32_t val,bool wait)209 arml2cc_cache_op(struct arml2cc_softc *sc, bus_size_t off, uint32_t val,
210 bool wait)
211 {
212 arml2cc_write_4(sc, off, val);
213 if (wait) {
214 while (arml2cc_read_4(sc, off) & 1) {
215 /* spin */
216 }
217 }
218 }
219
220 static inline void
arml2cc_cache_way_op(struct arml2cc_softc * sc,bus_size_t off,uint32_t way_mask)221 arml2cc_cache_way_op(struct arml2cc_softc *sc, bus_size_t off, uint32_t way_mask)
222 {
223 arml2cc_write_4(sc, off, way_mask);
224 while (arml2cc_read_4(sc, off) & way_mask) {
225 /* spin */
226 }
227 }
228
229 static inline void
arml2cc_cache_sync(struct arml2cc_softc * sc)230 arml2cc_cache_sync(struct arml2cc_softc *sc)
231 {
232 arml2cc_cache_op(sc, L2C_CACHE_SYNC, 0, true);
233 }
234
235 static inline void
arml2cc_disable(struct arml2cc_softc * sc)236 arml2cc_disable(struct arml2cc_softc *sc)
237 {
238 mutex_spin_enter(&sc->sc_lock);
239
240 arml2cc_cache_way_op(sc, L2C_CLEAN_INV_WAY, sc->sc_waymask);
241 arml2cc_cache_sync(sc);
242
243 arml2cc_write_4(sc, L2C_CTL, 0); // turn it off
244 mutex_spin_exit(&sc->sc_lock);
245 }
246
247 static inline void
arml2cc_enable(struct arml2cc_softc * sc)248 arml2cc_enable(struct arml2cc_softc *sc)
249 {
250 mutex_spin_enter(&sc->sc_lock);
251
252 arml2cc_cache_way_op(sc, L2C_INV_WAY, sc->sc_waymask);
253 arml2cc_cache_sync(sc);
254
255 arml2cc_write_4(sc, L2C_CTL, 1); // turn it on
256
257 mutex_spin_exit(&sc->sc_lock);
258 }
259
260 void
arml2cc_init(bus_space_tag_t bst,bus_space_handle_t bsh,bus_size_t o)261 arml2cc_init(bus_space_tag_t bst, bus_space_handle_t bsh, bus_size_t o)
262 {
263 struct arm_cache_info * const info = &arm_scache;
264
265 uint32_t cfg = bus_space_read_4(bst, bsh, o + L2C_CACHE_TYPE);
266
267 info->cache_type = __SHIFTOUT(cfg, CACHE_TYPE_CTYPE);
268 info->cache_unified = __SHIFTOUT(cfg, CACHE_TYPE_HARVARD) == 0;
269 u_int cfg_dsize = __SHIFTOUT(cfg, CACHE_TYPE_DSIZE);
270
271 u_int d_waysize = 8192 << __SHIFTOUT(cfg_dsize, CACHE_TYPE_xWAYSIZE);
272 info->dcache_ways = 8 << __SHIFTOUT(cfg_dsize, CACHE_TYPE_xASSOC);
273 info->dcache_line_size = 32 << __SHIFTOUT(cfg_dsize, CACHE_TYPE_xLINESIZE);
274 info->dcache_size = info->dcache_ways * d_waysize;
275 info->dcache_type = CACHE_TYPE_PIPT;
276 info->icache_type = CACHE_TYPE_PIPT;
277
278 if (info->cache_unified) {
279 info->icache_ways = info->dcache_ways;
280 info->icache_line_size = info->dcache_line_size;
281 info->icache_size = info->dcache_size;
282 } else {
283 u_int cfg_isize = __SHIFTOUT(cfg, CACHE_TYPE_ISIZE);
284 u_int i_waysize = 8192 << __SHIFTOUT(cfg_isize, CACHE_TYPE_xWAYSIZE);
285 info->icache_ways = 8 << __SHIFTOUT(cfg_isize, CACHE_TYPE_xASSOC);
286 info->icache_line_size = 32 << __SHIFTOUT(cfg_isize, CACHE_TYPE_xLINESIZE);
287 info->icache_size = i_waysize * info->icache_ways;
288 }
289 }
290
291 static void
arml2cc_cache_range_op(paddr_t pa,psize_t len,bus_size_t cache_op)292 arml2cc_cache_range_op(paddr_t pa, psize_t len, bus_size_t cache_op)
293 {
294 struct arml2cc_softc * const sc = arml2cc_sc;
295 const size_t line_size = arm_scache.dcache_line_size;
296 const size_t line_mask = line_size - 1;
297 size_t off = pa & line_mask;
298 if (off) {
299 len += off;
300 pa -= off;
301 }
302 len = roundup2(len, line_size);
303 mutex_spin_enter(&sc->sc_lock);
304 if (__predict_false(!sc->sc_enabled)) {
305 mutex_spin_exit(&sc->sc_lock);
306 return;
307 }
308 for (const paddr_t endpa = pa + len; pa < endpa; pa += line_size) {
309 arml2cc_cache_op(sc, cache_op, pa, false);
310 }
311 arml2cc_cache_sync(sc);
312 mutex_spin_exit(&sc->sc_lock);
313 }
314
315 static void
arml2cc_sdcache_inv_range(vaddr_t va,paddr_t pa,psize_t len)316 arml2cc_sdcache_inv_range(vaddr_t va, paddr_t pa, psize_t len)
317 {
318 atomic_inc_64(&arml2cc_sc->sc_ev_inv.ev_count);
319 arml2cc_cache_range_op(pa, len, L2C_INV_PA);
320 }
321
322 static void
arml2cc_sdcache_wb_range(vaddr_t va,paddr_t pa,psize_t len)323 arml2cc_sdcache_wb_range(vaddr_t va, paddr_t pa, psize_t len)
324 {
325 atomic_inc_64(&arml2cc_sc->sc_ev_wb.ev_count);
326 arml2cc_cache_range_op(pa, len, L2C_CLEAN_PA);
327 }
328
329 static void
arml2cc_sdcache_wbinv_range(vaddr_t va,paddr_t pa,psize_t len)330 arml2cc_sdcache_wbinv_range(vaddr_t va, paddr_t pa, psize_t len)
331 {
332 atomic_inc_64(&arml2cc_sc->sc_ev_wbinv.ev_count);
333 arml2cc_cache_range_op(pa, len, L2C_CLEAN_INV_PA);
334 }
335