1*34924089Sskrll /* $NetBSD: pl310.c,v 1.20 2021/10/02 20:52:09 skrll Exp $ */
2c2c35168Smatt
3c2c35168Smatt /*-
4c2c35168Smatt * Copyright (c) 2012 The NetBSD Foundation, Inc.
5c2c35168Smatt * All rights reserved.
6c2c35168Smatt *
7c2c35168Smatt * This code is derived from software contributed to The NetBSD Foundation
8c2c35168Smatt * by Matt Thomas
9c2c35168Smatt *
10c2c35168Smatt * Redistribution and use in source and binary forms, with or without
11c2c35168Smatt * modification, are permitted provided that the following conditions
12c2c35168Smatt * are met:
13c2c35168Smatt * 1. Redistributions of source code must retain the above copyright
14c2c35168Smatt * notice, this list of conditions and the following disclaimer.
15c2c35168Smatt * 2. Redistributions in binary form must reproduce the above copyright
16c2c35168Smatt * notice, this list of conditions and the following disclaimer in the
17c2c35168Smatt * documentation and/or other materials provided with the distribution.
18c2c35168Smatt *
19c2c35168Smatt * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS
20c2c35168Smatt * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
21c2c35168Smatt * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
22c2c35168Smatt * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS
23c2c35168Smatt * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
24c2c35168Smatt * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
25c2c35168Smatt * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
26c2c35168Smatt * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
27c2c35168Smatt * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
28c2c35168Smatt * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
29c2c35168Smatt * POSSIBILITY OF SUCH DAMAGE.
30c2c35168Smatt */
31c2c35168Smatt
32c2c35168Smatt #include <sys/cdefs.h>
33*34924089Sskrll __KERNEL_RCSID(0, "$NetBSD: pl310.c,v 1.20 2021/10/02 20:52:09 skrll Exp $");
34c2c35168Smatt
35c2c35168Smatt #include <sys/param.h>
36c2c35168Smatt #include <sys/bus.h>
37c2c35168Smatt #include <sys/cpu.h>
38c2c35168Smatt #include <sys/device.h>
3996c39f56Smatt #include <sys/atomic.h>
40c2c35168Smatt
41c3d10eb6Smatt #include <arm/locore.h>
42c3d10eb6Smatt
43c2c35168Smatt #include <arm/cortex/mpcore_var.h>
44c2c35168Smatt #include <arm/cortex/pl310_reg.h>
45162b9739Smatt #include <arm/cortex/pl310_var.h>
46c2c35168Smatt
47c2c35168Smatt static int arml2cc_match(device_t, cfdata_t, void *);
48c2c35168Smatt static void arml2cc_attach(device_t, device_t, void *);
49c2c35168Smatt
50c2c35168Smatt #define L2CC_SIZE 0x1000
51c2c35168Smatt
52c2c35168Smatt struct arml2cc_softc {
53c2c35168Smatt device_t sc_dev;
54c2c35168Smatt bus_space_tag_t sc_memt;
55c2c35168Smatt bus_space_handle_t sc_memh;
5696c39f56Smatt kmutex_t sc_lock;
5796c39f56Smatt uint32_t sc_waymask;
5896c39f56Smatt struct evcnt sc_ev_inv __aligned(8);
5996c39f56Smatt struct evcnt sc_ev_wb;
6096c39f56Smatt struct evcnt sc_ev_wbinv;
6196c39f56Smatt bool sc_enabled;
62c2c35168Smatt };
63c2c35168Smatt
6496c39f56Smatt __CTASSERT(offsetof(struct arml2cc_softc, sc_ev_inv.ev_count) % 8 == 0);
6596c39f56Smatt __CTASSERT(offsetof(struct arml2cc_softc, sc_ev_wb.ev_count) % 8 == 0);
6696c39f56Smatt __CTASSERT(offsetof(struct arml2cc_softc, sc_ev_wbinv.ev_count) % 8 == 0);
6796c39f56Smatt
68c2c35168Smatt CFATTACH_DECL_NEW(arml2cc, sizeof(struct arml2cc_softc),
69c2c35168Smatt arml2cc_match, arml2cc_attach, NULL, NULL);
70c2c35168Smatt
71374340c0Smatt static inline void arml2cc_disable(struct arml2cc_softc *);
72374340c0Smatt static inline void arml2cc_enable(struct arml2cc_softc *);
7396c39f56Smatt static void arml2cc_sdcache_wb_range(vaddr_t, paddr_t, psize_t);
7496c39f56Smatt static void arml2cc_sdcache_inv_range(vaddr_t, paddr_t, psize_t);
7596c39f56Smatt static void arml2cc_sdcache_wbinv_range(vaddr_t, paddr_t, psize_t);
76162b9739Smatt
7796c39f56Smatt static struct arml2cc_softc *arml2cc_sc;
78c2c35168Smatt
79c2c35168Smatt static inline uint32_t
arml2cc_read_4(struct arml2cc_softc * sc,bus_size_t o)80c2c35168Smatt arml2cc_read_4(struct arml2cc_softc *sc, bus_size_t o)
81c2c35168Smatt {
82c2c35168Smatt return bus_space_read_4(sc->sc_memt, sc->sc_memh, o);
83c2c35168Smatt }
84c2c35168Smatt
85c2c35168Smatt static inline void
arml2cc_write_4(struct arml2cc_softc * sc,bus_size_t o,uint32_t v)86c2c35168Smatt arml2cc_write_4(struct arml2cc_softc *sc, bus_size_t o, uint32_t v)
87c2c35168Smatt {
88c2c35168Smatt bus_space_write_4(sc->sc_memt, sc->sc_memh, o, v);
89c2c35168Smatt }
90c2c35168Smatt
91c2c35168Smatt
92c2c35168Smatt /* ARGSUSED */
93c2c35168Smatt static int
arml2cc_match(device_t parent,cfdata_t cf,void * aux)94c2c35168Smatt arml2cc_match(device_t parent, cfdata_t cf, void *aux)
95c2c35168Smatt {
96c2c35168Smatt struct mpcore_attach_args * const mpcaa = aux;
97c2c35168Smatt
9896c39f56Smatt if (arml2cc_sc)
99c2c35168Smatt return 0;
100c2c35168Smatt
101df5d72efSjmcneill if (!CPU_ID_CORTEX_A9_P(curcpu()->ci_arm_cpuid) &&
102df5d72efSjmcneill !CPU_ID_CORTEX_A5_P(curcpu()->ci_arm_cpuid))
103c2c35168Smatt return 0;
104c2c35168Smatt
105c2c35168Smatt if (strcmp(mpcaa->mpcaa_name, cf->cf_name) != 0)
106c2c35168Smatt return 0;
107c2c35168Smatt
108c2c35168Smatt /*
109c2c35168Smatt * This isn't present on UP A9s (since CBAR isn't present).
110c2c35168Smatt */
111c2c35168Smatt uint32_t mpidr = armreg_mpidr_read();
112c2c35168Smatt if (mpidr == 0 || (mpidr & MPIDR_U))
113c2c35168Smatt return 0;
114c2c35168Smatt
115c2c35168Smatt return 1;
116c2c35168Smatt }
117c2c35168Smatt
118c2c35168Smatt static const struct {
119c2c35168Smatt uint8_t rev;
120c2c35168Smatt uint8_t str[7];
121c2c35168Smatt } pl310_revs[] = {
122c2c35168Smatt { 0, " r0p0" },
123c2c35168Smatt { 2, " r1p0" },
124c2c35168Smatt { 4, " r2p0" },
125c2c35168Smatt { 5, " r3p0" },
126c2c35168Smatt { 6, " r3p1" },
127ac390955Smatt { 7, " r3p1a" },
128c2c35168Smatt { 8, " r3p2" },
129c2c35168Smatt { 9, " r3p3" },
130c2c35168Smatt };
131c2c35168Smatt
132c2c35168Smatt static void
arml2cc_attach(device_t parent,device_t self,void * aux)133c2c35168Smatt arml2cc_attach(device_t parent, device_t self, void *aux)
134c2c35168Smatt {
135c2c35168Smatt struct arml2cc_softc * const sc = device_private(self);
136c2c35168Smatt struct mpcore_attach_args * const mpcaa = aux;
13796c39f56Smatt const char * const xname = device_xname(self);
138ac390955Smatt prop_dictionary_t dict = device_properties(self);
139ac390955Smatt uint32_t off;
140ac390955Smatt
141df5d72efSjmcneill aprint_naive("\n");
142df5d72efSjmcneill
1434e0f51deSjmcneill if (!prop_dictionary_get_uint32(dict, "offset", &off))
144ef716440Shkenken off = mpcaa->mpcaa_off1;
145c2c35168Smatt
14696c39f56Smatt arml2cc_sc = sc;
147c2c35168Smatt sc->sc_dev = self;
148c2c35168Smatt sc->sc_memt = mpcaa->mpcaa_memt;
14996c39f56Smatt sc->sc_waymask = __BIT(arm_scache.dcache_ways) - 1;
15096c39f56Smatt
15196c39f56Smatt evcnt_attach_dynamic(&sc->sc_ev_inv, EVCNT_TYPE_MISC, NULL,
15296c39f56Smatt xname, "L2 inv requests");
15396c39f56Smatt evcnt_attach_dynamic(&sc->sc_ev_wb, EVCNT_TYPE_MISC, NULL,
15496c39f56Smatt xname, "L2 wb requests");
15596c39f56Smatt evcnt_attach_dynamic(&sc->sc_ev_wbinv, EVCNT_TYPE_MISC, NULL,
15696c39f56Smatt xname, "L2 wbinv requests");
15796c39f56Smatt
15896c39f56Smatt mutex_init(&sc->sc_lock, MUTEX_DEFAULT, IPL_HIGH);
159c2c35168Smatt
160c2c35168Smatt bus_space_subregion(sc->sc_memt, mpcaa->mpcaa_memh,
161ac390955Smatt off, L2CC_SIZE, &sc->sc_memh);
162c2c35168Smatt
163c2c35168Smatt uint32_t id = arml2cc_read_4(sc, L2C_CACHE_ID);
164c2c35168Smatt u_int rev = __SHIFTOUT(id, CACHE_ID_REV);
165c2c35168Smatt
166c2c35168Smatt const char *revstr = "";
167c2c35168Smatt for (size_t i = 0; i < __arraycount(pl310_revs); i++) {
168c2c35168Smatt if (rev == pl310_revs[i].rev) {
169c2c35168Smatt revstr = pl310_revs[i].str;
170c2c35168Smatt break;
171c2c35168Smatt }
172c2c35168Smatt }
173c2c35168Smatt
17456f0e3eaSmatt const bool enabled_p = arml2cc_read_4(sc, L2C_CTL) != 0;
17556f0e3eaSmatt
176162b9739Smatt aprint_normal(": ARM PL310%s L2 Cache Controller%s\n",
17756f0e3eaSmatt revstr, enabled_p ? "" : " (disabled)");
178c2c35168Smatt
17956f0e3eaSmatt if (enabled_p) {
1808aa761beSmatt if (device_cfdata(self)->cf_flags & 1) {
181162b9739Smatt arml2cc_disable(sc);
182162b9739Smatt aprint_normal_dev(self, "cache %s\n",
183162b9739Smatt arml2cc_read_4(sc, L2C_CTL) ? "enabled" : "disabled");
18496c39f56Smatt sc->sc_enabled = false;
18596c39f56Smatt } else {
18696c39f56Smatt cpufuncs.cf_sdcache_wb_range = arml2cc_sdcache_wb_range;
18796c39f56Smatt cpufuncs.cf_sdcache_inv_range = arml2cc_sdcache_inv_range;
18896c39f56Smatt cpufuncs.cf_sdcache_wbinv_range = arml2cc_sdcache_wbinv_range;
18996c39f56Smatt sc->sc_enabled = true;
190c2c35168Smatt }
1918aa761beSmatt } else if ((device_cfdata(self)->cf_flags & 1) == 0) {
19296c39f56Smatt if (!enabled_p) {
19396c39f56Smatt arml2cc_enable(sc);
19496c39f56Smatt aprint_normal_dev(self, "cache %s\n",
19596c39f56Smatt arml2cc_read_4(sc, L2C_CTL) ? "enabled" : "disabled");
19696c39f56Smatt }
19796c39f56Smatt cpufuncs.cf_sdcache_wb_range = arml2cc_sdcache_wb_range;
19896c39f56Smatt cpufuncs.cf_sdcache_inv_range = arml2cc_sdcache_inv_range;
19996c39f56Smatt cpufuncs.cf_sdcache_wbinv_range = arml2cc_sdcache_wbinv_range;
20096c39f56Smatt sc->sc_enabled = true;
2018aa761beSmatt }
202c2c35168Smatt
203b395ced9Smatt KASSERTMSG(arm_pcache.dcache_line_size == arm_scache.dcache_line_size,
204b395ced9Smatt "pcache %u scache %u",
205b395ced9Smatt arm_pcache.dcache_line_size, arm_scache.dcache_line_size);
20656f0e3eaSmatt }
20756f0e3eaSmatt
20896c39f56Smatt static inline void
arml2cc_cache_op(struct arml2cc_softc * sc,bus_size_t off,uint32_t val,bool wait)209445568dcSmatt arml2cc_cache_op(struct arml2cc_softc *sc, bus_size_t off, uint32_t val,
210445568dcSmatt bool wait)
21196c39f56Smatt {
21296c39f56Smatt arml2cc_write_4(sc, off, val);
213445568dcSmatt if (wait) {
21496c39f56Smatt while (arml2cc_read_4(sc, off) & 1) {
21596c39f56Smatt /* spin */
21696c39f56Smatt }
21796c39f56Smatt }
218445568dcSmatt }
21996c39f56Smatt
22096c39f56Smatt static inline void
arml2cc_cache_way_op(struct arml2cc_softc * sc,bus_size_t off,uint32_t way_mask)22196c39f56Smatt arml2cc_cache_way_op(struct arml2cc_softc *sc, bus_size_t off, uint32_t way_mask)
22296c39f56Smatt {
22396c39f56Smatt arml2cc_write_4(sc, off, way_mask);
22496c39f56Smatt while (arml2cc_read_4(sc, off) & way_mask) {
22596c39f56Smatt /* spin */
22696c39f56Smatt }
22796c39f56Smatt }
22896c39f56Smatt
22996c39f56Smatt static inline void
arml2cc_cache_sync(struct arml2cc_softc * sc)23096c39f56Smatt arml2cc_cache_sync(struct arml2cc_softc *sc)
23196c39f56Smatt {
232445568dcSmatt arml2cc_cache_op(sc, L2C_CACHE_SYNC, 0, true);
23396c39f56Smatt }
23496c39f56Smatt
23596c39f56Smatt static inline void
arml2cc_disable(struct arml2cc_softc * sc)236162b9739Smatt arml2cc_disable(struct arml2cc_softc *sc)
237162b9739Smatt {
23896c39f56Smatt mutex_spin_enter(&sc->sc_lock);
239162b9739Smatt
24096c39f56Smatt arml2cc_cache_way_op(sc, L2C_CLEAN_INV_WAY, sc->sc_waymask);
24196c39f56Smatt arml2cc_cache_sync(sc);
24296c39f56Smatt
24396c39f56Smatt arml2cc_write_4(sc, L2C_CTL, 0); // turn it off
24496c39f56Smatt mutex_spin_exit(&sc->sc_lock);
245162b9739Smatt }
246162b9739Smatt
24796c39f56Smatt static inline void
arml2cc_enable(struct arml2cc_softc * sc)24896c39f56Smatt arml2cc_enable(struct arml2cc_softc *sc)
24996c39f56Smatt {
25096c39f56Smatt mutex_spin_enter(&sc->sc_lock);
251162b9739Smatt
252aa1c874eSmatt arml2cc_cache_way_op(sc, L2C_INV_WAY, sc->sc_waymask);
25396c39f56Smatt arml2cc_cache_sync(sc);
25496c39f56Smatt
255487f016eSmatt arml2cc_write_4(sc, L2C_CTL, 1); // turn it on
256487f016eSmatt
25796c39f56Smatt mutex_spin_exit(&sc->sc_lock);
258162b9739Smatt }
259162b9739Smatt
260162b9739Smatt void
arml2cc_init(bus_space_tag_t bst,bus_space_handle_t bsh,bus_size_t o)261162b9739Smatt arml2cc_init(bus_space_tag_t bst, bus_space_handle_t bsh, bus_size_t o)
262162b9739Smatt {
263162b9739Smatt struct arm_cache_info * const info = &arm_scache;
264162b9739Smatt
265162b9739Smatt uint32_t cfg = bus_space_read_4(bst, bsh, o + L2C_CACHE_TYPE);
266162b9739Smatt
267162b9739Smatt info->cache_type = __SHIFTOUT(cfg, CACHE_TYPE_CTYPE);
268162b9739Smatt info->cache_unified = __SHIFTOUT(cfg, CACHE_TYPE_HARVARD) == 0;
269162b9739Smatt u_int cfg_dsize = __SHIFTOUT(cfg, CACHE_TYPE_DSIZE);
270162b9739Smatt
271162b9739Smatt u_int d_waysize = 8192 << __SHIFTOUT(cfg_dsize, CACHE_TYPE_xWAYSIZE);
272162b9739Smatt info->dcache_ways = 8 << __SHIFTOUT(cfg_dsize, CACHE_TYPE_xASSOC);
273162b9739Smatt info->dcache_line_size = 32 << __SHIFTOUT(cfg_dsize, CACHE_TYPE_xLINESIZE);
274162b9739Smatt info->dcache_size = info->dcache_ways * d_waysize;
27567b539efSmatt info->dcache_type = CACHE_TYPE_PIPT;
27667b539efSmatt info->icache_type = CACHE_TYPE_PIPT;
277162b9739Smatt
278162b9739Smatt if (info->cache_unified) {
279162b9739Smatt info->icache_ways = info->dcache_ways;
280162b9739Smatt info->icache_line_size = info->dcache_line_size;
281162b9739Smatt info->icache_size = info->dcache_size;
282162b9739Smatt } else {
283162b9739Smatt u_int cfg_isize = __SHIFTOUT(cfg, CACHE_TYPE_ISIZE);
284162b9739Smatt u_int i_waysize = 8192 << __SHIFTOUT(cfg_isize, CACHE_TYPE_xWAYSIZE);
285162b9739Smatt info->icache_ways = 8 << __SHIFTOUT(cfg_isize, CACHE_TYPE_xASSOC);
286162b9739Smatt info->icache_line_size = 32 << __SHIFTOUT(cfg_isize, CACHE_TYPE_xLINESIZE);
287162b9739Smatt info->icache_size = i_waysize * info->icache_ways;
288162b9739Smatt }
289c2c35168Smatt }
29056f0e3eaSmatt
29156f0e3eaSmatt static void
arml2cc_cache_range_op(paddr_t pa,psize_t len,bus_size_t cache_op)29296c39f56Smatt arml2cc_cache_range_op(paddr_t pa, psize_t len, bus_size_t cache_op)
29356f0e3eaSmatt {
29496c39f56Smatt struct arml2cc_softc * const sc = arml2cc_sc;
29596c39f56Smatt const size_t line_size = arm_scache.dcache_line_size;
29656f0e3eaSmatt const size_t line_mask = line_size - 1;
29796c39f56Smatt size_t off = pa & line_mask;
29856f0e3eaSmatt if (off) {
29956f0e3eaSmatt len += off;
30096c39f56Smatt pa -= off;
30156f0e3eaSmatt }
30296c39f56Smatt len = roundup2(len, line_size);
30396c39f56Smatt mutex_spin_enter(&sc->sc_lock);
304ecd33720Smatt if (__predict_false(!sc->sc_enabled)) {
305abeffea0Smatt mutex_spin_exit(&sc->sc_lock);
30696c39f56Smatt return;
307abeffea0Smatt }
308ecd33720Smatt for (const paddr_t endpa = pa + len; pa < endpa; pa += line_size) {
309445568dcSmatt arml2cc_cache_op(sc, cache_op, pa, false);
31096c39f56Smatt }
311ecd33720Smatt arml2cc_cache_sync(sc);
31296c39f56Smatt mutex_spin_exit(&sc->sc_lock);
31356f0e3eaSmatt }
31496c39f56Smatt
31596c39f56Smatt static void
arml2cc_sdcache_inv_range(vaddr_t va,paddr_t pa,psize_t len)31696c39f56Smatt arml2cc_sdcache_inv_range(vaddr_t va, paddr_t pa, psize_t len)
31796c39f56Smatt {
31896c39f56Smatt atomic_inc_64(&arml2cc_sc->sc_ev_inv.ev_count);
31996c39f56Smatt arml2cc_cache_range_op(pa, len, L2C_INV_PA);
32096c39f56Smatt }
32196c39f56Smatt
32296c39f56Smatt static void
arml2cc_sdcache_wb_range(vaddr_t va,paddr_t pa,psize_t len)32396c39f56Smatt arml2cc_sdcache_wb_range(vaddr_t va, paddr_t pa, psize_t len)
32496c39f56Smatt {
32596c39f56Smatt atomic_inc_64(&arml2cc_sc->sc_ev_wb.ev_count);
32696c39f56Smatt arml2cc_cache_range_op(pa, len, L2C_CLEAN_PA);
32796c39f56Smatt }
32896c39f56Smatt
32996c39f56Smatt static void
arml2cc_sdcache_wbinv_range(vaddr_t va,paddr_t pa,psize_t len)33096c39f56Smatt arml2cc_sdcache_wbinv_range(vaddr_t va, paddr_t pa, psize_t len)
33196c39f56Smatt {
33296c39f56Smatt atomic_inc_64(&arml2cc_sc->sc_ev_wbinv.ev_count);
33396c39f56Smatt arml2cc_cache_range_op(pa, len, L2C_CLEAN_INV_PA);
33496c39f56Smatt }
335