1 /* $NetBSD: booke_cache.c,v 1.5 2020/07/06 09:34:16 rin Exp $ */
2 /*-
3 * Copyright (c) 2010, 2011 The NetBSD Foundation, Inc.
4 * All rights reserved.
5 *
6 * This code is derived from software contributed to The NetBSD Foundation
7 * by Raytheon BBN Technologies Corp and Defense Advanced Research Projects
8 * Agency and which was developed by Matt Thomas of 3am Software Foundry.
9 *
10 * This material is based upon work supported by the Defense Advanced Research
11 * Projects Agency and Space and Naval Warfare Systems Center, Pacific, under
12 * Contract No. N66001-09-C-2073.
13 * Approved for Public Release, Distribution Unlimited
14 *
15 * Redistribution and use in source and binary forms, with or without
16 * modification, are permitted provided that the following conditions
17 * are met:
18 * 1. Redistributions of source code must retain the above copyright
19 * notice, this list of conditions and the following disclaimer.
20 * 2. Redistributions in binary form must reproduce the above copyright
21 * notice, this list of conditions and the following disclaimer in the
22 * documentation and/or other materials provided with the distribution.
23 *
24 * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS
25 * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
26 * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
27 * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS
28 * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
29 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
30 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
31 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
32 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
33 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
34 * POSSIBILITY OF SUCH DAMAGE.
35 */
36
37 #include <sys/cdefs.h>
38 __KERNEL_RCSID(0, "$NetBSD: booke_cache.c,v 1.5 2020/07/06 09:34:16 rin Exp $");
39
40 #include <sys/param.h>
41 #include <sys/cpu.h>
42 #include <sys/atomic.h>
43
44 enum cache_op { OP_DCBF, OP_DCBST, OP_DCBI, OP_DCBZ, OP_DCBA, OP_ICBI };
45
46 static void inline
dcbf(vaddr_t va,vsize_t off)47 dcbf(vaddr_t va, vsize_t off)
48 {
49 __asm volatile("dcbf\t%0,%1" : : "b" (va), "r" (off));
50 }
51
52 static void inline
dcbst(vaddr_t va,vsize_t off)53 dcbst(vaddr_t va, vsize_t off)
54 {
55 __asm volatile("dcbst\t%0,%1" : : "b" (va), "r" (off));
56 }
57
58 static void inline
dcbi(vaddr_t va,vsize_t off)59 dcbi(vaddr_t va, vsize_t off)
60 {
61 __asm volatile("dcbi\t%0,%1" : : "b" (va), "r" (off));
62 }
63
64 static void inline
dcbz(vaddr_t va,vsize_t off)65 dcbz(vaddr_t va, vsize_t off)
66 {
67 __asm volatile("dcbz\t%0,%1" : : "b" (va), "r" (off));
68 }
69
70 static void inline
dcba(vaddr_t va,vsize_t off)71 dcba(vaddr_t va, vsize_t off)
72 {
73 __asm volatile("dcba\t%0,%1" : : "b" (va), "r" (off));
74 }
75
76 static void inline
icbi(vaddr_t va,vsize_t off)77 icbi(vaddr_t va, vsize_t off)
78 {
79 __asm volatile("icbi\t%0,%1" : : "b" (va), "r" (off));
80 }
81
82 static inline void
cache_op(vaddr_t va,vsize_t len,vsize_t line_size,enum cache_op op)83 cache_op(vaddr_t va, vsize_t len, vsize_t line_size, enum cache_op op)
84 {
85 KASSERT(line_size > 0);
86
87 if (len == 0)
88 return;
89
90 /* Make sure we flush all cache lines */
91 len += va & (line_size - 1);
92 va &= -line_size;
93
94 for (vsize_t i = 0; i < len; i += line_size) {
95 switch (op) {
96 case OP_DCBF: dcbf(va, i); break;
97 case OP_DCBST: dcbst(va, i); break;
98 case OP_DCBI: dcbi(va, i); break;
99 case OP_DCBZ: dcbz(va, i); break;
100 case OP_DCBA: dcba(va, i); break;
101 case OP_ICBI: icbi(va, i); break;
102 }
103 }
104 if (op != OP_ICBI)
105 membar_producer();
106 }
107
108 void
dcache_wb_page(vaddr_t va)109 dcache_wb_page(vaddr_t va)
110 {
111 cache_op(va, PAGE_SIZE, curcpu()->ci_ci.dcache_line_size, OP_DCBST);
112 }
113
114 void
dcache_wbinv_page(vaddr_t va)115 dcache_wbinv_page(vaddr_t va)
116 {
117 cache_op(va, PAGE_SIZE, curcpu()->ci_ci.dcache_line_size, OP_DCBF);
118 }
119
120 void
dcache_inv_page(vaddr_t va)121 dcache_inv_page(vaddr_t va)
122 {
123 cache_op(va, PAGE_SIZE, curcpu()->ci_ci.dcache_line_size, OP_DCBI);
124 }
125
126 void
dcache_zero_page(vaddr_t va)127 dcache_zero_page(vaddr_t va)
128 {
129 cache_op(va, PAGE_SIZE, curcpu()->ci_ci.dcache_line_size, OP_DCBZ);
130 }
131
132 void
icache_inv_page(vaddr_t va)133 icache_inv_page(vaddr_t va)
134 {
135 membar_sync();
136 cache_op(va, PAGE_SIZE, curcpu()->ci_ci.icache_line_size, OP_ICBI);
137 membar_sync();
138 /* synchronizing instruction will be the rfi to user mode */
139 }
140
141 void
dcache_wb(vaddr_t va,vsize_t len)142 dcache_wb(vaddr_t va, vsize_t len)
143 {
144 cache_op(va, len, curcpu()->ci_ci.dcache_line_size, OP_DCBST);
145 }
146
147 void
dcache_wbinv(vaddr_t va,vsize_t len)148 dcache_wbinv(vaddr_t va, vsize_t len)
149 {
150 cache_op(va, len, curcpu()->ci_ci.dcache_line_size, OP_DCBF);
151 }
152
153 void
dcache_inv(vaddr_t va,vsize_t len)154 dcache_inv(vaddr_t va, vsize_t len)
155 {
156 cache_op(va, len, curcpu()->ci_ci.dcache_line_size, OP_DCBI);
157 }
158
159 void
icache_inv(vaddr_t va,vsize_t len)160 icache_inv(vaddr_t va, vsize_t len)
161 {
162 membar_sync();
163 cache_op(va, len, curcpu()->ci_ci.icache_line_size, OP_ICBI);
164 membar_sync();
165 }
166