1 /*
2 * SPDX-License-Identifier: BSD-2-Clause
3 *
4 * Copyright (c) 2019 Western Digital Corporation or its affiliates.
5 *
6 * Authors:
7 * Atish Patra <atish.patra@wdc.com>
8 * Anup Patel <anup.patel@wdc.com>
9 */
10
11 #include <sbi/riscv_asm.h>
12 #include <sbi/riscv_atomic.h>
13 #include <sbi/riscv_barrier.h>
14 #include <sbi/sbi_error.h>
15 #include <sbi/sbi_fifo.h>
16 #include <sbi/sbi_hart.h>
17 #include <sbi/sbi_ipi.h>
18 #include <sbi/sbi_scratch.h>
19 #include <sbi/sbi_tlb.h>
20 #include <sbi/sbi_hfence.h>
21 #include <sbi/sbi_string.h>
22 #include <sbi/sbi_console.h>
23 #include <sbi/sbi_platform.h>
24
25 static unsigned long tlb_sync_off;
26 static unsigned long tlb_fifo_off;
27 static unsigned long tlb_fifo_mem_off;
28 static unsigned long tlb_range_flush_limit;
29
sbi_tlb_flush_all(void)30 static void sbi_tlb_flush_all(void)
31 {
32 __asm__ __volatile("sfence.vma");
33 }
34
sbi_tlb_local_hfence_vvma(struct sbi_tlb_info * tinfo)35 void sbi_tlb_local_hfence_vvma(struct sbi_tlb_info *tinfo)
36 {
37 unsigned long start = tinfo->start;
38 unsigned long size = tinfo->size;
39 unsigned long vmid = tinfo->vmid;
40 unsigned long i, hgatp;
41
42 hgatp = csr_swap(CSR_HGATP,
43 (vmid << HGATP_VMID_SHIFT) & HGATP_VMID_MASK);
44
45 if ((start == 0 && size == 0) || (size == SBI_TLB_FLUSH_ALL)) {
46 __sbi_hfence_vvma_all();
47 goto done;
48 }
49
50 for (i = 0; i < size; i += PAGE_SIZE) {
51 __sbi_hfence_vvma_va(start+i);
52 }
53
54 done:
55 csr_write(CSR_HGATP, hgatp);
56 }
57
sbi_tlb_local_hfence_gvma(struct sbi_tlb_info * tinfo)58 void sbi_tlb_local_hfence_gvma(struct sbi_tlb_info *tinfo)
59 {
60 unsigned long start = tinfo->start;
61 unsigned long size = tinfo->size;
62 unsigned long i;
63
64 if ((start == 0 && size == 0) || (size == SBI_TLB_FLUSH_ALL)) {
65 __sbi_hfence_gvma_all();
66 return;
67 }
68
69 for (i = 0; i < size; i += PAGE_SIZE) {
70 __sbi_hfence_gvma_gpa(start+i);
71 }
72 }
73
sbi_tlb_local_sfence_vma(struct sbi_tlb_info * tinfo)74 void sbi_tlb_local_sfence_vma(struct sbi_tlb_info *tinfo)
75 {
76 unsigned long start = tinfo->start;
77 unsigned long size = tinfo->size;
78 unsigned long i;
79
80 if ((start == 0 && size == 0) || (size == SBI_TLB_FLUSH_ALL)) {
81 sbi_tlb_flush_all();
82 return;
83 }
84
85 for (i = 0; i < size; i += PAGE_SIZE) {
86 __asm__ __volatile__("sfence.vma %0"
87 :
88 : "r"(start + i)
89 : "memory");
90 }
91 }
92
sbi_tlb_local_hfence_vvma_asid(struct sbi_tlb_info * tinfo)93 void sbi_tlb_local_hfence_vvma_asid(struct sbi_tlb_info *tinfo)
94 {
95 unsigned long start = tinfo->start;
96 unsigned long size = tinfo->size;
97 unsigned long asid = tinfo->asid;
98 unsigned long vmid = tinfo->vmid;
99 unsigned long i, hgatp;
100
101 hgatp = csr_swap(CSR_HGATP,
102 (vmid << HGATP_VMID_SHIFT) & HGATP_VMID_MASK);
103
104 if (start == 0 && size == 0) {
105 __sbi_hfence_vvma_all();
106 goto done;
107 }
108
109 if (size == SBI_TLB_FLUSH_ALL) {
110 __sbi_hfence_vvma_asid(asid);
111 goto done;
112 }
113
114 for (i = 0; i < size; i += PAGE_SIZE) {
115 __sbi_hfence_vvma_asid_va(start + i, asid);
116 }
117
118 done:
119 csr_write(CSR_HGATP, hgatp);
120 }
121
sbi_tlb_local_hfence_gvma_vmid(struct sbi_tlb_info * tinfo)122 void sbi_tlb_local_hfence_gvma_vmid(struct sbi_tlb_info *tinfo)
123 {
124 unsigned long start = tinfo->start;
125 unsigned long size = tinfo->size;
126 unsigned long vmid = tinfo->vmid;
127 unsigned long i;
128
129 if (start == 0 && size == 0) {
130 __sbi_hfence_gvma_all();
131 return;
132 }
133
134 if (size == SBI_TLB_FLUSH_ALL) {
135 __sbi_hfence_gvma_vmid(vmid);
136 return;
137 }
138
139 for (i = 0; i < size; i += PAGE_SIZE) {
140 __sbi_hfence_gvma_vmid_gpa(start + i, vmid);
141 }
142 }
143
sbi_tlb_local_sfence_vma_asid(struct sbi_tlb_info * tinfo)144 void sbi_tlb_local_sfence_vma_asid(struct sbi_tlb_info *tinfo)
145 {
146 unsigned long start = tinfo->start;
147 unsigned long size = tinfo->size;
148 unsigned long asid = tinfo->asid;
149 unsigned long i;
150
151 if (start == 0 && size == 0) {
152 sbi_tlb_flush_all();
153 return;
154 }
155
156 /* Flush entire MM context for a given ASID */
157 if (size == SBI_TLB_FLUSH_ALL) {
158 __asm__ __volatile__("sfence.vma x0, %0"
159 :
160 : "r"(asid)
161 : "memory");
162 return;
163 }
164
165 for (i = 0; i < size; i += PAGE_SIZE) {
166 __asm__ __volatile__("sfence.vma %0, %1"
167 :
168 : "r"(start + i), "r"(asid)
169 : "memory");
170 }
171 }
172
sbi_tlb_local_fence_i(struct sbi_tlb_info * tinfo)173 void sbi_tlb_local_fence_i(struct sbi_tlb_info *tinfo)
174 {
175 __asm__ __volatile("fence.i");
176 }
177
sbi_tlb_entry_process(struct sbi_tlb_info * tinfo)178 static void sbi_tlb_entry_process(struct sbi_tlb_info *tinfo)
179 {
180 u32 rhartid;
181 struct sbi_scratch *rscratch = NULL;
182 unsigned long *rtlb_sync = NULL;
183
184 tinfo->local_fn(tinfo);
185
186 sbi_hartmask_for_each_hart(rhartid, &tinfo->smask) {
187 rscratch = sbi_hartid_to_scratch(rhartid);
188 if (!rscratch)
189 continue;
190
191 rtlb_sync = sbi_scratch_offset_ptr(rscratch, tlb_sync_off);
192 while (atomic_raw_xchg_ulong(rtlb_sync, 1)) ;
193 }
194 }
195
sbi_tlb_process_count(struct sbi_scratch * scratch,int count)196 static void sbi_tlb_process_count(struct sbi_scratch *scratch, int count)
197 {
198 struct sbi_tlb_info tinfo;
199 u32 deq_count = 0;
200 struct sbi_fifo *tlb_fifo =
201 sbi_scratch_offset_ptr(scratch, tlb_fifo_off);
202
203 while (!sbi_fifo_dequeue(tlb_fifo, &tinfo)) {
204 sbi_tlb_entry_process(&tinfo);
205 deq_count++;
206 if (deq_count > count)
207 break;
208
209 }
210 }
211
sbi_tlb_process(struct sbi_scratch * scratch)212 static void sbi_tlb_process(struct sbi_scratch *scratch)
213 {
214 struct sbi_tlb_info tinfo;
215 struct sbi_fifo *tlb_fifo =
216 sbi_scratch_offset_ptr(scratch, tlb_fifo_off);
217
218 while (!sbi_fifo_dequeue(tlb_fifo, &tinfo))
219 sbi_tlb_entry_process(&tinfo);
220 }
221
sbi_tlb_sync(struct sbi_scratch * scratch)222 static void sbi_tlb_sync(struct sbi_scratch *scratch)
223 {
224 unsigned long *tlb_sync =
225 sbi_scratch_offset_ptr(scratch, tlb_sync_off);
226
227 while (!atomic_raw_xchg_ulong(tlb_sync, 0)) {
228 /*
229 * While we are waiting for remote hart to set the sync,
230 * consume fifo requests to avoid deadlock.
231 */
232 sbi_tlb_process_count(scratch, 1);
233 }
234
235 return;
236 }
237
__sbi_tlb_range_check(struct sbi_tlb_info * curr,struct sbi_tlb_info * next)238 static inline int __sbi_tlb_range_check(struct sbi_tlb_info *curr,
239 struct sbi_tlb_info *next)
240 {
241 unsigned long curr_end;
242 unsigned long next_end;
243 int ret = SBI_FIFO_UNCHANGED;
244
245 if (!curr || !next)
246 return ret;
247
248 next_end = next->start + next->size;
249 curr_end = curr->start + curr->size;
250 if (next->start <= curr->start && next_end > curr_end) {
251 curr->start = next->start;
252 curr->size = next->size;
253 sbi_hartmask_or(&curr->smask, &curr->smask, &next->smask);
254 ret = SBI_FIFO_UPDATED;
255 } else if (next->start >= curr->start && next_end <= curr_end) {
256 sbi_hartmask_or(&curr->smask, &curr->smask, &next->smask);
257 ret = SBI_FIFO_SKIP;
258 }
259
260 return ret;
261 }
262
263 /**
264 * Call back to decide if an inplace fifo update is required or next entry can
265 * can be skipped. Here are the different cases that are being handled.
266 *
267 * Case1:
268 * if next flush request range lies within one of the existing entry, skip
269 * the next entry.
270 * Case2:
271 * if flush request range in current fifo entry lies within next flush
272 * request, update the current entry.
273 *
274 * Note:
275 * We can not issue a fifo reset anymore if a complete vma flush is requested.
276 * This is because we are queueing FENCE.I requests as well now.
277 * To ease up the pressure in enqueue/fifo sync path, try to dequeue 1 element
278 * before continuing the while loop. This method is preferred over wfi/ipi because
279 * of MMIO cost involved in later method.
280 */
sbi_tlb_update_cb(void * in,void * data)281 static int sbi_tlb_update_cb(void *in, void *data)
282 {
283 struct sbi_tlb_info *curr;
284 struct sbi_tlb_info *next;
285 int ret = SBI_FIFO_UNCHANGED;
286
287 if (!in || !data)
288 return ret;
289
290 curr = (struct sbi_tlb_info *)data;
291 next = (struct sbi_tlb_info *)in;
292
293 if (next->local_fn == sbi_tlb_local_sfence_vma_asid &&
294 curr->local_fn == sbi_tlb_local_sfence_vma_asid) {
295 if (next->asid == curr->asid)
296 ret = __sbi_tlb_range_check(curr, next);
297 } else if (next->local_fn == sbi_tlb_local_sfence_vma &&
298 curr->local_fn == sbi_tlb_local_sfence_vma) {
299 ret = __sbi_tlb_range_check(curr, next);
300 }
301
302 return ret;
303 }
304
sbi_tlb_update(struct sbi_scratch * scratch,struct sbi_scratch * remote_scratch,u32 remote_hartid,void * data)305 static int sbi_tlb_update(struct sbi_scratch *scratch,
306 struct sbi_scratch *remote_scratch,
307 u32 remote_hartid, void *data)
308 {
309 int ret;
310 struct sbi_fifo *tlb_fifo_r;
311 struct sbi_tlb_info *tinfo = data;
312 u32 curr_hartid = current_hartid();
313
314 /*
315 * If address range to flush is too big then simply
316 * upgrade it to flush all because we can only flush
317 * 4KB at a time.
318 */
319 if (tinfo->size > tlb_range_flush_limit) {
320 tinfo->start = 0;
321 tinfo->size = SBI_TLB_FLUSH_ALL;
322 }
323
324 /*
325 * If the request is to queue a tlb flush entry for itself
326 * then just do a local flush and return;
327 */
328 if (remote_hartid == curr_hartid) {
329 tinfo->local_fn(tinfo);
330 return -1;
331 }
332
333 tlb_fifo_r = sbi_scratch_offset_ptr(remote_scratch, tlb_fifo_off);
334
335 ret = sbi_fifo_inplace_update(tlb_fifo_r, data, sbi_tlb_update_cb);
336 if (ret != SBI_FIFO_UNCHANGED) {
337 return 1;
338 }
339
340 while (sbi_fifo_enqueue(tlb_fifo_r, data) < 0) {
341 /**
342 * For now, Busy loop until there is space in the fifo.
343 * There may be case where target hart is also
344 * enqueue in source hart's fifo. Both hart may busy
345 * loop leading to a deadlock.
346 * TODO: Introduce a wait/wakeup event mechanism to handle
347 * this properly.
348 */
349 sbi_tlb_process_count(scratch, 1);
350 sbi_dprintf("hart%d: hart%d tlb fifo full\n",
351 curr_hartid, remote_hartid);
352 }
353
354 return 0;
355 }
356
357 static struct sbi_ipi_event_ops tlb_ops = {
358 .name = "IPI_TLB",
359 .update = sbi_tlb_update,
360 .sync = sbi_tlb_sync,
361 .process = sbi_tlb_process,
362 };
363
364 static u32 tlb_event = SBI_IPI_EVENT_MAX;
365
sbi_tlb_request(ulong hmask,ulong hbase,struct sbi_tlb_info * tinfo)366 int sbi_tlb_request(ulong hmask, ulong hbase, struct sbi_tlb_info *tinfo)
367 {
368 if (!tinfo->local_fn)
369 return SBI_EINVAL;
370
371 return sbi_ipi_send_many(hmask, hbase, tlb_event, tinfo);
372 }
373
sbi_tlb_init(struct sbi_scratch * scratch,bool cold_boot)374 int sbi_tlb_init(struct sbi_scratch *scratch, bool cold_boot)
375 {
376 int ret;
377 void *tlb_mem;
378 unsigned long *tlb_sync;
379 struct sbi_fifo *tlb_q;
380 const struct sbi_platform *plat = sbi_platform_ptr(scratch);
381
382 if (cold_boot) {
383 tlb_sync_off = sbi_scratch_alloc_offset(sizeof(*tlb_sync),
384 "IPI_TLB_SYNC");
385 if (!tlb_sync_off)
386 return SBI_ENOMEM;
387 tlb_fifo_off = sbi_scratch_alloc_offset(sizeof(*tlb_q),
388 "IPI_TLB_FIFO");
389 if (!tlb_fifo_off) {
390 sbi_scratch_free_offset(tlb_sync_off);
391 return SBI_ENOMEM;
392 }
393 tlb_fifo_mem_off = sbi_scratch_alloc_offset(
394 SBI_TLB_FIFO_NUM_ENTRIES * SBI_TLB_INFO_SIZE,
395 "IPI_TLB_FIFO_MEM");
396 if (!tlb_fifo_mem_off) {
397 sbi_scratch_free_offset(tlb_fifo_off);
398 sbi_scratch_free_offset(tlb_sync_off);
399 return SBI_ENOMEM;
400 }
401 ret = sbi_ipi_event_create(&tlb_ops);
402 if (ret < 0) {
403 sbi_scratch_free_offset(tlb_fifo_mem_off);
404 sbi_scratch_free_offset(tlb_fifo_off);
405 sbi_scratch_free_offset(tlb_sync_off);
406 return ret;
407 }
408 tlb_event = ret;
409 tlb_range_flush_limit = sbi_platform_tlbr_flush_limit(plat);
410 } else {
411 if (!tlb_sync_off ||
412 !tlb_fifo_off ||
413 !tlb_fifo_mem_off)
414 return SBI_ENOMEM;
415 if (SBI_IPI_EVENT_MAX <= tlb_event)
416 return SBI_ENOSPC;
417 }
418
419 tlb_sync = sbi_scratch_offset_ptr(scratch, tlb_sync_off);
420 tlb_q = sbi_scratch_offset_ptr(scratch, tlb_fifo_off);
421 tlb_mem = sbi_scratch_offset_ptr(scratch, tlb_fifo_mem_off);
422
423 *tlb_sync = 0;
424
425 sbi_fifo_init(tlb_q, tlb_mem,
426 SBI_TLB_FIFO_NUM_ENTRIES, SBI_TLB_INFO_SIZE);
427
428 return 0;
429 }
430