xref: /linux/arch/parisc/kernel/pacache.S (revision 0be3ff0c) 
1/* SPDX-License-Identifier: GPL-2.0-or-later */
2/*
3 *  PARISC TLB and cache flushing support
4 *  Copyright (C) 2000-2001 Hewlett-Packard (John Marvin)
5 *  Copyright (C) 2001 Matthew Wilcox (willy at parisc-linux.org)
6 *  Copyright (C) 2002 Richard Hirst (rhirst with parisc-linux.org)
7 */
8
9/*
10 * NOTE: fdc,fic, and pdc instructions that use base register modification
11 *       should only use index and base registers that are not shadowed,
12 *       so that the fast path emulation in the non access miss handler
13 *       can be used.
14 */
15
16#ifdef CONFIG_64BIT
17	.level	2.0w
18#else
19	.level	2.0
20#endif
21
22#include <asm/psw.h>
23#include <asm/assembly.h>
24#include <asm/cache.h>
25#include <asm/ldcw.h>
26#include <asm/alternative.h>
27#include <linux/linkage.h>
28#include <linux/init.h>
29#include <linux/pgtable.h>
30
31	.section .text.hot
32	.align	16
33
34ENTRY_CFI(flush_tlb_all_local)
35	/*
36	 * The pitlbe and pdtlbe instructions should only be used to
37	 * flush the entire tlb. Also, there needs to be no intervening
38	 * tlb operations, e.g. tlb misses, so the operation needs
39	 * to happen in real mode with all interruptions disabled.
40	 */
41
42	/* pcxt_ssm_bug	- relied upon translation! PA 2.0 Arch. F-4 and F-5 */
43	rsm		PSW_SM_I, %r19		/* save I-bit state */
44	load32		PA(1f), %r1
45	nop
46	nop
47	nop
48	nop
49	nop
50
51	rsm		PSW_SM_Q, %r0		/* prep to load iia queue */
52	mtctl		%r0, %cr17		/* Clear IIASQ tail */
53	mtctl		%r0, %cr17		/* Clear IIASQ head */
54	mtctl		%r1, %cr18		/* IIAOQ head */
55	ldo		4(%r1), %r1
56	mtctl		%r1, %cr18		/* IIAOQ tail */
57	load32		REAL_MODE_PSW, %r1
58	mtctl           %r1, %ipsw
59	rfi
60	nop
61
621:      load32		PA(cache_info), %r1
63
64	/* Flush Instruction Tlb */
65
6688:	LDREG		ITLB_SID_BASE(%r1), %r20
67	LDREG		ITLB_SID_STRIDE(%r1), %r21
68	LDREG		ITLB_SID_COUNT(%r1), %r22
69	LDREG		ITLB_OFF_BASE(%r1), %arg0
70	LDREG		ITLB_OFF_STRIDE(%r1), %arg1
71	LDREG		ITLB_OFF_COUNT(%r1), %arg2
72	LDREG		ITLB_LOOP(%r1), %arg3
73
74	addib,COND(=)		-1, %arg3, fitoneloop	/* Preadjust and test */
75	movb,<,n	%arg3, %r31, fitdone	/* If loop < 0, skip */
76	copy		%arg0, %r28		/* Init base addr */
77
78fitmanyloop:					/* Loop if LOOP >= 2 */
79	mtsp		%r20, %sr1
80	add		%r21, %r20, %r20	/* increment space */
81	copy		%arg2, %r29		/* Init middle loop count */
82
83fitmanymiddle:					/* Loop if LOOP >= 2 */
84	addib,COND(>)		-1, %r31, fitmanymiddle	/* Adjusted inner loop decr */
85	pitlbe		%r0(%sr1, %r28)
86	pitlbe,m	%arg1(%sr1, %r28)	/* Last pitlbe and addr adjust */
87	addib,COND(>)		-1, %r29, fitmanymiddle	/* Middle loop decr */
88	copy		%arg3, %r31		/* Re-init inner loop count */
89
90	movb,tr		%arg0, %r28, fitmanyloop /* Re-init base addr */
91	addib,COND(<=),n	-1, %r22, fitdone	/* Outer loop count decr */
92
93fitoneloop:					/* Loop if LOOP = 1 */
94	mtsp		%r20, %sr1
95	copy		%arg0, %r28		/* init base addr */
96	copy		%arg2, %r29		/* init middle loop count */
97
98fitonemiddle:					/* Loop if LOOP = 1 */
99	addib,COND(>)		-1, %r29, fitonemiddle	/* Middle loop count decr */
100	pitlbe,m	%arg1(%sr1, %r28)	/* pitlbe for one loop */
101
102	addib,COND(>)		-1, %r22, fitoneloop	/* Outer loop count decr */
103	add		%r21, %r20, %r20		/* increment space */
104
105fitdone:
106	ALTERNATIVE(88b, fitdone, ALT_COND_NO_SPLIT_TLB, INSN_NOP)
107
108	/* Flush Data Tlb */
109
110	LDREG		DTLB_SID_BASE(%r1), %r20
111	LDREG		DTLB_SID_STRIDE(%r1), %r21
112	LDREG		DTLB_SID_COUNT(%r1), %r22
113	LDREG		DTLB_OFF_BASE(%r1), %arg0
114	LDREG		DTLB_OFF_STRIDE(%r1), %arg1
115	LDREG		DTLB_OFF_COUNT(%r1), %arg2
116	LDREG		DTLB_LOOP(%r1), %arg3
117
118	addib,COND(=)		-1, %arg3, fdtoneloop	/* Preadjust and test */
119	movb,<,n	%arg3, %r31, fdtdone	/* If loop < 0, skip */
120	copy		%arg0, %r28		/* Init base addr */
121
122fdtmanyloop:					/* Loop if LOOP >= 2 */
123	mtsp		%r20, %sr1
124	add		%r21, %r20, %r20	/* increment space */
125	copy		%arg2, %r29		/* Init middle loop count */
126
127fdtmanymiddle:					/* Loop if LOOP >= 2 */
128	addib,COND(>)		-1, %r31, fdtmanymiddle	/* Adjusted inner loop decr */
129	pdtlbe		%r0(%sr1, %r28)
130	pdtlbe,m	%arg1(%sr1, %r28)	/* Last pdtlbe and addr adjust */
131	addib,COND(>)		-1, %r29, fdtmanymiddle	/* Middle loop decr */
132	copy		%arg3, %r31		/* Re-init inner loop count */
133
134	movb,tr		%arg0, %r28, fdtmanyloop /* Re-init base addr */
135	addib,COND(<=),n	-1, %r22,fdtdone	/* Outer loop count decr */
136
137fdtoneloop:					/* Loop if LOOP = 1 */
138	mtsp		%r20, %sr1
139	copy		%arg0, %r28		/* init base addr */
140	copy		%arg2, %r29		/* init middle loop count */
141
142fdtonemiddle:					/* Loop if LOOP = 1 */
143	addib,COND(>)		-1, %r29, fdtonemiddle	/* Middle loop count decr */
144	pdtlbe,m	%arg1(%sr1, %r28)	/* pdtlbe for one loop */
145
146	addib,COND(>)		-1, %r22, fdtoneloop	/* Outer loop count decr */
147	add		%r21, %r20, %r20	/* increment space */
148
149
150fdtdone:
151	/*
152	 * Switch back to virtual mode
153	 */
154	/* pcxt_ssm_bug */
155	rsm		PSW_SM_I, %r0
156	load32		2f, %r1
157	nop
158	nop
159	nop
160	nop
161	nop
162
163	rsm		PSW_SM_Q, %r0		/* prep to load iia queue */
164	mtctl		%r0, %cr17		/* Clear IIASQ tail */
165	mtctl		%r0, %cr17		/* Clear IIASQ head */
166	mtctl		%r1, %cr18		/* IIAOQ head */
167	ldo		4(%r1), %r1
168	mtctl		%r1, %cr18		/* IIAOQ tail */
169	load32		KERNEL_PSW, %r1
170	or		%r1, %r19, %r1	/* I-bit to state on entry */
171	mtctl		%r1, %ipsw	/* restore I-bit (entire PSW) */
172	rfi
173	nop
174
1752:      bv		%r0(%r2)
176	nop
177
178	/*
179	 * When running in qemu, drop whole flush_tlb_all_local function and
180	 * replace by one pdtlbe instruction, for which QEMU will drop all
181	 * local TLB entries.
182	 */
1833:	pdtlbe		%r0(%sr1,%r0)
184	bv,n		%r0(%r2)
185	ALTERNATIVE_CODE(flush_tlb_all_local, 2, ALT_COND_RUN_ON_QEMU, 3b)
186ENDPROC_CFI(flush_tlb_all_local)
187
188	.import cache_info,data
189
190ENTRY_CFI(flush_instruction_cache_local)
19188:	load32		cache_info, %r1
192
193	/* Flush Instruction Cache */
194
195	LDREG		ICACHE_BASE(%r1), %arg0
196	LDREG		ICACHE_STRIDE(%r1), %arg1
197	LDREG		ICACHE_COUNT(%r1), %arg2
198	LDREG		ICACHE_LOOP(%r1), %arg3
199	rsm		PSW_SM_I, %r22		/* No mmgt ops during loop*/
200	mtsp		%r0, %sr1
201	addib,COND(=)		-1, %arg3, fioneloop	/* Preadjust and test */
202	movb,<,n	%arg3, %r31, fisync	/* If loop < 0, do sync */
203
204fimanyloop:					/* Loop if LOOP >= 2 */
205	addib,COND(>)		-1, %r31, fimanyloop	/* Adjusted inner loop decr */
206	fice            %r0(%sr1, %arg0)
207	fice,m		%arg1(%sr1, %arg0)	/* Last fice and addr adjust */
208	movb,tr		%arg3, %r31, fimanyloop	/* Re-init inner loop count */
209	addib,COND(<=),n	-1, %arg2, fisync	/* Outer loop decr */
210
211fioneloop:					/* Loop if LOOP = 1 */
212	/* Some implementations may flush with a single fice instruction */
213	cmpib,COND(>>=),n	15, %arg2, fioneloop2
214
215fioneloop1:
216	fice,m		%arg1(%sr1, %arg0)
217	fice,m		%arg1(%sr1, %arg0)
218	fice,m		%arg1(%sr1, %arg0)
219	fice,m		%arg1(%sr1, %arg0)
220	fice,m		%arg1(%sr1, %arg0)
221	fice,m		%arg1(%sr1, %arg0)
222	fice,m		%arg1(%sr1, %arg0)
223	fice,m		%arg1(%sr1, %arg0)
224	fice,m		%arg1(%sr1, %arg0)
225	fice,m		%arg1(%sr1, %arg0)
226	fice,m		%arg1(%sr1, %arg0)
227	fice,m		%arg1(%sr1, %arg0)
228	fice,m		%arg1(%sr1, %arg0)
229	fice,m		%arg1(%sr1, %arg0)
230	fice,m		%arg1(%sr1, %arg0)
231	addib,COND(>)	-16, %arg2, fioneloop1
232	fice,m		%arg1(%sr1, %arg0)
233
234	/* Check if done */
235	cmpb,COND(=),n	%arg2, %r0, fisync	/* Predict branch taken */
236
237fioneloop2:
238	addib,COND(>)	-1, %arg2, fioneloop2	/* Outer loop count decr */
239	fice,m		%arg1(%sr1, %arg0)	/* Fice for one loop */
240
241fisync:
242	sync
243	mtsm		%r22			/* restore I-bit */
24489:	ALTERNATIVE(88b, 89b, ALT_COND_NO_ICACHE, INSN_NOP)
245	bv		%r0(%r2)
246	nop
247ENDPROC_CFI(flush_instruction_cache_local)
248
249
250	.import cache_info, data
251ENTRY_CFI(flush_data_cache_local)
25288:	load32		cache_info, %r1
253
254	/* Flush Data Cache */
255
256	LDREG		DCACHE_BASE(%r1), %arg0
257	LDREG		DCACHE_STRIDE(%r1), %arg1
258	LDREG		DCACHE_COUNT(%r1), %arg2
259	LDREG		DCACHE_LOOP(%r1), %arg3
260	rsm		PSW_SM_I, %r22		/* No mmgt ops during loop*/
261	mtsp		%r0, %sr1
262	addib,COND(=)		-1, %arg3, fdoneloop	/* Preadjust and test */
263	movb,<,n	%arg3, %r31, fdsync	/* If loop < 0, do sync */
264
265fdmanyloop:					/* Loop if LOOP >= 2 */
266	addib,COND(>)		-1, %r31, fdmanyloop	/* Adjusted inner loop decr */
267	fdce		%r0(%sr1, %arg0)
268	fdce,m		%arg1(%sr1, %arg0)	/* Last fdce and addr adjust */
269	movb,tr		%arg3, %r31, fdmanyloop	/* Re-init inner loop count */
270	addib,COND(<=),n	-1, %arg2, fdsync	/* Outer loop decr */
271
272fdoneloop:					/* Loop if LOOP = 1 */
273	/* Some implementations may flush with a single fdce instruction */
274	cmpib,COND(>>=),n	15, %arg2, fdoneloop2
275
276fdoneloop1:
277	fdce,m		%arg1(%sr1, %arg0)
278	fdce,m		%arg1(%sr1, %arg0)
279	fdce,m		%arg1(%sr1, %arg0)
280	fdce,m		%arg1(%sr1, %arg0)
281	fdce,m		%arg1(%sr1, %arg0)
282	fdce,m		%arg1(%sr1, %arg0)
283	fdce,m		%arg1(%sr1, %arg0)
284	fdce,m		%arg1(%sr1, %arg0)
285	fdce,m		%arg1(%sr1, %arg0)
286	fdce,m		%arg1(%sr1, %arg0)
287	fdce,m		%arg1(%sr1, %arg0)
288	fdce,m		%arg1(%sr1, %arg0)
289	fdce,m		%arg1(%sr1, %arg0)
290	fdce,m		%arg1(%sr1, %arg0)
291	fdce,m		%arg1(%sr1, %arg0)
292	addib,COND(>)	-16, %arg2, fdoneloop1
293	fdce,m		%arg1(%sr1, %arg0)
294
295	/* Check if done */
296	cmpb,COND(=),n	%arg2, %r0, fdsync	/* Predict branch taken */
297
298fdoneloop2:
299	addib,COND(>)	-1, %arg2, fdoneloop2	/* Outer loop count decr */
300	fdce,m		%arg1(%sr1, %arg0)	/* Fdce for one loop */
301
302fdsync:
303	syncdma
304	sync
305	mtsm		%r22			/* restore I-bit */
30689:	ALTERNATIVE(88b, 89b, ALT_COND_NO_DCACHE, INSN_NOP)
307	bv		%r0(%r2)
308	nop
309ENDPROC_CFI(flush_data_cache_local)
310
311/* Clear page using kernel mapping.  */
312
313ENTRY_CFI(clear_page_asm)
314#ifdef CONFIG_64BIT
315
316	/* Unroll the loop.  */
317	ldi		(PAGE_SIZE / 128), %r1
318
3191:
320	std		%r0, 0(%r26)
321	std		%r0, 8(%r26)
322	std		%r0, 16(%r26)
323	std		%r0, 24(%r26)
324	std		%r0, 32(%r26)
325	std		%r0, 40(%r26)
326	std		%r0, 48(%r26)
327	std		%r0, 56(%r26)
328	std		%r0, 64(%r26)
329	std		%r0, 72(%r26)
330	std		%r0, 80(%r26)
331	std		%r0, 88(%r26)
332	std		%r0, 96(%r26)
333	std		%r0, 104(%r26)
334	std		%r0, 112(%r26)
335	std		%r0, 120(%r26)
336
337	/* Note reverse branch hint for addib is taken.  */
338	addib,COND(>),n	-1, %r1, 1b
339	ldo		128(%r26), %r26
340
341#else
342
343	/*
344	 * Note that until (if) we start saving the full 64-bit register
345	 * values on interrupt, we can't use std on a 32 bit kernel.
346	 */
347	ldi		(PAGE_SIZE / 64), %r1
348
3491:
350	stw		%r0, 0(%r26)
351	stw		%r0, 4(%r26)
352	stw		%r0, 8(%r26)
353	stw		%r0, 12(%r26)
354	stw		%r0, 16(%r26)
355	stw		%r0, 20(%r26)
356	stw		%r0, 24(%r26)
357	stw		%r0, 28(%r26)
358	stw		%r0, 32(%r26)
359	stw		%r0, 36(%r26)
360	stw		%r0, 40(%r26)
361	stw		%r0, 44(%r26)
362	stw		%r0, 48(%r26)
363	stw		%r0, 52(%r26)
364	stw		%r0, 56(%r26)
365	stw		%r0, 60(%r26)
366
367	addib,COND(>),n	-1, %r1, 1b
368	ldo		64(%r26), %r26
369#endif
370	bv		%r0(%r2)
371	nop
372ENDPROC_CFI(clear_page_asm)
373
374/* Copy page using kernel mapping.  */
375
376ENTRY_CFI(copy_page_asm)
377#ifdef CONFIG_64BIT
378	/* PA8x00 CPUs can consume 2 loads or 1 store per cycle.
379	 * Unroll the loop by hand and arrange insn appropriately.
380	 * Prefetch doesn't improve performance on rp3440.
381	 * GCC probably can do this just as well...
382	 */
383
384	ldi		(PAGE_SIZE / 128), %r1
385
3861:	ldd		0(%r25), %r19
387	ldd		8(%r25), %r20
388
389	ldd		16(%r25), %r21
390	ldd		24(%r25), %r22
391	std		%r19, 0(%r26)
392	std		%r20, 8(%r26)
393
394	ldd		32(%r25), %r19
395	ldd		40(%r25), %r20
396	std		%r21, 16(%r26)
397	std		%r22, 24(%r26)
398
399	ldd		48(%r25), %r21
400	ldd		56(%r25), %r22
401	std		%r19, 32(%r26)
402	std		%r20, 40(%r26)
403
404	ldd		64(%r25), %r19
405	ldd		72(%r25), %r20
406	std		%r21, 48(%r26)
407	std		%r22, 56(%r26)
408
409	ldd		80(%r25), %r21
410	ldd		88(%r25), %r22
411	std		%r19, 64(%r26)
412	std		%r20, 72(%r26)
413
414	ldd		 96(%r25), %r19
415	ldd		104(%r25), %r20
416	std		%r21, 80(%r26)
417	std		%r22, 88(%r26)
418
419	ldd		112(%r25), %r21
420	ldd		120(%r25), %r22
421	ldo		128(%r25), %r25
422	std		%r19, 96(%r26)
423	std		%r20, 104(%r26)
424
425	std		%r21, 112(%r26)
426	std		%r22, 120(%r26)
427
428	/* Note reverse branch hint for addib is taken.  */
429	addib,COND(>),n	-1, %r1, 1b
430	ldo		128(%r26), %r26
431
432#else
433
434	/*
435	 * This loop is optimized for PCXL/PCXL2 ldw/ldw and stw/stw
436	 * bundles (very restricted rules for bundling).
437	 * Note that until (if) we start saving
438	 * the full 64 bit register values on interrupt, we can't
439	 * use ldd/std on a 32 bit kernel.
440	 */
441	ldw		0(%r25), %r19
442	ldi		(PAGE_SIZE / 64), %r1
443
4441:
445	ldw		4(%r25), %r20
446	ldw		8(%r25), %r21
447	ldw		12(%r25), %r22
448	stw		%r19, 0(%r26)
449	stw		%r20, 4(%r26)
450	stw		%r21, 8(%r26)
451	stw		%r22, 12(%r26)
452	ldw		16(%r25), %r19
453	ldw		20(%r25), %r20
454	ldw		24(%r25), %r21
455	ldw		28(%r25), %r22
456	stw		%r19, 16(%r26)
457	stw		%r20, 20(%r26)
458	stw		%r21, 24(%r26)
459	stw		%r22, 28(%r26)
460	ldw		32(%r25), %r19
461	ldw		36(%r25), %r20
462	ldw		40(%r25), %r21
463	ldw		44(%r25), %r22
464	stw		%r19, 32(%r26)
465	stw		%r20, 36(%r26)
466	stw		%r21, 40(%r26)
467	stw		%r22, 44(%r26)
468	ldw		48(%r25), %r19
469	ldw		52(%r25), %r20
470	ldw		56(%r25), %r21
471	ldw		60(%r25), %r22
472	stw		%r19, 48(%r26)
473	stw		%r20, 52(%r26)
474	ldo		64(%r25), %r25
475	stw		%r21, 56(%r26)
476	stw		%r22, 60(%r26)
477	ldo		64(%r26), %r26
478	addib,COND(>),n	-1, %r1, 1b
479	ldw		0(%r25), %r19
480#endif
481	bv		%r0(%r2)
482	nop
483ENDPROC_CFI(copy_page_asm)
484
485/*
486 * NOTE: Code in clear_user_page has a hard coded dependency on the
487 *       maximum alias boundary being 4 Mb. We've been assured by the
488 *       parisc chip designers that there will not ever be a parisc
489 *       chip with a larger alias boundary (Never say never :-) ).
490 *
491 *       Subtle: the dtlb miss handlers support the temp alias region by
492 *       "knowing" that if a dtlb miss happens within the temp alias
493 *       region it must have occurred while in clear_user_page. Since
494 *       this routine makes use of processor local translations, we
495 *       don't want to insert them into the kernel page table. Instead,
496 *       we load up some general registers (they need to be registers
497 *       which aren't shadowed) with the physical page numbers (preshifted
498 *       for tlb insertion) needed to insert the translations. When we
499 *       miss on the translation, the dtlb miss handler inserts the
500 *       translation into the tlb using these values:
501 *
502 *          %r26 physical page (shifted for tlb insert) of "to" translation
503 *          %r23 physical page (shifted for tlb insert) of "from" translation
504 */
505
506        /* Drop prot bits and convert to page addr for iitlbt and idtlbt */
507        #define PAGE_ADD_SHIFT  (PAGE_SHIFT-12)
508        .macro          convert_phys_for_tlb_insert20  phys
509        extrd,u         \phys, 56-PAGE_ADD_SHIFT, 32-PAGE_ADD_SHIFT, \phys
510#if _PAGE_SIZE_ENCODING_DEFAULT
511        depdi           _PAGE_SIZE_ENCODING_DEFAULT, 63, (63-58), \phys
512#endif
513	.endm
514
515	/*
516	 * copy_user_page_asm() performs a page copy using mappings
517	 * equivalent to the user page mappings.  It can be used to
518	 * implement copy_user_page() but unfortunately both the `from'
519	 * and `to' pages need to be flushed through mappings equivalent
520	 * to the user mappings after the copy because the kernel accesses
521	 * the `from' page through the kmap kernel mapping and the `to'
522	 * page needs to be flushed since code can be copied.  As a
523	 * result, this implementation is less efficient than the simpler
524	 * copy using the kernel mapping.  It only needs the `from' page
525	 * to flushed via the user mapping.  The kunmap routines handle
526	 * the flushes needed for the kernel mapping.
527	 *
528	 * I'm still keeping this around because it may be possible to
529	 * use it if more information is passed into copy_user_page().
530	 * Have to do some measurements to see if it is worthwhile to
531	 * lobby for such a change.
532	 *
533	 */
534
535ENTRY_CFI(copy_user_page_asm)
536	/* Convert virtual `to' and `from' addresses to physical addresses.
537	   Move `from' physical address to non shadowed register.  */
538	ldil		L%(__PAGE_OFFSET), %r1
539	sub		%r26, %r1, %r26
540	sub		%r25, %r1, %r23
541
542	ldil		L%(TMPALIAS_MAP_START), %r28
543#ifdef CONFIG_64BIT
544#if (TMPALIAS_MAP_START >= 0x80000000)
545	depdi		0, 31,32, %r28		/* clear any sign extension */
546#endif
547	convert_phys_for_tlb_insert20 %r26	/* convert phys addr to tlb insert format */
548	convert_phys_for_tlb_insert20 %r23	/* convert phys addr to tlb insert format */
549	depd		%r24,63,22, %r28	/* Form aliased virtual address 'to' */
550	depdi		0, 63,PAGE_SHIFT, %r28	/* Clear any offset bits */
551	copy		%r28, %r29
552	depdi		1, 41,1, %r29		/* Form aliased virtual address 'from' */
553#else
554	extrw,u		%r26, 24,25, %r26	/* convert phys addr to tlb insert format */
555	extrw,u		%r23, 24,25, %r23	/* convert phys addr to tlb insert format */
556	depw		%r24, 31,22, %r28	/* Form aliased virtual address 'to' */
557	depwi		0, 31,PAGE_SHIFT, %r28	/* Clear any offset bits */
558	copy		%r28, %r29
559	depwi		1, 9,1, %r29		/* Form aliased virtual address 'from' */
560#endif
561
562	/* Purge any old translations */
563
564#ifdef CONFIG_PA20
565	pdtlb,l		%r0(%r28)
566	pdtlb,l		%r0(%r29)
567#else
5680:	pdtlb		%r0(%r28)
5691:	pdtlb		%r0(%r29)
570	ALTERNATIVE(0b, 0b+4, ALT_COND_NO_SMP, INSN_PxTLB)
571	ALTERNATIVE(1b, 1b+4, ALT_COND_NO_SMP, INSN_PxTLB)
572#endif
573
574#ifdef CONFIG_64BIT
575	/* PA8x00 CPUs can consume 2 loads or 1 store per cycle.
576	 * Unroll the loop by hand and arrange insn appropriately.
577	 * GCC probably can do this just as well.
578	 */
579
580	ldd		0(%r29), %r19
581	ldi		(PAGE_SIZE / 128), %r1
582
5831:	ldd		8(%r29), %r20
584
585	ldd		16(%r29), %r21
586	ldd		24(%r29), %r22
587	std		%r19, 0(%r28)
588	std		%r20, 8(%r28)
589
590	ldd		32(%r29), %r19
591	ldd		40(%r29), %r20
592	std		%r21, 16(%r28)
593	std		%r22, 24(%r28)
594
595	ldd		48(%r29), %r21
596	ldd		56(%r29), %r22
597	std		%r19, 32(%r28)
598	std		%r20, 40(%r28)
599
600	ldd		64(%r29), %r19
601	ldd		72(%r29), %r20
602	std		%r21, 48(%r28)
603	std		%r22, 56(%r28)
604
605	ldd		80(%r29), %r21
606	ldd		88(%r29), %r22
607	std		%r19, 64(%r28)
608	std		%r20, 72(%r28)
609
610	ldd		 96(%r29), %r19
611	ldd		104(%r29), %r20
612	std		%r21, 80(%r28)
613	std		%r22, 88(%r28)
614
615	ldd		112(%r29), %r21
616	ldd		120(%r29), %r22
617	std		%r19, 96(%r28)
618	std		%r20, 104(%r28)
619
620	ldo		128(%r29), %r29
621	std		%r21, 112(%r28)
622	std		%r22, 120(%r28)
623	ldo		128(%r28), %r28
624
625	/* conditional branches nullify on forward taken branch, and on
626	 * non-taken backward branch. Note that .+4 is a backwards branch.
627	 * The ldd should only get executed if the branch is taken.
628	 */
629	addib,COND(>),n	-1, %r1, 1b		/* bundle 10 */
630	ldd		0(%r29), %r19		/* start next loads */
631
632#else
633	ldi		(PAGE_SIZE / 64), %r1
634
635	/*
636	 * This loop is optimized for PCXL/PCXL2 ldw/ldw and stw/stw
637	 * bundles (very restricted rules for bundling). It probably
638	 * does OK on PCXU and better, but we could do better with
639	 * ldd/std instructions. Note that until (if) we start saving
640	 * the full 64 bit register values on interrupt, we can't
641	 * use ldd/std on a 32 bit kernel.
642	 */
643
6441:	ldw		0(%r29), %r19
645	ldw		4(%r29), %r20
646	ldw		8(%r29), %r21
647	ldw		12(%r29), %r22
648	stw		%r19, 0(%r28)
649	stw		%r20, 4(%r28)
650	stw		%r21, 8(%r28)
651	stw		%r22, 12(%r28)
652	ldw		16(%r29), %r19
653	ldw		20(%r29), %r20
654	ldw		24(%r29), %r21
655	ldw		28(%r29), %r22
656	stw		%r19, 16(%r28)
657	stw		%r20, 20(%r28)
658	stw		%r21, 24(%r28)
659	stw		%r22, 28(%r28)
660	ldw		32(%r29), %r19
661	ldw		36(%r29), %r20
662	ldw		40(%r29), %r21
663	ldw		44(%r29), %r22
664	stw		%r19, 32(%r28)
665	stw		%r20, 36(%r28)
666	stw		%r21, 40(%r28)
667	stw		%r22, 44(%r28)
668	ldw		48(%r29), %r19
669	ldw		52(%r29), %r20
670	ldw		56(%r29), %r21
671	ldw		60(%r29), %r22
672	stw		%r19, 48(%r28)
673	stw		%r20, 52(%r28)
674	stw		%r21, 56(%r28)
675	stw		%r22, 60(%r28)
676	ldo		64(%r28), %r28
677
678	addib,COND(>)		-1, %r1,1b
679	ldo		64(%r29), %r29
680#endif
681
682	bv		%r0(%r2)
683	nop
684ENDPROC_CFI(copy_user_page_asm)
685
686ENTRY_CFI(clear_user_page_asm)
687	tophys_r1	%r26
688
689	ldil		L%(TMPALIAS_MAP_START), %r28
690#ifdef CONFIG_64BIT
691#if (TMPALIAS_MAP_START >= 0x80000000)
692	depdi		0, 31,32, %r28		/* clear any sign extension */
693#endif
694	convert_phys_for_tlb_insert20 %r26	/* convert phys addr to tlb insert format */
695	depd		%r25, 63,22, %r28	/* Form aliased virtual address 'to' */
696	depdi		0, 63,PAGE_SHIFT, %r28	/* Clear any offset bits */
697#else
698	extrw,u		%r26, 24,25, %r26	/* convert phys addr to tlb insert format */
699	depw		%r25, 31,22, %r28	/* Form aliased virtual address 'to' */
700	depwi		0, 31,PAGE_SHIFT, %r28	/* Clear any offset bits */
701#endif
702
703	/* Purge any old translation */
704
705#ifdef CONFIG_PA20
706	pdtlb,l		%r0(%r28)
707#else
7080:	pdtlb		%r0(%r28)
709	ALTERNATIVE(0b, 0b+4, ALT_COND_NO_SMP, INSN_PxTLB)
710#endif
711
712#ifdef CONFIG_64BIT
713	ldi		(PAGE_SIZE / 128), %r1
714
715	/* PREFETCH (Write) has not (yet) been proven to help here */
716	/* #define	PREFETCHW_OP	ldd		256(%0), %r0 */
717
7181:	std		%r0, 0(%r28)
719	std		%r0, 8(%r28)
720	std		%r0, 16(%r28)
721	std		%r0, 24(%r28)
722	std		%r0, 32(%r28)
723	std		%r0, 40(%r28)
724	std		%r0, 48(%r28)
725	std		%r0, 56(%r28)
726	std		%r0, 64(%r28)
727	std		%r0, 72(%r28)
728	std		%r0, 80(%r28)
729	std		%r0, 88(%r28)
730	std		%r0, 96(%r28)
731	std		%r0, 104(%r28)
732	std		%r0, 112(%r28)
733	std		%r0, 120(%r28)
734	addib,COND(>)		-1, %r1, 1b
735	ldo		128(%r28), %r28
736
737#else	/* ! CONFIG_64BIT */
738	ldi		(PAGE_SIZE / 64), %r1
739
7401:	stw		%r0, 0(%r28)
741	stw		%r0, 4(%r28)
742	stw		%r0, 8(%r28)
743	stw		%r0, 12(%r28)
744	stw		%r0, 16(%r28)
745	stw		%r0, 20(%r28)
746	stw		%r0, 24(%r28)
747	stw		%r0, 28(%r28)
748	stw		%r0, 32(%r28)
749	stw		%r0, 36(%r28)
750	stw		%r0, 40(%r28)
751	stw		%r0, 44(%r28)
752	stw		%r0, 48(%r28)
753	stw		%r0, 52(%r28)
754	stw		%r0, 56(%r28)
755	stw		%r0, 60(%r28)
756	addib,COND(>)		-1, %r1, 1b
757	ldo		64(%r28), %r28
758#endif	/* CONFIG_64BIT */
759
760	bv		%r0(%r2)
761	nop
762ENDPROC_CFI(clear_user_page_asm)
763
764ENTRY_CFI(flush_dcache_page_asm)
765	ldil		L%(TMPALIAS_MAP_START), %r28
766#ifdef CONFIG_64BIT
767#if (TMPALIAS_MAP_START >= 0x80000000)
768	depdi		0, 31,32, %r28		/* clear any sign extension */
769#endif
770	convert_phys_for_tlb_insert20 %r26	/* convert phys addr to tlb insert format */
771	depd		%r25, 63,22, %r28	/* Form aliased virtual address 'to' */
772	depdi		0, 63,PAGE_SHIFT, %r28	/* Clear any offset bits */
773#else
774	extrw,u		%r26, 24,25, %r26	/* convert phys addr to tlb insert format */
775	depw		%r25, 31,22, %r28	/* Form aliased virtual address 'to' */
776	depwi		0, 31,PAGE_SHIFT, %r28	/* Clear any offset bits */
777#endif
778
779	/* Purge any old translation */
780
781#ifdef CONFIG_PA20
782	pdtlb,l		%r0(%r28)
783#else
7840:	pdtlb		%r0(%r28)
785	ALTERNATIVE(0b, 0b+4, ALT_COND_NO_SMP, INSN_PxTLB)
786#endif
787
78888:	ldil		L%dcache_stride, %r1
789	ldw		R%dcache_stride(%r1), r31
790
791#ifdef CONFIG_64BIT
792	depdi,z		1, 63-PAGE_SHIFT,1, %r25
793#else
794	depwi,z		1, 31-PAGE_SHIFT,1, %r25
795#endif
796	add		%r28, %r25, %r25
797	sub		%r25, r31, %r25
798
7991:	fdc,m		r31(%r28)
800	fdc,m		r31(%r28)
801	fdc,m		r31(%r28)
802	fdc,m		r31(%r28)
803	fdc,m		r31(%r28)
804	fdc,m		r31(%r28)
805	fdc,m		r31(%r28)
806	fdc,m		r31(%r28)
807	fdc,m		r31(%r28)
808	fdc,m		r31(%r28)
809	fdc,m		r31(%r28)
810	fdc,m		r31(%r28)
811	fdc,m		r31(%r28)
812	fdc,m		r31(%r28)
813	fdc,m		r31(%r28)
814	cmpb,COND(>>)	%r25, %r28, 1b /* predict taken */
815	fdc,m		r31(%r28)
816
81789:	ALTERNATIVE(88b, 89b, ALT_COND_NO_DCACHE, INSN_NOP)
818	sync
819	bv		%r0(%r2)
820	nop
821ENDPROC_CFI(flush_dcache_page_asm)
822
823ENTRY_CFI(purge_dcache_page_asm)
824	ldil		L%(TMPALIAS_MAP_START), %r28
825#ifdef CONFIG_64BIT
826#if (TMPALIAS_MAP_START >= 0x80000000)
827	depdi		0, 31,32, %r28		/* clear any sign extension */
828#endif
829	convert_phys_for_tlb_insert20 %r26	/* convert phys addr to tlb insert format */
830	depd		%r25, 63,22, %r28	/* Form aliased virtual address 'to' */
831	depdi		0, 63,PAGE_SHIFT, %r28	/* Clear any offset bits */
832#else
833	extrw,u		%r26, 24,25, %r26	/* convert phys addr to tlb insert format */
834	depw		%r25, 31,22, %r28	/* Form aliased virtual address 'to' */
835	depwi		0, 31,PAGE_SHIFT, %r28	/* Clear any offset bits */
836#endif
837
838	/* Purge any old translation */
839
840#ifdef CONFIG_PA20
841	pdtlb,l		%r0(%r28)
842#else
8430:	pdtlb		%r0(%r28)
844	ALTERNATIVE(0b, 0b+4, ALT_COND_NO_SMP, INSN_PxTLB)
845#endif
846
84788:	ldil		L%dcache_stride, %r1
848	ldw		R%dcache_stride(%r1), r31
849
850#ifdef CONFIG_64BIT
851	depdi,z		1, 63-PAGE_SHIFT,1, %r25
852#else
853	depwi,z		1, 31-PAGE_SHIFT,1, %r25
854#endif
855	add		%r28, %r25, %r25
856	sub		%r25, r31, %r25
857
8581:      pdc,m		r31(%r28)
859	pdc,m		r31(%r28)
860	pdc,m		r31(%r28)
861	pdc,m		r31(%r28)
862	pdc,m		r31(%r28)
863	pdc,m		r31(%r28)
864	pdc,m		r31(%r28)
865	pdc,m		r31(%r28)
866	pdc,m		r31(%r28)
867	pdc,m		r31(%r28)
868	pdc,m		r31(%r28)
869	pdc,m		r31(%r28)
870	pdc,m		r31(%r28)
871	pdc,m		r31(%r28)
872	pdc,m		r31(%r28)
873	cmpb,COND(>>)	%r25, %r28, 1b /* predict taken */
874	pdc,m		r31(%r28)
875
87689:	ALTERNATIVE(88b, 89b, ALT_COND_NO_DCACHE, INSN_NOP)
877	sync
878	bv		%r0(%r2)
879	nop
880ENDPROC_CFI(purge_dcache_page_asm)
881
882ENTRY_CFI(flush_icache_page_asm)
883	ldil		L%(TMPALIAS_MAP_START), %r28
884#ifdef CONFIG_64BIT
885#if (TMPALIAS_MAP_START >= 0x80000000)
886	depdi		0, 31,32, %r28		/* clear any sign extension */
887#endif
888	convert_phys_for_tlb_insert20 %r26	/* convert phys addr to tlb insert format */
889	depd		%r25, 63,22, %r28	/* Form aliased virtual address 'to' */
890	depdi		0, 63,PAGE_SHIFT, %r28	/* Clear any offset bits */
891#else
892	extrw,u		%r26, 24,25, %r26	/* convert phys addr to tlb insert format */
893	depw		%r25, 31,22, %r28	/* Form aliased virtual address 'to' */
894	depwi		0, 31,PAGE_SHIFT, %r28	/* Clear any offset bits */
895#endif
896
897	/* Purge any old translation.  Note that the FIC instruction
898	 * may use either the instruction or data TLB.  Given that we
899	 * have a flat address space, it's not clear which TLB will be
900	 * used.  So, we purge both entries.  */
901
902#ifdef CONFIG_PA20
903	pdtlb,l		%r0(%r28)
9041:	pitlb,l         %r0(%sr4,%r28)
905	ALTERNATIVE(1b, 1b+4, ALT_COND_NO_SPLIT_TLB, INSN_NOP)
906#else
9070:	pdtlb		%r0(%r28)
9081:	pitlb           %r0(%sr4,%r28)
909	ALTERNATIVE(0b, 0b+4, ALT_COND_NO_SMP, INSN_PxTLB)
910	ALTERNATIVE(1b, 1b+4, ALT_COND_NO_SMP, INSN_PxTLB)
911	ALTERNATIVE(1b, 1b+4, ALT_COND_NO_SPLIT_TLB, INSN_NOP)
912#endif
913
91488:	ldil		L%icache_stride, %r1
915	ldw		R%icache_stride(%r1), %r31
916
917#ifdef CONFIG_64BIT
918	depdi,z		1, 63-PAGE_SHIFT,1, %r25
919#else
920	depwi,z		1, 31-PAGE_SHIFT,1, %r25
921#endif
922	add		%r28, %r25, %r25
923	sub		%r25, %r31, %r25
924
925	/* fic only has the type 26 form on PA1.1, requiring an
926	 * explicit space specification, so use %sr4 */
9271:      fic,m		%r31(%sr4,%r28)
928	fic,m		%r31(%sr4,%r28)
929	fic,m		%r31(%sr4,%r28)
930	fic,m		%r31(%sr4,%r28)
931	fic,m		%r31(%sr4,%r28)
932	fic,m		%r31(%sr4,%r28)
933	fic,m		%r31(%sr4,%r28)
934	fic,m		%r31(%sr4,%r28)
935	fic,m		%r31(%sr4,%r28)
936	fic,m		%r31(%sr4,%r28)
937	fic,m		%r31(%sr4,%r28)
938	fic,m		%r31(%sr4,%r28)
939	fic,m		%r31(%sr4,%r28)
940	fic,m		%r31(%sr4,%r28)
941	fic,m		%r31(%sr4,%r28)
942	cmpb,COND(>>)	%r25, %r28, 1b /* predict taken */
943	fic,m		%r31(%sr4,%r28)
944
94589:	ALTERNATIVE(88b, 89b, ALT_COND_NO_ICACHE, INSN_NOP)
946	sync
947	bv		%r0(%r2)
948	nop
949ENDPROC_CFI(flush_icache_page_asm)
950
951ENTRY_CFI(flush_kernel_dcache_page_asm)
95288:	ldil		L%dcache_stride, %r1
953	ldw		R%dcache_stride(%r1), %r23
954
955#ifdef CONFIG_64BIT
956	depdi,z		1, 63-PAGE_SHIFT,1, %r25
957#else
958	depwi,z		1, 31-PAGE_SHIFT,1, %r25
959#endif
960	add		%r26, %r25, %r25
961	sub		%r25, %r23, %r25
962
9631:      fdc,m		%r23(%r26)
964	fdc,m		%r23(%r26)
965	fdc,m		%r23(%r26)
966	fdc,m		%r23(%r26)
967	fdc,m		%r23(%r26)
968	fdc,m		%r23(%r26)
969	fdc,m		%r23(%r26)
970	fdc,m		%r23(%r26)
971	fdc,m		%r23(%r26)
972	fdc,m		%r23(%r26)
973	fdc,m		%r23(%r26)
974	fdc,m		%r23(%r26)
975	fdc,m		%r23(%r26)
976	fdc,m		%r23(%r26)
977	fdc,m		%r23(%r26)
978	cmpb,COND(>>)	%r25, %r26, 1b /* predict taken */
979	fdc,m		%r23(%r26)
980
98189:	ALTERNATIVE(88b, 89b, ALT_COND_NO_DCACHE, INSN_NOP)
982	sync
983	bv		%r0(%r2)
984	nop
985ENDPROC_CFI(flush_kernel_dcache_page_asm)
986
987ENTRY_CFI(purge_kernel_dcache_page_asm)
98888:	ldil		L%dcache_stride, %r1
989	ldw		R%dcache_stride(%r1), %r23
990
991#ifdef CONFIG_64BIT
992	depdi,z		1, 63-PAGE_SHIFT,1, %r25
993#else
994	depwi,z		1, 31-PAGE_SHIFT,1, %r25
995#endif
996	add		%r26, %r25, %r25
997	sub		%r25, %r23, %r25
998
9991:      pdc,m		%r23(%r26)
1000	pdc,m		%r23(%r26)
1001	pdc,m		%r23(%r26)
1002	pdc,m		%r23(%r26)
1003	pdc,m		%r23(%r26)
1004	pdc,m		%r23(%r26)
1005	pdc,m		%r23(%r26)
1006	pdc,m		%r23(%r26)
1007	pdc,m		%r23(%r26)
1008	pdc,m		%r23(%r26)
1009	pdc,m		%r23(%r26)
1010	pdc,m		%r23(%r26)
1011	pdc,m		%r23(%r26)
1012	pdc,m		%r23(%r26)
1013	pdc,m		%r23(%r26)
1014	cmpb,COND(>>)	%r25, %r26, 1b /* predict taken */
1015	pdc,m		%r23(%r26)
1016
101789:	ALTERNATIVE(88b, 89b, ALT_COND_NO_DCACHE, INSN_NOP)
1018	sync
1019	bv		%r0(%r2)
1020	nop
1021ENDPROC_CFI(purge_kernel_dcache_page_asm)
1022
1023ENTRY_CFI(flush_user_dcache_range_asm)
102488:	ldil		L%dcache_stride, %r1
1025	ldw		R%dcache_stride(%r1), %r23
1026	ldo		-1(%r23), %r21
1027	ANDCM		%r26, %r21, %r26
1028
1029#ifdef CONFIG_64BIT
1030	depd,z		%r23, 59, 60, %r21
1031#else
1032	depw,z		%r23, 27, 28, %r21
1033#endif
1034	add		%r26, %r21, %r22
1035	cmpb,COND(>>),n	%r22, %r25, 2f /* predict not taken */
10361:	add		%r22, %r21, %r22
1037	fdc,m		%r23(%sr3, %r26)
1038	fdc,m		%r23(%sr3, %r26)
1039	fdc,m		%r23(%sr3, %r26)
1040	fdc,m		%r23(%sr3, %r26)
1041	fdc,m		%r23(%sr3, %r26)
1042	fdc,m		%r23(%sr3, %r26)
1043	fdc,m		%r23(%sr3, %r26)
1044	fdc,m		%r23(%sr3, %r26)
1045	fdc,m		%r23(%sr3, %r26)
1046	fdc,m		%r23(%sr3, %r26)
1047	fdc,m		%r23(%sr3, %r26)
1048	fdc,m		%r23(%sr3, %r26)
1049	fdc,m		%r23(%sr3, %r26)
1050	fdc,m		%r23(%sr3, %r26)
1051	fdc,m		%r23(%sr3, %r26)
1052	cmpb,COND(<<=)	%r22, %r25, 1b /* predict taken */
1053	fdc,m		%r23(%sr3, %r26)
1054
10552:	cmpb,COND(>>),n	%r25, %r26, 2b
1056	fdc,m		%r23(%sr3, %r26)
1057
105889:	ALTERNATIVE(88b, 89b, ALT_COND_NO_DCACHE, INSN_NOP)
1059	sync
1060	bv		%r0(%r2)
1061	nop
1062ENDPROC_CFI(flush_user_dcache_range_asm)
1063
1064ENTRY_CFI(flush_kernel_dcache_range_asm)
106588:	ldil		L%dcache_stride, %r1
1066	ldw		R%dcache_stride(%r1), %r23
1067	ldo		-1(%r23), %r21
1068	ANDCM		%r26, %r21, %r26
1069
1070#ifdef CONFIG_64BIT
1071	depd,z		%r23, 59, 60, %r21
1072#else
1073	depw,z		%r23, 27, 28, %r21
1074#endif
1075	add		%r26, %r21, %r22
1076	cmpb,COND(>>),n	%r22, %r25, 2f /* predict not taken */
10771:	add		%r22, %r21, %r22
1078	fdc,m		%r23(%r26)
1079	fdc,m		%r23(%r26)
1080	fdc,m		%r23(%r26)
1081	fdc,m		%r23(%r26)
1082	fdc,m		%r23(%r26)
1083	fdc,m		%r23(%r26)
1084	fdc,m		%r23(%r26)
1085	fdc,m		%r23(%r26)
1086	fdc,m		%r23(%r26)
1087	fdc,m		%r23(%r26)
1088	fdc,m		%r23(%r26)
1089	fdc,m		%r23(%r26)
1090	fdc,m		%r23(%r26)
1091	fdc,m		%r23(%r26)
1092	fdc,m		%r23(%r26)
1093	cmpb,COND(<<=)	%r22, %r25, 1b /* predict taken */
1094	fdc,m		%r23(%r26)
1095
10962:	cmpb,COND(>>),n	%r25, %r26, 2b /* predict taken */
1097	fdc,m		%r23(%r26)
1098
1099	sync
110089:	ALTERNATIVE(88b, 89b, ALT_COND_NO_DCACHE, INSN_NOP)
1101	syncdma
1102	bv		%r0(%r2)
1103	nop
1104ENDPROC_CFI(flush_kernel_dcache_range_asm)
1105
1106ENTRY_CFI(purge_kernel_dcache_range_asm)
110788:	ldil		L%dcache_stride, %r1
1108	ldw		R%dcache_stride(%r1), %r23
1109	ldo		-1(%r23), %r21
1110	ANDCM		%r26, %r21, %r26
1111
1112#ifdef CONFIG_64BIT
1113	depd,z		%r23, 59, 60, %r21
1114#else
1115	depw,z		%r23, 27, 28, %r21
1116#endif
1117	add		%r26, %r21, %r22
1118	cmpb,COND(>>),n	%r22, %r25, 2f /* predict not taken */
11191:	add		%r22, %r21, %r22
1120	pdc,m		%r23(%r26)
1121	pdc,m		%r23(%r26)
1122	pdc,m		%r23(%r26)
1123	pdc,m		%r23(%r26)
1124	pdc,m		%r23(%r26)
1125	pdc,m		%r23(%r26)
1126	pdc,m		%r23(%r26)
1127	pdc,m		%r23(%r26)
1128	pdc,m		%r23(%r26)
1129	pdc,m		%r23(%r26)
1130	pdc,m		%r23(%r26)
1131	pdc,m		%r23(%r26)
1132	pdc,m		%r23(%r26)
1133	pdc,m		%r23(%r26)
1134	pdc,m		%r23(%r26)
1135	cmpb,COND(<<=)	%r22, %r25, 1b /* predict taken */
1136	pdc,m		%r23(%r26)
1137
11382:	cmpb,COND(>>),n	%r25, %r26, 2b /* predict taken */
1139	pdc,m		%r23(%r26)
1140
1141	sync
114289:	ALTERNATIVE(88b, 89b, ALT_COND_NO_DCACHE, INSN_NOP)
1143	syncdma
1144	bv		%r0(%r2)
1145	nop
1146ENDPROC_CFI(purge_kernel_dcache_range_asm)
1147
1148ENTRY_CFI(flush_user_icache_range_asm)
114988:	ldil		L%icache_stride, %r1
1150	ldw		R%icache_stride(%r1), %r23
1151	ldo		-1(%r23), %r21
1152	ANDCM		%r26, %r21, %r26
1153
1154#ifdef CONFIG_64BIT
1155	depd,z		%r23, 59, 60, %r21
1156#else
1157	depw,z		%r23, 27, 28, %r21
1158#endif
1159	add		%r26, %r21, %r22
1160	cmpb,COND(>>),n	%r22, %r25, 2f /* predict not taken */
11611:	add		%r22, %r21, %r22
1162	fic,m		%r23(%sr3, %r26)
1163	fic,m		%r23(%sr3, %r26)
1164	fic,m		%r23(%sr3, %r26)
1165	fic,m		%r23(%sr3, %r26)
1166	fic,m		%r23(%sr3, %r26)
1167	fic,m		%r23(%sr3, %r26)
1168	fic,m		%r23(%sr3, %r26)
1169	fic,m		%r23(%sr3, %r26)
1170	fic,m		%r23(%sr3, %r26)
1171	fic,m		%r23(%sr3, %r26)
1172	fic,m		%r23(%sr3, %r26)
1173	fic,m		%r23(%sr3, %r26)
1174	fic,m		%r23(%sr3, %r26)
1175	fic,m		%r23(%sr3, %r26)
1176	fic,m		%r23(%sr3, %r26)
1177	cmpb,COND(<<=)	%r22, %r25, 1b /* predict taken */
1178	fic,m		%r23(%sr3, %r26)
1179
11802:	cmpb,COND(>>),n	%r25, %r26, 2b
1181	fic,m		%r23(%sr3, %r26)
1182
118389:	ALTERNATIVE(88b, 89b, ALT_COND_NO_ICACHE, INSN_NOP)
1184	sync
1185	bv		%r0(%r2)
1186	nop
1187ENDPROC_CFI(flush_user_icache_range_asm)
1188
1189ENTRY_CFI(flush_kernel_icache_page)
119088:	ldil		L%icache_stride, %r1
1191	ldw		R%icache_stride(%r1), %r23
1192
1193#ifdef CONFIG_64BIT
1194	depdi,z		1, 63-PAGE_SHIFT,1, %r25
1195#else
1196	depwi,z		1, 31-PAGE_SHIFT,1, %r25
1197#endif
1198	add		%r26, %r25, %r25
1199	sub		%r25, %r23, %r25
1200
1201
12021:      fic,m		%r23(%sr4, %r26)
1203	fic,m		%r23(%sr4, %r26)
1204	fic,m		%r23(%sr4, %r26)
1205	fic,m		%r23(%sr4, %r26)
1206	fic,m		%r23(%sr4, %r26)
1207	fic,m		%r23(%sr4, %r26)
1208	fic,m		%r23(%sr4, %r26)
1209	fic,m		%r23(%sr4, %r26)
1210	fic,m		%r23(%sr4, %r26)
1211	fic,m		%r23(%sr4, %r26)
1212	fic,m		%r23(%sr4, %r26)
1213	fic,m		%r23(%sr4, %r26)
1214	fic,m		%r23(%sr4, %r26)
1215	fic,m		%r23(%sr4, %r26)
1216	fic,m		%r23(%sr4, %r26)
1217	cmpb,COND(>>)	%r25, %r26, 1b /* predict taken */
1218	fic,m		%r23(%sr4, %r26)
1219
122089:	ALTERNATIVE(88b, 89b, ALT_COND_NO_ICACHE, INSN_NOP)
1221	sync
1222	bv		%r0(%r2)
1223	nop
1224ENDPROC_CFI(flush_kernel_icache_page)
1225
1226ENTRY_CFI(flush_kernel_icache_range_asm)
122788:	ldil		L%icache_stride, %r1
1228	ldw		R%icache_stride(%r1), %r23
1229	ldo		-1(%r23), %r21
1230	ANDCM		%r26, %r21, %r26
1231
1232#ifdef CONFIG_64BIT
1233	depd,z		%r23, 59, 60, %r21
1234#else
1235	depw,z		%r23, 27, 28, %r21
1236#endif
1237	add		%r26, %r21, %r22
1238	cmpb,COND(>>),n	%r22, %r25, 2f /* predict not taken */
12391:	add		%r22, %r21, %r22
1240	fic,m		%r23(%sr4, %r26)
1241	fic,m		%r23(%sr4, %r26)
1242	fic,m		%r23(%sr4, %r26)
1243	fic,m		%r23(%sr4, %r26)
1244	fic,m		%r23(%sr4, %r26)
1245	fic,m		%r23(%sr4, %r26)
1246	fic,m		%r23(%sr4, %r26)
1247	fic,m		%r23(%sr4, %r26)
1248	fic,m		%r23(%sr4, %r26)
1249	fic,m		%r23(%sr4, %r26)
1250	fic,m		%r23(%sr4, %r26)
1251	fic,m		%r23(%sr4, %r26)
1252	fic,m		%r23(%sr4, %r26)
1253	fic,m		%r23(%sr4, %r26)
1254	fic,m		%r23(%sr4, %r26)
1255	cmpb,COND(<<=)	%r22, %r25, 1b /* predict taken */
1256	fic,m		%r23(%sr4, %r26)
1257
12582:	cmpb,COND(>>),n	%r25, %r26, 2b /* predict taken */
1259	fic,m		%r23(%sr4, %r26)
1260
126189:	ALTERNATIVE(88b, 89b, ALT_COND_NO_ICACHE, INSN_NOP)
1262	sync
1263	bv		%r0(%r2)
1264	nop
1265ENDPROC_CFI(flush_kernel_icache_range_asm)
1266
1267	.text
1268
1269	/* align should cover use of rfi in disable_sr_hashing_asm and
1270	 * srdis_done.
1271	 */
1272	.align	256
1273ENTRY_CFI(disable_sr_hashing_asm)
1274	/*
1275	 * Switch to real mode
1276	 */
1277	/* pcxt_ssm_bug */
1278	rsm		PSW_SM_I, %r0
1279	load32		PA(1f), %r1
1280	nop
1281	nop
1282	nop
1283	nop
1284	nop
1285
1286	rsm		PSW_SM_Q, %r0		/* prep to load iia queue */
1287	mtctl		%r0, %cr17		/* Clear IIASQ tail */
1288	mtctl		%r0, %cr17		/* Clear IIASQ head */
1289	mtctl		%r1, %cr18		/* IIAOQ head */
1290	ldo		4(%r1), %r1
1291	mtctl		%r1, %cr18		/* IIAOQ tail */
1292	load32		REAL_MODE_PSW, %r1
1293	mtctl		%r1, %ipsw
1294	rfi
1295	nop
1296
12971:      cmpib,=,n	SRHASH_PCXST, %r26,srdis_pcxs
1298	cmpib,=,n	SRHASH_PCXL, %r26,srdis_pcxl
1299	cmpib,=,n	SRHASH_PA20, %r26,srdis_pa20
1300	b,n		srdis_done
1301
1302srdis_pcxs:
1303
1304	/* Disable Space Register Hashing for PCXS,PCXT,PCXT' */
1305
1306	.word		0x141c1a00		/* mfdiag %dr0, %r28 */
1307	.word		0x141c1a00		/* must issue twice */
1308	depwi		0,18,1, %r28		/* Clear DHE (dcache hash enable) */
1309	depwi		0,20,1, %r28		/* Clear IHE (icache hash enable) */
1310	.word		0x141c1600		/* mtdiag %r28, %dr0 */
1311	.word		0x141c1600		/* must issue twice */
1312	b,n		srdis_done
1313
1314srdis_pcxl:
1315
1316	/* Disable Space Register Hashing for PCXL */
1317
1318	.word		0x141c0600		/* mfdiag %dr0, %r28 */
1319	depwi           0,28,2, %r28		/* Clear DHASH_EN & IHASH_EN */
1320	.word		0x141c0240		/* mtdiag %r28, %dr0 */
1321	b,n		srdis_done
1322
1323srdis_pa20:
1324
1325	/* Disable Space Register Hashing for PCXU,PCXU+,PCXW,PCXW+,PCXW2 */
1326
1327	.word		0x144008bc		/* mfdiag %dr2, %r28 */
1328	depdi		0, 54,1, %r28		/* clear DIAG_SPHASH_ENAB (bit 54) */
1329	.word		0x145c1840		/* mtdiag %r28, %dr2 */
1330
1331
1332srdis_done:
1333	/* Switch back to virtual mode */
1334	rsm		PSW_SM_I, %r0		/* prep to load iia queue */
1335	load32 	   	2f, %r1
1336	nop
1337	nop
1338	nop
1339	nop
1340	nop
1341
1342	rsm		PSW_SM_Q, %r0		/* prep to load iia queue */
1343	mtctl		%r0, %cr17		/* Clear IIASQ tail */
1344	mtctl		%r0, %cr17		/* Clear IIASQ head */
1345	mtctl		%r1, %cr18		/* IIAOQ head */
1346	ldo		4(%r1), %r1
1347	mtctl		%r1, %cr18		/* IIAOQ tail */
1348	load32		KERNEL_PSW, %r1
1349	mtctl		%r1, %ipsw
1350	rfi
1351	nop
1352
13532:      bv		%r0(%r2)
1354	nop
1355ENDPROC_CFI(disable_sr_hashing_asm)
1356
1357	.end
1358