xref: /linux/arch/mips/kernel/cps-vec.S (revision b1264ad8) 
1/* SPDX-License-Identifier: GPL-2.0-or-later */
2/*
3 * Copyright (C) 2013 Imagination Technologies
4 * Author: Paul Burton <paul.burton@mips.com>
5 */
6
7#include <linux/init.h>
8#include <asm/addrspace.h>
9#include <asm/asm.h>
10#include <asm/asm-offsets.h>
11#include <asm/asmmacro.h>
12#include <asm/cacheops.h>
13#include <asm/eva.h>
14#include <asm/mipsregs.h>
15#include <asm/mipsmtregs.h>
16#include <asm/pm.h>
17#include <asm/smp-cps.h>
18
19#define GCR_CPC_BASE_OFS	0x0088
20#define GCR_CL_COHERENCE_OFS	0x2008
21#define GCR_CL_ID_OFS		0x2028
22
23#define CPC_CL_VC_STOP_OFS	0x2020
24#define CPC_CL_VC_RUN_OFS	0x2028
25
26.extern mips_cm_base
27
28.set noreorder
29
30#ifdef CONFIG_64BIT
31# define STATUS_BITDEPS		ST0_KX
32#else
33# define STATUS_BITDEPS		0
34#endif
35
36#ifdef CONFIG_MIPS_CPS_NS16550
37
38#define DUMP_EXCEP(name)		\
39	PTR_LA	a0, 8f;			\
40	jal	mips_cps_bev_dump;	\
41	 nop;				\
42	TEXT(name)
43
44#else /* !CONFIG_MIPS_CPS_NS16550 */
45
46#define DUMP_EXCEP(name)
47
48#endif /* !CONFIG_MIPS_CPS_NS16550 */
49
50	/*
51	 * Set dest to non-zero if the core supports the MT ASE, else zero. If
52	 * MT is not supported then branch to nomt.
53	 */
54	.macro	has_mt	dest, nomt
55	mfc0	\dest, CP0_CONFIG, 1
56	bgez	\dest, \nomt
57	 mfc0	\dest, CP0_CONFIG, 2
58	bgez	\dest, \nomt
59	 mfc0	\dest, CP0_CONFIG, 3
60	andi	\dest, \dest, MIPS_CONF3_MT
61	beqz	\dest, \nomt
62	 nop
63	.endm
64
65	/*
66	 * Set dest to non-zero if the core supports MIPSr6 multithreading
67	 * (ie. VPs), else zero. If MIPSr6 multithreading is not supported then
68	 * branch to nomt.
69	 */
70	.macro	has_vp	dest, nomt
71	mfc0	\dest, CP0_CONFIG, 1
72	bgez	\dest, \nomt
73	 mfc0	\dest, CP0_CONFIG, 2
74	bgez	\dest, \nomt
75	 mfc0	\dest, CP0_CONFIG, 3
76	bgez	\dest, \nomt
77	 mfc0	\dest, CP0_CONFIG, 4
78	bgez	\dest, \nomt
79	 mfc0	\dest, CP0_CONFIG, 5
80	andi	\dest, \dest, MIPS_CONF5_VP
81	beqz	\dest, \nomt
82	 nop
83	.endm
84
85
86LEAF(mips_cps_core_boot)
87	/* Save  CCA and GCR base */
88	move   s0, a0
89	move   s1, a1
90
91	/* We don't know how to do coherence setup on earlier ISA */
92#if MIPS_ISA_REV > 0
93	/* Skip cache & coherence setup if we're already coherent */
94	lw	s7, GCR_CL_COHERENCE_OFS(s1)
95	bnez	s7, 1f
96	 nop
97
98	/* Initialize the L1 caches */
99	jal	mips_cps_cache_init
100	 nop
101
102	/* Enter the coherent domain */
103	li	t0, 0xff
104	sw	t0, GCR_CL_COHERENCE_OFS(s1)
105	ehb
106#endif /* MIPS_ISA_REV > 0 */
107
108	/* Set Kseg0 CCA to that in s0 */
1091:	mfc0	t0, CP0_CONFIG
110	ori	t0, 0x7
111	xori	t0, 0x7
112	or	t0, t0, s0
113	mtc0	t0, CP0_CONFIG
114	ehb
115
116	/* Jump to kseg0 */
117	PTR_LA	t0, 1f
118	jr	t0
119	 nop
120
121	/*
122	 * We're up, cached & coherent. Perform any EVA initialization necessary
123	 * before we access memory.
124	 */
1251:	eva_init
126
127	/* Retrieve boot configuration pointers */
128	jal	mips_cps_get_bootcfg
129	 nop
130
131	/* Skip core-level init if we started up coherent */
132	bnez	s7, 1f
133	 nop
134
135	/* Perform any further required core-level initialisation */
136	jal	mips_cps_core_init
137	 nop
138
139	/*
140	 * Boot any other VPEs within this core that should be online, and
141	 * deactivate this VPE if it should be offline.
142	 */
143	move	a1, t9
144	jal	mips_cps_boot_vpes
145	 move	a0, v0
146
147	/* Off we go! */
1481:	PTR_L	t1, VPEBOOTCFG_PC(v1)
149	PTR_L	gp, VPEBOOTCFG_GP(v1)
150	PTR_L	sp, VPEBOOTCFG_SP(v1)
151	jr	t1
152	 nop
153	END(mips_cps_core_boot)
154
155	__INIT
156LEAF(excep_tlbfill)
157	DUMP_EXCEP("TLB Fill")
158	b	.
159	 nop
160	END(excep_tlbfill)
161
162LEAF(excep_xtlbfill)
163	DUMP_EXCEP("XTLB Fill")
164	b	.
165	 nop
166	END(excep_xtlbfill)
167
168LEAF(excep_cache)
169	DUMP_EXCEP("Cache")
170	b	.
171	 nop
172	END(excep_cache)
173
174LEAF(excep_genex)
175	DUMP_EXCEP("General")
176	b	.
177	 nop
178	END(excep_genex)
179
180LEAF(excep_intex)
181	DUMP_EXCEP("Interrupt")
182	b	.
183	 nop
184	END(excep_intex)
185
186LEAF(excep_ejtag)
187	PTR_LA	k0, ejtag_debug_handler
188	jr	k0
189	 nop
190	END(excep_ejtag)
191	__FINIT
192
193LEAF(mips_cps_core_init)
194#ifdef CONFIG_MIPS_MT_SMP
195	/* Check that the core implements the MT ASE */
196	has_mt	t0, 3f
197
198	.set	push
199	.set	MIPS_ISA_LEVEL_RAW
200	.set	mt
201
202	/* Only allow 1 TC per VPE to execute... */
203	dmt
204
205	/* ...and for the moment only 1 VPE */
206	dvpe
207	PTR_LA	t1, 1f
208	jr.hb	t1
209	 nop
210
211	/* Enter VPE configuration state */
2121:	mfc0	t0, CP0_MVPCONTROL
213	ori	t0, t0, MVPCONTROL_VPC
214	mtc0	t0, CP0_MVPCONTROL
215
216	/* Retrieve the number of VPEs within the core */
217	mfc0	t0, CP0_MVPCONF0
218	srl	t0, t0, MVPCONF0_PVPE_SHIFT
219	andi	t0, t0, (MVPCONF0_PVPE >> MVPCONF0_PVPE_SHIFT)
220	addiu	ta3, t0, 1
221
222	/* If there's only 1, we're done */
223	beqz	t0, 2f
224	 nop
225
226	/* Loop through each VPE within this core */
227	li	ta1, 1
228
2291:	/* Operate on the appropriate TC */
230	mtc0	ta1, CP0_VPECONTROL
231	ehb
232
233	/* Bind TC to VPE (1:1 TC:VPE mapping) */
234	mttc0	ta1, CP0_TCBIND
235
236	/* Set exclusive TC, non-active, master */
237	li	t0, VPECONF0_MVP
238	sll	t1, ta1, VPECONF0_XTC_SHIFT
239	or	t0, t0, t1
240	mttc0	t0, CP0_VPECONF0
241
242	/* Set TC non-active, non-allocatable */
243	mttc0	zero, CP0_TCSTATUS
244
245	/* Set TC halted */
246	li	t0, TCHALT_H
247	mttc0	t0, CP0_TCHALT
248
249	/* Next VPE */
250	addiu	ta1, ta1, 1
251	slt	t0, ta1, ta3
252	bnez	t0, 1b
253	 nop
254
255	/* Leave VPE configuration state */
2562:	mfc0	t0, CP0_MVPCONTROL
257	xori	t0, t0, MVPCONTROL_VPC
258	mtc0	t0, CP0_MVPCONTROL
259
2603:	.set	pop
261#endif
262	jr	ra
263	 nop
264	END(mips_cps_core_init)
265
266/**
267 * mips_cps_get_bootcfg() - retrieve boot configuration pointers
268 *
269 * Returns: pointer to struct core_boot_config in v0, pointer to
270 *          struct vpe_boot_config in v1, VPE ID in t9
271 */
272LEAF(mips_cps_get_bootcfg)
273	/* Calculate a pointer to this cores struct core_boot_config */
274	lw	t0, GCR_CL_ID_OFS(s1)
275	li	t1, COREBOOTCFG_SIZE
276	mul	t0, t0, t1
277	PTR_LA	t1, mips_cps_core_bootcfg
278	PTR_L	t1, 0(t1)
279	PTR_ADDU v0, t0, t1
280
281	/* Calculate this VPEs ID. If the core doesn't support MT use 0 */
282	li	t9, 0
283#if defined(CONFIG_CPU_MIPSR6)
284	has_vp	ta2, 1f
285
286	/*
287	 * Assume non-contiguous numbering. Perhaps some day we'll need
288	 * to handle contiguous VP numbering, but no such systems yet
289	 * exist.
290	 */
291	mfc0	t9, CP0_GLOBALNUMBER
292	andi	t9, t9, MIPS_GLOBALNUMBER_VP
293#elif defined(CONFIG_MIPS_MT_SMP)
294	has_mt	ta2, 1f
295
296	/* Find the number of VPEs present in the core */
297	mfc0	t1, CP0_MVPCONF0
298	srl	t1, t1, MVPCONF0_PVPE_SHIFT
299	andi	t1, t1, MVPCONF0_PVPE >> MVPCONF0_PVPE_SHIFT
300	addiu	t1, t1, 1
301
302	/* Calculate a mask for the VPE ID from EBase.CPUNum */
303	clz	t1, t1
304	li	t2, 31
305	subu	t1, t2, t1
306	li	t2, 1
307	sll	t1, t2, t1
308	addiu	t1, t1, -1
309
310	/* Retrieve the VPE ID from EBase.CPUNum */
311	mfc0	t9, $15, 1
312	and	t9, t9, t1
313#endif
314
3151:	/* Calculate a pointer to this VPEs struct vpe_boot_config */
316	li	t1, VPEBOOTCFG_SIZE
317	mul	v1, t9, t1
318	PTR_L	ta3, COREBOOTCFG_VPECONFIG(v0)
319	PTR_ADDU v1, v1, ta3
320
321	jr	ra
322	 nop
323	END(mips_cps_get_bootcfg)
324
325LEAF(mips_cps_boot_vpes)
326	lw	ta2, COREBOOTCFG_VPEMASK(a0)
327	PTR_L	ta3, COREBOOTCFG_VPECONFIG(a0)
328
329#if defined(CONFIG_CPU_MIPSR6)
330
331	has_vp	t0, 5f
332
333	/* Find base address of CPC */
334	PTR_LA	t1, mips_gcr_base
335	PTR_L	t1, 0(t1)
336	PTR_L	t1, GCR_CPC_BASE_OFS(t1)
337	PTR_LI	t2, ~0x7fff
338	and	t1, t1, t2
339	PTR_LI	t2, UNCAC_BASE
340	PTR_ADD	t1, t1, t2
341
342	/* Start any other VPs that ought to be running */
343	PTR_S	ta2, CPC_CL_VC_RUN_OFS(t1)
344
345	/* Ensure this VP stops running if it shouldn't be */
346	not	ta2
347	PTR_S	ta2, CPC_CL_VC_STOP_OFS(t1)
348	ehb
349
350#elif defined(CONFIG_MIPS_MT)
351
352	/* If the core doesn't support MT then return */
353	has_mt	t0, 5f
354
355	/* Enter VPE configuration state */
356	.set	push
357	.set	MIPS_ISA_LEVEL_RAW
358	.set	mt
359	dvpe
360	.set	pop
361
362	PTR_LA	t1, 1f
363	jr.hb	t1
364	 nop
3651:	mfc0	t1, CP0_MVPCONTROL
366	ori	t1, t1, MVPCONTROL_VPC
367	mtc0	t1, CP0_MVPCONTROL
368	ehb
369
370	/* Loop through each VPE */
371	move	t8, ta2
372	li	ta1, 0
373
374	/* Check whether the VPE should be running. If not, skip it */
3751:	andi	t0, ta2, 1
376	beqz	t0, 2f
377	 nop
378
379	/* Operate on the appropriate TC */
380	mfc0	t0, CP0_VPECONTROL
381	ori	t0, t0, VPECONTROL_TARGTC
382	xori	t0, t0, VPECONTROL_TARGTC
383	or	t0, t0, ta1
384	mtc0	t0, CP0_VPECONTROL
385	ehb
386
387	.set	push
388	.set	MIPS_ISA_LEVEL_RAW
389	.set	mt
390
391	/* Skip the VPE if its TC is not halted */
392	mftc0	t0, CP0_TCHALT
393	beqz	t0, 2f
394	 nop
395
396	/* Calculate a pointer to the VPEs struct vpe_boot_config */
397	li	t0, VPEBOOTCFG_SIZE
398	mul	t0, t0, ta1
399	PTR_ADDU t0, t0, ta3
400
401	/* Set the TC restart PC */
402	lw	t1, VPEBOOTCFG_PC(t0)
403	mttc0	t1, CP0_TCRESTART
404
405	/* Set the TC stack pointer */
406	lw	t1, VPEBOOTCFG_SP(t0)
407	mttgpr	t1, sp
408
409	/* Set the TC global pointer */
410	lw	t1, VPEBOOTCFG_GP(t0)
411	mttgpr	t1, gp
412
413	/* Copy config from this VPE */
414	mfc0	t0, CP0_CONFIG
415	mttc0	t0, CP0_CONFIG
416
417	/*
418	 * Copy the EVA config from this VPE if the CPU supports it.
419	 * CONFIG3 must exist to be running MT startup - just read it.
420	 */
421	mfc0	t0, CP0_CONFIG, 3
422	and	t0, t0, MIPS_CONF3_SC
423	beqz	t0, 3f
424	 nop
425	mfc0    t0, CP0_SEGCTL0
426	mttc0	t0, CP0_SEGCTL0
427	mfc0    t0, CP0_SEGCTL1
428	mttc0	t0, CP0_SEGCTL1
429	mfc0    t0, CP0_SEGCTL2
430	mttc0	t0, CP0_SEGCTL2
4313:
432	/* Ensure no software interrupts are pending */
433	mttc0	zero, CP0_CAUSE
434	mttc0	zero, CP0_STATUS
435
436	/* Set TC active, not interrupt exempt */
437	mftc0	t0, CP0_TCSTATUS
438	li	t1, ~TCSTATUS_IXMT
439	and	t0, t0, t1
440	ori	t0, t0, TCSTATUS_A
441	mttc0	t0, CP0_TCSTATUS
442
443	/* Clear the TC halt bit */
444	mttc0	zero, CP0_TCHALT
445
446	/* Set VPE active */
447	mftc0	t0, CP0_VPECONF0
448	ori	t0, t0, VPECONF0_VPA
449	mttc0	t0, CP0_VPECONF0
450
451	/* Next VPE */
4522:	srl	ta2, ta2, 1
453	addiu	ta1, ta1, 1
454	bnez	ta2, 1b
455	 nop
456
457	/* Leave VPE configuration state */
458	mfc0	t1, CP0_MVPCONTROL
459	xori	t1, t1, MVPCONTROL_VPC
460	mtc0	t1, CP0_MVPCONTROL
461	ehb
462	evpe
463
464	.set	pop
465
466	/* Check whether this VPE is meant to be running */
467	li	t0, 1
468	sll	t0, t0, a1
469	and	t0, t0, t8
470	bnez	t0, 2f
471	 nop
472
473	/* This VPE should be offline, halt the TC */
474	li	t0, TCHALT_H
475	mtc0	t0, CP0_TCHALT
476	PTR_LA	t0, 1f
4771:	jr.hb	t0
478	 nop
479
4802:
481
482#endif /* CONFIG_MIPS_MT_SMP */
483
484	/* Return */
4855:	jr	ra
486	 nop
487	END(mips_cps_boot_vpes)
488
489#if MIPS_ISA_REV > 0
490LEAF(mips_cps_cache_init)
491	/*
492	 * Clear the bits used to index the caches. Note that the architecture
493	 * dictates that writing to any of TagLo or TagHi selects 0 or 2 should
494	 * be valid for all MIPS32 CPUs, even those for which said writes are
495	 * unnecessary.
496	 */
497	mtc0	zero, CP0_TAGLO, 0
498	mtc0	zero, CP0_TAGHI, 0
499	mtc0	zero, CP0_TAGLO, 2
500	mtc0	zero, CP0_TAGHI, 2
501	ehb
502
503	/* Primary cache configuration is indicated by Config1 */
504	mfc0	v0, CP0_CONFIG, 1
505
506	/* Detect I-cache line size */
507	_EXT	t0, v0, MIPS_CONF1_IL_SHF, MIPS_CONF1_IL_SZ
508	beqz	t0, icache_done
509	 li	t1, 2
510	sllv	t0, t1, t0
511
512	/* Detect I-cache size */
513	_EXT	t1, v0, MIPS_CONF1_IS_SHF, MIPS_CONF1_IS_SZ
514	xori	t2, t1, 0x7
515	beqz	t2, 1f
516	 li	t3, 32
517	addiu	t1, t1, 1
518	sllv	t1, t3, t1
5191:	/* At this point t1 == I-cache sets per way */
520	_EXT	t2, v0, MIPS_CONF1_IA_SHF, MIPS_CONF1_IA_SZ
521	addiu	t2, t2, 1
522	mul	t1, t1, t0
523	mul	t1, t1, t2
524
525	li	a0, CKSEG0
526	PTR_ADD	a1, a0, t1
5271:	cache	Index_Store_Tag_I, 0(a0)
528	PTR_ADD	a0, a0, t0
529	bne	a0, a1, 1b
530	 nop
531icache_done:
532
533	/* Detect D-cache line size */
534	_EXT	t0, v0, MIPS_CONF1_DL_SHF, MIPS_CONF1_DL_SZ
535	beqz	t0, dcache_done
536	 li	t1, 2
537	sllv	t0, t1, t0
538
539	/* Detect D-cache size */
540	_EXT	t1, v0, MIPS_CONF1_DS_SHF, MIPS_CONF1_DS_SZ
541	xori	t2, t1, 0x7
542	beqz	t2, 1f
543	 li	t3, 32
544	addiu	t1, t1, 1
545	sllv	t1, t3, t1
5461:	/* At this point t1 == D-cache sets per way */
547	_EXT	t2, v0, MIPS_CONF1_DA_SHF, MIPS_CONF1_DA_SZ
548	addiu	t2, t2, 1
549	mul	t1, t1, t0
550	mul	t1, t1, t2
551
552	li	a0, CKSEG0
553	PTR_ADDU a1, a0, t1
554	PTR_SUBU a1, a1, t0
5551:	cache	Index_Store_Tag_D, 0(a0)
556	bne	a0, a1, 1b
557	 PTR_ADD a0, a0, t0
558dcache_done:
559
560	jr	ra
561	 nop
562	END(mips_cps_cache_init)
563#endif /* MIPS_ISA_REV > 0 */
564
565#if defined(CONFIG_MIPS_CPS_PM) && defined(CONFIG_CPU_PM)
566
567	/* Calculate a pointer to this CPUs struct mips_static_suspend_state */
568	.macro	psstate	dest
569	.set	push
570	.set	noat
571	lw	$1, TI_CPU(gp)
572	sll	$1, $1, LONGLOG
573	PTR_LA	\dest, __per_cpu_offset
574	PTR_ADDU $1, $1, \dest
575	lw	$1, 0($1)
576	PTR_LA	\dest, cps_cpu_state
577	PTR_ADDU \dest, \dest, $1
578	.set	pop
579	.endm
580
581LEAF(mips_cps_pm_save)
582	/* Save CPU state */
583	SUSPEND_SAVE_REGS
584	psstate	t1
585	SUSPEND_SAVE_STATIC
586	jr	v0
587	 nop
588	END(mips_cps_pm_save)
589
590LEAF(mips_cps_pm_restore)
591	/* Restore CPU state */
592	psstate	t1
593	RESUME_RESTORE_STATIC
594	RESUME_RESTORE_REGS_RETURN
595	END(mips_cps_pm_restore)
596
597#endif /* CONFIG_MIPS_CPS_PM && CONFIG_CPU_PM */
598