1 // SPDX-License-Identifier: GPL-2.0-only
2 /*
3 * linux/arch/arm/kernel/setup.c
4 *
5 * Copyright (C) 1995-2001 Russell King
6 */
7 #include <linux/efi.h>
8 #include <linux/export.h>
9 #include <linux/kernel.h>
10 #include <linux/stddef.h>
11 #include <linux/ioport.h>
12 #include <linux/delay.h>
13 #include <linux/utsname.h>
14 #include <linux/initrd.h>
15 #include <linux/console.h>
16 #include <linux/seq_file.h>
17 #include <linux/screen_info.h>
18 #include <linux/of_platform.h>
19 #include <linux/init.h>
20 #include <linux/kexec.h>
21 #include <linux/libfdt.h>
22 #include <linux/of_fdt.h>
23 #include <linux/cpu.h>
24 #include <linux/interrupt.h>
25 #include <linux/smp.h>
26 #include <linux/proc_fs.h>
27 #include <linux/memblock.h>
28 #include <linux/bug.h>
29 #include <linux/compiler.h>
30 #include <linux/sort.h>
31 #include <linux/psci.h>
32
33 #include <asm/unified.h>
34 #include <asm/cp15.h>
35 #include <asm/cpu.h>
36 #include <asm/cputype.h>
37 #include <asm/efi.h>
38 #include <asm/elf.h>
39 #include <asm/early_ioremap.h>
40 #include <asm/fixmap.h>
41 #include <asm/procinfo.h>
42 #include <asm/psci.h>
43 #include <asm/sections.h>
44 #include <asm/setup.h>
45 #include <asm/smp_plat.h>
46 #include <asm/mach-types.h>
47 #include <asm/cacheflush.h>
48 #include <asm/cachetype.h>
49 #include <asm/tlbflush.h>
50 #include <asm/xen/hypervisor.h>
51
52 #include <asm/prom.h>
53 #include <asm/mach/arch.h>
54 #include <asm/mach/irq.h>
55 #include <asm/mach/time.h>
56 #include <asm/system_info.h>
57 #include <asm/system_misc.h>
58 #include <asm/traps.h>
59 #include <asm/unwind.h>
60 #include <asm/memblock.h>
61 #include <asm/virt.h>
62 #include <asm/kasan.h>
63
64 #include "atags.h"
65
66
67 #if defined(CONFIG_FPE_NWFPE) || defined(CONFIG_FPE_FASTFPE)
68 char fpe_type[8];
69
fpe_setup(char * line)70 static int __init fpe_setup(char *line)
71 {
72 memcpy(fpe_type, line, 8);
73 return 1;
74 }
75
76 __setup("fpe=", fpe_setup);
77 #endif
78
79 extern void init_default_cache_policy(unsigned long);
80 extern void paging_init(const struct machine_desc *desc);
81 extern void early_mm_init(const struct machine_desc *);
82 extern void adjust_lowmem_bounds(void);
83 extern enum reboot_mode reboot_mode;
84 extern void setup_dma_zone(const struct machine_desc *desc);
85
86 unsigned int processor_id;
87 EXPORT_SYMBOL(processor_id);
88 unsigned int __machine_arch_type __read_mostly;
89 EXPORT_SYMBOL(__machine_arch_type);
90 unsigned int cacheid __read_mostly;
91 EXPORT_SYMBOL(cacheid);
92
93 unsigned int __atags_pointer __initdata;
94
95 unsigned int system_rev;
96 EXPORT_SYMBOL(system_rev);
97
98 const char *system_serial;
99 EXPORT_SYMBOL(system_serial);
100
101 unsigned int system_serial_low;
102 EXPORT_SYMBOL(system_serial_low);
103
104 unsigned int system_serial_high;
105 EXPORT_SYMBOL(system_serial_high);
106
107 unsigned int elf_hwcap __read_mostly;
108 EXPORT_SYMBOL(elf_hwcap);
109
110 unsigned int elf_hwcap2 __read_mostly;
111 EXPORT_SYMBOL(elf_hwcap2);
112
113
114 #ifdef MULTI_CPU
115 struct processor processor __ro_after_init;
116 #if defined(CONFIG_BIG_LITTLE) && defined(CONFIG_HARDEN_BRANCH_PREDICTOR)
117 struct processor *cpu_vtable[NR_CPUS] = {
118 [0] = &processor,
119 };
120 #endif
121 #endif
122 #ifdef MULTI_TLB
123 struct cpu_tlb_fns cpu_tlb __ro_after_init;
124 #endif
125 #ifdef MULTI_USER
126 struct cpu_user_fns cpu_user __ro_after_init;
127 #endif
128 #ifdef MULTI_CACHE
129 struct cpu_cache_fns cpu_cache __ro_after_init;
130 #endif
131 #ifdef CONFIG_OUTER_CACHE
132 struct outer_cache_fns outer_cache __ro_after_init;
133 EXPORT_SYMBOL(outer_cache);
134 #endif
135
136 /*
137 * Cached cpu_architecture() result for use by assembler code.
138 * C code should use the cpu_architecture() function instead of accessing this
139 * variable directly.
140 */
141 int __cpu_architecture __read_mostly = CPU_ARCH_UNKNOWN;
142
143 struct stack {
144 u32 irq[3];
145 u32 abt[3];
146 u32 und[3];
147 u32 fiq[3];
148 } ____cacheline_aligned;
149
150 #ifndef CONFIG_CPU_V7M
151 static struct stack stacks[NR_CPUS];
152 #endif
153
154 char elf_platform[ELF_PLATFORM_SIZE];
155 EXPORT_SYMBOL(elf_platform);
156
157 static const char *cpu_name;
158 static const char *machine_name;
159 static char __initdata cmd_line[COMMAND_LINE_SIZE];
160 const struct machine_desc *machine_desc __initdata;
161
162 static union { char c[4]; unsigned long l; } endian_test __initdata = { { 'l', '?', '?', 'b' } };
163 #define ENDIANNESS ((char)endian_test.l)
164
165 DEFINE_PER_CPU(struct cpuinfo_arm, cpu_data);
166
167 /*
168 * Standard memory resources
169 */
170 static struct resource mem_res[] = {
171 {
172 .name = "Video RAM",
173 .start = 0,
174 .end = 0,
175 .flags = IORESOURCE_MEM
176 },
177 {
178 .name = "Kernel code",
179 .start = 0,
180 .end = 0,
181 .flags = IORESOURCE_SYSTEM_RAM
182 },
183 {
184 .name = "Kernel data",
185 .start = 0,
186 .end = 0,
187 .flags = IORESOURCE_SYSTEM_RAM
188 }
189 };
190
191 #define video_ram mem_res[0]
192 #define kernel_code mem_res[1]
193 #define kernel_data mem_res[2]
194
195 static struct resource io_res[] = {
196 {
197 .name = "reserved",
198 .start = 0x3bc,
199 .end = 0x3be,
200 .flags = IORESOURCE_IO | IORESOURCE_BUSY
201 },
202 {
203 .name = "reserved",
204 .start = 0x378,
205 .end = 0x37f,
206 .flags = IORESOURCE_IO | IORESOURCE_BUSY
207 },
208 {
209 .name = "reserved",
210 .start = 0x278,
211 .end = 0x27f,
212 .flags = IORESOURCE_IO | IORESOURCE_BUSY
213 }
214 };
215
216 #define lp0 io_res[0]
217 #define lp1 io_res[1]
218 #define lp2 io_res[2]
219
220 static const char *proc_arch[] = {
221 "undefined/unknown",
222 "3",
223 "4",
224 "4T",
225 "5",
226 "5T",
227 "5TE",
228 "5TEJ",
229 "6TEJ",
230 "7",
231 "7M",
232 "?(12)",
233 "?(13)",
234 "?(14)",
235 "?(15)",
236 "?(16)",
237 "?(17)",
238 };
239
240 #ifdef CONFIG_CPU_V7M
__get_cpu_architecture(void)241 static int __get_cpu_architecture(void)
242 {
243 return CPU_ARCH_ARMv7M;
244 }
245 #else
__get_cpu_architecture(void)246 static int __get_cpu_architecture(void)
247 {
248 int cpu_arch;
249
250 if ((read_cpuid_id() & 0x0008f000) == 0) {
251 cpu_arch = CPU_ARCH_UNKNOWN;
252 } else if ((read_cpuid_id() & 0x0008f000) == 0x00007000) {
253 cpu_arch = (read_cpuid_id() & (1 << 23)) ? CPU_ARCH_ARMv4T : CPU_ARCH_ARMv3;
254 } else if ((read_cpuid_id() & 0x00080000) == 0x00000000) {
255 cpu_arch = (read_cpuid_id() >> 16) & 7;
256 if (cpu_arch)
257 cpu_arch += CPU_ARCH_ARMv3;
258 } else if ((read_cpuid_id() & 0x000f0000) == 0x000f0000) {
259 /* Revised CPUID format. Read the Memory Model Feature
260 * Register 0 and check for VMSAv7 or PMSAv7 */
261 unsigned int mmfr0 = read_cpuid_ext(CPUID_EXT_MMFR0);
262 if ((mmfr0 & 0x0000000f) >= 0x00000003 ||
263 (mmfr0 & 0x000000f0) >= 0x00000030)
264 cpu_arch = CPU_ARCH_ARMv7;
265 else if ((mmfr0 & 0x0000000f) == 0x00000002 ||
266 (mmfr0 & 0x000000f0) == 0x00000020)
267 cpu_arch = CPU_ARCH_ARMv6;
268 else
269 cpu_arch = CPU_ARCH_UNKNOWN;
270 } else
271 cpu_arch = CPU_ARCH_UNKNOWN;
272
273 return cpu_arch;
274 }
275 #endif
276
cpu_architecture(void)277 int __pure cpu_architecture(void)
278 {
279 BUG_ON(__cpu_architecture == CPU_ARCH_UNKNOWN);
280
281 return __cpu_architecture;
282 }
283
cpu_has_aliasing_icache(unsigned int arch)284 static int cpu_has_aliasing_icache(unsigned int arch)
285 {
286 int aliasing_icache;
287 unsigned int id_reg, num_sets, line_size;
288
289 /* PIPT caches never alias. */
290 if (icache_is_pipt())
291 return 0;
292
293 /* arch specifies the register format */
294 switch (arch) {
295 case CPU_ARCH_ARMv7:
296 set_csselr(CSSELR_ICACHE | CSSELR_L1);
297 isb();
298 id_reg = read_ccsidr();
299 line_size = 4 << ((id_reg & 0x7) + 2);
300 num_sets = ((id_reg >> 13) & 0x7fff) + 1;
301 aliasing_icache = (line_size * num_sets) > PAGE_SIZE;
302 break;
303 case CPU_ARCH_ARMv6:
304 aliasing_icache = read_cpuid_cachetype() & (1 << 11);
305 break;
306 default:
307 /* I-cache aliases will be handled by D-cache aliasing code */
308 aliasing_icache = 0;
309 }
310
311 return aliasing_icache;
312 }
313
cacheid_init(void)314 static void __init cacheid_init(void)
315 {
316 unsigned int arch = cpu_architecture();
317
318 if (arch >= CPU_ARCH_ARMv6) {
319 unsigned int cachetype = read_cpuid_cachetype();
320
321 if ((arch == CPU_ARCH_ARMv7M) && !(cachetype & 0xf000f)) {
322 cacheid = 0;
323 } else if ((cachetype & (7 << 29)) == 4 << 29) {
324 /* ARMv7 register format */
325 arch = CPU_ARCH_ARMv7;
326 cacheid = CACHEID_VIPT_NONALIASING;
327 switch (cachetype & (3 << 14)) {
328 case (1 << 14):
329 cacheid |= CACHEID_ASID_TAGGED;
330 break;
331 case (3 << 14):
332 cacheid |= CACHEID_PIPT;
333 break;
334 }
335 } else {
336 arch = CPU_ARCH_ARMv6;
337 if (cachetype & (1 << 23))
338 cacheid = CACHEID_VIPT_ALIASING;
339 else
340 cacheid = CACHEID_VIPT_NONALIASING;
341 }
342 if (cpu_has_aliasing_icache(arch))
343 cacheid |= CACHEID_VIPT_I_ALIASING;
344 } else {
345 cacheid = CACHEID_VIVT;
346 }
347
348 pr_info("CPU: %s data cache, %s instruction cache\n",
349 cache_is_vivt() ? "VIVT" :
350 cache_is_vipt_aliasing() ? "VIPT aliasing" :
351 cache_is_vipt_nonaliasing() ? "PIPT / VIPT nonaliasing" : "unknown",
352 cache_is_vivt() ? "VIVT" :
353 icache_is_vivt_asid_tagged() ? "VIVT ASID tagged" :
354 icache_is_vipt_aliasing() ? "VIPT aliasing" :
355 icache_is_pipt() ? "PIPT" :
356 cache_is_vipt_nonaliasing() ? "VIPT nonaliasing" : "unknown");
357 }
358
359 /*
360 * These functions re-use the assembly code in head.S, which
361 * already provide the required functionality.
362 */
363 extern struct proc_info_list *lookup_processor_type(unsigned int);
364
early_print(const char * str,...)365 void __init early_print(const char *str, ...)
366 {
367 extern void printascii(const char *);
368 char buf[256];
369 va_list ap;
370
371 va_start(ap, str);
372 vsnprintf(buf, sizeof(buf), str, ap);
373 va_end(ap);
374
375 #ifdef CONFIG_DEBUG_LL
376 printascii(buf);
377 #endif
378 printk("%s", buf);
379 }
380
381 #ifdef CONFIG_ARM_PATCH_IDIV
382
sdiv_instruction(void)383 static inline u32 __attribute_const__ sdiv_instruction(void)
384 {
385 if (IS_ENABLED(CONFIG_THUMB2_KERNEL)) {
386 /* "sdiv r0, r0, r1" */
387 u32 insn = __opcode_thumb32_compose(0xfb90, 0xf0f1);
388 return __opcode_to_mem_thumb32(insn);
389 }
390
391 /* "sdiv r0, r0, r1" */
392 return __opcode_to_mem_arm(0xe710f110);
393 }
394
udiv_instruction(void)395 static inline u32 __attribute_const__ udiv_instruction(void)
396 {
397 if (IS_ENABLED(CONFIG_THUMB2_KERNEL)) {
398 /* "udiv r0, r0, r1" */
399 u32 insn = __opcode_thumb32_compose(0xfbb0, 0xf0f1);
400 return __opcode_to_mem_thumb32(insn);
401 }
402
403 /* "udiv r0, r0, r1" */
404 return __opcode_to_mem_arm(0xe730f110);
405 }
406
bx_lr_instruction(void)407 static inline u32 __attribute_const__ bx_lr_instruction(void)
408 {
409 if (IS_ENABLED(CONFIG_THUMB2_KERNEL)) {
410 /* "bx lr; nop" */
411 u32 insn = __opcode_thumb32_compose(0x4770, 0x46c0);
412 return __opcode_to_mem_thumb32(insn);
413 }
414
415 /* "bx lr" */
416 return __opcode_to_mem_arm(0xe12fff1e);
417 }
418
patch_aeabi_idiv(void)419 static void __init patch_aeabi_idiv(void)
420 {
421 extern void __aeabi_uidiv(void);
422 extern void __aeabi_idiv(void);
423 uintptr_t fn_addr;
424 unsigned int mask;
425
426 mask = IS_ENABLED(CONFIG_THUMB2_KERNEL) ? HWCAP_IDIVT : HWCAP_IDIVA;
427 if (!(elf_hwcap & mask))
428 return;
429
430 pr_info("CPU: div instructions available: patching division code\n");
431
432 fn_addr = ((uintptr_t)&__aeabi_uidiv) & ~1;
433 asm ("" : "+g" (fn_addr));
434 ((u32 *)fn_addr)[0] = udiv_instruction();
435 ((u32 *)fn_addr)[1] = bx_lr_instruction();
436 flush_icache_range(fn_addr, fn_addr + 8);
437
438 fn_addr = ((uintptr_t)&__aeabi_idiv) & ~1;
439 asm ("" : "+g" (fn_addr));
440 ((u32 *)fn_addr)[0] = sdiv_instruction();
441 ((u32 *)fn_addr)[1] = bx_lr_instruction();
442 flush_icache_range(fn_addr, fn_addr + 8);
443 }
444
445 #else
patch_aeabi_idiv(void)446 static inline void patch_aeabi_idiv(void) { }
447 #endif
448
cpuid_init_hwcaps(void)449 static void __init cpuid_init_hwcaps(void)
450 {
451 int block;
452 u32 isar5;
453
454 if (cpu_architecture() < CPU_ARCH_ARMv7)
455 return;
456
457 block = cpuid_feature_extract(CPUID_EXT_ISAR0, 24);
458 if (block >= 2)
459 elf_hwcap |= HWCAP_IDIVA;
460 if (block >= 1)
461 elf_hwcap |= HWCAP_IDIVT;
462
463 /* LPAE implies atomic ldrd/strd instructions */
464 block = cpuid_feature_extract(CPUID_EXT_MMFR0, 0);
465 if (block >= 5)
466 elf_hwcap |= HWCAP_LPAE;
467
468 /* check for supported v8 Crypto instructions */
469 isar5 = read_cpuid_ext(CPUID_EXT_ISAR5);
470
471 block = cpuid_feature_extract_field(isar5, 4);
472 if (block >= 2)
473 elf_hwcap2 |= HWCAP2_PMULL;
474 if (block >= 1)
475 elf_hwcap2 |= HWCAP2_AES;
476
477 block = cpuid_feature_extract_field(isar5, 8);
478 if (block >= 1)
479 elf_hwcap2 |= HWCAP2_SHA1;
480
481 block = cpuid_feature_extract_field(isar5, 12);
482 if (block >= 1)
483 elf_hwcap2 |= HWCAP2_SHA2;
484
485 block = cpuid_feature_extract_field(isar5, 16);
486 if (block >= 1)
487 elf_hwcap2 |= HWCAP2_CRC32;
488 }
489
elf_hwcap_fixup(void)490 static void __init elf_hwcap_fixup(void)
491 {
492 unsigned id = read_cpuid_id();
493
494 /*
495 * HWCAP_TLS is available only on 1136 r1p0 and later,
496 * see also kuser_get_tls_init.
497 */
498 if (read_cpuid_part() == ARM_CPU_PART_ARM1136 &&
499 ((id >> 20) & 3) == 0) {
500 elf_hwcap &= ~HWCAP_TLS;
501 return;
502 }
503
504 /* Verify if CPUID scheme is implemented */
505 if ((id & 0x000f0000) != 0x000f0000)
506 return;
507
508 /*
509 * If the CPU supports LDREX/STREX and LDREXB/STREXB,
510 * avoid advertising SWP; it may not be atomic with
511 * multiprocessing cores.
512 */
513 if (cpuid_feature_extract(CPUID_EXT_ISAR3, 12) > 1 ||
514 (cpuid_feature_extract(CPUID_EXT_ISAR3, 12) == 1 &&
515 cpuid_feature_extract(CPUID_EXT_ISAR4, 20) >= 3))
516 elf_hwcap &= ~HWCAP_SWP;
517 }
518
519 /*
520 * cpu_init - initialise one CPU.
521 *
522 * cpu_init sets up the per-CPU stacks.
523 */
cpu_init(void)524 void notrace cpu_init(void)
525 {
526 #ifndef CONFIG_CPU_V7M
527 unsigned int cpu = smp_processor_id();
528 struct stack *stk = &stacks[cpu];
529
530 if (cpu >= NR_CPUS) {
531 pr_crit("CPU%u: bad primary CPU number\n", cpu);
532 BUG();
533 }
534
535 /*
536 * This only works on resume and secondary cores. For booting on the
537 * boot cpu, smp_prepare_boot_cpu is called after percpu area setup.
538 */
539 set_my_cpu_offset(per_cpu_offset(cpu));
540
541 cpu_proc_init();
542
543 /*
544 * Define the placement constraint for the inline asm directive below.
545 * In Thumb-2, msr with an immediate value is not allowed.
546 */
547 #ifdef CONFIG_THUMB2_KERNEL
548 #define PLC "r"
549 #else
550 #define PLC "I"
551 #endif
552
553 /*
554 * setup stacks for re-entrant exception handlers
555 */
556 __asm__ (
557 "msr cpsr_c, %1\n\t"
558 "add r14, %0, %2\n\t"
559 "mov sp, r14\n\t"
560 "msr cpsr_c, %3\n\t"
561 "add r14, %0, %4\n\t"
562 "mov sp, r14\n\t"
563 "msr cpsr_c, %5\n\t"
564 "add r14, %0, %6\n\t"
565 "mov sp, r14\n\t"
566 "msr cpsr_c, %7\n\t"
567 "add r14, %0, %8\n\t"
568 "mov sp, r14\n\t"
569 "msr cpsr_c, %9"
570 :
571 : "r" (stk),
572 PLC (PSR_F_BIT | PSR_I_BIT | IRQ_MODE),
573 "I" (offsetof(struct stack, irq[0])),
574 PLC (PSR_F_BIT | PSR_I_BIT | ABT_MODE),
575 "I" (offsetof(struct stack, abt[0])),
576 PLC (PSR_F_BIT | PSR_I_BIT | UND_MODE),
577 "I" (offsetof(struct stack, und[0])),
578 PLC (PSR_F_BIT | PSR_I_BIT | FIQ_MODE),
579 "I" (offsetof(struct stack, fiq[0])),
580 PLC (PSR_F_BIT | PSR_I_BIT | SVC_MODE)
581 : "r14");
582 #endif
583 }
584
585 u32 __cpu_logical_map[NR_CPUS] = { [0 ... NR_CPUS-1] = MPIDR_INVALID };
586
smp_setup_processor_id(void)587 void __init smp_setup_processor_id(void)
588 {
589 int i;
590 u32 mpidr = is_smp() ? read_cpuid_mpidr() & MPIDR_HWID_BITMASK : 0;
591 u32 cpu = MPIDR_AFFINITY_LEVEL(mpidr, 0);
592
593 cpu_logical_map(0) = cpu;
594 for (i = 1; i < nr_cpu_ids; ++i)
595 cpu_logical_map(i) = i == cpu ? 0 : i;
596
597 /*
598 * clear __my_cpu_offset on boot CPU to avoid hang caused by
599 * using percpu variable early, for example, lockdep will
600 * access percpu variable inside lock_release
601 */
602 set_my_cpu_offset(0);
603
604 pr_info("Booting Linux on physical CPU 0x%x\n", mpidr);
605 }
606
607 struct mpidr_hash mpidr_hash;
608 #ifdef CONFIG_SMP
609 /**
610 * smp_build_mpidr_hash - Pre-compute shifts required at each affinity
611 * level in order to build a linear index from an
612 * MPIDR value. Resulting algorithm is a collision
613 * free hash carried out through shifting and ORing
614 */
smp_build_mpidr_hash(void)615 static void __init smp_build_mpidr_hash(void)
616 {
617 u32 i, affinity;
618 u32 fs[3], bits[3], ls, mask = 0;
619 /*
620 * Pre-scan the list of MPIDRS and filter out bits that do
621 * not contribute to affinity levels, ie they never toggle.
622 */
623 for_each_possible_cpu(i)
624 mask |= (cpu_logical_map(i) ^ cpu_logical_map(0));
625 pr_debug("mask of set bits 0x%x\n", mask);
626 /*
627 * Find and stash the last and first bit set at all affinity levels to
628 * check how many bits are required to represent them.
629 */
630 for (i = 0; i < 3; i++) {
631 affinity = MPIDR_AFFINITY_LEVEL(mask, i);
632 /*
633 * Find the MSB bit and LSB bits position
634 * to determine how many bits are required
635 * to express the affinity level.
636 */
637 ls = fls(affinity);
638 fs[i] = affinity ? ffs(affinity) - 1 : 0;
639 bits[i] = ls - fs[i];
640 }
641 /*
642 * An index can be created from the MPIDR by isolating the
643 * significant bits at each affinity level and by shifting
644 * them in order to compress the 24 bits values space to a
645 * compressed set of values. This is equivalent to hashing
646 * the MPIDR through shifting and ORing. It is a collision free
647 * hash though not minimal since some levels might contain a number
648 * of CPUs that is not an exact power of 2 and their bit
649 * representation might contain holes, eg MPIDR[7:0] = {0x2, 0x80}.
650 */
651 mpidr_hash.shift_aff[0] = fs[0];
652 mpidr_hash.shift_aff[1] = MPIDR_LEVEL_BITS + fs[1] - bits[0];
653 mpidr_hash.shift_aff[2] = 2*MPIDR_LEVEL_BITS + fs[2] -
654 (bits[1] + bits[0]);
655 mpidr_hash.mask = mask;
656 mpidr_hash.bits = bits[2] + bits[1] + bits[0];
657 pr_debug("MPIDR hash: aff0[%u] aff1[%u] aff2[%u] mask[0x%x] bits[%u]\n",
658 mpidr_hash.shift_aff[0],
659 mpidr_hash.shift_aff[1],
660 mpidr_hash.shift_aff[2],
661 mpidr_hash.mask,
662 mpidr_hash.bits);
663 /*
664 * 4x is an arbitrary value used to warn on a hash table much bigger
665 * than expected on most systems.
666 */
667 if (mpidr_hash_size() > 4 * num_possible_cpus())
668 pr_warn("Large number of MPIDR hash buckets detected\n");
669 sync_cache_w(&mpidr_hash);
670 }
671 #endif
672
673 /*
674 * locate processor in the list of supported processor types. The linker
675 * builds this table for us from the entries in arch/arm/mm/proc-*.S
676 */
lookup_processor(u32 midr)677 struct proc_info_list *lookup_processor(u32 midr)
678 {
679 struct proc_info_list *list = lookup_processor_type(midr);
680
681 if (!list) {
682 pr_err("CPU%u: configuration botched (ID %08x), CPU halted\n",
683 smp_processor_id(), midr);
684 while (1)
685 /* can't use cpu_relax() here as it may require MMU setup */;
686 }
687
688 return list;
689 }
690
setup_processor(void)691 static void __init setup_processor(void)
692 {
693 unsigned int midr = read_cpuid_id();
694 struct proc_info_list *list = lookup_processor(midr);
695
696 cpu_name = list->cpu_name;
697 __cpu_architecture = __get_cpu_architecture();
698
699 init_proc_vtable(list->proc);
700 #ifdef MULTI_TLB
701 cpu_tlb = *list->tlb;
702 #endif
703 #ifdef MULTI_USER
704 cpu_user = *list->user;
705 #endif
706 #ifdef MULTI_CACHE
707 cpu_cache = *list->cache;
708 #endif
709
710 pr_info("CPU: %s [%08x] revision %d (ARMv%s), cr=%08lx\n",
711 list->cpu_name, midr, midr & 15,
712 proc_arch[cpu_architecture()], get_cr());
713
714 snprintf(init_utsname()->machine, __NEW_UTS_LEN + 1, "%s%c",
715 list->arch_name, ENDIANNESS);
716 snprintf(elf_platform, ELF_PLATFORM_SIZE, "%s%c",
717 list->elf_name, ENDIANNESS);
718 elf_hwcap = list->elf_hwcap;
719
720 cpuid_init_hwcaps();
721 patch_aeabi_idiv();
722
723 #ifndef CONFIG_ARM_THUMB
724 elf_hwcap &= ~(HWCAP_THUMB | HWCAP_IDIVT);
725 #endif
726 #ifdef CONFIG_MMU
727 init_default_cache_policy(list->__cpu_mm_mmu_flags);
728 #endif
729 erratum_a15_798181_init();
730
731 elf_hwcap_fixup();
732
733 cacheid_init();
734 cpu_init();
735 }
736
dump_machine_table(void)737 void __init dump_machine_table(void)
738 {
739 const struct machine_desc *p;
740
741 early_print("Available machine support:\n\nID (hex)\tNAME\n");
742 for_each_machine_desc(p)
743 early_print("%08x\t%s\n", p->nr, p->name);
744
745 early_print("\nPlease check your kernel config and/or bootloader.\n");
746
747 while (true)
748 /* can't use cpu_relax() here as it may require MMU setup */;
749 }
750
arm_add_memory(u64 start,u64 size)751 int __init arm_add_memory(u64 start, u64 size)
752 {
753 u64 aligned_start;
754
755 /*
756 * Ensure that start/size are aligned to a page boundary.
757 * Size is rounded down, start is rounded up.
758 */
759 aligned_start = PAGE_ALIGN(start);
760 if (aligned_start > start + size)
761 size = 0;
762 else
763 size -= aligned_start - start;
764
765 #ifndef CONFIG_PHYS_ADDR_T_64BIT
766 if (aligned_start > ULONG_MAX) {
767 pr_crit("Ignoring memory at 0x%08llx outside 32-bit physical address space\n",
768 start);
769 return -EINVAL;
770 }
771
772 if (aligned_start + size > ULONG_MAX) {
773 pr_crit("Truncating memory at 0x%08llx to fit in 32-bit physical address space\n",
774 (long long)start);
775 /*
776 * To ensure bank->start + bank->size is representable in
777 * 32 bits, we use ULONG_MAX as the upper limit rather than 4GB.
778 * This means we lose a page after masking.
779 */
780 size = ULONG_MAX - aligned_start;
781 }
782 #endif
783
784 if (aligned_start < PHYS_OFFSET) {
785 if (aligned_start + size <= PHYS_OFFSET) {
786 pr_info("Ignoring memory below PHYS_OFFSET: 0x%08llx-0x%08llx\n",
787 aligned_start, aligned_start + size);
788 return -EINVAL;
789 }
790
791 pr_info("Ignoring memory below PHYS_OFFSET: 0x%08llx-0x%08llx\n",
792 aligned_start, (u64)PHYS_OFFSET);
793
794 size -= PHYS_OFFSET - aligned_start;
795 aligned_start = PHYS_OFFSET;
796 }
797
798 start = aligned_start;
799 size = size & ~(phys_addr_t)(PAGE_SIZE - 1);
800
801 /*
802 * Check whether this memory region has non-zero size or
803 * invalid node number.
804 */
805 if (size == 0)
806 return -EINVAL;
807
808 memblock_add(start, size);
809 return 0;
810 }
811
812 /*
813 * Pick out the memory size. We look for mem=size@start,
814 * where start and size are "size[KkMm]"
815 */
816
early_mem(char * p)817 static int __init early_mem(char *p)
818 {
819 static int usermem __initdata = 0;
820 u64 size;
821 u64 start;
822 char *endp;
823
824 /*
825 * If the user specifies memory size, we
826 * blow away any automatically generated
827 * size.
828 */
829 if (usermem == 0) {
830 usermem = 1;
831 memblock_remove(memblock_start_of_DRAM(),
832 memblock_end_of_DRAM() - memblock_start_of_DRAM());
833 }
834
835 start = PHYS_OFFSET;
836 size = memparse(p, &endp);
837 if (*endp == '@')
838 start = memparse(endp + 1, NULL);
839
840 arm_add_memory(start, size);
841
842 return 0;
843 }
844 early_param("mem", early_mem);
845
request_standard_resources(const struct machine_desc * mdesc)846 static void __init request_standard_resources(const struct machine_desc *mdesc)
847 {
848 phys_addr_t start, end, res_end;
849 struct resource *res;
850 u64 i;
851
852 kernel_code.start = virt_to_phys(_text);
853 kernel_code.end = virt_to_phys(__init_begin - 1);
854 kernel_data.start = virt_to_phys(_sdata);
855 kernel_data.end = virt_to_phys(_end - 1);
856
857 for_each_mem_range(i, &start, &end) {
858 unsigned long boot_alias_start;
859
860 /*
861 * In memblock, end points to the first byte after the
862 * range while in resourses, end points to the last byte in
863 * the range.
864 */
865 res_end = end - 1;
866
867 /*
868 * Some systems have a special memory alias which is only
869 * used for booting. We need to advertise this region to
870 * kexec-tools so they know where bootable RAM is located.
871 */
872 boot_alias_start = phys_to_idmap(start);
873 if (arm_has_idmap_alias() && boot_alias_start != IDMAP_INVALID_ADDR) {
874 res = memblock_alloc(sizeof(*res), SMP_CACHE_BYTES);
875 if (!res)
876 panic("%s: Failed to allocate %zu bytes\n",
877 __func__, sizeof(*res));
878 res->name = "System RAM (boot alias)";
879 res->start = boot_alias_start;
880 res->end = phys_to_idmap(res_end);
881 res->flags = IORESOURCE_MEM | IORESOURCE_BUSY;
882 request_resource(&iomem_resource, res);
883 }
884
885 res = memblock_alloc(sizeof(*res), SMP_CACHE_BYTES);
886 if (!res)
887 panic("%s: Failed to allocate %zu bytes\n", __func__,
888 sizeof(*res));
889 res->name = "System RAM";
890 res->start = start;
891 res->end = res_end;
892 res->flags = IORESOURCE_SYSTEM_RAM | IORESOURCE_BUSY;
893
894 request_resource(&iomem_resource, res);
895
896 if (kernel_code.start >= res->start &&
897 kernel_code.end <= res->end)
898 request_resource(res, &kernel_code);
899 if (kernel_data.start >= res->start &&
900 kernel_data.end <= res->end)
901 request_resource(res, &kernel_data);
902 }
903
904 if (mdesc->video_start) {
905 video_ram.start = mdesc->video_start;
906 video_ram.end = mdesc->video_end;
907 request_resource(&iomem_resource, &video_ram);
908 }
909
910 /*
911 * Some machines don't have the possibility of ever
912 * possessing lp0, lp1 or lp2
913 */
914 if (mdesc->reserve_lp0)
915 request_resource(&ioport_resource, &lp0);
916 if (mdesc->reserve_lp1)
917 request_resource(&ioport_resource, &lp1);
918 if (mdesc->reserve_lp2)
919 request_resource(&ioport_resource, &lp2);
920 }
921
922 #if defined(CONFIG_VGA_CONSOLE) || defined(CONFIG_DUMMY_CONSOLE) || \
923 defined(CONFIG_EFI)
924 struct screen_info screen_info = {
925 .orig_video_lines = 30,
926 .orig_video_cols = 80,
927 .orig_video_mode = 0,
928 .orig_video_ega_bx = 0,
929 .orig_video_isVGA = 1,
930 .orig_video_points = 8
931 };
932 #endif
933
customize_machine(void)934 static int __init customize_machine(void)
935 {
936 /*
937 * customizes platform devices, or adds new ones
938 * On DT based machines, we fall back to populating the
939 * machine from the device tree, if no callback is provided,
940 * otherwise we would always need an init_machine callback.
941 */
942 if (machine_desc->init_machine)
943 machine_desc->init_machine();
944
945 return 0;
946 }
947 arch_initcall(customize_machine);
948
init_machine_late(void)949 static int __init init_machine_late(void)
950 {
951 struct device_node *root;
952 int ret;
953
954 if (machine_desc->init_late)
955 machine_desc->init_late();
956
957 root = of_find_node_by_path("/");
958 if (root) {
959 ret = of_property_read_string(root, "serial-number",
960 &system_serial);
961 if (ret)
962 system_serial = NULL;
963 }
964
965 if (!system_serial)
966 system_serial = kasprintf(GFP_KERNEL, "%08x%08x",
967 system_serial_high,
968 system_serial_low);
969
970 return 0;
971 }
972 late_initcall(init_machine_late);
973
974 #ifdef CONFIG_KEXEC
975 /*
976 * The crash region must be aligned to 128MB to avoid
977 * zImage relocating below the reserved region.
978 */
979 #define CRASH_ALIGN (128 << 20)
980
get_total_mem(void)981 static inline unsigned long long get_total_mem(void)
982 {
983 unsigned long total;
984
985 total = max_low_pfn - min_low_pfn;
986 return total << PAGE_SHIFT;
987 }
988
989 /**
990 * reserve_crashkernel() - reserves memory are for crash kernel
991 *
992 * This function reserves memory area given in "crashkernel=" kernel command
993 * line parameter. The memory reserved is used by a dump capture kernel when
994 * primary kernel is crashing.
995 */
reserve_crashkernel(void)996 static void __init reserve_crashkernel(void)
997 {
998 unsigned long long crash_size, crash_base;
999 unsigned long long total_mem;
1000 int ret;
1001
1002 total_mem = get_total_mem();
1003 ret = parse_crashkernel(boot_command_line, total_mem,
1004 &crash_size, &crash_base);
1005 if (ret)
1006 return;
1007
1008 if (crash_base <= 0) {
1009 unsigned long long crash_max = idmap_to_phys((u32)~0);
1010 unsigned long long lowmem_max = __pa(high_memory - 1) + 1;
1011 if (crash_max > lowmem_max)
1012 crash_max = lowmem_max;
1013 crash_base = memblock_find_in_range(CRASH_ALIGN, crash_max,
1014 crash_size, CRASH_ALIGN);
1015 if (!crash_base) {
1016 pr_err("crashkernel reservation failed - No suitable area found.\n");
1017 return;
1018 }
1019 } else {
1020 unsigned long long start;
1021
1022 start = memblock_find_in_range(crash_base,
1023 crash_base + crash_size,
1024 crash_size, SECTION_SIZE);
1025 if (start != crash_base) {
1026 pr_err("crashkernel reservation failed - memory is in use.\n");
1027 return;
1028 }
1029 }
1030
1031 ret = memblock_reserve(crash_base, crash_size);
1032 if (ret < 0) {
1033 pr_warn("crashkernel reservation failed - memory is in use (0x%lx)\n",
1034 (unsigned long)crash_base);
1035 return;
1036 }
1037
1038 pr_info("Reserving %ldMB of memory at %ldMB for crashkernel (System RAM: %ldMB)\n",
1039 (unsigned long)(crash_size >> 20),
1040 (unsigned long)(crash_base >> 20),
1041 (unsigned long)(total_mem >> 20));
1042
1043 /* The crashk resource must always be located in normal mem */
1044 crashk_res.start = crash_base;
1045 crashk_res.end = crash_base + crash_size - 1;
1046 insert_resource(&iomem_resource, &crashk_res);
1047
1048 if (arm_has_idmap_alias()) {
1049 /*
1050 * If we have a special RAM alias for use at boot, we
1051 * need to advertise to kexec tools where the alias is.
1052 */
1053 static struct resource crashk_boot_res = {
1054 .name = "Crash kernel (boot alias)",
1055 .flags = IORESOURCE_BUSY | IORESOURCE_MEM,
1056 };
1057
1058 crashk_boot_res.start = phys_to_idmap(crash_base);
1059 crashk_boot_res.end = crashk_boot_res.start + crash_size - 1;
1060 insert_resource(&iomem_resource, &crashk_boot_res);
1061 }
1062 }
1063 #else
reserve_crashkernel(void)1064 static inline void reserve_crashkernel(void) {}
1065 #endif /* CONFIG_KEXEC */
1066
hyp_mode_check(void)1067 void __init hyp_mode_check(void)
1068 {
1069 #ifdef CONFIG_ARM_VIRT_EXT
1070 sync_boot_mode();
1071
1072 if (is_hyp_mode_available()) {
1073 pr_info("CPU: All CPU(s) started in HYP mode.\n");
1074 pr_info("CPU: Virtualization extensions available.\n");
1075 } else if (is_hyp_mode_mismatched()) {
1076 pr_warn("CPU: WARNING: CPU(s) started in wrong/inconsistent modes (primary CPU mode 0x%x)\n",
1077 __boot_cpu_mode & MODE_MASK);
1078 pr_warn("CPU: This may indicate a broken bootloader or firmware.\n");
1079 } else
1080 pr_info("CPU: All CPU(s) started in SVC mode.\n");
1081 #endif
1082 }
1083
setup_arch(char ** cmdline_p)1084 void __init setup_arch(char **cmdline_p)
1085 {
1086 const struct machine_desc *mdesc = NULL;
1087 void *atags_vaddr = NULL;
1088
1089 if (__atags_pointer)
1090 atags_vaddr = FDT_VIRT_BASE(__atags_pointer);
1091
1092 setup_processor();
1093 if (atags_vaddr) {
1094 mdesc = setup_machine_fdt(atags_vaddr);
1095 if (mdesc)
1096 memblock_reserve(__atags_pointer,
1097 fdt_totalsize(atags_vaddr));
1098 }
1099 if (!mdesc)
1100 mdesc = setup_machine_tags(atags_vaddr, __machine_arch_type);
1101 if (!mdesc) {
1102 early_print("\nError: invalid dtb and unrecognized/unsupported machine ID\n");
1103 early_print(" r1=0x%08x, r2=0x%08x\n", __machine_arch_type,
1104 __atags_pointer);
1105 if (__atags_pointer)
1106 early_print(" r2[]=%*ph\n", 16, atags_vaddr);
1107 dump_machine_table();
1108 }
1109
1110 machine_desc = mdesc;
1111 machine_name = mdesc->name;
1112 dump_stack_set_arch_desc("%s", mdesc->name);
1113
1114 if (mdesc->reboot_mode != REBOOT_HARD)
1115 reboot_mode = mdesc->reboot_mode;
1116
1117 init_mm.start_code = (unsigned long) _text;
1118 init_mm.end_code = (unsigned long) _etext;
1119 init_mm.end_data = (unsigned long) _edata;
1120 init_mm.brk = (unsigned long) _end;
1121
1122 /* populate cmd_line too for later use, preserving boot_command_line */
1123 strlcpy(cmd_line, boot_command_line, COMMAND_LINE_SIZE);
1124 *cmdline_p = cmd_line;
1125
1126 early_fixmap_init();
1127 early_ioremap_init();
1128
1129 parse_early_param();
1130
1131 #ifdef CONFIG_MMU
1132 early_mm_init(mdesc);
1133 #endif
1134 setup_dma_zone(mdesc);
1135 xen_early_init();
1136 efi_init();
1137 /*
1138 * Make sure the calculation for lowmem/highmem is set appropriately
1139 * before reserving/allocating any memory
1140 */
1141 adjust_lowmem_bounds();
1142 arm_memblock_init(mdesc);
1143 /* Memory may have been removed so recalculate the bounds. */
1144 adjust_lowmem_bounds();
1145
1146 early_ioremap_reset();
1147
1148 paging_init(mdesc);
1149 kasan_init();
1150 request_standard_resources(mdesc);
1151
1152 if (mdesc->restart)
1153 arm_pm_restart = mdesc->restart;
1154
1155 unflatten_device_tree();
1156
1157 arm_dt_init_cpu_maps();
1158 psci_dt_init();
1159 #ifdef CONFIG_SMP
1160 if (is_smp()) {
1161 if (!mdesc->smp_init || !mdesc->smp_init()) {
1162 if (psci_smp_available())
1163 smp_set_ops(&psci_smp_ops);
1164 else if (mdesc->smp)
1165 smp_set_ops(mdesc->smp);
1166 }
1167 smp_init_cpus();
1168 smp_build_mpidr_hash();
1169 }
1170 #endif
1171
1172 if (!is_smp())
1173 hyp_mode_check();
1174
1175 reserve_crashkernel();
1176
1177 #ifdef CONFIG_GENERIC_IRQ_MULTI_HANDLER
1178 handle_arch_irq = mdesc->handle_irq;
1179 #endif
1180
1181 #ifdef CONFIG_VT
1182 #if defined(CONFIG_VGA_CONSOLE)
1183 conswitchp = &vga_con;
1184 #endif
1185 #endif
1186
1187 if (mdesc->init_early)
1188 mdesc->init_early();
1189 }
1190
1191
topology_init(void)1192 static int __init topology_init(void)
1193 {
1194 int cpu;
1195
1196 for_each_possible_cpu(cpu) {
1197 struct cpuinfo_arm *cpuinfo = &per_cpu(cpu_data, cpu);
1198 cpuinfo->cpu.hotpluggable = platform_can_hotplug_cpu(cpu);
1199 register_cpu(&cpuinfo->cpu, cpu);
1200 }
1201
1202 return 0;
1203 }
1204 subsys_initcall(topology_init);
1205
1206 #ifdef CONFIG_HAVE_PROC_CPU
proc_cpu_init(void)1207 static int __init proc_cpu_init(void)
1208 {
1209 struct proc_dir_entry *res;
1210
1211 res = proc_mkdir("cpu", NULL);
1212 if (!res)
1213 return -ENOMEM;
1214 return 0;
1215 }
1216 fs_initcall(proc_cpu_init);
1217 #endif
1218
1219 static const char *hwcap_str[] = {
1220 "swp",
1221 "half",
1222 "thumb",
1223 "26bit",
1224 "fastmult",
1225 "fpa",
1226 "vfp",
1227 "edsp",
1228 "java",
1229 "iwmmxt",
1230 "crunch",
1231 "thumbee",
1232 "neon",
1233 "vfpv3",
1234 "vfpv3d16",
1235 "tls",
1236 "vfpv4",
1237 "idiva",
1238 "idivt",
1239 "vfpd32",
1240 "lpae",
1241 "evtstrm",
1242 NULL
1243 };
1244
1245 static const char *hwcap2_str[] = {
1246 "aes",
1247 "pmull",
1248 "sha1",
1249 "sha2",
1250 "crc32",
1251 NULL
1252 };
1253
c_show(struct seq_file * m,void * v)1254 static int c_show(struct seq_file *m, void *v)
1255 {
1256 int i, j;
1257 u32 cpuid;
1258
1259 for_each_online_cpu(i) {
1260 /*
1261 * glibc reads /proc/cpuinfo to determine the number of
1262 * online processors, looking for lines beginning with
1263 * "processor". Give glibc what it expects.
1264 */
1265 seq_printf(m, "processor\t: %d\n", i);
1266 cpuid = is_smp() ? per_cpu(cpu_data, i).cpuid : read_cpuid_id();
1267 seq_printf(m, "model name\t: %s rev %d (%s)\n",
1268 cpu_name, cpuid & 15, elf_platform);
1269
1270 #if defined(CONFIG_SMP)
1271 seq_printf(m, "BogoMIPS\t: %lu.%02lu\n",
1272 per_cpu(cpu_data, i).loops_per_jiffy / (500000UL/HZ),
1273 (per_cpu(cpu_data, i).loops_per_jiffy / (5000UL/HZ)) % 100);
1274 #else
1275 seq_printf(m, "BogoMIPS\t: %lu.%02lu\n",
1276 loops_per_jiffy / (500000/HZ),
1277 (loops_per_jiffy / (5000/HZ)) % 100);
1278 #endif
1279 /* dump out the processor features */
1280 seq_puts(m, "Features\t: ");
1281
1282 for (j = 0; hwcap_str[j]; j++)
1283 if (elf_hwcap & (1 << j))
1284 seq_printf(m, "%s ", hwcap_str[j]);
1285
1286 for (j = 0; hwcap2_str[j]; j++)
1287 if (elf_hwcap2 & (1 << j))
1288 seq_printf(m, "%s ", hwcap2_str[j]);
1289
1290 seq_printf(m, "\nCPU implementer\t: 0x%02x\n", cpuid >> 24);
1291 seq_printf(m, "CPU architecture: %s\n",
1292 proc_arch[cpu_architecture()]);
1293
1294 if ((cpuid & 0x0008f000) == 0x00000000) {
1295 /* pre-ARM7 */
1296 seq_printf(m, "CPU part\t: %07x\n", cpuid >> 4);
1297 } else {
1298 if ((cpuid & 0x0008f000) == 0x00007000) {
1299 /* ARM7 */
1300 seq_printf(m, "CPU variant\t: 0x%02x\n",
1301 (cpuid >> 16) & 127);
1302 } else {
1303 /* post-ARM7 */
1304 seq_printf(m, "CPU variant\t: 0x%x\n",
1305 (cpuid >> 20) & 15);
1306 }
1307 seq_printf(m, "CPU part\t: 0x%03x\n",
1308 (cpuid >> 4) & 0xfff);
1309 }
1310 seq_printf(m, "CPU revision\t: %d\n\n", cpuid & 15);
1311 }
1312
1313 seq_printf(m, "Hardware\t: %s\n", machine_name);
1314 seq_printf(m, "Revision\t: %04x\n", system_rev);
1315 seq_printf(m, "Serial\t\t: %s\n", system_serial);
1316
1317 return 0;
1318 }
1319
c_start(struct seq_file * m,loff_t * pos)1320 static void *c_start(struct seq_file *m, loff_t *pos)
1321 {
1322 return *pos < 1 ? (void *)1 : NULL;
1323 }
1324
c_next(struct seq_file * m,void * v,loff_t * pos)1325 static void *c_next(struct seq_file *m, void *v, loff_t *pos)
1326 {
1327 ++*pos;
1328 return NULL;
1329 }
1330
c_stop(struct seq_file * m,void * v)1331 static void c_stop(struct seq_file *m, void *v)
1332 {
1333 }
1334
1335 const struct seq_operations cpuinfo_op = {
1336 .start = c_start,
1337 .next = c_next,
1338 .stop = c_stop,
1339 .show = c_show
1340 };
1341