1 /*-
2 * Copyright (c) 2014 Andrew Turner
3 * Copyright (c) 2014 The FreeBSD Foundation
4 * All rights reserved.
5 *
6 * Portions of this software were developed by Semihalf
7 * under sponsorship of the FreeBSD Foundation.
8 *
9 * Redistribution and use in source and binary forms, with or without
10 * modification, are permitted provided that the following conditions
11 * are met:
12 * 1. Redistributions of source code must retain the above copyright
13 * notice, this list of conditions and the following disclaimer.
14 * 2. Redistributions in binary form must reproduce the above copyright
15 * notice, this list of conditions and the following disclaimer in the
16 * documentation and/or other materials provided with the distribution.
17 *
18 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
19 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
20 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
21 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
22 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
23 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
24 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
25 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
26 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
27 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
28 * SUCH DAMAGE.
29 *
30 */
31
32 #include <sys/param.h>
33 #include <sys/kernel.h>
34 #include <sys/malloc.h>
35 #include <sys/proc.h>
36 #include <sys/pcpu.h>
37 #include <sys/sbuf.h>
38 #include <sys/smp.h>
39 #include <sys/sysctl.h>
40 #include <sys/sysent.h>
41 #include <sys/systm.h>
42
43 #include <machine/atomic.h>
44 #include <machine/cpu.h>
45 #include <machine/cpufunc.h>
46 #include <machine/elf.h>
47 #include <machine/md_var.h>
48 #include <machine/undefined.h>
49
50 static MALLOC_DEFINE(M_IDENTCPU, "CPU ID", "arm64 CPU identification memory");
51
52 struct cpu_desc;
53
54 static void print_cpu_midr(struct sbuf *sb, u_int cpu);
55 static void print_cpu_features(u_int cpu, struct cpu_desc *desc,
56 struct cpu_desc *prev_desc);
57 static void print_cpu_caches(struct sbuf *sb, struct cpu_desc *desc);
58 #ifdef COMPAT_FREEBSD32
59 static u_long parse_cpu_features_hwcap32(void);
60 #endif
61
62 char machine[] = "arm64";
63
64 #ifdef SCTL_MASK32
65 extern int adaptive_machine_arch;
66 #endif
67
68 static SYSCTL_NODE(_machdep, OID_AUTO, cache, CTLFLAG_RD | CTLFLAG_MPSAFE, 0,
69 "Cache management tuning");
70
71 static int allow_dic = 1;
72 SYSCTL_INT(_machdep_cache, OID_AUTO, allow_dic, CTLFLAG_RDTUN, &allow_dic, 0,
73 "Allow optimizations based on the DIC cache bit");
74
75 static int allow_idc = 1;
76 SYSCTL_INT(_machdep_cache, OID_AUTO, allow_idc, CTLFLAG_RDTUN, &allow_idc, 0,
77 "Allow optimizations based on the IDC cache bit");
78
79 static void check_cpu_regs(u_int cpu, struct cpu_desc *desc,
80 struct cpu_desc *prev_desc);
81
82 /*
83 * The default implementation of I-cache sync assumes we have an
84 * aliasing cache until we know otherwise.
85 */
86 void (*arm64_icache_sync_range)(void *, vm_size_t) =
87 &arm64_aliasing_icache_sync_range;
88
89 static int
sysctl_hw_machine(SYSCTL_HANDLER_ARGS)90 sysctl_hw_machine(SYSCTL_HANDLER_ARGS)
91 {
92 #ifdef SCTL_MASK32
93 static const char machine32[] = "arm";
94 #endif
95 int error;
96
97 #ifdef SCTL_MASK32
98 if ((req->flags & SCTL_MASK32) != 0 && adaptive_machine_arch)
99 error = SYSCTL_OUT(req, machine32, sizeof(machine32));
100 else
101 #endif
102 error = SYSCTL_OUT(req, machine, sizeof(machine));
103 return (error);
104 }
105
106 SYSCTL_PROC(_hw, HW_MACHINE, machine, CTLTYPE_STRING | CTLFLAG_RD |
107 CTLFLAG_CAPRD | CTLFLAG_MPSAFE, NULL, 0, sysctl_hw_machine, "A",
108 "Machine class");
109
110 static char cpu_model[64];
111 SYSCTL_STRING(_hw, HW_MODEL, model, CTLFLAG_RD | CTLFLAG_CAPRD,
112 cpu_model, sizeof(cpu_model), "Machine model");
113
114 #define MAX_CACHES 8 /* Maximum number of caches supported
115 architecturally. */
116 /*
117 * Per-CPU affinity as provided in MPIDR_EL1
118 * Indexed by CPU number in logical order selected by the system.
119 * Relevant fields can be extracted using CPU_AFFn macros,
120 * Aff3.Aff2.Aff1.Aff0 construct a unique CPU address in the system.
121 *
122 * Fields used by us:
123 * Aff1 - Cluster number
124 * Aff0 - CPU number in Aff1 cluster
125 */
126 uint64_t __cpu_affinity[MAXCPU];
127 static u_int cpu_aff_levels;
128
129 struct cpu_desc {
130 uint64_t mpidr;
131 uint64_t id_aa64afr0;
132 uint64_t id_aa64afr1;
133 uint64_t id_aa64dfr0;
134 uint64_t id_aa64dfr1;
135 uint64_t id_aa64isar0;
136 uint64_t id_aa64isar1;
137 uint64_t id_aa64isar2;
138 uint64_t id_aa64mmfr0;
139 uint64_t id_aa64mmfr1;
140 uint64_t id_aa64mmfr2;
141 uint64_t id_aa64mmfr3;
142 uint64_t id_aa64mmfr4;
143 uint64_t id_aa64pfr0;
144 uint64_t id_aa64pfr1;
145 uint64_t id_aa64pfr2;
146 uint64_t id_aa64zfr0;
147 uint64_t ctr;
148 #ifdef COMPAT_FREEBSD32
149 uint64_t id_isar5;
150 uint64_t mvfr0;
151 uint64_t mvfr1;
152 #endif
153 uint64_t clidr;
154 uint32_t ccsidr[MAX_CACHES][2]; /* 2 possible types. */
155 bool have_sve;
156 };
157
158 static struct cpu_desc cpu_desc0;
159 static struct cpu_desc *cpu_desc;
160 static struct cpu_desc kern_cpu_desc;
161 static struct cpu_desc user_cpu_desc;
162 static struct cpu_desc l_user_cpu_desc;
163
164 static struct cpu_desc *
get_cpu_desc(u_int cpu)165 get_cpu_desc(u_int cpu)
166 {
167 /* The cpu_desc for CPU 0 is used before the allocator is ready. */
168 if (cpu == 0)
169 return (&cpu_desc0);
170
171 MPASS(cpu_desc != NULL);
172 return (&cpu_desc[cpu - 1]);
173 }
174
175 struct cpu_parts {
176 u_int part_id;
177 const char *part_name;
178 };
179 #define CPU_PART_NONE { 0, NULL }
180
181 struct cpu_implementers {
182 u_int impl_id;
183 const char *impl_name;
184 /*
185 * Part number is implementation defined
186 * so each vendor will have its own set of values and names.
187 */
188 const struct cpu_parts *cpu_parts;
189 };
190 #define CPU_IMPLEMENTER_NONE { 0, NULL, NULL }
191
192 /*
193 * Per-implementer table of (PartNum, CPU Name) pairs.
194 */
195 /* ARM Ltd. */
196 static const struct cpu_parts cpu_parts_arm[] = {
197 { CPU_PART_AEM_V8, "AEMv8" },
198 { CPU_PART_FOUNDATION, "Foundation-Model" },
199 { CPU_PART_CORTEX_A34, "Cortex-A34" },
200 { CPU_PART_CORTEX_A35, "Cortex-A35" },
201 { CPU_PART_CORTEX_A53, "Cortex-A53" },
202 { CPU_PART_CORTEX_A55, "Cortex-A55" },
203 { CPU_PART_CORTEX_A57, "Cortex-A57" },
204 { CPU_PART_CORTEX_A65, "Cortex-A65" },
205 { CPU_PART_CORTEX_A65AE, "Cortex-A65AE" },
206 { CPU_PART_CORTEX_A72, "Cortex-A72" },
207 { CPU_PART_CORTEX_A73, "Cortex-A73" },
208 { CPU_PART_CORTEX_A75, "Cortex-A75" },
209 { CPU_PART_CORTEX_A76, "Cortex-A76" },
210 { CPU_PART_CORTEX_A76AE, "Cortex-A76AE" },
211 { CPU_PART_CORTEX_A77, "Cortex-A77" },
212 { CPU_PART_CORTEX_A78, "Cortex-A78" },
213 { CPU_PART_CORTEX_A78C, "Cortex-A78C" },
214 { CPU_PART_CORTEX_A510, "Cortex-A510" },
215 { CPU_PART_CORTEX_A710, "Cortex-A710" },
216 { CPU_PART_CORTEX_A715, "Cortex-A715" },
217 { CPU_PART_CORTEX_X1, "Cortex-X1" },
218 { CPU_PART_CORTEX_X1C, "Cortex-X1C" },
219 { CPU_PART_CORTEX_X2, "Cortex-X2" },
220 { CPU_PART_CORTEX_X3, "Cortex-X3" },
221 { CPU_PART_NEOVERSE_E1, "Neoverse-E1" },
222 { CPU_PART_NEOVERSE_N1, "Neoverse-N1" },
223 { CPU_PART_NEOVERSE_N2, "Neoverse-N2" },
224 { CPU_PART_NEOVERSE_V1, "Neoverse-V1" },
225 { CPU_PART_NEOVERSE_V2, "Neoverse-V2" },
226 CPU_PART_NONE,
227 };
228
229 /* Cavium */
230 static const struct cpu_parts cpu_parts_cavium[] = {
231 { CPU_PART_THUNDERX, "ThunderX" },
232 { CPU_PART_THUNDERX2, "ThunderX2" },
233 CPU_PART_NONE,
234 };
235
236 /* APM / Ampere */
237 static const struct cpu_parts cpu_parts_apm[] = {
238 { CPU_PART_EMAG8180, "eMAG 8180" },
239 CPU_PART_NONE,
240 };
241
242 /* Qualcomm */
243 static const struct cpu_parts cpu_parts_qcom[] = {
244 { CPU_PART_KRYO400_GOLD, "Kryo 400 Gold" },
245 { CPU_PART_KRYO400_SILVER, "Kryo 400 Silver" },
246 CPU_PART_NONE,
247 };
248
249 /* Apple */
250 static const struct cpu_parts cpu_parts_apple[] = {
251 { CPU_PART_M1_ICESTORM, "M1 Icestorm" },
252 { CPU_PART_M1_FIRESTORM, "M1 Firestorm" },
253 { CPU_PART_M1_ICESTORM_PRO, "M1 Pro Icestorm" },
254 { CPU_PART_M1_FIRESTORM_PRO, "M1 Pro Firestorm" },
255 { CPU_PART_M1_ICESTORM_MAX, "M1 Max Icestorm" },
256 { CPU_PART_M1_FIRESTORM_MAX, "M1 Max Firestorm" },
257 { CPU_PART_M2_BLIZZARD, "M2 Blizzard" },
258 { CPU_PART_M2_AVALANCHE, "M2 Avalanche" },
259 { CPU_PART_M2_BLIZZARD_PRO, "M2 Pro Blizzard" },
260 { CPU_PART_M2_AVALANCHE_PRO, "M2 Pro Avalanche" },
261 { CPU_PART_M2_BLIZZARD_MAX, "M2 Max Blizzard" },
262 { CPU_PART_M2_AVALANCHE_MAX, "M2 Max Avalanche" },
263 CPU_PART_NONE,
264 };
265
266 /* Unknown */
267 static const struct cpu_parts cpu_parts_none[] = {
268 CPU_PART_NONE,
269 };
270
271 /*
272 * Implementers table.
273 */
274 const struct cpu_implementers cpu_implementers[] = {
275 { CPU_IMPL_AMPERE, "Ampere", cpu_parts_none },
276 { CPU_IMPL_APPLE, "Apple", cpu_parts_apple },
277 { CPU_IMPL_APM, "APM", cpu_parts_apm },
278 { CPU_IMPL_ARM, "ARM", cpu_parts_arm },
279 { CPU_IMPL_BROADCOM, "Broadcom", cpu_parts_none },
280 { CPU_IMPL_CAVIUM, "Cavium", cpu_parts_cavium },
281 { CPU_IMPL_DEC, "DEC", cpu_parts_none },
282 { CPU_IMPL_FREESCALE, "Freescale", cpu_parts_none },
283 { CPU_IMPL_FUJITSU, "Fujitsu", cpu_parts_none },
284 { CPU_IMPL_INFINEON, "IFX", cpu_parts_none },
285 { CPU_IMPL_INTEL, "Intel", cpu_parts_none },
286 { CPU_IMPL_MARVELL, "Marvell", cpu_parts_none },
287 { CPU_IMPL_NVIDIA, "NVIDIA", cpu_parts_none },
288 { CPU_IMPL_QUALCOMM, "Qualcomm", cpu_parts_qcom },
289 CPU_IMPLEMENTER_NONE,
290 };
291
292 #define MRS_TYPE_MASK 0xf
293 #define MRS_TYPE_FBSD_SHIFT 0
294 #define MRS_TYPE_LNX_SHIFT 8
295 #define MRS_INVALID 0
296 #define MRS_EXACT 1
297 #define MRS_EXACT_VAL(x) (MRS_EXACT | ((x) << 4))
298 #define MRS_EXACT_FIELD(x) (((x) >> 4) & 0xf)
299 #define MRS_LOWER 2
300
301 struct mrs_field_value {
302 uint64_t value;
303 const char *desc;
304 };
305
306 #define MRS_FIELD_VALUE(_value, _desc) \
307 { \
308 .value = (_value), \
309 .desc = (_desc), \
310 }
311
312 #define MRS_FIELD_VALUE_NONE_IMPL(_reg, _field, _none, _impl) \
313 MRS_FIELD_VALUE(_reg ## _ ## _field ## _ ## _none, ""), \
314 MRS_FIELD_VALUE(_reg ## _ ## _field ## _ ## _impl, #_field)
315
316 #define MRS_FIELD_VALUE_COUNT(_reg, _field, _desc) \
317 MRS_FIELD_VALUE(0ul << _reg ## _ ## _field ## _SHIFT, "1 " _desc), \
318 MRS_FIELD_VALUE(1ul << _reg ## _ ## _field ## _SHIFT, "2 " _desc "s"), \
319 MRS_FIELD_VALUE(2ul << _reg ## _ ## _field ## _SHIFT, "3 " _desc "s"), \
320 MRS_FIELD_VALUE(3ul << _reg ## _ ## _field ## _SHIFT, "4 " _desc "s"), \
321 MRS_FIELD_VALUE(4ul << _reg ## _ ## _field ## _SHIFT, "5 " _desc "s"), \
322 MRS_FIELD_VALUE(5ul << _reg ## _ ## _field ## _SHIFT, "6 " _desc "s"), \
323 MRS_FIELD_VALUE(6ul << _reg ## _ ## _field ## _SHIFT, "7 " _desc "s"), \
324 MRS_FIELD_VALUE(7ul << _reg ## _ ## _field ## _SHIFT, "8 " _desc "s"), \
325 MRS_FIELD_VALUE(8ul << _reg ## _ ## _field ## _SHIFT, "9 " _desc "s"), \
326 MRS_FIELD_VALUE(9ul << _reg ## _ ## _field ## _SHIFT, "10 "_desc "s"), \
327 MRS_FIELD_VALUE(10ul<< _reg ## _ ## _field ## _SHIFT, "11 "_desc "s"), \
328 MRS_FIELD_VALUE(11ul<< _reg ## _ ## _field ## _SHIFT, "12 "_desc "s"), \
329 MRS_FIELD_VALUE(12ul<< _reg ## _ ## _field ## _SHIFT, "13 "_desc "s"), \
330 MRS_FIELD_VALUE(13ul<< _reg ## _ ## _field ## _SHIFT, "14 "_desc "s"), \
331 MRS_FIELD_VALUE(14ul<< _reg ## _ ## _field ## _SHIFT, "15 "_desc "s"), \
332 MRS_FIELD_VALUE(15ul<< _reg ## _ ## _field ## _SHIFT, "16 "_desc "s")
333
334 #define MRS_FIELD_VALUE_END { .desc = NULL }
335
336 struct mrs_field_hwcap {
337 uint64_t min;
338 u_long hwcap_val;
339 u_int hwcap_id;
340 };
341
342 #define MRS_HWCAP(_hwcap_id, _val, _min) \
343 { \
344 .hwcap_id = (_hwcap_id), \
345 .hwcap_val = (_val), \
346 .min = (_min), \
347 }
348
349 #define MRS_HWCAP_END { .hwcap_id = 0 }
350
351 struct mrs_field {
352 const char *name;
353 const struct mrs_field_value *values;
354 const struct mrs_field_hwcap *hwcaps;
355 uint64_t mask;
356 bool sign;
357 u_int type;
358 u_int shift;
359 };
360
361 #define MRS_FIELD_HWCAP_SPLIT(_register, _name, _sign, _fbsd_type, \
362 _lnx_type, _values, _hwcap) \
363 { \
364 .name = #_name, \
365 .sign = (_sign), \
366 .type = ((_fbsd_type) << MRS_TYPE_FBSD_SHIFT) | \
367 ((_lnx_type) << MRS_TYPE_LNX_SHIFT), \
368 .shift = _register ## _ ## _name ## _SHIFT, \
369 .mask = _register ## _ ## _name ## _MASK, \
370 .values = (_values), \
371 .hwcaps = (_hwcap), \
372 }
373
374 #define MRS_FIELD_HWCAP(_register, _name, _sign, _type, _values, _hwcap) \
375 MRS_FIELD_HWCAP_SPLIT(_register, _name, _sign, _type, _type, \
376 _values, _hwcap)
377
378 #define MRS_FIELD(_register, _name, _sign, _type, _values) \
379 MRS_FIELD_HWCAP(_register, _name, _sign, _type, _values, NULL)
380
381 #define MRS_FIELD_END { .type = MRS_INVALID, }
382
383 /* ID_AA64AFR0_EL1 */
384 static const struct mrs_field id_aa64afr0_fields[] = {
385 MRS_FIELD_END,
386 };
387
388
389 /* ID_AA64AFR1_EL1 */
390 static const struct mrs_field id_aa64afr1_fields[] = {
391 MRS_FIELD_END,
392 };
393
394
395 /* ID_AA64DFR0_EL1 */
396 static const struct mrs_field_value id_aa64dfr0_hpmn0[] = {
397 MRS_FIELD_VALUE_NONE_IMPL(ID_AA64DFR0, HPMN0, CONSTR, DEFINED),
398 MRS_FIELD_VALUE_END,
399 };
400
401 static const struct mrs_field_value id_aa64dfr0_brbe[] = {
402 MRS_FIELD_VALUE_NONE_IMPL(ID_AA64DFR0, BRBE, NONE, IMPL),
403 MRS_FIELD_VALUE(ID_AA64DFR0_BRBE_EL3, "BRBE EL3"),
404 MRS_FIELD_VALUE_END,
405 };
406
407 static const struct mrs_field_value id_aa64dfr0_mtpmu[] = {
408 MRS_FIELD_VALUE_NONE_IMPL(ID_AA64DFR0, MTPMU, NONE, IMPL),
409 MRS_FIELD_VALUE(ID_AA64DFR0_MTPMU_NONE_MT_RES0, "MTPMU res0"),
410 MRS_FIELD_VALUE_END,
411 };
412
413 static const struct mrs_field_value id_aa64dfr0_tracebuffer[] = {
414 MRS_FIELD_VALUE_NONE_IMPL(ID_AA64DFR0, TraceBuffer, NONE, IMPL),
415 MRS_FIELD_VALUE_END,
416 };
417
418 static const struct mrs_field_value id_aa64dfr0_tracefilt[] = {
419 MRS_FIELD_VALUE(ID_AA64DFR0_TraceFilt_NONE, ""),
420 MRS_FIELD_VALUE(ID_AA64DFR0_TraceFilt_8_4, "Trace v8.4"),
421 MRS_FIELD_VALUE_END,
422 };
423
424 static const struct mrs_field_value id_aa64dfr0_doublelock[] = {
425 MRS_FIELD_VALUE(ID_AA64DFR0_DoubleLock_IMPL, "DoubleLock"),
426 MRS_FIELD_VALUE(ID_AA64DFR0_DoubleLock_NONE, ""),
427 MRS_FIELD_VALUE_END,
428 };
429
430 static const struct mrs_field_value id_aa64dfr0_pmsver[] = {
431 MRS_FIELD_VALUE(ID_AA64DFR0_PMSVer_NONE, ""),
432 MRS_FIELD_VALUE(ID_AA64DFR0_PMSVer_SPE, "SPE"),
433 MRS_FIELD_VALUE(ID_AA64DFR0_PMSVer_SPE_1_1, "SPEv1p1"),
434 MRS_FIELD_VALUE(ID_AA64DFR0_PMSVer_SPE_1_2, "SPEv1p2"),
435 MRS_FIELD_VALUE(ID_AA64DFR0_PMSVer_SPE_1_3, "SPEv1p3"),
436 MRS_FIELD_VALUE_END,
437 };
438
439 static const struct mrs_field_value id_aa64dfr0_ctx_cmps[] = {
440 MRS_FIELD_VALUE_COUNT(ID_AA64DFR0, CTX_CMPs, "CTX BKPT"),
441 MRS_FIELD_VALUE_END,
442 };
443
444 static const struct mrs_field_value id_aa64dfr0_wrps[] = {
445 MRS_FIELD_VALUE_COUNT(ID_AA64DFR0, WRPs, "Watchpoint"),
446 MRS_FIELD_VALUE_END,
447 };
448
449 static const struct mrs_field_value id_aa64dfr0_brps[] = {
450 MRS_FIELD_VALUE_COUNT(ID_AA64DFR0, BRPs, "Breakpoint"),
451 MRS_FIELD_VALUE_END,
452 };
453
454 static const struct mrs_field_value id_aa64dfr0_pmuver[] = {
455 MRS_FIELD_VALUE(ID_AA64DFR0_PMUVer_NONE, ""),
456 MRS_FIELD_VALUE(ID_AA64DFR0_PMUVer_3, "PMUv3"),
457 MRS_FIELD_VALUE(ID_AA64DFR0_PMUVer_3_1, "PMUv3p1"),
458 MRS_FIELD_VALUE(ID_AA64DFR0_PMUVer_3_4, "PMUv3p4"),
459 MRS_FIELD_VALUE(ID_AA64DFR0_PMUVer_3_5, "PMUv3p5"),
460 MRS_FIELD_VALUE(ID_AA64DFR0_PMUVer_3_7, "PMUv3p7"),
461 MRS_FIELD_VALUE(ID_AA64DFR0_PMUVer_3_8, "PMUv3p8"),
462 MRS_FIELD_VALUE(ID_AA64DFR0_PMUVer_IMPL, "IMPL PMU"),
463 MRS_FIELD_VALUE_END,
464 };
465
466 static const struct mrs_field_value id_aa64dfr0_tracever[] = {
467 MRS_FIELD_VALUE(ID_AA64DFR0_TraceVer_NONE, ""),
468 MRS_FIELD_VALUE(ID_AA64DFR0_TraceVer_IMPL, "Trace"),
469 MRS_FIELD_VALUE_END,
470 };
471
472 static const struct mrs_field_value id_aa64dfr0_debugver[] = {
473 MRS_FIELD_VALUE(ID_AA64DFR0_DebugVer_8, "Debugv8"),
474 MRS_FIELD_VALUE(ID_AA64DFR0_DebugVer_8_VHE, "Debugv8_VHE"),
475 MRS_FIELD_VALUE(ID_AA64DFR0_DebugVer_8_2, "Debugv8p2"),
476 MRS_FIELD_VALUE(ID_AA64DFR0_DebugVer_8_4, "Debugv8p4"),
477 MRS_FIELD_VALUE(ID_AA64DFR0_DebugVer_8_8, "Debugv8p8"),
478 MRS_FIELD_VALUE_END,
479 };
480
481 static const struct mrs_field id_aa64dfr0_fields[] = {
482 MRS_FIELD(ID_AA64DFR0, HPMN0, false, MRS_EXACT, id_aa64dfr0_hpmn0),
483 MRS_FIELD(ID_AA64DFR0, BRBE, false, MRS_EXACT, id_aa64dfr0_brbe),
484 MRS_FIELD(ID_AA64DFR0, MTPMU, true, MRS_EXACT, id_aa64dfr0_mtpmu),
485 MRS_FIELD(ID_AA64DFR0, TraceBuffer, false, MRS_EXACT,
486 id_aa64dfr0_tracebuffer),
487 MRS_FIELD(ID_AA64DFR0, TraceFilt, false, MRS_EXACT,
488 id_aa64dfr0_tracefilt),
489 MRS_FIELD(ID_AA64DFR0, DoubleLock, false, MRS_EXACT,
490 id_aa64dfr0_doublelock),
491 MRS_FIELD(ID_AA64DFR0, PMSVer, false, MRS_EXACT, id_aa64dfr0_pmsver),
492 MRS_FIELD(ID_AA64DFR0, CTX_CMPs, false, MRS_EXACT,
493 id_aa64dfr0_ctx_cmps),
494 MRS_FIELD(ID_AA64DFR0, WRPs, false, MRS_LOWER, id_aa64dfr0_wrps),
495 MRS_FIELD(ID_AA64DFR0, BRPs, false, MRS_LOWER, id_aa64dfr0_brps),
496 MRS_FIELD(ID_AA64DFR0, PMUVer, false, MRS_EXACT, id_aa64dfr0_pmuver),
497 MRS_FIELD(ID_AA64DFR0, TraceVer, false, MRS_EXACT,
498 id_aa64dfr0_tracever),
499 MRS_FIELD(ID_AA64DFR0, DebugVer, false, MRS_EXACT_VAL(0x6),
500 id_aa64dfr0_debugver),
501 MRS_FIELD_END,
502 };
503
504
505 /* ID_AA64DFR1_EL1 */
506 static const struct mrs_field id_aa64dfr1_fields[] = {
507 MRS_FIELD_END,
508 };
509
510
511 /* ID_AA64ISAR0_EL1 */
512 static const struct mrs_field_value id_aa64isar0_rndr[] = {
513 MRS_FIELD_VALUE(ID_AA64ISAR0_RNDR_NONE, ""),
514 MRS_FIELD_VALUE(ID_AA64ISAR0_RNDR_IMPL, "RNG"),
515 MRS_FIELD_VALUE_END,
516 };
517
518 static const struct mrs_field_hwcap id_aa64isar0_rndr_caps[] = {
519 MRS_HWCAP(2, HWCAP2_RNG, ID_AA64ISAR0_RNDR_IMPL),
520 MRS_HWCAP_END
521 };
522
523 static const struct mrs_field_value id_aa64isar0_tlb[] = {
524 MRS_FIELD_VALUE(ID_AA64ISAR0_TLB_NONE, ""),
525 MRS_FIELD_VALUE(ID_AA64ISAR0_TLB_TLBIOS, "TLBI-OS"),
526 MRS_FIELD_VALUE(ID_AA64ISAR0_TLB_TLBIOSR, "TLBI-OSR"),
527 MRS_FIELD_VALUE_END,
528 };
529
530 static const struct mrs_field_value id_aa64isar0_ts[] = {
531 MRS_FIELD_VALUE(ID_AA64ISAR0_TS_NONE, ""),
532 MRS_FIELD_VALUE(ID_AA64ISAR0_TS_CondM_8_4, "CondM-8.4"),
533 MRS_FIELD_VALUE(ID_AA64ISAR0_TS_CondM_8_5, "CondM-8.5"),
534 MRS_FIELD_VALUE_END,
535 };
536
537 static const struct mrs_field_hwcap id_aa64isar0_ts_caps[] = {
538 MRS_HWCAP(1, HWCAP_FLAGM, ID_AA64ISAR0_TS_CondM_8_4),
539 MRS_HWCAP(2, HWCAP2_FLAGM2, ID_AA64ISAR0_TS_CondM_8_5),
540 MRS_HWCAP_END
541 };
542
543 static const struct mrs_field_value id_aa64isar0_fhm[] = {
544 MRS_FIELD_VALUE_NONE_IMPL(ID_AA64ISAR0, FHM, NONE, IMPL),
545 MRS_FIELD_VALUE_END,
546 };
547
548 static const struct mrs_field_hwcap id_aa64isar0_fhm_caps[] = {
549 MRS_HWCAP(1, HWCAP_ASIMDFHM, ID_AA64ISAR0_FHM_IMPL),
550 MRS_HWCAP_END
551 };
552
553 static const struct mrs_field_value id_aa64isar0_dp[] = {
554 MRS_FIELD_VALUE_NONE_IMPL(ID_AA64ISAR0, DP, NONE, IMPL),
555 MRS_FIELD_VALUE_END,
556 };
557
558 static const struct mrs_field_hwcap id_aa64isar0_dp_caps[] = {
559 MRS_HWCAP(1, HWCAP_ASIMDDP, ID_AA64ISAR0_DP_IMPL),
560 MRS_HWCAP_END
561 };
562
563 static const struct mrs_field_value id_aa64isar0_sm4[] = {
564 MRS_FIELD_VALUE_NONE_IMPL(ID_AA64ISAR0, SM4, NONE, IMPL),
565 MRS_FIELD_VALUE_END,
566 };
567
568 static const struct mrs_field_hwcap id_aa64isar0_sm4_caps[] = {
569 MRS_HWCAP(1, HWCAP_SM4, ID_AA64ISAR0_SM4_IMPL),
570 MRS_HWCAP_END
571 };
572
573 static const struct mrs_field_value id_aa64isar0_sm3[] = {
574 MRS_FIELD_VALUE_NONE_IMPL(ID_AA64ISAR0, SM3, NONE, IMPL),
575 MRS_FIELD_VALUE_END,
576 };
577
578 static const struct mrs_field_hwcap id_aa64isar0_sm3_caps[] = {
579 MRS_HWCAP(1, HWCAP_SM3, ID_AA64ISAR0_SM3_IMPL),
580 MRS_HWCAP_END
581 };
582
583 static const struct mrs_field_value id_aa64isar0_sha3[] = {
584 MRS_FIELD_VALUE_NONE_IMPL(ID_AA64ISAR0, SHA3, NONE, IMPL),
585 MRS_FIELD_VALUE_END,
586 };
587
588 static const struct mrs_field_hwcap id_aa64isar0_sha3_caps[] = {
589 MRS_HWCAP(1, HWCAP_SHA3, ID_AA64ISAR0_SHA3_IMPL),
590 MRS_HWCAP_END
591 };
592
593 static const struct mrs_field_value id_aa64isar0_rdm[] = {
594 MRS_FIELD_VALUE_NONE_IMPL(ID_AA64ISAR0, RDM, NONE, IMPL),
595 MRS_FIELD_VALUE_END,
596 };
597
598 static const struct mrs_field_hwcap id_aa64isar0_rdm_caps[] = {
599 MRS_HWCAP(1, HWCAP_ASIMDRDM, ID_AA64ISAR0_RDM_IMPL),
600 MRS_HWCAP_END
601 };
602
603 static const struct mrs_field_value id_aa64isar0_tme[] = {
604 MRS_FIELD_VALUE_NONE_IMPL(ID_AA64ISAR0, TME, NONE, IMPL),
605 MRS_FIELD_VALUE_END,
606 };
607
608 static const struct mrs_field_value id_aa64isar0_atomic[] = {
609 MRS_FIELD_VALUE_NONE_IMPL(ID_AA64ISAR0, Atomic, NONE, IMPL),
610 MRS_FIELD_VALUE_END,
611 };
612
613 static const struct mrs_field_hwcap id_aa64isar0_atomic_caps[] = {
614 MRS_HWCAP(1, HWCAP_ATOMICS, ID_AA64ISAR0_Atomic_IMPL),
615 MRS_HWCAP_END
616 };
617
618 static const struct mrs_field_value id_aa64isar0_crc32[] = {
619 MRS_FIELD_VALUE_NONE_IMPL(ID_AA64ISAR0, CRC32, NONE, BASE),
620 MRS_FIELD_VALUE_END,
621 };
622
623 static const struct mrs_field_hwcap id_aa64isar0_crc32_caps[] = {
624 MRS_HWCAP(1, HWCAP_CRC32, ID_AA64ISAR0_CRC32_BASE),
625 MRS_HWCAP_END
626 };
627
628 static const struct mrs_field_value id_aa64isar0_sha2[] = {
629 MRS_FIELD_VALUE_NONE_IMPL(ID_AA64ISAR0, SHA2, NONE, BASE),
630 MRS_FIELD_VALUE(ID_AA64ISAR0_SHA2_512, "SHA2+SHA512"),
631 MRS_FIELD_VALUE_END,
632 };
633
634 static const struct mrs_field_hwcap id_aa64isar0_sha2_caps[] = {
635 MRS_HWCAP(1, HWCAP_SHA2, ID_AA64ISAR0_SHA2_BASE),
636 MRS_HWCAP(1, HWCAP_SHA512, ID_AA64ISAR0_SHA2_512),
637 MRS_HWCAP_END
638 };
639
640 static const struct mrs_field_value id_aa64isar0_sha1[] = {
641 MRS_FIELD_VALUE_NONE_IMPL(ID_AA64ISAR0, SHA1, NONE, BASE),
642 MRS_FIELD_VALUE_END,
643 };
644
645 static const struct mrs_field_hwcap id_aa64isar0_sha1_caps[] = {
646 MRS_HWCAP(1, HWCAP_SHA1, ID_AA64ISAR0_SHA1_BASE),
647 MRS_HWCAP_END
648 };
649
650 static const struct mrs_field_value id_aa64isar0_aes[] = {
651 MRS_FIELD_VALUE_NONE_IMPL(ID_AA64ISAR0, AES, NONE, BASE),
652 MRS_FIELD_VALUE(ID_AA64ISAR0_AES_PMULL, "AES+PMULL"),
653 MRS_FIELD_VALUE_END,
654 };
655
656 static const struct mrs_field_hwcap id_aa64isar0_aes_caps[] = {
657 MRS_HWCAP(1, HWCAP_AES, ID_AA64ISAR0_AES_BASE),
658 MRS_HWCAP(1, HWCAP_PMULL, ID_AA64ISAR0_AES_PMULL),
659 MRS_HWCAP_END
660 };
661
662 static const struct mrs_field id_aa64isar0_fields[] = {
663 MRS_FIELD_HWCAP(ID_AA64ISAR0, RNDR, false, MRS_LOWER,
664 id_aa64isar0_rndr, id_aa64isar0_rndr_caps),
665 MRS_FIELD(ID_AA64ISAR0, TLB, false, MRS_EXACT, id_aa64isar0_tlb),
666 MRS_FIELD_HWCAP(ID_AA64ISAR0, TS, false, MRS_LOWER, id_aa64isar0_ts,
667 id_aa64isar0_ts_caps),
668 MRS_FIELD_HWCAP(ID_AA64ISAR0, FHM, false, MRS_LOWER, id_aa64isar0_fhm,
669 id_aa64isar0_fhm_caps),
670 MRS_FIELD_HWCAP(ID_AA64ISAR0, DP, false, MRS_LOWER, id_aa64isar0_dp,
671 id_aa64isar0_dp_caps),
672 MRS_FIELD_HWCAP(ID_AA64ISAR0, SM4, false, MRS_LOWER, id_aa64isar0_sm4,
673 id_aa64isar0_sm4_caps),
674 MRS_FIELD_HWCAP(ID_AA64ISAR0, SM3, false, MRS_LOWER, id_aa64isar0_sm3,
675 id_aa64isar0_sm3_caps),
676 MRS_FIELD_HWCAP(ID_AA64ISAR0, SHA3, false, MRS_LOWER, id_aa64isar0_sha3,
677 id_aa64isar0_sha3_caps),
678 MRS_FIELD_HWCAP(ID_AA64ISAR0, RDM, false, MRS_LOWER, id_aa64isar0_rdm,
679 id_aa64isar0_rdm_caps),
680 MRS_FIELD(ID_AA64ISAR0, TME, false, MRS_EXACT, id_aa64isar0_tme),
681 MRS_FIELD_HWCAP(ID_AA64ISAR0, Atomic, false, MRS_LOWER,
682 id_aa64isar0_atomic, id_aa64isar0_atomic_caps),
683 MRS_FIELD_HWCAP(ID_AA64ISAR0, CRC32, false, MRS_LOWER,
684 id_aa64isar0_crc32, id_aa64isar0_crc32_caps),
685 MRS_FIELD_HWCAP(ID_AA64ISAR0, SHA2, false, MRS_LOWER, id_aa64isar0_sha2,
686 id_aa64isar0_sha2_caps),
687 MRS_FIELD_HWCAP(ID_AA64ISAR0, SHA1, false, MRS_LOWER,
688 id_aa64isar0_sha1, id_aa64isar0_sha1_caps),
689 MRS_FIELD_HWCAP(ID_AA64ISAR0, AES, false, MRS_LOWER, id_aa64isar0_aes,
690 id_aa64isar0_aes_caps),
691 MRS_FIELD_END,
692 };
693
694
695 /* ID_AA64ISAR1_EL1 */
696 static const struct mrs_field_value id_aa64isar1_ls64[] = {
697 MRS_FIELD_VALUE_NONE_IMPL(ID_AA64ISAR1, LS64, NONE, IMPL),
698 MRS_FIELD_VALUE(ID_AA64ISAR1_LS64_V, "LS64v"),
699 MRS_FIELD_VALUE(ID_AA64ISAR1_LS64_ACCDATA, "LS64+ACCDATA"),
700 MRS_FIELD_VALUE_END,
701 };
702
703 static const struct mrs_field_value id_aa64isar1_xs[] = {
704 MRS_FIELD_VALUE_NONE_IMPL(ID_AA64ISAR1, XS, NONE, IMPL),
705 MRS_FIELD_VALUE_END,
706 };
707
708 static const struct mrs_field_value id_aa64isar1_i8mm[] = {
709 MRS_FIELD_VALUE_NONE_IMPL(ID_AA64ISAR1, I8MM, NONE, IMPL),
710 MRS_FIELD_VALUE_END,
711 };
712
713 static const struct mrs_field_hwcap id_aa64isar1_i8mm_caps[] = {
714 MRS_HWCAP(2, HWCAP2_I8MM, ID_AA64ISAR1_I8MM_IMPL),
715 MRS_HWCAP_END
716 };
717
718 static const struct mrs_field_value id_aa64isar1_dgh[] = {
719 MRS_FIELD_VALUE_NONE_IMPL(ID_AA64ISAR1, DGH, NONE, IMPL),
720 MRS_FIELD_VALUE_END,
721 };
722
723 static const struct mrs_field_hwcap id_aa64isar1_dgh_caps[] = {
724 MRS_HWCAP(2, HWCAP2_DGH, ID_AA64ISAR1_DGH_IMPL),
725 MRS_HWCAP_END
726 };
727
728 static const struct mrs_field_value id_aa64isar1_bf16[] = {
729 MRS_FIELD_VALUE_NONE_IMPL(ID_AA64ISAR1, BF16, NONE, IMPL),
730 MRS_FIELD_VALUE(ID_AA64ISAR1_BF16_EBF, "EBF16"),
731 MRS_FIELD_VALUE_END,
732 };
733
734 static const struct mrs_field_hwcap id_aa64isar1_bf16_caps[] = {
735 MRS_HWCAP(2, HWCAP2_BF16, ID_AA64ISAR1_BF16_IMPL),
736 MRS_HWCAP_END
737 };
738
739 static const struct mrs_field_value id_aa64isar1_specres[] = {
740 MRS_FIELD_VALUE(ID_AA64ISAR1_SPECRES_NONE, ""),
741 MRS_FIELD_VALUE(ID_AA64ISAR1_SPECRES_IMPL, "PredInv"),
742 MRS_FIELD_VALUE_END,
743 };
744
745 static const struct mrs_field_value id_aa64isar1_sb[] = {
746 MRS_FIELD_VALUE_NONE_IMPL(ID_AA64ISAR1, SB, NONE, IMPL),
747 MRS_FIELD_VALUE_END,
748 };
749
750 static const struct mrs_field_hwcap id_aa64isar1_sb_caps[] = {
751 MRS_HWCAP(1, HWCAP_SB, ID_AA64ISAR1_SB_IMPL),
752 MRS_HWCAP_END
753 };
754
755 static const struct mrs_field_value id_aa64isar1_frintts[] = {
756 MRS_FIELD_VALUE_NONE_IMPL(ID_AA64ISAR1, FRINTTS, NONE, IMPL),
757 MRS_FIELD_VALUE_END,
758 };
759
760 static const struct mrs_field_hwcap id_aa64isar1_frintts_caps[] = {
761 MRS_HWCAP(2, HWCAP2_FRINT, ID_AA64ISAR1_FRINTTS_IMPL),
762 MRS_HWCAP_END
763 };
764
765 static const struct mrs_field_value id_aa64isar1_gpi[] = {
766 MRS_FIELD_VALUE_NONE_IMPL(ID_AA64ISAR1, GPI, NONE, IMPL),
767 MRS_FIELD_VALUE_END,
768 };
769
770 static const struct mrs_field_hwcap id_aa64isar1_gpi_caps[] = {
771 MRS_HWCAP(1, HWCAP_PACG, ID_AA64ISAR1_GPI_IMPL),
772 MRS_HWCAP_END
773 };
774
775 static const struct mrs_field_value id_aa64isar1_gpa[] = {
776 MRS_FIELD_VALUE_NONE_IMPL(ID_AA64ISAR1, GPA, NONE, IMPL),
777 MRS_FIELD_VALUE_END,
778 };
779
780 static const struct mrs_field_hwcap id_aa64isar1_gpa_caps[] = {
781 MRS_HWCAP(1, HWCAP_PACG, ID_AA64ISAR1_GPA_IMPL),
782 MRS_HWCAP_END
783 };
784
785 static const struct mrs_field_value id_aa64isar1_lrcpc[] = {
786 MRS_FIELD_VALUE(ID_AA64ISAR1_LRCPC_NONE, ""),
787 MRS_FIELD_VALUE(ID_AA64ISAR1_LRCPC_RCPC_8_3, "RCPC-8.3"),
788 MRS_FIELD_VALUE(ID_AA64ISAR1_LRCPC_RCPC_8_4, "RCPC-8.4"),
789 MRS_FIELD_VALUE_END,
790 };
791
792 static const struct mrs_field_hwcap id_aa64isar1_lrcpc_caps[] = {
793 MRS_HWCAP(1, HWCAP_LRCPC, ID_AA64ISAR1_LRCPC_RCPC_8_3),
794 MRS_HWCAP(1, HWCAP_ILRCPC, ID_AA64ISAR1_LRCPC_RCPC_8_4),
795 MRS_HWCAP_END
796 };
797
798 static const struct mrs_field_value id_aa64isar1_fcma[] = {
799 MRS_FIELD_VALUE_NONE_IMPL(ID_AA64ISAR1, FCMA, NONE, IMPL),
800 MRS_FIELD_VALUE_END,
801 };
802
803 static const struct mrs_field_hwcap id_aa64isar1_fcma_caps[] = {
804 MRS_HWCAP(1, HWCAP_FCMA, ID_AA64ISAR1_FCMA_IMPL),
805 MRS_HWCAP_END
806 };
807
808 static const struct mrs_field_value id_aa64isar1_jscvt[] = {
809 MRS_FIELD_VALUE_NONE_IMPL(ID_AA64ISAR1, JSCVT, NONE, IMPL),
810 MRS_FIELD_VALUE_END,
811 };
812
813 static const struct mrs_field_hwcap id_aa64isar1_jscvt_caps[] = {
814 MRS_HWCAP(1, HWCAP_JSCVT, ID_AA64ISAR1_JSCVT_IMPL),
815 MRS_HWCAP_END
816 };
817
818 static const struct mrs_field_value id_aa64isar1_api[] = {
819 MRS_FIELD_VALUE(ID_AA64ISAR1_API_NONE, ""),
820 MRS_FIELD_VALUE(ID_AA64ISAR1_API_PAC, "API PAC"),
821 MRS_FIELD_VALUE(ID_AA64ISAR1_API_EPAC, "API EPAC"),
822 MRS_FIELD_VALUE(ID_AA64ISAR1_API_EPAC2, "Impl PAuth+EPAC2"),
823 MRS_FIELD_VALUE(ID_AA64ISAR1_API_FPAC, "Impl PAuth+FPAC"),
824 MRS_FIELD_VALUE(ID_AA64ISAR1_API_FPAC_COMBINED,
825 "Impl PAuth+FPAC+Combined"),
826 MRS_FIELD_VALUE_END,
827 };
828
829 static const struct mrs_field_hwcap id_aa64isar1_api_caps[] = {
830 MRS_HWCAP(1, HWCAP_PACA, ID_AA64ISAR1_API_PAC),
831 MRS_HWCAP_END
832 };
833
834 static const struct mrs_field_value id_aa64isar1_apa[] = {
835 MRS_FIELD_VALUE(ID_AA64ISAR1_APA_NONE, ""),
836 MRS_FIELD_VALUE(ID_AA64ISAR1_APA_PAC, "APA PAC"),
837 MRS_FIELD_VALUE(ID_AA64ISAR1_APA_EPAC, "APA EPAC"),
838 MRS_FIELD_VALUE(ID_AA64ISAR1_APA_EPAC2, "APA EPAC2"),
839 MRS_FIELD_VALUE(ID_AA64ISAR1_APA_FPAC, "APA FPAC"),
840 MRS_FIELD_VALUE(ID_AA64ISAR1_APA_FPAC_COMBINED,
841 "APA FPAC+Combined"),
842 MRS_FIELD_VALUE_END,
843 };
844
845 static const struct mrs_field_hwcap id_aa64isar1_apa_caps[] = {
846 MRS_HWCAP(1, HWCAP_PACA, ID_AA64ISAR1_APA_PAC),
847 MRS_HWCAP_END
848 };
849
850 static const struct mrs_field_value id_aa64isar1_dpb[] = {
851 MRS_FIELD_VALUE(ID_AA64ISAR1_DPB_NONE, ""),
852 MRS_FIELD_VALUE(ID_AA64ISAR1_DPB_DCCVAP, "DCPoP"),
853 MRS_FIELD_VALUE(ID_AA64ISAR1_DPB_DCCVADP, "DCCVADP"),
854 MRS_FIELD_VALUE_END,
855 };
856
857 static const struct mrs_field_hwcap id_aa64isar1_dpb_caps[] = {
858 MRS_HWCAP(1, HWCAP_DCPOP, ID_AA64ISAR1_DPB_DCCVAP),
859 MRS_HWCAP(2, HWCAP2_DCPODP, ID_AA64ISAR1_DPB_DCCVADP),
860 MRS_HWCAP_END
861 };
862
863 static const struct mrs_field id_aa64isar1_fields[] = {
864 MRS_FIELD(ID_AA64ISAR1, LS64, false, MRS_EXACT, id_aa64isar1_ls64),
865 MRS_FIELD(ID_AA64ISAR1, XS, false, MRS_EXACT, id_aa64isar1_xs),
866 MRS_FIELD_HWCAP(ID_AA64ISAR1, I8MM, false, MRS_LOWER,
867 id_aa64isar1_i8mm, id_aa64isar1_i8mm_caps),
868 MRS_FIELD_HWCAP(ID_AA64ISAR1, DGH, false, MRS_LOWER, id_aa64isar1_dgh,
869 id_aa64isar1_dgh_caps),
870 MRS_FIELD_HWCAP(ID_AA64ISAR1, BF16, false, MRS_LOWER,
871 id_aa64isar1_bf16, id_aa64isar1_bf16_caps),
872 MRS_FIELD(ID_AA64ISAR1, SPECRES, false, MRS_EXACT,
873 id_aa64isar1_specres),
874 MRS_FIELD_HWCAP(ID_AA64ISAR1, SB, false, MRS_LOWER, id_aa64isar1_sb,
875 id_aa64isar1_sb_caps),
876 MRS_FIELD_HWCAP(ID_AA64ISAR1, FRINTTS, false, MRS_LOWER,
877 id_aa64isar1_frintts, id_aa64isar1_frintts_caps),
878 MRS_FIELD_HWCAP(ID_AA64ISAR1, GPI, false, MRS_EXACT, id_aa64isar1_gpi,
879 id_aa64isar1_gpi_caps),
880 MRS_FIELD_HWCAP(ID_AA64ISAR1, GPA, false, MRS_EXACT, id_aa64isar1_gpa,
881 id_aa64isar1_gpa_caps),
882 MRS_FIELD_HWCAP(ID_AA64ISAR1, LRCPC, false, MRS_LOWER,
883 id_aa64isar1_lrcpc, id_aa64isar1_lrcpc_caps),
884 MRS_FIELD_HWCAP(ID_AA64ISAR1, FCMA, false, MRS_LOWER,
885 id_aa64isar1_fcma, id_aa64isar1_fcma_caps),
886 MRS_FIELD_HWCAP(ID_AA64ISAR1, JSCVT, false, MRS_LOWER,
887 id_aa64isar1_jscvt, id_aa64isar1_jscvt_caps),
888 MRS_FIELD_HWCAP(ID_AA64ISAR1, API, false, MRS_EXACT, id_aa64isar1_api,
889 id_aa64isar1_api_caps),
890 MRS_FIELD_HWCAP(ID_AA64ISAR1, APA, false, MRS_EXACT, id_aa64isar1_apa,
891 id_aa64isar1_apa_caps),
892 MRS_FIELD_HWCAP(ID_AA64ISAR1, DPB, false, MRS_LOWER, id_aa64isar1_dpb,
893 id_aa64isar1_dpb_caps),
894 MRS_FIELD_END,
895 };
896
897
898 /* ID_AA64ISAR2_EL1 */
899 static const struct mrs_field_value id_aa64isar2_pac_frac[] = {
900 MRS_FIELD_VALUE_NONE_IMPL(ID_AA64ISAR2, PAC_frac, NONE, IMPL),
901 MRS_FIELD_VALUE_END,
902 };
903
904 static const struct mrs_field_value id_aa64isar2_bc[] = {
905 MRS_FIELD_VALUE_NONE_IMPL(ID_AA64ISAR2, BC, NONE, IMPL),
906 MRS_FIELD_VALUE_END,
907 };
908
909 static const struct mrs_field_value id_aa64isar2_mops[] = {
910 MRS_FIELD_VALUE_NONE_IMPL(ID_AA64ISAR2, MOPS, NONE, IMPL),
911 MRS_FIELD_VALUE_END,
912 };
913
914 static const struct mrs_field_value id_aa64isar2_apa3[] = {
915 MRS_FIELD_VALUE(ID_AA64ISAR2_APA3_NONE, ""),
916 MRS_FIELD_VALUE(ID_AA64ISAR2_APA3_PAC, "APA3 PAC"),
917 MRS_FIELD_VALUE(ID_AA64ISAR2_APA3_EPAC, "APA3 EPAC"),
918 MRS_FIELD_VALUE(ID_AA64ISAR2_APA3_EPAC2, "APA3 EPAC2"),
919 MRS_FIELD_VALUE(ID_AA64ISAR2_APA3_FPAC, "APA3 FPAC"),
920 MRS_FIELD_VALUE(ID_AA64ISAR2_APA3_FPAC_COMBINED,
921 "APA3 FPAC+Combined"),
922 MRS_FIELD_VALUE_END,
923 };
924
925 static const struct mrs_field_hwcap id_aa64isar2_apa3_caps[] = {
926 MRS_HWCAP(1, HWCAP_PACA, ID_AA64ISAR2_APA3_PAC),
927 MRS_HWCAP_END
928 };
929
930 static const struct mrs_field_value id_aa64isar2_gpa3[] = {
931 MRS_FIELD_VALUE_NONE_IMPL(ID_AA64ISAR2, GPA3, NONE, IMPL),
932 MRS_FIELD_VALUE_END,
933 };
934
935 static const struct mrs_field_hwcap id_aa64isar2_gpa3_caps[] = {
936 MRS_HWCAP(1, HWCAP_PACG, ID_AA64ISAR2_GPA3_IMPL),
937 MRS_HWCAP_END
938 };
939
940 static const struct mrs_field_value id_aa64isar2_rpres[] = {
941 MRS_FIELD_VALUE_NONE_IMPL(ID_AA64ISAR2, RPRES, NONE, IMPL),
942 MRS_FIELD_VALUE_END,
943 };
944
945 static const struct mrs_field_value id_aa64isar2_wfxt[] = {
946 MRS_FIELD_VALUE_NONE_IMPL(ID_AA64ISAR2, WFxT, NONE, IMPL),
947 MRS_FIELD_VALUE_END,
948 };
949
950 static const struct mrs_field id_aa64isar2_fields[] = {
951 MRS_FIELD(ID_AA64ISAR2, PAC_frac, false, MRS_EXACT,
952 id_aa64isar2_pac_frac),
953 MRS_FIELD(ID_AA64ISAR2, BC, false, MRS_EXACT, id_aa64isar2_bc),
954 MRS_FIELD(ID_AA64ISAR2, MOPS, false, MRS_EXACT, id_aa64isar2_mops),
955 MRS_FIELD_HWCAP(ID_AA64ISAR2, APA3, false, MRS_EXACT,
956 id_aa64isar2_apa3, id_aa64isar2_apa3_caps),
957 MRS_FIELD_HWCAP(ID_AA64ISAR2, GPA3, false, MRS_EXACT,
958 id_aa64isar2_gpa3, id_aa64isar2_gpa3_caps),
959 MRS_FIELD(ID_AA64ISAR2, RPRES, false, MRS_EXACT, id_aa64isar2_rpres),
960 MRS_FIELD(ID_AA64ISAR2, WFxT, false, MRS_EXACT, id_aa64isar2_wfxt),
961 MRS_FIELD_END,
962 };
963
964
965 /* ID_AA64MMFR0_EL1 */
966 static const struct mrs_field_value id_aa64mmfr0_ecv[] = {
967 MRS_FIELD_VALUE_NONE_IMPL(ID_AA64MMFR0, ECV, NONE, IMPL),
968 MRS_FIELD_VALUE(ID_AA64MMFR0_ECV_CNTHCTL, "ECV+CNTHCTL"),
969 MRS_FIELD_VALUE_END,
970 };
971
972 static const struct mrs_field_value id_aa64mmfr0_fgt[] = {
973 MRS_FIELD_VALUE_NONE_IMPL(ID_AA64MMFR0, FGT, NONE, IMPL),
974 MRS_FIELD_VALUE_END,
975 };
976
977 static const struct mrs_field_value id_aa64mmfr0_exs[] = {
978 MRS_FIELD_VALUE_NONE_IMPL(ID_AA64MMFR0, ExS, ALL, IMPL),
979 MRS_FIELD_VALUE_END,
980 };
981
982 static const struct mrs_field_value id_aa64mmfr0_tgran4_2[] = {
983 MRS_FIELD_VALUE(ID_AA64MMFR0_TGran4_2_TGran4, ""),
984 MRS_FIELD_VALUE(ID_AA64MMFR0_TGran4_2_NONE, "No S2 TGran4"),
985 MRS_FIELD_VALUE(ID_AA64MMFR0_TGran4_2_IMPL, "S2 TGran4"),
986 MRS_FIELD_VALUE(ID_AA64MMFR0_TGran4_2_LPA2, "S2 TGran4+LPA2"),
987 MRS_FIELD_VALUE_END,
988 };
989
990 static const struct mrs_field_value id_aa64mmfr0_tgran64_2[] = {
991 MRS_FIELD_VALUE(ID_AA64MMFR0_TGran64_2_TGran64, ""),
992 MRS_FIELD_VALUE(ID_AA64MMFR0_TGran64_2_NONE, "No S2 TGran64"),
993 MRS_FIELD_VALUE(ID_AA64MMFR0_TGran64_2_IMPL, "S2 TGran64"),
994 MRS_FIELD_VALUE_END,
995 };
996
997 static const struct mrs_field_value id_aa64mmfr0_tgran16_2[] = {
998 MRS_FIELD_VALUE(ID_AA64MMFR0_TGran16_2_TGran16, ""),
999 MRS_FIELD_VALUE(ID_AA64MMFR0_TGran16_2_NONE, "No S2 TGran16"),
1000 MRS_FIELD_VALUE(ID_AA64MMFR0_TGran16_2_IMPL, "S2 TGran16"),
1001 MRS_FIELD_VALUE(ID_AA64MMFR0_TGran16_2_LPA2, "S2 TGran16+LPA2"),
1002 MRS_FIELD_VALUE_END,
1003 };
1004
1005 static const struct mrs_field_value id_aa64mmfr0_tgran4[] = {
1006 MRS_FIELD_VALUE_NONE_IMPL(ID_AA64MMFR0, TGran4, NONE, IMPL),
1007 MRS_FIELD_VALUE(ID_AA64MMFR0_TGran4_LPA2, "TGran4+LPA2"),
1008 MRS_FIELD_VALUE_END,
1009 };
1010
1011 static const struct mrs_field_value id_aa64mmfr0_tgran64[] = {
1012 MRS_FIELD_VALUE_NONE_IMPL(ID_AA64MMFR0, TGran64, NONE, IMPL),
1013 MRS_FIELD_VALUE_END,
1014 };
1015
1016 static const struct mrs_field_value id_aa64mmfr0_tgran16[] = {
1017 MRS_FIELD_VALUE_NONE_IMPL(ID_AA64MMFR0, TGran16, NONE, IMPL),
1018 MRS_FIELD_VALUE(ID_AA64MMFR0_TGran16_LPA2, "TGran16+LPA2"),
1019 MRS_FIELD_VALUE_END,
1020 };
1021
1022 static const struct mrs_field_value id_aa64mmfr0_bigendel0[] = {
1023 MRS_FIELD_VALUE_NONE_IMPL(ID_AA64MMFR0, BigEndEL0, FIXED, MIXED),
1024 MRS_FIELD_VALUE_END,
1025 };
1026
1027 static const struct mrs_field_value id_aa64mmfr0_snsmem[] = {
1028 MRS_FIELD_VALUE_NONE_IMPL(ID_AA64MMFR0, SNSMem, NONE, DISTINCT),
1029 MRS_FIELD_VALUE_END,
1030 };
1031
1032 static const struct mrs_field_value id_aa64mmfr0_bigend[] = {
1033 MRS_FIELD_VALUE_NONE_IMPL(ID_AA64MMFR0, BigEnd, FIXED, MIXED),
1034 MRS_FIELD_VALUE_END,
1035 };
1036
1037 static const struct mrs_field_value id_aa64mmfr0_asidbits[] = {
1038 MRS_FIELD_VALUE(ID_AA64MMFR0_ASIDBits_8, "8bit ASID"),
1039 MRS_FIELD_VALUE(ID_AA64MMFR0_ASIDBits_16, "16bit ASID"),
1040 MRS_FIELD_VALUE_END,
1041 };
1042
1043 static const struct mrs_field_value id_aa64mmfr0_parange[] = {
1044 MRS_FIELD_VALUE(ID_AA64MMFR0_PARange_4G, "4GB PA"),
1045 MRS_FIELD_VALUE(ID_AA64MMFR0_PARange_64G, "64GB PA"),
1046 MRS_FIELD_VALUE(ID_AA64MMFR0_PARange_1T, "1TB PA"),
1047 MRS_FIELD_VALUE(ID_AA64MMFR0_PARange_4T, "4TB PA"),
1048 MRS_FIELD_VALUE(ID_AA64MMFR0_PARange_16T, "16TB PA"),
1049 MRS_FIELD_VALUE(ID_AA64MMFR0_PARange_256T, "256TB PA"),
1050 MRS_FIELD_VALUE(ID_AA64MMFR0_PARange_4P, "4PB PA"),
1051 MRS_FIELD_VALUE_END,
1052 };
1053
1054 static const struct mrs_field id_aa64mmfr0_fields[] = {
1055 MRS_FIELD(ID_AA64MMFR0, ECV, false, MRS_EXACT, id_aa64mmfr0_ecv),
1056 MRS_FIELD(ID_AA64MMFR0, FGT, false, MRS_EXACT, id_aa64mmfr0_fgt),
1057 MRS_FIELD(ID_AA64MMFR0, ExS, false, MRS_EXACT, id_aa64mmfr0_exs),
1058 MRS_FIELD(ID_AA64MMFR0, TGran4_2, false, MRS_EXACT,
1059 id_aa64mmfr0_tgran4_2),
1060 MRS_FIELD(ID_AA64MMFR0, TGran64_2, false, MRS_EXACT,
1061 id_aa64mmfr0_tgran64_2),
1062 MRS_FIELD(ID_AA64MMFR0, TGran16_2, false, MRS_EXACT,
1063 id_aa64mmfr0_tgran16_2),
1064 MRS_FIELD(ID_AA64MMFR0, TGran4, false, MRS_EXACT, id_aa64mmfr0_tgran4),
1065 MRS_FIELD(ID_AA64MMFR0, TGran64, false, MRS_EXACT,
1066 id_aa64mmfr0_tgran64),
1067 MRS_FIELD(ID_AA64MMFR0, TGran16, false, MRS_EXACT,
1068 id_aa64mmfr0_tgran16),
1069 MRS_FIELD(ID_AA64MMFR0, BigEndEL0, false, MRS_EXACT,
1070 id_aa64mmfr0_bigendel0),
1071 MRS_FIELD(ID_AA64MMFR0, SNSMem, false, MRS_EXACT, id_aa64mmfr0_snsmem),
1072 MRS_FIELD(ID_AA64MMFR0, BigEnd, false, MRS_EXACT, id_aa64mmfr0_bigend),
1073 MRS_FIELD(ID_AA64MMFR0, ASIDBits, false, MRS_EXACT,
1074 id_aa64mmfr0_asidbits),
1075 MRS_FIELD(ID_AA64MMFR0, PARange, false, MRS_EXACT,
1076 id_aa64mmfr0_parange),
1077 MRS_FIELD_END,
1078 };
1079
1080
1081 /* ID_AA64MMFR1_EL1 */
1082 static const struct mrs_field_value id_aa64mmfr1_cmovw[] = {
1083 MRS_FIELD_VALUE_NONE_IMPL(ID_AA64MMFR1, CMOVW, NONE, IMPL),
1084 MRS_FIELD_VALUE_END,
1085 };
1086
1087 static const struct mrs_field_value id_aa64mmfr1_tidcp1[] = {
1088 MRS_FIELD_VALUE_NONE_IMPL(ID_AA64MMFR1, TIDCP1, NONE, IMPL),
1089 MRS_FIELD_VALUE_END,
1090 };
1091
1092 static const struct mrs_field_value id_aa64mmfr1_ntlbpa[] = {
1093 MRS_FIELD_VALUE_NONE_IMPL(ID_AA64MMFR1, nTLBPA, NONE, IMPL),
1094 MRS_FIELD_VALUE_END,
1095 };
1096
1097 static const struct mrs_field_value id_aa64mmfr1_afp[] = {
1098 MRS_FIELD_VALUE_NONE_IMPL(ID_AA64MMFR1, AFP, NONE, IMPL),
1099 MRS_FIELD_VALUE_END,
1100 };
1101
1102 static const struct mrs_field_value id_aa64mmfr1_hcx[] = {
1103 MRS_FIELD_VALUE_NONE_IMPL(ID_AA64MMFR1, HCX, NONE, IMPL),
1104 MRS_FIELD_VALUE_END,
1105 };
1106
1107 static const struct mrs_field_value id_aa64mmfr1_ets[] = {
1108 MRS_FIELD_VALUE_NONE_IMPL(ID_AA64MMFR1, ETS, NONE, IMPL),
1109 MRS_FIELD_VALUE_END,
1110 };
1111
1112 static const struct mrs_field_value id_aa64mmfr1_twed[] = {
1113 MRS_FIELD_VALUE_NONE_IMPL(ID_AA64MMFR1, TWED, NONE, IMPL),
1114 MRS_FIELD_VALUE_END,
1115 };
1116
1117 static const struct mrs_field_value id_aa64mmfr1_xnx[] = {
1118 MRS_FIELD_VALUE_NONE_IMPL(ID_AA64MMFR1, XNX, NONE, IMPL),
1119 MRS_FIELD_VALUE_END,
1120 };
1121
1122 static const struct mrs_field_value id_aa64mmfr1_specsei[] = {
1123 MRS_FIELD_VALUE_NONE_IMPL(ID_AA64MMFR1, SpecSEI, NONE, IMPL),
1124 MRS_FIELD_VALUE_END,
1125 };
1126
1127 static const struct mrs_field_value id_aa64mmfr1_pan[] = {
1128 MRS_FIELD_VALUE_NONE_IMPL(ID_AA64MMFR1, PAN, NONE, IMPL),
1129 MRS_FIELD_VALUE(ID_AA64MMFR1_PAN_ATS1E1, "PAN+ATS1E1"),
1130 MRS_FIELD_VALUE(ID_AA64MMFR1_PAN_EPAN, "EPAN"),
1131 MRS_FIELD_VALUE_END,
1132 };
1133
1134 static const struct mrs_field_value id_aa64mmfr1_lo[] = {
1135 MRS_FIELD_VALUE_NONE_IMPL(ID_AA64MMFR1, LO, NONE, IMPL),
1136 MRS_FIELD_VALUE_END,
1137 };
1138
1139 static const struct mrs_field_value id_aa64mmfr1_hpds[] = {
1140 MRS_FIELD_VALUE(ID_AA64MMFR1_HPDS_NONE, ""),
1141 MRS_FIELD_VALUE(ID_AA64MMFR1_HPDS_HPD, "HPD"),
1142 MRS_FIELD_VALUE(ID_AA64MMFR1_HPDS_TTPBHA, "HPD+TTPBHA"),
1143 MRS_FIELD_VALUE_END,
1144 };
1145
1146 static const struct mrs_field_value id_aa64mmfr1_vh[] = {
1147 MRS_FIELD_VALUE_NONE_IMPL(ID_AA64MMFR1, VH, NONE, IMPL),
1148 MRS_FIELD_VALUE_END,
1149 };
1150
1151 static const struct mrs_field_value id_aa64mmfr1_vmidbits[] = {
1152 MRS_FIELD_VALUE(ID_AA64MMFR1_VMIDBits_8, "8bit VMID"),
1153 MRS_FIELD_VALUE(ID_AA64MMFR1_VMIDBits_16, "16bit VMID"),
1154 MRS_FIELD_VALUE_END,
1155 };
1156
1157 static const struct mrs_field_value id_aa64mmfr1_hafdbs[] = {
1158 MRS_FIELD_VALUE(ID_AA64MMFR1_HAFDBS_NONE, ""),
1159 MRS_FIELD_VALUE(ID_AA64MMFR1_HAFDBS_AF, "HAF"),
1160 MRS_FIELD_VALUE(ID_AA64MMFR1_HAFDBS_AF_DBS, "HAF+DS"),
1161 MRS_FIELD_VALUE_END,
1162 };
1163
1164 static const struct mrs_field id_aa64mmfr1_fields[] = {
1165 MRS_FIELD(ID_AA64MMFR1, CMOVW, false, MRS_EXACT, id_aa64mmfr1_cmovw),
1166 MRS_FIELD(ID_AA64MMFR1, TIDCP1, false, MRS_EXACT, id_aa64mmfr1_tidcp1),
1167 MRS_FIELD(ID_AA64MMFR1, nTLBPA, false, MRS_EXACT, id_aa64mmfr1_ntlbpa),
1168 MRS_FIELD(ID_AA64MMFR1, AFP, false, MRS_EXACT, id_aa64mmfr1_afp),
1169 MRS_FIELD(ID_AA64MMFR1, HCX, false, MRS_EXACT, id_aa64mmfr1_hcx),
1170 MRS_FIELD(ID_AA64MMFR1, ETS, false, MRS_EXACT, id_aa64mmfr1_ets),
1171 MRS_FIELD(ID_AA64MMFR1, TWED, false, MRS_EXACT, id_aa64mmfr1_twed),
1172 MRS_FIELD(ID_AA64MMFR1, XNX, false, MRS_EXACT, id_aa64mmfr1_xnx),
1173 MRS_FIELD(ID_AA64MMFR1, SpecSEI, false, MRS_EXACT,
1174 id_aa64mmfr1_specsei),
1175 MRS_FIELD(ID_AA64MMFR1, PAN, false, MRS_EXACT, id_aa64mmfr1_pan),
1176 MRS_FIELD(ID_AA64MMFR1, LO, false, MRS_EXACT, id_aa64mmfr1_lo),
1177 MRS_FIELD(ID_AA64MMFR1, HPDS, false, MRS_EXACT, id_aa64mmfr1_hpds),
1178 MRS_FIELD(ID_AA64MMFR1, VH, false, MRS_EXACT, id_aa64mmfr1_vh),
1179 MRS_FIELD(ID_AA64MMFR1, VMIDBits, false, MRS_EXACT,
1180 id_aa64mmfr1_vmidbits),
1181 MRS_FIELD(ID_AA64MMFR1, HAFDBS, false, MRS_EXACT, id_aa64mmfr1_hafdbs),
1182 MRS_FIELD_END,
1183 };
1184
1185
1186 /* ID_AA64MMFR2_EL1 */
1187 static const struct mrs_field_value id_aa64mmfr2_e0pd[] = {
1188 MRS_FIELD_VALUE_NONE_IMPL(ID_AA64MMFR2, E0PD, NONE, IMPL),
1189 MRS_FIELD_VALUE_END,
1190 };
1191
1192 static const struct mrs_field_value id_aa64mmfr2_evt[] = {
1193 MRS_FIELD_VALUE(ID_AA64MMFR2_EVT_NONE, ""),
1194 MRS_FIELD_VALUE(ID_AA64MMFR2_EVT_8_2, "EVT-8.2"),
1195 MRS_FIELD_VALUE(ID_AA64MMFR2_EVT_8_5, "EVT-8.5"),
1196 MRS_FIELD_VALUE_END,
1197 };
1198
1199 static const struct mrs_field_value id_aa64mmfr2_bbm[] = {
1200 MRS_FIELD_VALUE(ID_AA64MMFR2_BBM_LEVEL0, ""),
1201 MRS_FIELD_VALUE(ID_AA64MMFR2_BBM_LEVEL1, "BBM level 1"),
1202 MRS_FIELD_VALUE(ID_AA64MMFR2_BBM_LEVEL2, "BBM level 2"),
1203 MRS_FIELD_VALUE_END,
1204 };
1205
1206 static const struct mrs_field_value id_aa64mmfr2_ttl[] = {
1207 MRS_FIELD_VALUE_NONE_IMPL(ID_AA64MMFR2, TTL, NONE, IMPL),
1208 MRS_FIELD_VALUE_END,
1209 };
1210
1211 static const struct mrs_field_value id_aa64mmfr2_fwb[] = {
1212 MRS_FIELD_VALUE_NONE_IMPL(ID_AA64MMFR2, FWB, NONE, IMPL),
1213 MRS_FIELD_VALUE_END,
1214 };
1215
1216 static const struct mrs_field_value id_aa64mmfr2_ids[] = {
1217 MRS_FIELD_VALUE_NONE_IMPL(ID_AA64MMFR2, IDS, NONE, IMPL),
1218 MRS_FIELD_VALUE_END,
1219 };
1220
1221 static const struct mrs_field_value id_aa64mmfr2_at[] = {
1222 MRS_FIELD_VALUE_NONE_IMPL(ID_AA64MMFR2, AT, NONE, IMPL),
1223 MRS_FIELD_VALUE_END,
1224 };
1225
1226 static const struct mrs_field_hwcap id_aa64mmfr2_at_caps[] = {
1227 MRS_HWCAP(1, HWCAP_USCAT, ID_AA64MMFR2_AT_IMPL),
1228 MRS_HWCAP_END
1229 };
1230
1231 static const struct mrs_field_value id_aa64mmfr2_st[] = {
1232 MRS_FIELD_VALUE_NONE_IMPL(ID_AA64MMFR2, ST, NONE, IMPL),
1233 MRS_FIELD_VALUE_END,
1234 };
1235
1236 static const struct mrs_field_value id_aa64mmfr2_nv[] = {
1237 MRS_FIELD_VALUE_NONE_IMPL(ID_AA64MMFR2, NV, NONE, 8_3),
1238 MRS_FIELD_VALUE(ID_AA64MMFR2_NV_8_4, "NV v8.4"),
1239 MRS_FIELD_VALUE_END,
1240 };
1241
1242 static const struct mrs_field_value id_aa64mmfr2_ccidx[] = {
1243 MRS_FIELD_VALUE(ID_AA64MMFR2_CCIDX_32, "32bit CCIDX"),
1244 MRS_FIELD_VALUE(ID_AA64MMFR2_CCIDX_64, "64bit CCIDX"),
1245 MRS_FIELD_VALUE_END,
1246 };
1247
1248 static const struct mrs_field_value id_aa64mmfr2_varange[] = {
1249 MRS_FIELD_VALUE(ID_AA64MMFR2_VARange_48, "48bit VA"),
1250 MRS_FIELD_VALUE(ID_AA64MMFR2_VARange_52, "52bit VA"),
1251 MRS_FIELD_VALUE_END,
1252 };
1253
1254 static const struct mrs_field_value id_aa64mmfr2_iesb[] = {
1255 MRS_FIELD_VALUE_NONE_IMPL(ID_AA64MMFR2, IESB, NONE, IMPL),
1256 MRS_FIELD_VALUE_END,
1257 };
1258
1259 static const struct mrs_field_value id_aa64mmfr2_lsm[] = {
1260 MRS_FIELD_VALUE_NONE_IMPL(ID_AA64MMFR2, LSM, NONE, IMPL),
1261 MRS_FIELD_VALUE_END,
1262 };
1263
1264 static const struct mrs_field_value id_aa64mmfr2_uao[] = {
1265 MRS_FIELD_VALUE_NONE_IMPL(ID_AA64MMFR2, UAO, NONE, IMPL),
1266 MRS_FIELD_VALUE_END,
1267 };
1268
1269 static const struct mrs_field_value id_aa64mmfr2_cnp[] = {
1270 MRS_FIELD_VALUE_NONE_IMPL(ID_AA64MMFR2, CnP, NONE, IMPL),
1271 MRS_FIELD_VALUE_END,
1272 };
1273
1274 static const struct mrs_field id_aa64mmfr2_fields[] = {
1275 MRS_FIELD(ID_AA64MMFR2, E0PD, false, MRS_EXACT, id_aa64mmfr2_e0pd),
1276 MRS_FIELD(ID_AA64MMFR2, EVT, false, MRS_EXACT, id_aa64mmfr2_evt),
1277 MRS_FIELD(ID_AA64MMFR2, BBM, false, MRS_EXACT, id_aa64mmfr2_bbm),
1278 MRS_FIELD(ID_AA64MMFR2, TTL, false, MRS_EXACT, id_aa64mmfr2_ttl),
1279 MRS_FIELD(ID_AA64MMFR2, FWB, false, MRS_EXACT, id_aa64mmfr2_fwb),
1280 MRS_FIELD(ID_AA64MMFR2, IDS, false, MRS_EXACT, id_aa64mmfr2_ids),
1281 MRS_FIELD_HWCAP(ID_AA64MMFR2, AT, false, MRS_LOWER, id_aa64mmfr2_at,
1282 id_aa64mmfr2_at_caps),
1283 MRS_FIELD(ID_AA64MMFR2, ST, false, MRS_EXACT, id_aa64mmfr2_st),
1284 MRS_FIELD(ID_AA64MMFR2, NV, false, MRS_EXACT, id_aa64mmfr2_nv),
1285 MRS_FIELD(ID_AA64MMFR2, CCIDX, false, MRS_EXACT, id_aa64mmfr2_ccidx),
1286 MRS_FIELD(ID_AA64MMFR2, VARange, false, MRS_EXACT,
1287 id_aa64mmfr2_varange),
1288 MRS_FIELD(ID_AA64MMFR2, IESB, false, MRS_EXACT, id_aa64mmfr2_iesb),
1289 MRS_FIELD(ID_AA64MMFR2, LSM, false, MRS_EXACT, id_aa64mmfr2_lsm),
1290 MRS_FIELD(ID_AA64MMFR2, UAO, false, MRS_EXACT, id_aa64mmfr2_uao),
1291 MRS_FIELD(ID_AA64MMFR2, CnP, false, MRS_EXACT, id_aa64mmfr2_cnp),
1292 MRS_FIELD_END,
1293 };
1294
1295
1296 /* ID_AA64MMFR2_EL1 */
1297 static const struct mrs_field_value id_aa64mmfr3_spec_fpacc[] = {
1298 MRS_FIELD_VALUE_NONE_IMPL(ID_AA64MMFR3, Spec_FPACC, NONE, IMPL),
1299 MRS_FIELD_VALUE_END,
1300 };
1301
1302 static const struct mrs_field_value id_aa64mmfr3_mec[] = {
1303 MRS_FIELD_VALUE_NONE_IMPL(ID_AA64MMFR3, MEC, NONE, IMPL),
1304 MRS_FIELD_VALUE_END,
1305 };
1306
1307 static const struct mrs_field_value id_aa64mmfr3_sctlrx[] = {
1308 MRS_FIELD_VALUE_NONE_IMPL(ID_AA64MMFR3, SCTLRX, NONE, IMPL),
1309 MRS_FIELD_VALUE_END,
1310 };
1311
1312 static const struct mrs_field_value id_aa64mmfr3_tcrx[] = {
1313 MRS_FIELD_VALUE_NONE_IMPL(ID_AA64MMFR3, TCRX, NONE, IMPL),
1314 MRS_FIELD_VALUE_END,
1315 };
1316
1317 static const struct mrs_field id_aa64mmfr3_fields[] = {
1318 MRS_FIELD(ID_AA64MMFR3, Spec_FPACC, false, MRS_EXACT,
1319 id_aa64mmfr3_spec_fpacc),
1320 MRS_FIELD(ID_AA64MMFR3, MEC, false, MRS_EXACT, id_aa64mmfr3_mec),
1321 MRS_FIELD(ID_AA64MMFR3, SCTLRX, false, MRS_EXACT, id_aa64mmfr3_sctlrx),
1322 MRS_FIELD(ID_AA64MMFR3, TCRX, false, MRS_EXACT, id_aa64mmfr3_tcrx),
1323 MRS_FIELD_END,
1324 };
1325
1326
1327 /* ID_AA64MMFR4_EL1 */
1328 static const struct mrs_field id_aa64mmfr4_fields[] = {
1329 MRS_FIELD_END,
1330 };
1331
1332
1333 /* ID_AA64PFR0_EL1 */
1334 static const struct mrs_field_value id_aa64pfr0_csv3[] = {
1335 MRS_FIELD_VALUE(ID_AA64PFR0_CSV3_NONE, ""),
1336 MRS_FIELD_VALUE(ID_AA64PFR0_CSV3_ISOLATED, "CSV3"),
1337 MRS_FIELD_VALUE_END,
1338 };
1339
1340 static const struct mrs_field_value id_aa64pfr0_csv2[] = {
1341 MRS_FIELD_VALUE(ID_AA64PFR0_CSV2_NONE, ""),
1342 MRS_FIELD_VALUE(ID_AA64PFR0_CSV2_ISOLATED, "CSV2"),
1343 MRS_FIELD_VALUE(ID_AA64PFR0_CSV2_SCXTNUM, "CSV2_2"),
1344 MRS_FIELD_VALUE(ID_AA64PFR0_CSV2_3, "CSV2_3"),
1345 MRS_FIELD_VALUE_END,
1346 };
1347
1348 static const struct mrs_field_value id_aa64pfr0_rme[] = {
1349 MRS_FIELD_VALUE_NONE_IMPL(ID_AA64PFR0, RME, NONE, IMPL),
1350 MRS_FIELD_VALUE_END,
1351 };
1352
1353 static const struct mrs_field_value id_aa64pfr0_dit[] = {
1354 MRS_FIELD_VALUE(ID_AA64PFR0_DIT_NONE, ""),
1355 MRS_FIELD_VALUE(ID_AA64PFR0_DIT_PSTATE, "PSTATE.DIT"),
1356 MRS_FIELD_VALUE_END,
1357 };
1358
1359 static const struct mrs_field_hwcap id_aa64pfr0_dit_caps[] = {
1360 MRS_HWCAP(1, HWCAP_DIT, ID_AA64PFR0_DIT_PSTATE),
1361 MRS_HWCAP_END
1362 };
1363
1364 static const struct mrs_field_value id_aa64pfr0_amu[] = {
1365 MRS_FIELD_VALUE(ID_AA64PFR0_AMU_NONE, ""),
1366 MRS_FIELD_VALUE(ID_AA64PFR0_AMU_V1, "AMUv1"),
1367 MRS_FIELD_VALUE(ID_AA64PFR0_AMU_V1_1, "AMUv1p1"),
1368 MRS_FIELD_VALUE_END,
1369 };
1370
1371 static const struct mrs_field_value id_aa64pfr0_mpam[] = {
1372 MRS_FIELD_VALUE_NONE_IMPL(ID_AA64PFR0, MPAM, NONE, IMPL),
1373 MRS_FIELD_VALUE_END,
1374 };
1375
1376 static const struct mrs_field_value id_aa64pfr0_sel2[] = {
1377 MRS_FIELD_VALUE_NONE_IMPL(ID_AA64PFR0, SEL2, NONE, IMPL),
1378 MRS_FIELD_VALUE_END,
1379 };
1380
1381 static const struct mrs_field_value id_aa64pfr0_sve[] = {
1382 MRS_FIELD_VALUE_NONE_IMPL(ID_AA64PFR0, SVE, NONE, IMPL),
1383 MRS_FIELD_VALUE_END,
1384 };
1385
1386 #if 0
1387 /* Enable when we add SVE support */
1388 static const struct mrs_field_hwcap id_aa64pfr0_sve_caps[] = {
1389 MRS_HWCAP(1, HWCAP_SVE, ID_AA64PFR0_SVE_IMPL),
1390 MRS_HWCAP_END
1391 };
1392 #endif
1393
1394 static const struct mrs_field_value id_aa64pfr0_ras[] = {
1395 MRS_FIELD_VALUE(ID_AA64PFR0_RAS_NONE, ""),
1396 MRS_FIELD_VALUE(ID_AA64PFR0_RAS_IMPL, "RAS"),
1397 MRS_FIELD_VALUE(ID_AA64PFR0_RAS_8_4, "RAS v8.4"),
1398 MRS_FIELD_VALUE_END,
1399 };
1400
1401 static const struct mrs_field_value id_aa64pfr0_gic[] = {
1402 MRS_FIELD_VALUE_NONE_IMPL(ID_AA64PFR0, GIC, CPUIF_NONE, CPUIF_EN),
1403 MRS_FIELD_VALUE(ID_AA64PFR0_GIC_CPUIF_NONE, ""),
1404 MRS_FIELD_VALUE(ID_AA64PFR0_GIC_CPUIF_EN, "GIC"),
1405 MRS_FIELD_VALUE(ID_AA64PFR0_GIC_CPUIF_4_1, "GIC 4.1"),
1406 MRS_FIELD_VALUE_END,
1407 };
1408
1409 static const struct mrs_field_value id_aa64pfr0_advsimd[] = {
1410 MRS_FIELD_VALUE_NONE_IMPL(ID_AA64PFR0, AdvSIMD, NONE, IMPL),
1411 MRS_FIELD_VALUE(ID_AA64PFR0_AdvSIMD_HP, "AdvSIMD+HP"),
1412 MRS_FIELD_VALUE_END,
1413 };
1414
1415 static const struct mrs_field_hwcap id_aa64pfr0_advsimd_caps[] = {
1416 MRS_HWCAP(1, HWCAP_ASIMD, ID_AA64PFR0_AdvSIMD_IMPL),
1417 MRS_HWCAP(1, HWCAP_ASIMDHP, ID_AA64PFR0_AdvSIMD_HP),
1418 MRS_HWCAP_END
1419 };
1420
1421 static const struct mrs_field_value id_aa64pfr0_fp[] = {
1422 MRS_FIELD_VALUE_NONE_IMPL(ID_AA64PFR0, FP, NONE, IMPL),
1423 MRS_FIELD_VALUE(ID_AA64PFR0_FP_HP, "FP+HP"),
1424 MRS_FIELD_VALUE_END,
1425 };
1426
1427 static const struct mrs_field_hwcap id_aa64pfr0_fp_caps[] = {
1428 MRS_HWCAP(1, HWCAP_FP, ID_AA64PFR0_FP_IMPL),
1429 MRS_HWCAP(1, HWCAP_FPHP, ID_AA64PFR0_FP_HP),
1430 MRS_HWCAP_END
1431 };
1432
1433 static const struct mrs_field_value id_aa64pfr0_el3[] = {
1434 MRS_FIELD_VALUE_NONE_IMPL(ID_AA64PFR0, EL3, NONE, 64),
1435 MRS_FIELD_VALUE(ID_AA64PFR0_EL3_64_32, "EL3 32"),
1436 MRS_FIELD_VALUE_END,
1437 };
1438
1439 static const struct mrs_field_value id_aa64pfr0_el2[] = {
1440 MRS_FIELD_VALUE_NONE_IMPL(ID_AA64PFR0, EL2, NONE, 64),
1441 MRS_FIELD_VALUE(ID_AA64PFR0_EL2_64_32, "EL2 32"),
1442 MRS_FIELD_VALUE_END,
1443 };
1444
1445 static const struct mrs_field_value id_aa64pfr0_el1[] = {
1446 MRS_FIELD_VALUE(ID_AA64PFR0_EL1_64, "EL1"),
1447 MRS_FIELD_VALUE(ID_AA64PFR0_EL1_64_32, "EL1 32"),
1448 MRS_FIELD_VALUE_END,
1449 };
1450
1451 static const struct mrs_field_value id_aa64pfr0_el0[] = {
1452 MRS_FIELD_VALUE(ID_AA64PFR0_EL0_64, "EL0"),
1453 MRS_FIELD_VALUE(ID_AA64PFR0_EL0_64_32, "EL0 32"),
1454 MRS_FIELD_VALUE_END,
1455 };
1456
1457 static const struct mrs_field id_aa64pfr0_fields[] = {
1458 MRS_FIELD(ID_AA64PFR0, CSV3, false, MRS_EXACT, id_aa64pfr0_csv3),
1459 MRS_FIELD(ID_AA64PFR0, CSV2, false, MRS_EXACT, id_aa64pfr0_csv2),
1460 MRS_FIELD(ID_AA64PFR0, RME, false, MRS_EXACT, id_aa64pfr0_rme),
1461 MRS_FIELD_HWCAP(ID_AA64PFR0, DIT, false, MRS_LOWER, id_aa64pfr0_dit,
1462 id_aa64pfr0_dit_caps),
1463 MRS_FIELD(ID_AA64PFR0, AMU, false, MRS_EXACT, id_aa64pfr0_amu),
1464 MRS_FIELD(ID_AA64PFR0, MPAM, false, MRS_EXACT, id_aa64pfr0_mpam),
1465 MRS_FIELD(ID_AA64PFR0, SEL2, false, MRS_EXACT, id_aa64pfr0_sel2),
1466 MRS_FIELD(ID_AA64PFR0, SVE, false, MRS_EXACT, id_aa64pfr0_sve),
1467 MRS_FIELD(ID_AA64PFR0, RAS, false, MRS_EXACT, id_aa64pfr0_ras),
1468 MRS_FIELD(ID_AA64PFR0, GIC, false, MRS_EXACT, id_aa64pfr0_gic),
1469 MRS_FIELD_HWCAP(ID_AA64PFR0, AdvSIMD, true, MRS_LOWER,
1470 id_aa64pfr0_advsimd, id_aa64pfr0_advsimd_caps),
1471 MRS_FIELD_HWCAP(ID_AA64PFR0, FP, true, MRS_LOWER, id_aa64pfr0_fp,
1472 id_aa64pfr0_fp_caps),
1473 MRS_FIELD(ID_AA64PFR0, EL3, false, MRS_EXACT, id_aa64pfr0_el3),
1474 MRS_FIELD(ID_AA64PFR0, EL2, false, MRS_EXACT, id_aa64pfr0_el2),
1475 MRS_FIELD(ID_AA64PFR0, EL1, false, MRS_LOWER, id_aa64pfr0_el1),
1476 MRS_FIELD(ID_AA64PFR0, EL0, false, MRS_LOWER, id_aa64pfr0_el0),
1477 MRS_FIELD_END,
1478 };
1479
1480
1481 /* ID_AA64PFR1_EL1 */
1482 static const struct mrs_field_value id_aa64pfr1_nmi[] = {
1483 MRS_FIELD_VALUE_NONE_IMPL(ID_AA64PFR1, NMI, NONE, IMPL),
1484 MRS_FIELD_VALUE_END,
1485 };
1486
1487 static const struct mrs_field_value id_aa64pfr1_csv2_frac[] = {
1488 MRS_FIELD_VALUE(ID_AA64PFR1_CSV2_frac_p0, ""),
1489 MRS_FIELD_VALUE(ID_AA64PFR1_CSV2_frac_p1, "CSV2 p1"),
1490 MRS_FIELD_VALUE(ID_AA64PFR1_CSV2_frac_p2, "CSV2 p2"),
1491 MRS_FIELD_VALUE_END,
1492 };
1493
1494 static const struct mrs_field_value id_aa64pfr1_rndr_trap[] = {
1495 MRS_FIELD_VALUE_NONE_IMPL(ID_AA64PFR1, RNDR_trap, NONE, IMPL),
1496 MRS_FIELD_VALUE_END,
1497 };
1498
1499 static const struct mrs_field_value id_aa64pfr1_sme[] = {
1500 MRS_FIELD_VALUE(ID_AA64PFR1_SME_NONE, ""),
1501 MRS_FIELD_VALUE(ID_AA64PFR1_SME_SME, "SME"),
1502 MRS_FIELD_VALUE(ID_AA64PFR1_SME_SME2, "SME2"),
1503 MRS_FIELD_VALUE_END,
1504 };
1505
1506 static const struct mrs_field_value id_aa64pfr1_mpam_frac[] = {
1507 MRS_FIELD_VALUE(ID_AA64PFR1_MPAM_frac_p0, ""),
1508 MRS_FIELD_VALUE(ID_AA64PFR1_MPAM_frac_p1, "MPAM p1"),
1509 MRS_FIELD_VALUE_END,
1510 };
1511
1512 static const struct mrs_field_value id_aa64pfr1_ras_frac[] = {
1513 MRS_FIELD_VALUE(ID_AA64PFR1_RAS_frac_p0, ""),
1514 MRS_FIELD_VALUE(ID_AA64PFR1_RAS_frac_p1, "RAS p1"),
1515 MRS_FIELD_VALUE_END,
1516 };
1517
1518 static const struct mrs_field_value id_aa64pfr1_mte[] = {
1519 MRS_FIELD_VALUE(ID_AA64PFR1_MTE_NONE, ""),
1520 MRS_FIELD_VALUE(ID_AA64PFR1_MTE_MTE, "MTE"),
1521 MRS_FIELD_VALUE(ID_AA64PFR1_MTE_MTE2, "MTE2"),
1522 MRS_FIELD_VALUE(ID_AA64PFR1_MTE_MTE3, "MTE3"),
1523 MRS_FIELD_VALUE_END,
1524 };
1525
1526 static const struct mrs_field_value id_aa64pfr1_ssbs[] = {
1527 MRS_FIELD_VALUE(ID_AA64PFR1_SSBS_NONE, ""),
1528 MRS_FIELD_VALUE(ID_AA64PFR1_SSBS_PSTATE, "PSTATE.SSBS"),
1529 MRS_FIELD_VALUE(ID_AA64PFR1_SSBS_PSTATE_MSR, "PSTATE.SSBS MSR"),
1530 MRS_FIELD_VALUE_END,
1531 };
1532
1533 static const struct mrs_field_hwcap id_aa64pfr1_ssbs_caps[] = {
1534 MRS_HWCAP(1, HWCAP_SSBS, ID_AA64PFR1_SSBS_PSTATE),
1535 MRS_HWCAP_END
1536 };
1537
1538 static const struct mrs_field_value id_aa64pfr1_bt[] = {
1539 MRS_FIELD_VALUE(ID_AA64PFR1_BT_NONE, ""),
1540 MRS_FIELD_VALUE(ID_AA64PFR1_BT_IMPL, "BTI"),
1541 MRS_FIELD_VALUE_END,
1542 };
1543
1544 static const struct mrs_field_hwcap id_aa64pfr1_bt_caps[] = {
1545 MRS_HWCAP(2, HWCAP2_BTI, ID_AA64PFR1_BT_IMPL),
1546 MRS_HWCAP_END
1547 };
1548
1549 static const struct mrs_field id_aa64pfr1_fields[] = {
1550 MRS_FIELD(ID_AA64PFR1, NMI, false, MRS_EXACT, id_aa64pfr1_nmi),
1551 MRS_FIELD(ID_AA64PFR1, CSV2_frac, false, MRS_EXACT,
1552 id_aa64pfr1_csv2_frac),
1553 MRS_FIELD(ID_AA64PFR1, RNDR_trap, false, MRS_EXACT,
1554 id_aa64pfr1_rndr_trap),
1555 MRS_FIELD(ID_AA64PFR1, SME, false, MRS_EXACT, id_aa64pfr1_sme),
1556 MRS_FIELD(ID_AA64PFR1, MPAM_frac, false, MRS_EXACT,
1557 id_aa64pfr1_mpam_frac),
1558 MRS_FIELD(ID_AA64PFR1, RAS_frac, false, MRS_EXACT,
1559 id_aa64pfr1_ras_frac),
1560 MRS_FIELD(ID_AA64PFR1, MTE, false, MRS_EXACT, id_aa64pfr1_mte),
1561 MRS_FIELD_HWCAP(ID_AA64PFR1, SSBS, false, MRS_LOWER, id_aa64pfr1_ssbs,
1562 id_aa64pfr1_ssbs_caps),
1563 MRS_FIELD_HWCAP_SPLIT(ID_AA64PFR1, BT, false, MRS_LOWER, MRS_EXACT,
1564 id_aa64pfr1_bt, id_aa64pfr1_bt_caps),
1565 MRS_FIELD_END,
1566 };
1567
1568
1569 /* ID_AA64PFR2_EL1 */
1570 static const struct mrs_field id_aa64pfr2_fields[] = {
1571 MRS_FIELD_END,
1572 };
1573
1574
1575 /* ID_AA64ZFR0_EL1 */
1576 static const struct mrs_field_value id_aa64zfr0_f64mm[] = {
1577 MRS_FIELD_VALUE_NONE_IMPL(ID_AA64ZFR0, F64MM, NONE, IMPL),
1578 MRS_FIELD_VALUE_END,
1579 };
1580
1581 static const struct mrs_field_value id_aa64zfr0_f32mm[] = {
1582 MRS_FIELD_VALUE_NONE_IMPL(ID_AA64ZFR0, F32MM, NONE, IMPL),
1583 MRS_FIELD_VALUE_END,
1584 };
1585
1586 static const struct mrs_field_value id_aa64zfr0_i8mm[] = {
1587 MRS_FIELD_VALUE_NONE_IMPL(ID_AA64ZFR0, I8MM, NONE, IMPL),
1588 MRS_FIELD_VALUE_END,
1589 };
1590
1591 static const struct mrs_field_value id_aa64zfr0_sm4[] = {
1592 MRS_FIELD_VALUE_NONE_IMPL(ID_AA64ZFR0, SM4, NONE, IMPL),
1593 MRS_FIELD_VALUE_END,
1594 };
1595
1596 static const struct mrs_field_value id_aa64zfr0_sha3[] = {
1597 MRS_FIELD_VALUE_NONE_IMPL(ID_AA64ZFR0, SHA3, NONE, IMPL),
1598 MRS_FIELD_VALUE_END,
1599 };
1600
1601 static const struct mrs_field_value id_aa64zfr0_bf16[] = {
1602 MRS_FIELD_VALUE_NONE_IMPL(ID_AA64ZFR0, BF16, NONE, BASE),
1603 MRS_FIELD_VALUE(ID_AA64ZFR0_BF16_EBF, "BF16+EBF"),
1604 MRS_FIELD_VALUE_END,
1605 };
1606
1607 static const struct mrs_field_value id_aa64zfr0_bitperm[] = {
1608 MRS_FIELD_VALUE_NONE_IMPL(ID_AA64ZFR0, BitPerm, NONE, IMPL),
1609 MRS_FIELD_VALUE_END,
1610 };
1611
1612 static const struct mrs_field_value id_aa64zfr0_aes[] = {
1613 MRS_FIELD_VALUE_NONE_IMPL(ID_AA64ZFR0, AES, NONE, BASE),
1614 MRS_FIELD_VALUE(ID_AA64ZFR0_AES_PMULL, "AES+PMULL"),
1615 MRS_FIELD_VALUE_END,
1616 };
1617
1618 static const struct mrs_field_value id_aa64zfr0_svever[] = {
1619 MRS_FIELD_VALUE(ID_AA64ZFR0_SVEver_SVE1, "SVE1"),
1620 MRS_FIELD_VALUE(ID_AA64ZFR0_SVEver_SVE2, "SVE2"),
1621 MRS_FIELD_VALUE_END,
1622 };
1623
1624 static const struct mrs_field id_aa64zfr0_fields[] = {
1625 MRS_FIELD(ID_AA64ZFR0, F64MM, false, MRS_EXACT, id_aa64zfr0_f64mm),
1626 MRS_FIELD(ID_AA64ZFR0, F32MM, false, MRS_EXACT, id_aa64zfr0_f32mm),
1627 MRS_FIELD(ID_AA64ZFR0, I8MM, false, MRS_EXACT, id_aa64zfr0_i8mm),
1628 MRS_FIELD(ID_AA64ZFR0, SM4, false, MRS_EXACT, id_aa64zfr0_sm4),
1629 MRS_FIELD(ID_AA64ZFR0, SHA3, false, MRS_EXACT, id_aa64zfr0_sha3),
1630 MRS_FIELD(ID_AA64ZFR0, BF16, false, MRS_EXACT, id_aa64zfr0_bf16),
1631 MRS_FIELD(ID_AA64ZFR0, BitPerm, false, MRS_EXACT, id_aa64zfr0_bitperm),
1632 MRS_FIELD(ID_AA64ZFR0, AES, false, MRS_EXACT, id_aa64zfr0_aes),
1633 MRS_FIELD(ID_AA64ZFR0, SVEver, false, MRS_EXACT, id_aa64zfr0_svever),
1634 MRS_FIELD_END,
1635 };
1636
1637
1638 #ifdef COMPAT_FREEBSD32
1639 /* ID_ISAR5_EL1 */
1640 static const struct mrs_field_value id_isar5_vcma[] = {
1641 MRS_FIELD_VALUE_NONE_IMPL(ID_ISAR5, VCMA, NONE, IMPL),
1642 MRS_FIELD_VALUE_END,
1643 };
1644
1645 static const struct mrs_field_value id_isar5_rdm[] = {
1646 MRS_FIELD_VALUE_NONE_IMPL(ID_ISAR5, RDM, NONE, IMPL),
1647 MRS_FIELD_VALUE_END,
1648 };
1649
1650 static const struct mrs_field_value id_isar5_crc32[] = {
1651 MRS_FIELD_VALUE_NONE_IMPL(ID_ISAR5, CRC32, NONE, IMPL),
1652 MRS_FIELD_VALUE_END,
1653 };
1654
1655 static const struct mrs_field_hwcap id_isar5_crc32_caps[] = {
1656 MRS_HWCAP(2, HWCAP32_2_CRC32, ID_ISAR5_CRC32_IMPL),
1657 MRS_HWCAP_END
1658 };
1659
1660 static const struct mrs_field_value id_isar5_sha2[] = {
1661 MRS_FIELD_VALUE_NONE_IMPL(ID_ISAR5, SHA2, NONE, IMPL),
1662 MRS_FIELD_VALUE_END,
1663 };
1664
1665 static const struct mrs_field_hwcap id_isar5_sha2_caps[] = {
1666 MRS_HWCAP(2, HWCAP32_2_SHA2, ID_ISAR5_SHA2_IMPL),
1667 MRS_HWCAP_END
1668 };
1669
1670 static const struct mrs_field_value id_isar5_sha1[] = {
1671 MRS_FIELD_VALUE_NONE_IMPL(ID_ISAR5, SHA1, NONE, IMPL),
1672 MRS_FIELD_VALUE_END,
1673 };
1674
1675 static const struct mrs_field_hwcap id_isar5_sha1_caps[] = {
1676 MRS_HWCAP(2, HWCAP32_2_SHA1, ID_ISAR5_SHA1_IMPL),
1677 MRS_HWCAP_END
1678 };
1679
1680 static const struct mrs_field_value id_isar5_aes[] = {
1681 MRS_FIELD_VALUE_NONE_IMPL(ID_ISAR5, AES, NONE, BASE),
1682 MRS_FIELD_VALUE(ID_ISAR5_AES_VMULL, "AES+VMULL"),
1683 MRS_FIELD_VALUE_END,
1684 };
1685
1686 static const struct mrs_field_hwcap id_isar5_aes_caps[] = {
1687 MRS_HWCAP(2, HWCAP32_2_AES, ID_ISAR5_AES_BASE),
1688 MRS_HWCAP(2, HWCAP32_2_PMULL, ID_ISAR5_AES_VMULL),
1689 MRS_HWCAP_END
1690 };
1691
1692 static const struct mrs_field_value id_isar5_sevl[] = {
1693 MRS_FIELD_VALUE_NONE_IMPL(ID_ISAR5, SEVL, NOP, IMPL),
1694 MRS_FIELD_VALUE_END,
1695 };
1696
1697 static const struct mrs_field id_isar5_fields[] = {
1698 MRS_FIELD(ID_ISAR5, VCMA, false, MRS_LOWER, id_isar5_vcma),
1699 MRS_FIELD(ID_ISAR5, RDM, false, MRS_LOWER, id_isar5_rdm),
1700 MRS_FIELD_HWCAP(ID_ISAR5, CRC32, false, MRS_LOWER, id_isar5_crc32,
1701 id_isar5_crc32_caps),
1702 MRS_FIELD_HWCAP(ID_ISAR5, SHA2, false, MRS_LOWER, id_isar5_sha2,
1703 id_isar5_sha2_caps),
1704 MRS_FIELD_HWCAP(ID_ISAR5, SHA1, false, MRS_LOWER, id_isar5_sha1,
1705 id_isar5_sha1_caps),
1706 MRS_FIELD_HWCAP(ID_ISAR5, AES, false, MRS_LOWER, id_isar5_aes,
1707 id_isar5_aes_caps),
1708 MRS_FIELD(ID_ISAR5, SEVL, false, MRS_LOWER, id_isar5_sevl),
1709 MRS_FIELD_END,
1710 };
1711
1712 /* MVFR0 */
1713 static const struct mrs_field_value mvfr0_fpround[] = {
1714 MRS_FIELD_VALUE_NONE_IMPL(MVFR0, FPRound, NONE, IMPL),
1715 MRS_FIELD_VALUE_END,
1716 };
1717
1718 static const struct mrs_field_value mvfr0_fpsqrt[] = {
1719 MRS_FIELD_VALUE_NONE_IMPL(MVFR0, FPSqrt, NONE, IMPL),
1720 MRS_FIELD_VALUE_END,
1721 };
1722
1723 static const struct mrs_field_value mvfr0_fpdivide[] = {
1724 MRS_FIELD_VALUE_NONE_IMPL(MVFR0, FPDivide, NONE, IMPL),
1725 MRS_FIELD_VALUE_END,
1726 };
1727
1728 static const struct mrs_field_value mvfr0_fptrap[] = {
1729 MRS_FIELD_VALUE_NONE_IMPL(MVFR0, FPTrap, NONE, IMPL),
1730 MRS_FIELD_VALUE_END,
1731 };
1732
1733 static const struct mrs_field_value mvfr0_fpdp[] = {
1734 MRS_FIELD_VALUE(MVFR0_FPDP_NONE, ""),
1735 MRS_FIELD_VALUE(MVFR0_FPDP_VFP_v2, "DP VFPv2"),
1736 MRS_FIELD_VALUE(MVFR0_FPDP_VFP_v3_v4, "DP VFPv3+v4"),
1737 MRS_FIELD_VALUE_END,
1738 };
1739
1740 static const struct mrs_field_hwcap mvfr0_fpdp_caps[] = {
1741 MRS_HWCAP(1, HWCAP32_VFP, MVFR0_FPDP_VFP_v2),
1742 MRS_HWCAP(1, HWCAP32_VFPv3, MVFR0_FPDP_VFP_v3_v4),
1743 MRS_HWCAP_END
1744 };
1745
1746 static const struct mrs_field_value mvfr0_fpsp[] = {
1747 MRS_FIELD_VALUE(MVFR0_FPSP_NONE, ""),
1748 MRS_FIELD_VALUE(MVFR0_FPSP_VFP_v2, "SP VFPv2"),
1749 MRS_FIELD_VALUE(MVFR0_FPSP_VFP_v3_v4, "SP VFPv3+v4"),
1750 MRS_FIELD_VALUE_END,
1751 };
1752
1753 static const struct mrs_field_value mvfr0_simdreg[] = {
1754 MRS_FIELD_VALUE(MVFR0_SIMDReg_NONE, ""),
1755 MRS_FIELD_VALUE(MVFR0_SIMDReg_FP, "FP 16x64"),
1756 MRS_FIELD_VALUE(MVFR0_SIMDReg_AdvSIMD, "AdvSIMD"),
1757 MRS_FIELD_VALUE_END,
1758 };
1759
1760 static const struct mrs_field mvfr0_fields[] = {
1761 MRS_FIELD(MVFR0, FPRound, false, MRS_LOWER, mvfr0_fpround),
1762 MRS_FIELD(MVFR0, FPSqrt, false, MRS_LOWER, mvfr0_fpsqrt),
1763 MRS_FIELD(MVFR0, FPDivide, false, MRS_LOWER, mvfr0_fpdivide),
1764 MRS_FIELD(MVFR0, FPTrap, false, MRS_LOWER, mvfr0_fptrap),
1765 MRS_FIELD_HWCAP(MVFR0, FPDP, false, MRS_LOWER, mvfr0_fpdp,
1766 mvfr0_fpdp_caps),
1767 MRS_FIELD(MVFR0, FPSP, false, MRS_LOWER, mvfr0_fpsp),
1768 MRS_FIELD(MVFR0, SIMDReg, false, MRS_LOWER, mvfr0_simdreg),
1769 MRS_FIELD_END,
1770 };
1771
1772 /* MVFR1 */
1773 static const struct mrs_field_value mvfr1_simdfmac[] = {
1774 MRS_FIELD_VALUE_NONE_IMPL(MVFR1, SIMDFMAC, NONE, IMPL),
1775 MRS_FIELD_VALUE_END,
1776 };
1777
1778 static const struct mrs_field_hwcap mvfr1_simdfmac_caps[] = {
1779 MRS_HWCAP(1, HWCAP32_VFPv4, MVFR1_SIMDFMAC_IMPL),
1780 MRS_HWCAP_END
1781 };
1782
1783 static const struct mrs_field_value mvfr1_fphp[] = {
1784 MRS_FIELD_VALUE(MVFR1_FPHP_NONE, ""),
1785 MRS_FIELD_VALUE(MVFR1_FPHP_CONV_SP, "FPHP SP Conv"),
1786 MRS_FIELD_VALUE(MVFR1_FPHP_CONV_DP, "FPHP DP Conv"),
1787 MRS_FIELD_VALUE(MVFR1_FPHP_ARITH, "FPHP Arith"),
1788 MRS_FIELD_VALUE_END,
1789 };
1790
1791 static const struct mrs_field_value mvfr1_simdhp[] = {
1792 MRS_FIELD_VALUE(MVFR1_SIMDHP_NONE, ""),
1793 MRS_FIELD_VALUE(MVFR1_SIMDHP_CONV_SP, "SIMDHP SP Conv"),
1794 MRS_FIELD_VALUE(MVFR1_SIMDHP_ARITH, "SIMDHP Arith"),
1795 MRS_FIELD_VALUE_END,
1796 };
1797
1798 static const struct mrs_field_value mvfr1_simdsp[] = {
1799 MRS_FIELD_VALUE_NONE_IMPL(MVFR1, SIMDSP, NONE, IMPL),
1800 MRS_FIELD_VALUE_END,
1801 };
1802
1803 static const struct mrs_field_value mvfr1_simdint[] = {
1804 MRS_FIELD_VALUE_NONE_IMPL(MVFR1, SIMDInt, NONE, IMPL),
1805 MRS_FIELD_VALUE_END,
1806 };
1807
1808 static const struct mrs_field_value mvfr1_simdls[] = {
1809 MRS_FIELD_VALUE_NONE_IMPL(MVFR1, SIMDLS, NONE, IMPL),
1810 MRS_FIELD_VALUE_END,
1811 };
1812
1813 static const struct mrs_field_hwcap mvfr1_simdls_caps[] = {
1814 MRS_HWCAP(1, HWCAP32_VFPv4, MVFR1_SIMDFMAC_IMPL),
1815 MRS_HWCAP_END
1816 };
1817
1818 static const struct mrs_field_value mvfr1_fpdnan[] = {
1819 MRS_FIELD_VALUE_NONE_IMPL(MVFR1, FPDNaN, NONE, IMPL),
1820 MRS_FIELD_VALUE_END,
1821 };
1822
1823 static const struct mrs_field_value mvfr1_fpftz[] = {
1824 MRS_FIELD_VALUE_NONE_IMPL(MVFR1, FPFtZ, NONE, IMPL),
1825 MRS_FIELD_VALUE_END,
1826 };
1827
1828 static const struct mrs_field mvfr1_fields[] = {
1829 MRS_FIELD_HWCAP(MVFR1, SIMDFMAC, false, MRS_LOWER, mvfr1_simdfmac,
1830 mvfr1_simdfmac_caps),
1831 MRS_FIELD(MVFR1, FPHP, false, MRS_LOWER, mvfr1_fphp),
1832 MRS_FIELD(MVFR1, SIMDHP, false, MRS_LOWER, mvfr1_simdhp),
1833 MRS_FIELD(MVFR1, SIMDSP, false, MRS_LOWER, mvfr1_simdsp),
1834 MRS_FIELD(MVFR1, SIMDInt, false, MRS_LOWER, mvfr1_simdint),
1835 MRS_FIELD_HWCAP(MVFR1, SIMDLS, false, MRS_LOWER, mvfr1_simdls,
1836 mvfr1_simdls_caps),
1837 MRS_FIELD(MVFR1, FPDNaN, false, MRS_LOWER, mvfr1_fpdnan),
1838 MRS_FIELD(MVFR1, FPFtZ, false, MRS_LOWER, mvfr1_fpftz),
1839 MRS_FIELD_END,
1840 };
1841 #endif /* COMPAT_FREEBSD32 */
1842
1843 struct mrs_user_reg {
1844 u_int reg;
1845 u_int CRm;
1846 u_int Op2;
1847 bool is64bit;
1848 size_t offset;
1849 const struct mrs_field *fields;
1850 };
1851
1852 #define USER_REG(name, field_name, _is64bit) \
1853 { \
1854 .reg = name, \
1855 .CRm = name##_CRm, \
1856 .Op2 = name##_op2, \
1857 .offset = __offsetof(struct cpu_desc, field_name), \
1858 .fields = field_name##_fields, \
1859 .is64bit = _is64bit, \
1860 }
1861 static const struct mrs_user_reg user_regs[] = {
1862 USER_REG(ID_AA64AFR0_EL1, id_aa64afr0, true),
1863 USER_REG(ID_AA64AFR1_EL1, id_aa64afr1, true),
1864
1865 USER_REG(ID_AA64DFR0_EL1, id_aa64dfr0, true),
1866 USER_REG(ID_AA64DFR1_EL1, id_aa64dfr1, true),
1867
1868 USER_REG(ID_AA64ISAR0_EL1, id_aa64isar0, true),
1869 USER_REG(ID_AA64ISAR1_EL1, id_aa64isar1, true),
1870 USER_REG(ID_AA64ISAR2_EL1, id_aa64isar2, true),
1871
1872 USER_REG(ID_AA64MMFR0_EL1, id_aa64mmfr0, true),
1873 USER_REG(ID_AA64MMFR1_EL1, id_aa64mmfr1, true),
1874 USER_REG(ID_AA64MMFR2_EL1, id_aa64mmfr2, true),
1875 USER_REG(ID_AA64MMFR3_EL1, id_aa64mmfr3, true),
1876 USER_REG(ID_AA64MMFR4_EL1, id_aa64mmfr4, true),
1877
1878 USER_REG(ID_AA64PFR0_EL1, id_aa64pfr0, true),
1879 USER_REG(ID_AA64PFR1_EL1, id_aa64pfr1, true),
1880 USER_REG(ID_AA64PFR2_EL1, id_aa64pfr2, true),
1881
1882 USER_REG(ID_AA64ZFR0_EL1, id_aa64zfr0, true),
1883
1884 #ifdef COMPAT_FREEBSD32
1885 USER_REG(ID_ISAR5_EL1, id_isar5, false),
1886
1887 USER_REG(MVFR0_EL1, mvfr0, false),
1888 USER_REG(MVFR1_EL1, mvfr1, false),
1889 #endif /* COMPAT_FREEBSD32 */
1890 };
1891
1892 #define CPU_DESC_FIELD(desc, idx) \
1893 *(uint64_t *)((char *)&(desc) + user_regs[(idx)].offset)
1894
1895 static int
user_mrs_handler(vm_offset_t va,uint32_t insn,struct trapframe * frame,uint32_t esr)1896 user_mrs_handler(vm_offset_t va, uint32_t insn, struct trapframe *frame,
1897 uint32_t esr)
1898 {
1899 uint64_t value;
1900 int CRm, Op2, i, reg;
1901
1902 if ((insn & MRS_MASK) != MRS_VALUE)
1903 return (0);
1904
1905 /*
1906 * We only emulate Op0 == 3, Op1 == 0, CRn == 0, CRm == {0, 4-7}.
1907 * These are in the EL1 CPU identification space.
1908 * CRm == 0 holds MIDR_EL1, MPIDR_EL1, and REVID_EL1.
1909 * CRm == {4-7} holds the ID_AA64 registers.
1910 *
1911 * For full details see the ARMv8 ARM (ARM DDI 0487C.a)
1912 * Table D9-2 System instruction encodings for non-Debug System
1913 * register accesses.
1914 */
1915 if (mrs_Op0(insn) != 3 || mrs_Op1(insn) != 0 || mrs_CRn(insn) != 0)
1916 return (0);
1917
1918 CRm = mrs_CRm(insn);
1919 if (CRm > 7 || (CRm < 4 && CRm != 0))
1920 return (0);
1921
1922 Op2 = mrs_Op2(insn);
1923 value = 0;
1924
1925 for (i = 0; i < nitems(user_regs); i++) {
1926 if (user_regs[i].CRm == CRm && user_regs[i].Op2 == Op2) {
1927 if (SV_CURPROC_ABI() == SV_ABI_FREEBSD)
1928 value = CPU_DESC_FIELD(user_cpu_desc, i);
1929 else
1930 value = CPU_DESC_FIELD(l_user_cpu_desc, i);
1931 break;
1932 }
1933 }
1934
1935 if (CRm == 0) {
1936 switch (Op2) {
1937 case 0:
1938 value = READ_SPECIALREG(midr_el1);
1939 break;
1940 case 5:
1941 value = READ_SPECIALREG(mpidr_el1);
1942 break;
1943 case 6:
1944 value = READ_SPECIALREG(revidr_el1);
1945 break;
1946 default:
1947 return (0);
1948 }
1949 }
1950
1951 /*
1952 * We will handle this instruction, move to the next so we
1953 * don't trap here again.
1954 */
1955 frame->tf_elr += INSN_SIZE;
1956
1957 reg = MRS_REGISTER(insn);
1958 /* If reg is 31 then write to xzr, i.e. do nothing */
1959 if (reg == 31)
1960 return (1);
1961
1962 if (reg < nitems(frame->tf_x))
1963 frame->tf_x[reg] = value;
1964 else if (reg == 30)
1965 frame->tf_lr = value;
1966
1967 return (1);
1968 }
1969
1970 /*
1971 * Compares two field values that may be signed or unsigned.
1972 * Returns:
1973 * < 0 when a is less than b
1974 * = 0 when a equals b
1975 * > 0 when a is greater than b
1976 */
1977 static int
mrs_field_cmp(uint64_t a,uint64_t b,u_int shift,int width,bool sign)1978 mrs_field_cmp(uint64_t a, uint64_t b, u_int shift, int width, bool sign)
1979 {
1980 uint64_t mask;
1981
1982 KASSERT(width > 0 && width < 64, ("%s: Invalid width %d", __func__,
1983 width));
1984
1985 mask = (1ul << width) - 1;
1986 /* Move the field to the lower bits */
1987 a = (a >> shift) & mask;
1988 b = (b >> shift) & mask;
1989
1990 if (sign) {
1991 /*
1992 * The field is signed. Toggle the upper bit so the comparison
1993 * works on unsigned values as this makes positive numbers,
1994 * i.e. those with a 0 bit, larger than negative numbers,
1995 * i.e. those with a 1 bit, in an unsigned comparison.
1996 */
1997 a ^= 1ul << (width - 1);
1998 b ^= 1ul << (width - 1);
1999 }
2000
2001 return (a - b);
2002 }
2003
2004 static uint64_t
update_lower_register(uint64_t val,uint64_t new_val,u_int shift,int width,bool sign)2005 update_lower_register(uint64_t val, uint64_t new_val, u_int shift,
2006 int width, bool sign)
2007 {
2008 uint64_t mask;
2009
2010 KASSERT(width > 0 && width < 64, ("%s: Invalid width %d", __func__,
2011 width));
2012
2013 /*
2014 * If the new value is less than the existing value update it.
2015 */
2016 if (mrs_field_cmp(new_val, val, shift, width, sign) < 0) {
2017 mask = (1ul << width) - 1;
2018 val &= ~(mask << shift);
2019 val |= new_val & (mask << shift);
2020 }
2021
2022 return (val);
2023 }
2024
2025 bool
extract_user_id_field(u_int reg,u_int field_shift,uint8_t * val)2026 extract_user_id_field(u_int reg, u_int field_shift, uint8_t *val)
2027 {
2028 uint64_t value;
2029 int i;
2030
2031 for (i = 0; i < nitems(user_regs); i++) {
2032 if (user_regs[i].reg == reg) {
2033 value = CPU_DESC_FIELD(user_cpu_desc, i);
2034 *val = value >> field_shift;
2035 return (true);
2036 }
2037 }
2038
2039 return (false);
2040 }
2041
2042 bool
get_kernel_reg(u_int reg,uint64_t * val)2043 get_kernel_reg(u_int reg, uint64_t *val)
2044 {
2045 int i;
2046
2047 for (i = 0; i < nitems(user_regs); i++) {
2048 if (user_regs[i].reg == reg) {
2049 *val = CPU_DESC_FIELD(kern_cpu_desc, i);
2050 return (true);
2051 }
2052 }
2053
2054 return (false);
2055 }
2056
2057 /*
2058 * Fetch the specified register's value, ensuring that individual field values
2059 * do not exceed those in the mask.
2060 */
2061 bool
get_kernel_reg_masked(u_int reg,uint64_t * valp,uint64_t mask)2062 get_kernel_reg_masked(u_int reg, uint64_t *valp, uint64_t mask)
2063 {
2064 const struct mrs_field *fields;
2065 uint64_t val;
2066
2067 for (int i = 0; i < nitems(user_regs); i++) {
2068 if (user_regs[i].reg == reg) {
2069 val = CPU_DESC_FIELD(kern_cpu_desc, i);
2070 fields = user_regs[i].fields;
2071 for (int j = 0; fields[j].type != 0; j++) {
2072 mask = update_lower_register(mask, val,
2073 fields[j].shift, 4, fields[j].sign);
2074 }
2075 *valp = mask;
2076 return (true);
2077 }
2078 }
2079
2080 return (false);
2081 }
2082
2083 static uint64_t
update_special_reg_field(uint64_t user_reg,u_int type,uint64_t value,u_int shift,bool sign)2084 update_special_reg_field(uint64_t user_reg, u_int type, uint64_t value,
2085 u_int shift, bool sign)
2086 {
2087 switch (type & MRS_TYPE_MASK) {
2088 case MRS_EXACT:
2089 user_reg &= ~(0xful << shift);
2090 user_reg |= (uint64_t)MRS_EXACT_FIELD(type) << shift;
2091 break;
2092 case MRS_LOWER:
2093 user_reg = update_lower_register(user_reg, value, shift, 4,
2094 sign);
2095 break;
2096 default:
2097 panic("Invalid field type: %d", type);
2098 }
2099
2100 return (user_reg);
2101 }
2102
2103 void
update_special_regs(u_int cpu)2104 update_special_regs(u_int cpu)
2105 {
2106 struct cpu_desc *desc;
2107 const struct mrs_field *fields;
2108 uint64_t l_user_reg, user_reg, kern_reg, value;
2109 int i, j;
2110
2111 if (cpu == 0) {
2112 /* Create a user visible cpu description with safe values */
2113 memset(&user_cpu_desc, 0, sizeof(user_cpu_desc));
2114 /* Safe values for these registers */
2115 user_cpu_desc.id_aa64pfr0 = ID_AA64PFR0_AdvSIMD_NONE |
2116 ID_AA64PFR0_FP_NONE | ID_AA64PFR0_EL1_64 |
2117 ID_AA64PFR0_EL0_64;
2118 user_cpu_desc.id_aa64dfr0 = ID_AA64DFR0_DebugVer_8;
2119 /* Create the Linux user visible cpu description */
2120 memcpy(&l_user_cpu_desc, &user_cpu_desc, sizeof(user_cpu_desc));
2121 }
2122
2123 desc = get_cpu_desc(cpu);
2124 for (i = 0; i < nitems(user_regs); i++) {
2125 value = CPU_DESC_FIELD(*desc, i);
2126 if (cpu == 0) {
2127 kern_reg = value;
2128 user_reg = value;
2129 l_user_reg = value;
2130 } else {
2131 kern_reg = CPU_DESC_FIELD(kern_cpu_desc, i);
2132 user_reg = CPU_DESC_FIELD(user_cpu_desc, i);
2133 l_user_reg = CPU_DESC_FIELD(l_user_cpu_desc, i);
2134 }
2135
2136 fields = user_regs[i].fields;
2137 for (j = 0; fields[j].type != 0; j++) {
2138 /* Update the FreeBSD userspace ID register view */
2139 user_reg = update_special_reg_field(user_reg,
2140 fields[j].type >> MRS_TYPE_FBSD_SHIFT, value,
2141 fields[j].shift, fields[j].sign);
2142
2143 /* Update the Linux userspace ID register view */
2144 l_user_reg = update_special_reg_field(l_user_reg,
2145 fields[j].type >> MRS_TYPE_LNX_SHIFT, value,
2146 fields[j].shift, fields[j].sign);
2147
2148 /* Update the kernel ID register view */
2149 kern_reg = update_lower_register(kern_reg, value,
2150 fields[j].shift, 4, fields[j].sign);
2151 }
2152
2153 CPU_DESC_FIELD(kern_cpu_desc, i) = kern_reg;
2154 CPU_DESC_FIELD(user_cpu_desc, i) = user_reg;
2155 CPU_DESC_FIELD(l_user_cpu_desc, i) = l_user_reg;
2156 }
2157 }
2158
2159 void
cpu_desc_init(void)2160 cpu_desc_init(void)
2161 {
2162 if (mp_ncpus == 1)
2163 return;
2164
2165 /*
2166 * Allocate memory for the non-boot CPUs to store their registers.
2167 * As this is indexed by CPU ID we need to allocate space for CPUs
2168 * 1 to mp_maxid. Because of this mp_maxid is already the correct
2169 * number of elements.
2170 */
2171 cpu_desc = mallocarray(mp_maxid, sizeof(*cpu_desc), M_IDENTCPU,
2172 M_ZERO | M_WAITOK);
2173 }
2174
2175 /* HWCAP */
2176 bool __read_frequently lse_supported = false;
2177
2178 bool __read_frequently icache_aliasing = false;
2179 bool __read_frequently icache_vmid = false;
2180
2181 int64_t dcache_line_size; /* The minimum D cache line size */
2182 int64_t icache_line_size; /* The minimum I cache line size */
2183 int64_t idcache_line_size; /* The minimum cache line size */
2184
2185 /*
2186 * Find the values to export to userspace as AT_HWCAP and AT_HWCAP2.
2187 */
2188 static void
parse_cpu_features(bool is64bit,struct cpu_desc * cpu_desc,u_long * hwcap,u_long * hwcap2)2189 parse_cpu_features(bool is64bit, struct cpu_desc *cpu_desc, u_long *hwcap,
2190 u_long *hwcap2)
2191 {
2192 const struct mrs_field_hwcap *hwcaps;
2193 const struct mrs_field *fields;
2194 uint64_t min, reg;
2195 u_long *cur_hwcap;
2196 int i, j, k;
2197
2198 for (i = 0; i < nitems(user_regs); i++) {
2199 if (user_regs[i].is64bit != is64bit)
2200 continue;
2201
2202 reg = CPU_DESC_FIELD(*cpu_desc, i);
2203 fields = user_regs[i].fields;
2204 for (j = 0; fields[j].type != 0; j++) {
2205 hwcaps = fields[j].hwcaps;
2206 if (hwcaps == NULL)
2207 continue;
2208
2209 for (k = 0; hwcaps[k].hwcap_id != 0; k++) {
2210 KASSERT(hwcaps[k].hwcap_id == 1 ||
2211 hwcaps[k].hwcap_id == 2,
2212 ("%s: Invalid HWCAP ID %d", __func__,
2213 hwcaps[k].hwcap_id));
2214
2215 cur_hwcap = hwcaps[k].hwcap_id == 1 ?
2216 hwcap : hwcap2;
2217 min = hwcaps[k].min;
2218
2219 /*
2220 * If the field is greater than the minimum
2221 * value we can set the hwcap;
2222 */
2223 if (mrs_field_cmp(reg, min, fields[j].shift,
2224 4, fields[j].sign) >= 0) {
2225 *cur_hwcap |= hwcaps[k].hwcap_val;
2226 }
2227 }
2228 }
2229 }
2230 }
2231
2232 static void
identify_cpu_sysinit(void * dummy __unused)2233 identify_cpu_sysinit(void *dummy __unused)
2234 {
2235 struct cpu_desc *desc, *prev_desc;
2236 int cpu;
2237 bool dic, idc;
2238
2239 dic = (allow_dic != 0);
2240 idc = (allow_idc != 0);
2241
2242 prev_desc = NULL;
2243 CPU_FOREACH(cpu) {
2244 desc = get_cpu_desc(cpu);
2245 if (cpu != 0) {
2246 check_cpu_regs(cpu, desc, prev_desc);
2247 update_special_regs(cpu);
2248 }
2249
2250 if (CTR_DIC_VAL(desc->ctr) == 0)
2251 dic = false;
2252 if (CTR_IDC_VAL(desc->ctr) == 0)
2253 idc = false;
2254 prev_desc = desc;
2255 }
2256
2257 /* Find the values to export to userspace as AT_HWCAP and AT_HWCAP2 */
2258 parse_cpu_features(true, &user_cpu_desc, &elf_hwcap, &elf_hwcap2);
2259 parse_cpu_features(true, &l_user_cpu_desc, &linux_elf_hwcap,
2260 &linux_elf_hwcap2);
2261 #ifdef COMPAT_FREEBSD32
2262 parse_cpu_features(false, &user_cpu_desc, &elf32_hwcap, &elf32_hwcap2);
2263 #endif
2264
2265 /* We export the CPUID registers */
2266 elf_hwcap |= HWCAP_CPUID;
2267 linux_elf_hwcap |= HWCAP_CPUID;
2268
2269 #ifdef COMPAT_FREEBSD32
2270 /* Set the default caps and any that need to check multiple fields */
2271 elf32_hwcap |= parse_cpu_features_hwcap32();
2272 #endif
2273
2274 if (dic && idc) {
2275 arm64_icache_sync_range = &arm64_dic_idc_icache_sync_range;
2276 if (bootverbose)
2277 printf("Enabling DIC & IDC ICache sync\n");
2278 } else if (idc) {
2279 arm64_icache_sync_range = &arm64_idc_aliasing_icache_sync_range;
2280 if (bootverbose)
2281 printf("Enabling IDC ICache sync\n");
2282 }
2283
2284 if ((elf_hwcap & HWCAP_ATOMICS) != 0) {
2285 lse_supported = true;
2286 if (bootverbose)
2287 printf("Enabling LSE atomics in the kernel\n");
2288 }
2289 #ifdef LSE_ATOMICS
2290 if (!lse_supported)
2291 panic("CPU does not support LSE atomic instructions");
2292 #endif
2293
2294 install_undef_handler(true, user_mrs_handler);
2295 }
2296 SYSINIT(identify_cpu, SI_SUB_CPU, SI_ORDER_MIDDLE, identify_cpu_sysinit, NULL);
2297
2298 static void
cpu_features_sysinit(void * dummy __unused)2299 cpu_features_sysinit(void *dummy __unused)
2300 {
2301 struct sbuf sb;
2302 struct cpu_desc *desc, *prev_desc;
2303 u_int cpu;
2304
2305 prev_desc = NULL;
2306 CPU_FOREACH(cpu) {
2307 desc = get_cpu_desc(cpu);
2308 print_cpu_features(cpu, desc, prev_desc);
2309 prev_desc = desc;
2310 }
2311
2312 /* Fill in cpu_model for the hw.model sysctl */
2313 sbuf_new(&sb, cpu_model, sizeof(cpu_model), SBUF_FIXEDLEN);
2314 print_cpu_midr(&sb, 0);
2315
2316 sbuf_finish(&sb);
2317 sbuf_delete(&sb);
2318
2319 free(cpu_desc, M_IDENTCPU);
2320 }
2321 /* Log features before APs are released and start printing to the dmesg. */
2322 SYSINIT(cpu_features, SI_SUB_SMP - 1, SI_ORDER_ANY, cpu_features_sysinit, NULL);
2323
2324 static void
tcr_set_e0pd1(void * arg __unused)2325 tcr_set_e0pd1(void *arg __unused)
2326 {
2327 uint64_t tcr;
2328
2329 tcr = READ_SPECIALREG(tcr_el1);
2330 tcr |= TCR_E0PD1;
2331 WRITE_SPECIALREG(tcr_el1, tcr);
2332 isb();
2333 }
2334
2335 /* Enable support for more recent architecture features */
2336 static void
cpu_feat_support(void * arg __unused)2337 cpu_feat_support(void *arg __unused)
2338 {
2339 /*
2340 * If FEAT_E0PD is supported use it to cause faults without a page
2341 * table walk if userspace tries to access kernel memory.
2342 */
2343 if (ID_AA64MMFR2_E0PD_VAL(kern_cpu_desc.id_aa64mmfr2) !=
2344 ID_AA64MMFR2_E0PD_NONE)
2345 smp_rendezvous(NULL, tcr_set_e0pd1, NULL, NULL);
2346 }
2347 SYSINIT(cpu_feat_support, SI_SUB_SMP, SI_ORDER_ANY, cpu_feat_support, NULL);
2348
2349 #ifdef COMPAT_FREEBSD32
2350 static u_long
parse_cpu_features_hwcap32(void)2351 parse_cpu_features_hwcap32(void)
2352 {
2353 u_long hwcap = HWCAP32_DEFAULT;
2354
2355 if ((MVFR1_SIMDLS_VAL(user_cpu_desc.mvfr1) >=
2356 MVFR1_SIMDLS_IMPL) &&
2357 (MVFR1_SIMDInt_VAL(user_cpu_desc.mvfr1) >=
2358 MVFR1_SIMDInt_IMPL) &&
2359 (MVFR1_SIMDSP_VAL(user_cpu_desc.mvfr1) >=
2360 MVFR1_SIMDSP_IMPL))
2361 hwcap |= HWCAP32_NEON;
2362
2363 return (hwcap);
2364 }
2365 #endif /* COMPAT_FREEBSD32 */
2366
2367 static void
print_ctr_fields(struct sbuf * sb,uint64_t reg,const void * arg __unused)2368 print_ctr_fields(struct sbuf *sb, uint64_t reg, const void *arg __unused)
2369 {
2370
2371 sbuf_printf(sb, "%u byte D-cacheline,", CTR_DLINE_SIZE(reg));
2372 sbuf_printf(sb, "%u byte I-cacheline,", CTR_ILINE_SIZE(reg));
2373 reg &= ~(CTR_DLINE_MASK | CTR_ILINE_MASK);
2374
2375 switch(CTR_L1IP_VAL(reg)) {
2376 case CTR_L1IP_VPIPT:
2377 sbuf_printf(sb, "VPIPT");
2378 break;
2379 case CTR_L1IP_AIVIVT:
2380 sbuf_printf(sb, "AIVIVT");
2381 break;
2382 case CTR_L1IP_VIPT:
2383 sbuf_printf(sb, "VIPT");
2384 break;
2385 case CTR_L1IP_PIPT:
2386 sbuf_printf(sb, "PIPT");
2387 break;
2388 }
2389 sbuf_printf(sb, " ICache,");
2390 reg &= ~CTR_L1IP_MASK;
2391
2392 sbuf_printf(sb, "%d byte ERG,", CTR_ERG_SIZE(reg));
2393 sbuf_printf(sb, "%d byte CWG", CTR_CWG_SIZE(reg));
2394 reg &= ~(CTR_ERG_MASK | CTR_CWG_MASK);
2395
2396 if (CTR_IDC_VAL(reg) != 0)
2397 sbuf_printf(sb, ",IDC");
2398 if (CTR_DIC_VAL(reg) != 0)
2399 sbuf_printf(sb, ",DIC");
2400 reg &= ~(CTR_IDC_MASK | CTR_DIC_MASK);
2401 reg &= ~CTR_RES1;
2402
2403 if (reg != 0)
2404 sbuf_printf(sb, ",%lx", reg);
2405 }
2406
2407 static void
print_register(struct sbuf * sb,const char * reg_name,uint64_t reg,void (* print_fields)(struct sbuf *,uint64_t,const void *),const void * arg)2408 print_register(struct sbuf *sb, const char *reg_name, uint64_t reg,
2409 void (*print_fields)(struct sbuf *, uint64_t, const void *),
2410 const void *arg)
2411 {
2412
2413 sbuf_printf(sb, "%29s = <", reg_name);
2414
2415 print_fields(sb, reg, arg);
2416
2417 sbuf_finish(sb);
2418 printf("%s>\n", sbuf_data(sb));
2419 sbuf_clear(sb);
2420 }
2421
2422 static void
print_id_fields(struct sbuf * sb,uint64_t reg,const void * arg)2423 print_id_fields(struct sbuf *sb, uint64_t reg, const void *arg)
2424 {
2425 const struct mrs_field *fields = arg;
2426 const struct mrs_field_value *fv;
2427 int field, i, j, printed;
2428
2429 #define SEP_STR ((printed++) == 0) ? "" : ","
2430 printed = 0;
2431 for (i = 0; fields[i].type != 0; i++) {
2432 fv = fields[i].values;
2433
2434 /* TODO: Handle with an unknown message */
2435 if (fv == NULL)
2436 continue;
2437
2438 field = (reg & fields[i].mask) >> fields[i].shift;
2439 for (j = 0; fv[j].desc != NULL; j++) {
2440 if ((fv[j].value >> fields[i].shift) != field)
2441 continue;
2442
2443 if (fv[j].desc[0] != '\0')
2444 sbuf_printf(sb, "%s%s", SEP_STR, fv[j].desc);
2445 break;
2446 }
2447 if (fv[j].desc == NULL)
2448 sbuf_printf(sb, "%sUnknown %s(%x)", SEP_STR,
2449 fields[i].name, field);
2450
2451 reg &= ~(0xful << fields[i].shift);
2452 }
2453
2454 if (reg != 0)
2455 sbuf_printf(sb, "%s%#lx", SEP_STR, reg);
2456 #undef SEP_STR
2457 }
2458
2459 static void
print_id_register(struct sbuf * sb,const char * reg_name,uint64_t reg,const struct mrs_field * fields)2460 print_id_register(struct sbuf *sb, const char *reg_name, uint64_t reg,
2461 const struct mrs_field *fields)
2462 {
2463
2464 print_register(sb, reg_name, reg, print_id_fields, fields);
2465 }
2466
2467 static void
print_cpu_midr(struct sbuf * sb,u_int cpu)2468 print_cpu_midr(struct sbuf *sb, u_int cpu)
2469 {
2470 const struct cpu_parts *cpu_partsp;
2471 const char *cpu_impl_name;
2472 const char *cpu_part_name;
2473 u_int midr;
2474 u_int impl_id;
2475 u_int part_id;
2476
2477 midr = pcpu_find(cpu)->pc_midr;
2478
2479 cpu_impl_name = NULL;
2480 cpu_partsp = NULL;
2481 impl_id = CPU_IMPL(midr);
2482 for (int i = 0; cpu_implementers[i].impl_name != NULL; i++) {
2483 if (impl_id == cpu_implementers[i].impl_id) {
2484 cpu_impl_name = cpu_implementers[i].impl_name;
2485 cpu_partsp = cpu_implementers[i].cpu_parts;
2486 break;
2487 }
2488 }
2489 /* Unknown implementer, so unknown part */
2490 if (cpu_impl_name == NULL) {
2491 sbuf_printf(sb, "Unknown Implementer (midr: %08x)", midr);
2492 return;
2493 }
2494
2495 KASSERT(cpu_partsp != NULL, ("%s: No parts table for implementer %s",
2496 __func__, cpu_impl_name));
2497
2498 cpu_part_name = NULL;
2499 part_id = CPU_PART(midr);
2500 for (int i = 0; cpu_partsp[i].part_name != NULL; i++) {
2501 if (part_id == cpu_partsp[i].part_id) {
2502 cpu_part_name = cpu_partsp[i].part_name;
2503 break;
2504 }
2505 }
2506 /* Known Implementer, Unknown part */
2507 if (cpu_part_name == NULL) {
2508 sbuf_printf(sb, "%s Unknown CPU r%dp%d (midr: %08x)",
2509 cpu_impl_name, CPU_VAR(midr), CPU_REV(midr), midr);
2510 return;
2511 }
2512
2513 sbuf_printf(sb, "%s %s r%dp%d", cpu_impl_name,
2514 cpu_part_name, CPU_VAR(midr), CPU_REV(midr));
2515 }
2516
2517 static void
print_cpu_cache(struct cpu_desc * desc,struct sbuf * sb,uint64_t ccs,bool icache,bool unified)2518 print_cpu_cache(struct cpu_desc *desc, struct sbuf *sb, uint64_t ccs,
2519 bool icache, bool unified)
2520 {
2521 size_t cache_size;
2522 size_t line_size;
2523
2524 /* LineSize is Log2(S) - 4. */
2525 line_size = 1 << ((ccs & CCSIDR_LineSize_MASK) + 4);
2526 /*
2527 * Calculate cache size (sets * ways * line size). There are different
2528 * formats depending on the FEAT_CCIDX bit in ID_AA64MMFR2 feature
2529 * register.
2530 */
2531 if ((desc->id_aa64mmfr2 & ID_AA64MMFR2_CCIDX_64))
2532 cache_size = (CCSIDR_NSETS_64(ccs) + 1) *
2533 (CCSIDR_ASSOC_64(ccs) + 1);
2534 else
2535 cache_size = (CCSIDR_NSETS(ccs) + 1) * (CCSIDR_ASSOC(ccs) + 1);
2536
2537 cache_size *= line_size;
2538 sbuf_printf(sb, "%zuKB (%s)", cache_size / 1024,
2539 icache ? "instruction" : unified ? "unified" : "data");
2540 }
2541
2542 static void
print_cpu_caches(struct sbuf * sb,struct cpu_desc * desc)2543 print_cpu_caches(struct sbuf *sb, struct cpu_desc *desc)
2544 {
2545 /* Print out each cache combination */
2546 uint64_t clidr;
2547 int i = 1;
2548 clidr = desc->clidr;
2549
2550 for (i = 0; (clidr & CLIDR_CTYPE_MASK) != 0; i++, clidr >>= 3) {
2551 int j = 0;
2552 int ctype_m = (clidr & CLIDR_CTYPE_MASK);
2553
2554 sbuf_printf(sb, " L%d cache: ", i + 1);
2555 if ((clidr & CLIDR_CTYPE_IO)) {
2556 print_cpu_cache(desc, sb, desc->ccsidr[i][j++], true,
2557 false);
2558 /* If there's more, add to the line. */
2559 if ((ctype_m & ~CLIDR_CTYPE_IO) != 0)
2560 sbuf_printf(sb, ", ");
2561 }
2562 if ((ctype_m & ~CLIDR_CTYPE_IO) != 0) {
2563 print_cpu_cache(desc, sb, desc->ccsidr[i][j], false,
2564 (clidr & CLIDR_CTYPE_UNIFIED));
2565 }
2566 sbuf_printf(sb, "\n");
2567
2568 }
2569 sbuf_finish(sb);
2570 printf("%s", sbuf_data(sb));
2571 }
2572
2573 static void
print_cpu_features(u_int cpu,struct cpu_desc * desc,struct cpu_desc * prev_desc)2574 print_cpu_features(u_int cpu, struct cpu_desc *desc,
2575 struct cpu_desc *prev_desc)
2576 {
2577 struct sbuf *sb;
2578
2579 sb = sbuf_new_auto();
2580 sbuf_printf(sb, "CPU%3u: ", cpu);
2581 print_cpu_midr(sb, cpu);
2582
2583 sbuf_cat(sb, " affinity:");
2584 switch(cpu_aff_levels) {
2585 default:
2586 case 4:
2587 sbuf_printf(sb, " %2d", CPU_AFF3(desc->mpidr));
2588 /* FALLTHROUGH */
2589 case 3:
2590 sbuf_printf(sb, " %2d", CPU_AFF2(desc->mpidr));
2591 /* FALLTHROUGH */
2592 case 2:
2593 sbuf_printf(sb, " %2d", CPU_AFF1(desc->mpidr));
2594 /* FALLTHROUGH */
2595 case 1:
2596 case 0: /* On UP this will be zero */
2597 sbuf_printf(sb, " %2d", CPU_AFF0(desc->mpidr));
2598 break;
2599 }
2600 sbuf_finish(sb);
2601 printf("%s\n", sbuf_data(sb));
2602 sbuf_clear(sb);
2603
2604 /*
2605 * There is a hardware errata where, if one CPU is performing a TLB
2606 * invalidation while another is performing a store-exclusive the
2607 * store-exclusive may return the wrong status. A workaround seems
2608 * to be to use an IPI to invalidate on each CPU, however given the
2609 * limited number of affected units (pass 1.1 is the evaluation
2610 * hardware revision), and the lack of information from Cavium
2611 * this has not been implemented.
2612 *
2613 * At the time of writing this the only information is from:
2614 * https://lkml.org/lkml/2016/8/4/722
2615 */
2616 /*
2617 * XXX: CPU_MATCH_ERRATA_CAVIUM_THUNDERX_1_1 on its own also
2618 * triggers on pass 2.0+.
2619 */
2620 if (cpu == 0 && CPU_VAR(PCPU_GET(midr)) == 0 &&
2621 CPU_MATCH_ERRATA_CAVIUM_THUNDERX_1_1)
2622 printf("WARNING: ThunderX Pass 1.1 detected.\nThis has known "
2623 "hardware bugs that may cause the incorrect operation of "
2624 "atomic operations.\n");
2625
2626 #define SHOULD_PRINT_REG(_reg) \
2627 (prev_desc == NULL || desc->_reg != prev_desc->_reg)
2628
2629 /* Cache Type Register */
2630 if (SHOULD_PRINT_REG(ctr)) {
2631 print_register(sb, "Cache Type",
2632 desc->ctr, print_ctr_fields, NULL);
2633 }
2634
2635 /* AArch64 Instruction Set Attribute Register 0 */
2636 if (SHOULD_PRINT_REG(id_aa64isar0))
2637 print_id_register(sb, "Instruction Set Attributes 0",
2638 desc->id_aa64isar0, id_aa64isar0_fields);
2639
2640 /* AArch64 Instruction Set Attribute Register 1 */
2641 if (SHOULD_PRINT_REG(id_aa64isar1))
2642 print_id_register(sb, "Instruction Set Attributes 1",
2643 desc->id_aa64isar1, id_aa64isar1_fields);
2644
2645 /* AArch64 Instruction Set Attribute Register 2 */
2646 if (SHOULD_PRINT_REG(id_aa64isar2))
2647 print_id_register(sb, "Instruction Set Attributes 2",
2648 desc->id_aa64isar2, id_aa64isar2_fields);
2649
2650 /* AArch64 Processor Feature Register 0 */
2651 if (SHOULD_PRINT_REG(id_aa64pfr0))
2652 print_id_register(sb, "Processor Features 0",
2653 desc->id_aa64pfr0, id_aa64pfr0_fields);
2654
2655 /* AArch64 Processor Feature Register 1 */
2656 if (SHOULD_PRINT_REG(id_aa64pfr1))
2657 print_id_register(sb, "Processor Features 1",
2658 desc->id_aa64pfr1, id_aa64pfr1_fields);
2659
2660 /* AArch64 Processor Feature Register 2 */
2661 if (SHOULD_PRINT_REG(id_aa64pfr2))
2662 print_id_register(sb, "Processor Features 2",
2663 desc->id_aa64pfr2, id_aa64pfr2_fields);
2664
2665 /* AArch64 Memory Model Feature Register 0 */
2666 if (SHOULD_PRINT_REG(id_aa64mmfr0))
2667 print_id_register(sb, "Memory Model Features 0",
2668 desc->id_aa64mmfr0, id_aa64mmfr0_fields);
2669
2670 /* AArch64 Memory Model Feature Register 1 */
2671 if (SHOULD_PRINT_REG(id_aa64mmfr1))
2672 print_id_register(sb, "Memory Model Features 1",
2673 desc->id_aa64mmfr1, id_aa64mmfr1_fields);
2674
2675 /* AArch64 Memory Model Feature Register 2 */
2676 if (SHOULD_PRINT_REG(id_aa64mmfr2))
2677 print_id_register(sb, "Memory Model Features 2",
2678 desc->id_aa64mmfr2, id_aa64mmfr2_fields);
2679
2680 /* AArch64 Memory Model Feature Register 3 */
2681 if (SHOULD_PRINT_REG(id_aa64mmfr3))
2682 print_id_register(sb, "Memory Model Features 3",
2683 desc->id_aa64mmfr3, id_aa64mmfr3_fields);
2684
2685 /* AArch64 Memory Model Feature Register 4 */
2686 if (SHOULD_PRINT_REG(id_aa64mmfr4))
2687 print_id_register(sb, "Memory Model Features 4",
2688 desc->id_aa64mmfr4, id_aa64mmfr4_fields);
2689
2690 /* AArch64 Debug Feature Register 0 */
2691 if (SHOULD_PRINT_REG(id_aa64dfr0))
2692 print_id_register(sb, "Debug Features 0",
2693 desc->id_aa64dfr0, id_aa64dfr0_fields);
2694
2695 /* AArch64 Memory Model Feature Register 1 */
2696 if (SHOULD_PRINT_REG(id_aa64dfr1))
2697 print_id_register(sb, "Debug Features 1",
2698 desc->id_aa64dfr1, id_aa64dfr1_fields);
2699
2700 /* AArch64 Auxiliary Feature Register 0 */
2701 if (SHOULD_PRINT_REG(id_aa64afr0))
2702 print_id_register(sb, "Auxiliary Features 0",
2703 desc->id_aa64afr0, id_aa64afr0_fields);
2704
2705 /* AArch64 Auxiliary Feature Register 1 */
2706 if (SHOULD_PRINT_REG(id_aa64afr1))
2707 print_id_register(sb, "Auxiliary Features 1",
2708 desc->id_aa64afr1, id_aa64afr1_fields);
2709
2710 /* AArch64 SVE Feature Register 0 */
2711 if (desc->have_sve) {
2712 if (SHOULD_PRINT_REG(id_aa64zfr0) ||
2713 !prev_desc->have_sve) {
2714 print_id_register(sb, "SVE Features 0",
2715 desc->id_aa64zfr0, id_aa64zfr0_fields);
2716 }
2717 }
2718
2719 #ifdef COMPAT_FREEBSD32
2720 /* AArch32 Instruction Set Attribute Register 5 */
2721 if (SHOULD_PRINT_REG(id_isar5))
2722 print_id_register(sb, "AArch32 Instruction Set Attributes 5",
2723 desc->id_isar5, id_isar5_fields);
2724
2725 /* AArch32 Media and VFP Feature Register 0 */
2726 if (SHOULD_PRINT_REG(mvfr0))
2727 print_id_register(sb, "AArch32 Media and VFP Features 0",
2728 desc->mvfr0, mvfr0_fields);
2729
2730 /* AArch32 Media and VFP Feature Register 1 */
2731 if (SHOULD_PRINT_REG(mvfr1))
2732 print_id_register(sb, "AArch32 Media and VFP Features 1",
2733 desc->mvfr1, mvfr1_fields);
2734 #endif
2735 if (bootverbose)
2736 print_cpu_caches(sb, desc);
2737
2738 sbuf_delete(sb);
2739 sb = NULL;
2740 #undef SHOULD_PRINT_REG
2741 #undef SEP_STR
2742 }
2743
2744 void
identify_cache(uint64_t ctr)2745 identify_cache(uint64_t ctr)
2746 {
2747
2748 /* Identify the L1 cache type */
2749 switch (CTR_L1IP_VAL(ctr)) {
2750 case CTR_L1IP_PIPT:
2751 break;
2752 case CTR_L1IP_VPIPT:
2753 icache_vmid = true;
2754 break;
2755 default:
2756 case CTR_L1IP_VIPT:
2757 icache_aliasing = true;
2758 break;
2759 }
2760
2761 if (dcache_line_size == 0) {
2762 KASSERT(icache_line_size == 0, ("%s: i-cacheline size set: %ld",
2763 __func__, icache_line_size));
2764
2765 /* Get the D cache line size */
2766 dcache_line_size = CTR_DLINE_SIZE(ctr);
2767 /* And the same for the I cache */
2768 icache_line_size = CTR_ILINE_SIZE(ctr);
2769
2770 idcache_line_size = MIN(dcache_line_size, icache_line_size);
2771 }
2772
2773 if (dcache_line_size != CTR_DLINE_SIZE(ctr)) {
2774 printf("WARNING: D-cacheline size mismatch %ld != %d\n",
2775 dcache_line_size, CTR_DLINE_SIZE(ctr));
2776 }
2777
2778 if (icache_line_size != CTR_ILINE_SIZE(ctr)) {
2779 printf("WARNING: I-cacheline size mismatch %ld != %d\n",
2780 icache_line_size, CTR_ILINE_SIZE(ctr));
2781 }
2782 }
2783
2784 void
identify_cpu(u_int cpu)2785 identify_cpu(u_int cpu)
2786 {
2787 struct cpu_desc *desc;
2788 uint64_t clidr;
2789
2790 desc = get_cpu_desc(cpu);
2791 /* Save affinity for current CPU */
2792 desc->mpidr = get_mpidr();
2793 CPU_AFFINITY(cpu) = desc->mpidr & CPU_AFF_MASK;
2794
2795 desc->ctr = READ_SPECIALREG(ctr_el0);
2796 desc->id_aa64dfr0 = READ_SPECIALREG(ID_AA64DFR0_EL1_REG);
2797 desc->id_aa64dfr1 = READ_SPECIALREG(ID_AA64DFR1_EL1_REG);
2798 desc->id_aa64isar0 = READ_SPECIALREG(ID_AA64ISAR0_EL1_REG);
2799 desc->id_aa64isar1 = READ_SPECIALREG(ID_AA64ISAR1_EL1_REG);
2800 desc->id_aa64isar2 = READ_SPECIALREG(ID_AA64ISAR2_EL1_REG);
2801 desc->id_aa64mmfr0 = READ_SPECIALREG(ID_AA64MMFR0_EL1_REG);
2802 desc->id_aa64mmfr1 = READ_SPECIALREG(ID_AA64MMFR1_EL1_REG);
2803 desc->id_aa64mmfr2 = READ_SPECIALREG(ID_AA64MMFR2_EL1_REG);
2804 desc->id_aa64mmfr3 = READ_SPECIALREG(ID_AA64MMFR3_EL1_REG);
2805 desc->id_aa64mmfr4 = READ_SPECIALREG(ID_AA64MMFR4_EL1_REG);
2806 desc->id_aa64pfr0 = READ_SPECIALREG(ID_AA64PFR0_EL1_REG);
2807 desc->id_aa64pfr1 = READ_SPECIALREG(ID_AA64PFR1_EL1_REG);
2808 desc->id_aa64pfr2 = READ_SPECIALREG(ID_AA64PFR2_EL1_REG);
2809
2810 /*
2811 * ID_AA64ZFR0_EL1 is only valid when at least one of:
2812 * - ID_AA64PFR0_EL1.SVE is non-zero
2813 * - ID_AA64PFR1_EL1.SME is non-zero
2814 * In other cases it is zero, but still safe to read
2815 */
2816 desc->have_sve =
2817 (ID_AA64PFR0_SVE_VAL(desc->id_aa64pfr0) != 0);
2818 desc->id_aa64zfr0 = READ_SPECIALREG(ID_AA64ZFR0_EL1_REG);
2819
2820 desc->clidr = READ_SPECIALREG(clidr_el1);
2821
2822 clidr = desc->clidr;
2823
2824 for (int i = 0; (clidr & CLIDR_CTYPE_MASK) != 0; i++, clidr >>= 3) {
2825 int j = 0;
2826 if ((clidr & CLIDR_CTYPE_IO)) {
2827 WRITE_SPECIALREG(csselr_el1,
2828 CSSELR_Level(i) | CSSELR_InD);
2829 desc->ccsidr[i][j++] =
2830 READ_SPECIALREG(ccsidr_el1);
2831 }
2832 if ((clidr & ~CLIDR_CTYPE_IO) == 0)
2833 continue;
2834 WRITE_SPECIALREG(csselr_el1, CSSELR_Level(i));
2835 desc->ccsidr[i][j] = READ_SPECIALREG(ccsidr_el1);
2836 }
2837
2838 #ifdef COMPAT_FREEBSD32
2839 /* Only read aarch32 SRs if EL0-32 is available */
2840 if (ID_AA64PFR0_EL0_VAL(desc->id_aa64pfr0) == ID_AA64PFR0_EL0_64_32) {
2841 desc->id_isar5 = READ_SPECIALREG(id_isar5_el1);
2842 desc->mvfr0 = READ_SPECIALREG(mvfr0_el1);
2843 desc->mvfr1 = READ_SPECIALREG(mvfr1_el1);
2844 }
2845 #endif
2846 }
2847
2848 static void
check_cpu_regs(u_int cpu,struct cpu_desc * desc,struct cpu_desc * prev_desc)2849 check_cpu_regs(u_int cpu, struct cpu_desc *desc, struct cpu_desc *prev_desc)
2850 {
2851 switch (cpu_aff_levels) {
2852 case 0:
2853 if (CPU_AFF0(desc->mpidr) != CPU_AFF0(prev_desc->mpidr))
2854 cpu_aff_levels = 1;
2855 /* FALLTHROUGH */
2856 case 1:
2857 if (CPU_AFF1(desc->mpidr) != CPU_AFF1(prev_desc->mpidr))
2858 cpu_aff_levels = 2;
2859 /* FALLTHROUGH */
2860 case 2:
2861 if (CPU_AFF2(desc->mpidr) != CPU_AFF2(prev_desc->mpidr))
2862 cpu_aff_levels = 3;
2863 /* FALLTHROUGH */
2864 case 3:
2865 if (CPU_AFF3(desc->mpidr) != CPU_AFF3(prev_desc->mpidr))
2866 cpu_aff_levels = 4;
2867 break;
2868 }
2869
2870 if (desc->ctr != prev_desc->ctr) {
2871 /*
2872 * If the cache type register is different we may
2873 * have a different l1 cache type.
2874 */
2875 identify_cache(desc->ctr);
2876 }
2877 }
2878