1/* 2 * Copyright (c) 2014-2020, ARM Limited and Contributors. All rights reserved. 3 * Copyright (c) 2020, NVIDIA Corporation. All rights reserved. 4 * 5 * SPDX-License-Identifier: BSD-3-Clause 6 */ 7#include <arch.h> 8#include <asm_macros.S> 9#include <assert_macros.S> 10#include <common/bl_common.h> 11#include <common/debug.h> 12#include <cortex_a57.h> 13#include <cpu_macros.S> 14#include <plat_macros.S> 15 16 /* --------------------------------------------- 17 * Disable L1 data cache and unified L2 cache 18 * --------------------------------------------- 19 */ 20func cortex_a57_disable_dcache 21 mrs x1, sctlr_el3 22 bic x1, x1, #SCTLR_C_BIT 23 msr sctlr_el3, x1 24 isb 25 ret 26endfunc cortex_a57_disable_dcache 27 28 /* --------------------------------------------- 29 * Disable all types of L2 prefetches. 30 * --------------------------------------------- 31 */ 32func cortex_a57_disable_l2_prefetch 33 mrs x0, CORTEX_A57_ECTLR_EL1 34 orr x0, x0, #CORTEX_A57_ECTLR_DIS_TWD_ACC_PFTCH_BIT 35 mov x1, #CORTEX_A57_ECTLR_L2_IPFTCH_DIST_MASK 36 orr x1, x1, #CORTEX_A57_ECTLR_L2_DPFTCH_DIST_MASK 37 bic x0, x0, x1 38 msr CORTEX_A57_ECTLR_EL1, x0 39 isb 40 dsb ish 41 ret 42endfunc cortex_a57_disable_l2_prefetch 43 44 /* --------------------------------------------- 45 * Disable intra-cluster coherency 46 * --------------------------------------------- 47 */ 48func cortex_a57_disable_smp 49 mrs x0, CORTEX_A57_ECTLR_EL1 50 bic x0, x0, #CORTEX_A57_ECTLR_SMP_BIT 51 msr CORTEX_A57_ECTLR_EL1, x0 52 ret 53endfunc cortex_a57_disable_smp 54 55 /* --------------------------------------------- 56 * Disable debug interfaces 57 * --------------------------------------------- 58 */ 59func cortex_a57_disable_ext_debug 60 mov x0, #1 61 msr osdlr_el1, x0 62 isb 63#if ERRATA_A57_817169 64 /* 65 * Invalidate any TLB address 66 */ 67 mov x0, #0 68 tlbi vae3, x0 69#endif 70 dsb sy 71 ret 72endfunc cortex_a57_disable_ext_debug 73 74 /* -------------------------------------------------- 75 * Errata Workaround for Cortex A57 Errata #806969. 76 * This applies only to revision r0p0 of Cortex A57. 77 * Inputs: 78 * x0: variant[4:7] and revision[0:3] of current cpu. 79 * Shall clobber: x0-x17 80 * -------------------------------------------------- 81 */ 82func errata_a57_806969_wa 83 /* 84 * Compare x0 against revision r0p0 85 */ 86 mov x17, x30 87 bl check_errata_806969 88 cbz x0, 1f 89 mrs x1, CORTEX_A57_CPUACTLR_EL1 90 orr x1, x1, #CORTEX_A57_CPUACTLR_EL1_NO_ALLOC_WBWA 91 msr CORTEX_A57_CPUACTLR_EL1, x1 921: 93 ret x17 94endfunc errata_a57_806969_wa 95 96func check_errata_806969 97 mov x1, #0x00 98 b cpu_rev_var_ls 99endfunc check_errata_806969 100 101 /* --------------------------------------------------- 102 * Errata Workaround for Cortex A57 Errata #813419. 103 * This applies only to revision r0p0 of Cortex A57. 104 * --------------------------------------------------- 105 */ 106func check_errata_813419 107 /* 108 * Even though this is only needed for revision r0p0, it 109 * is always applied due to limitations of the current 110 * errata framework. 111 */ 112 mov x0, #ERRATA_APPLIES 113 ret 114endfunc check_errata_813419 115 116 /* --------------------------------------------------- 117 * Errata Workaround for Cortex A57 Errata #813420. 118 * This applies only to revision r0p0 of Cortex A57. 119 * Inputs: 120 * x0: variant[4:7] and revision[0:3] of current cpu. 121 * Shall clobber: x0-x17 122 * --------------------------------------------------- 123 */ 124func errata_a57_813420_wa 125 /* 126 * Compare x0 against revision r0p0 127 */ 128 mov x17, x30 129 bl check_errata_813420 130 cbz x0, 1f 131 mrs x1, CORTEX_A57_CPUACTLR_EL1 132 orr x1, x1, #CORTEX_A57_CPUACTLR_EL1_DCC_AS_DCCI 133 msr CORTEX_A57_CPUACTLR_EL1, x1 1341: 135 ret x17 136endfunc errata_a57_813420_wa 137 138func check_errata_813420 139 mov x1, #0x00 140 b cpu_rev_var_ls 141endfunc check_errata_813420 142 143 /* --------------------------------------------------- 144 * Errata Workaround for Cortex A57 Errata #814670. 145 * This applies only to revision r0p0 of Cortex A57. 146 * Inputs: 147 * x0: variant[4:7] and revision[0:3] of current cpu. 148 * Shall clobber: x0-x17 149 * --------------------------------------------------- 150 */ 151func errata_a57_814670_wa 152 /* 153 * Compare x0 against revision r0p0 154 */ 155 mov x17, x30 156 bl check_errata_814670 157 cbz x0, 1f 158 mrs x1, CORTEX_A57_CPUACTLR_EL1 159 orr x1, x1, #CORTEX_A57_CPUACTLR_EL1_DIS_DMB_NULLIFICATION 160 msr CORTEX_A57_CPUACTLR_EL1, x1 161 isb 1621: 163 ret x17 164endfunc errata_a57_814670_wa 165 166func check_errata_814670 167 mov x1, #0x00 168 b cpu_rev_var_ls 169endfunc check_errata_814670 170 171 /* ---------------------------------------------------- 172 * Errata Workaround for Cortex A57 Errata #817169. 173 * This applies only to revision <= r0p1 of Cortex A57. 174 * ---------------------------------------------------- 175 */ 176func check_errata_817169 177 /* 178 * Even though this is only needed for revision <= r0p1, it 179 * is always applied because of the low cost of the workaround. 180 */ 181 mov x0, #ERRATA_APPLIES 182 ret 183endfunc check_errata_817169 184 185 /* -------------------------------------------------------------------- 186 * Disable the over-read from the LDNP instruction. 187 * 188 * This applies to all revisions <= r1p2. The performance degradation 189 * observed with LDNP/STNP has been fixed on r1p3 and onwards. 190 * 191 * Inputs: 192 * x0: variant[4:7] and revision[0:3] of current cpu. 193 * Shall clobber: x0-x17 194 * --------------------------------------------------------------------- 195 */ 196func a57_disable_ldnp_overread 197 /* 198 * Compare x0 against revision r1p2 199 */ 200 mov x17, x30 201 bl check_errata_disable_ldnp_overread 202 cbz x0, 1f 203 mrs x1, CORTEX_A57_CPUACTLR_EL1 204 orr x1, x1, #CORTEX_A57_CPUACTLR_EL1_DIS_OVERREAD 205 msr CORTEX_A57_CPUACTLR_EL1, x1 2061: 207 ret x17 208endfunc a57_disable_ldnp_overread 209 210func check_errata_disable_ldnp_overread 211 mov x1, #0x12 212 b cpu_rev_var_ls 213endfunc check_errata_disable_ldnp_overread 214 215 /* --------------------------------------------------- 216 * Errata Workaround for Cortex A57 Errata #826974. 217 * This applies only to revision <= r1p1 of Cortex A57. 218 * Inputs: 219 * x0: variant[4:7] and revision[0:3] of current cpu. 220 * Shall clobber: x0-x17 221 * --------------------------------------------------- 222 */ 223func errata_a57_826974_wa 224 /* 225 * Compare x0 against revision r1p1 226 */ 227 mov x17, x30 228 bl check_errata_826974 229 cbz x0, 1f 230 mrs x1, CORTEX_A57_CPUACTLR_EL1 231 orr x1, x1, #CORTEX_A57_CPUACTLR_EL1_DIS_LOAD_PASS_DMB 232 msr CORTEX_A57_CPUACTLR_EL1, x1 2331: 234 ret x17 235endfunc errata_a57_826974_wa 236 237func check_errata_826974 238 mov x1, #0x11 239 b cpu_rev_var_ls 240endfunc check_errata_826974 241 242 /* --------------------------------------------------- 243 * Errata Workaround for Cortex A57 Errata #826977. 244 * This applies only to revision <= r1p1 of Cortex A57. 245 * Inputs: 246 * x0: variant[4:7] and revision[0:3] of current cpu. 247 * Shall clobber: x0-x17 248 * --------------------------------------------------- 249 */ 250func errata_a57_826977_wa 251 /* 252 * Compare x0 against revision r1p1 253 */ 254 mov x17, x30 255 bl check_errata_826977 256 cbz x0, 1f 257 mrs x1, CORTEX_A57_CPUACTLR_EL1 258 orr x1, x1, #CORTEX_A57_CPUACTLR_EL1_GRE_NGRE_AS_NGNRE 259 msr CORTEX_A57_CPUACTLR_EL1, x1 2601: 261 ret x17 262endfunc errata_a57_826977_wa 263 264func check_errata_826977 265 mov x1, #0x11 266 b cpu_rev_var_ls 267endfunc check_errata_826977 268 269 /* --------------------------------------------------- 270 * Errata Workaround for Cortex A57 Errata #828024. 271 * This applies only to revision <= r1p1 of Cortex A57. 272 * Inputs: 273 * x0: variant[4:7] and revision[0:3] of current cpu. 274 * Shall clobber: x0-x17 275 * --------------------------------------------------- 276 */ 277func errata_a57_828024_wa 278 /* 279 * Compare x0 against revision r1p1 280 */ 281 mov x17, x30 282 bl check_errata_828024 283 cbz x0, 1f 284 mrs x1, CORTEX_A57_CPUACTLR_EL1 285 /* 286 * Setting the relevant bits in CPUACTLR_EL1 has to be done in 2 287 * instructions here because the resulting bitmask doesn't fit in a 288 * 16-bit value so it cannot be encoded in a single instruction. 289 */ 290 orr x1, x1, #CORTEX_A57_CPUACTLR_EL1_NO_ALLOC_WBWA 291 orr x1, x1, #(CORTEX_A57_CPUACTLR_EL1_DIS_L1_STREAMING | \ 292 CORTEX_A57_CPUACTLR_EL1_DIS_STREAMING) 293 msr CORTEX_A57_CPUACTLR_EL1, x1 2941: 295 ret x17 296endfunc errata_a57_828024_wa 297 298func check_errata_828024 299 mov x1, #0x11 300 b cpu_rev_var_ls 301endfunc check_errata_828024 302 303 /* --------------------------------------------------- 304 * Errata Workaround for Cortex A57 Errata #829520. 305 * This applies only to revision <= r1p2 of Cortex A57. 306 * Inputs: 307 * x0: variant[4:7] and revision[0:3] of current cpu. 308 * Shall clobber: x0-x17 309 * --------------------------------------------------- 310 */ 311func errata_a57_829520_wa 312 /* 313 * Compare x0 against revision r1p2 314 */ 315 mov x17, x30 316 bl check_errata_829520 317 cbz x0, 1f 318 mrs x1, CORTEX_A57_CPUACTLR_EL1 319 orr x1, x1, #CORTEX_A57_CPUACTLR_EL1_DIS_INDIRECT_PREDICTOR 320 msr CORTEX_A57_CPUACTLR_EL1, x1 3211: 322 ret x17 323endfunc errata_a57_829520_wa 324 325func check_errata_829520 326 mov x1, #0x12 327 b cpu_rev_var_ls 328endfunc check_errata_829520 329 330 /* --------------------------------------------------- 331 * Errata Workaround for Cortex A57 Errata #833471. 332 * This applies only to revision <= r1p2 of Cortex A57. 333 * Inputs: 334 * x0: variant[4:7] and revision[0:3] of current cpu. 335 * Shall clobber: x0-x17 336 * --------------------------------------------------- 337 */ 338func errata_a57_833471_wa 339 /* 340 * Compare x0 against revision r1p2 341 */ 342 mov x17, x30 343 bl check_errata_833471 344 cbz x0, 1f 345 mrs x1, CORTEX_A57_CPUACTLR_EL1 346 orr x1, x1, #CORTEX_A57_CPUACTLR_EL1_FORCE_FPSCR_FLUSH 347 msr CORTEX_A57_CPUACTLR_EL1, x1 3481: 349 ret x17 350endfunc errata_a57_833471_wa 351 352func check_errata_833471 353 mov x1, #0x12 354 b cpu_rev_var_ls 355endfunc check_errata_833471 356 357 /* -------------------------------------------------- 358 * Errata Workaround for Cortex A57 Errata #859972. 359 * This applies only to revision <= r1p3 of Cortex A57. 360 * Inputs: 361 * x0: variant[4:7] and revision[0:3] of current cpu. 362 * Shall clobber: 363 * -------------------------------------------------- 364 */ 365func errata_a57_859972_wa 366 mov x17, x30 367 bl check_errata_859972 368 cbz x0, 1f 369 mrs x1, CORTEX_A57_CPUACTLR_EL1 370 orr x1, x1, #CORTEX_A57_CPUACTLR_EL1_DIS_INSTR_PREFETCH 371 msr CORTEX_A57_CPUACTLR_EL1, x1 3721: 373 ret x17 374endfunc errata_a57_859972_wa 375 376func check_errata_859972 377 mov x1, #0x13 378 b cpu_rev_var_ls 379endfunc check_errata_859972 380 381func check_errata_cve_2017_5715 382#if WORKAROUND_CVE_2017_5715 383 mov x0, #ERRATA_APPLIES 384#else 385 mov x0, #ERRATA_MISSING 386#endif 387 ret 388endfunc check_errata_cve_2017_5715 389 390func check_errata_cve_2018_3639 391#if WORKAROUND_CVE_2018_3639 392 mov x0, #ERRATA_APPLIES 393#else 394 mov x0, #ERRATA_MISSING 395#endif 396 ret 397endfunc check_errata_cve_2018_3639 398 399 /* -------------------------------------------------- 400 * Errata workaround for Cortex A57 Errata #1319537. 401 * This applies to all revisions of Cortex A57. 402 * -------------------------------------------------- 403 */ 404func check_errata_1319537 405#if ERRATA_A57_1319537 406 mov x0, #ERRATA_APPLIES 407#else 408 mov x0, #ERRATA_MISSING 409#endif 410 ret 411endfunc check_errata_1319537 412 413 /* ------------------------------------------------- 414 * The CPU Ops reset function for Cortex-A57. 415 * Shall clobber: x0-x19 416 * ------------------------------------------------- 417 */ 418func cortex_a57_reset_func 419 mov x19, x30 420 bl cpu_get_rev_var 421 mov x18, x0 422 423#if ERRATA_A57_806969 424 mov x0, x18 425 bl errata_a57_806969_wa 426#endif 427 428#if ERRATA_A57_813420 429 mov x0, x18 430 bl errata_a57_813420_wa 431#endif 432 433#if ERRATA_A57_814670 434 mov x0, x18 435 bl errata_a57_814670_wa 436#endif 437 438#if A57_DISABLE_NON_TEMPORAL_HINT 439 mov x0, x18 440 bl a57_disable_ldnp_overread 441#endif 442 443#if ERRATA_A57_826974 444 mov x0, x18 445 bl errata_a57_826974_wa 446#endif 447 448#if ERRATA_A57_826977 449 mov x0, x18 450 bl errata_a57_826977_wa 451#endif 452 453#if ERRATA_A57_828024 454 mov x0, x18 455 bl errata_a57_828024_wa 456#endif 457 458#if ERRATA_A57_829520 459 mov x0, x18 460 bl errata_a57_829520_wa 461#endif 462 463#if ERRATA_A57_833471 464 mov x0, x18 465 bl errata_a57_833471_wa 466#endif 467 468#if ERRATA_A57_859972 469 mov x0, x18 470 bl errata_a57_859972_wa 471#endif 472 473#if IMAGE_BL31 && WORKAROUND_CVE_2017_5715 474 adr x0, wa_cve_2017_5715_mmu_vbar 475 msr vbar_el3, x0 476 /* isb will be performed before returning from this function */ 477#endif 478 479#if WORKAROUND_CVE_2018_3639 480 mrs x0, CORTEX_A57_CPUACTLR_EL1 481 orr x0, x0, #CORTEX_A57_CPUACTLR_EL1_DIS_LOAD_PASS_STORE 482 msr CORTEX_A57_CPUACTLR_EL1, x0 483 isb 484 dsb sy 485#endif 486 487#if A57_ENABLE_NONCACHEABLE_LOAD_FWD 488 /* --------------------------------------------- 489 * Enable higher performance non-cacheable load 490 * forwarding 491 * --------------------------------------------- 492 */ 493 mrs x0, CORTEX_A57_CPUACTLR_EL1 494 orr x0, x0, #CORTEX_A57_CPUACTLR_EL1_EN_NC_LOAD_FWD 495 msr CORTEX_A57_CPUACTLR_EL1, x0 496#endif 497 498 /* --------------------------------------------- 499 * Enable the SMP bit. 500 * --------------------------------------------- 501 */ 502 mrs x0, CORTEX_A57_ECTLR_EL1 503 orr x0, x0, #CORTEX_A57_ECTLR_SMP_BIT 504 msr CORTEX_A57_ECTLR_EL1, x0 505 isb 506 ret x19 507endfunc cortex_a57_reset_func 508 509 /* ---------------------------------------------------- 510 * The CPU Ops core power down function for Cortex-A57. 511 * ---------------------------------------------------- 512 */ 513func cortex_a57_core_pwr_dwn 514 mov x18, x30 515 516 /* --------------------------------------------- 517 * Turn off caches. 518 * --------------------------------------------- 519 */ 520 bl cortex_a57_disable_dcache 521 522 /* --------------------------------------------- 523 * Disable the L2 prefetches. 524 * --------------------------------------------- 525 */ 526 bl cortex_a57_disable_l2_prefetch 527 528 /* --------------------------------------------- 529 * Flush L1 caches. 530 * --------------------------------------------- 531 */ 532 mov x0, #DCCISW 533 bl dcsw_op_level1 534 535 /* --------------------------------------------- 536 * Come out of intra cluster coherency 537 * --------------------------------------------- 538 */ 539 bl cortex_a57_disable_smp 540 541 /* --------------------------------------------- 542 * Force the debug interfaces to be quiescent 543 * --------------------------------------------- 544 */ 545 mov x30, x18 546 b cortex_a57_disable_ext_debug 547endfunc cortex_a57_core_pwr_dwn 548 549 /* ------------------------------------------------------- 550 * The CPU Ops cluster power down function for Cortex-A57. 551 * ------------------------------------------------------- 552 */ 553func cortex_a57_cluster_pwr_dwn 554 mov x18, x30 555 556 /* --------------------------------------------- 557 * Turn off caches. 558 * --------------------------------------------- 559 */ 560 bl cortex_a57_disable_dcache 561 562 /* --------------------------------------------- 563 * Disable the L2 prefetches. 564 * --------------------------------------------- 565 */ 566 bl cortex_a57_disable_l2_prefetch 567 568#if !SKIP_A57_L1_FLUSH_PWR_DWN 569 /* ------------------------------------------------- 570 * Flush the L1 caches. 571 * ------------------------------------------------- 572 */ 573 mov x0, #DCCISW 574 bl dcsw_op_level1 575#endif 576 /* --------------------------------------------- 577 * Disable the optional ACP. 578 * --------------------------------------------- 579 */ 580 bl plat_disable_acp 581 582 /* ------------------------------------------------- 583 * Flush the L2 caches. 584 * ------------------------------------------------- 585 */ 586 mov x0, #DCCISW 587 bl dcsw_op_level2 588 589 /* --------------------------------------------- 590 * Come out of intra cluster coherency 591 * --------------------------------------------- 592 */ 593 bl cortex_a57_disable_smp 594 595 /* --------------------------------------------- 596 * Force the debug interfaces to be quiescent 597 * --------------------------------------------- 598 */ 599 mov x30, x18 600 b cortex_a57_disable_ext_debug 601endfunc cortex_a57_cluster_pwr_dwn 602 603#if REPORT_ERRATA 604/* 605 * Errata printing function for Cortex A57. Must follow AAPCS. 606 */ 607func cortex_a57_errata_report 608 stp x8, x30, [sp, #-16]! 609 610 bl cpu_get_rev_var 611 mov x8, x0 612 613 /* 614 * Report all errata. The revision-variant information is passed to 615 * checking functions of each errata. 616 */ 617 report_errata ERRATA_A57_806969, cortex_a57, 806969 618 report_errata ERRATA_A57_813419, cortex_a57, 813419 619 report_errata ERRATA_A57_813420, cortex_a57, 813420 620 report_errata ERRATA_A57_814670, cortex_a57, 814670 621 report_errata ERRATA_A57_817169, cortex_a57, 817169 622 report_errata A57_DISABLE_NON_TEMPORAL_HINT, cortex_a57, \ 623 disable_ldnp_overread 624 report_errata ERRATA_A57_826974, cortex_a57, 826974 625 report_errata ERRATA_A57_826977, cortex_a57, 826977 626 report_errata ERRATA_A57_828024, cortex_a57, 828024 627 report_errata ERRATA_A57_829520, cortex_a57, 829520 628 report_errata ERRATA_A57_833471, cortex_a57, 833471 629 report_errata ERRATA_A57_859972, cortex_a57, 859972 630 report_errata ERRATA_A57_1319537, cortex_a57, 1319537 631 report_errata WORKAROUND_CVE_2017_5715, cortex_a57, cve_2017_5715 632 report_errata WORKAROUND_CVE_2018_3639, cortex_a57, cve_2018_3639 633 634 ldp x8, x30, [sp], #16 635 ret 636endfunc cortex_a57_errata_report 637#endif 638 639 /* --------------------------------------------- 640 * This function provides cortex_a57 specific 641 * register information for crash reporting. 642 * It needs to return with x6 pointing to 643 * a list of register names in ascii and 644 * x8 - x15 having values of registers to be 645 * reported. 646 * --------------------------------------------- 647 */ 648.section .rodata.cortex_a57_regs, "aS" 649cortex_a57_regs: /* The ascii list of register names to be reported */ 650 .asciz "cpuectlr_el1", "cpumerrsr_el1", "l2merrsr_el1", "" 651 652func cortex_a57_cpu_reg_dump 653 adr x6, cortex_a57_regs 654 mrs x8, CORTEX_A57_ECTLR_EL1 655 mrs x9, CORTEX_A57_MERRSR_EL1 656 mrs x10, CORTEX_A57_L2MERRSR_EL1 657 ret 658endfunc cortex_a57_cpu_reg_dump 659 660declare_cpu_ops_wa cortex_a57, CORTEX_A57_MIDR, \ 661 cortex_a57_reset_func, \ 662 check_errata_cve_2017_5715, \ 663 CPU_NO_EXTRA2_FUNC, \ 664 cortex_a57_core_pwr_dwn, \ 665 cortex_a57_cluster_pwr_dwn 666