1;; Machine description for AArch64 architecture. 2;; Copyright (C) 2009-2021 Free Software Foundation, Inc. 3;; Contributed by ARM Ltd. 4;; 5;; This file is part of GCC. 6;; 7;; GCC is free software; you can redistribute it and/or modify it 8;; under the terms of the GNU General Public License as published by 9;; the Free Software Foundation; either version 3, or (at your option) 10;; any later version. 11;; 12;; GCC is distributed in the hope that it will be useful, but 13;; WITHOUT ANY WARRANTY; without even the implied warranty of 14;; MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU 15;; General Public License for more details. 16;; 17;; You should have received a copy of the GNU General Public License 18;; along with GCC; see the file COPYING3. If not see 19;; <http://www.gnu.org/licenses/>. 20 21;; Register numbers 22(define_constants 23 [ 24 (R0_REGNUM 0) 25 (R1_REGNUM 1) 26 (R2_REGNUM 2) 27 (R3_REGNUM 3) 28 (R4_REGNUM 4) 29 (R5_REGNUM 5) 30 (R6_REGNUM 6) 31 (R7_REGNUM 7) 32 (R8_REGNUM 8) 33 (R9_REGNUM 9) 34 (R10_REGNUM 10) 35 (R11_REGNUM 11) 36 (R12_REGNUM 12) 37 (R13_REGNUM 13) 38 (R14_REGNUM 14) 39 (R15_REGNUM 15) 40 (R16_REGNUM 16) 41 (R17_REGNUM 17) 42 (R18_REGNUM 18) 43 (R19_REGNUM 19) 44 (R20_REGNUM 20) 45 (R21_REGNUM 21) 46 (R22_REGNUM 22) 47 (R23_REGNUM 23) 48 (R24_REGNUM 24) 49 (R25_REGNUM 25) 50 (R26_REGNUM 26) 51 (R27_REGNUM 27) 52 (R28_REGNUM 28) 53 (R29_REGNUM 29) 54 (R30_REGNUM 30) 55 (SP_REGNUM 31) 56 (V0_REGNUM 32) 57 (V1_REGNUM 33) 58 (V2_REGNUM 34) 59 (V3_REGNUM 35) 60 (V4_REGNUM 36) 61 (V5_REGNUM 37) 62 (V6_REGNUM 38) 63 (V7_REGNUM 39) 64 (V8_REGNUM 40) 65 (V9_REGNUM 41) 66 (V10_REGNUM 42) 67 (V11_REGNUM 43) 68 (V12_REGNUM 44) 69 (V13_REGNUM 45) 70 (V14_REGNUM 46) 71 (V15_REGNUM 47) 72 (V16_REGNUM 48) 73 (V17_REGNUM 49) 74 (V18_REGNUM 50) 75 (V19_REGNUM 51) 76 (V20_REGNUM 52) 77 (V21_REGNUM 53) 78 (V22_REGNUM 54) 79 (V23_REGNUM 55) 80 (V24_REGNUM 56) 81 (V25_REGNUM 57) 82 (V26_REGNUM 58) 83 (V27_REGNUM 59) 84 (V28_REGNUM 60) 85 (V29_REGNUM 61) 86 (V30_REGNUM 62) 87 (V31_REGNUM 63) 88 (SFP_REGNUM 64) 89 (AP_REGNUM 65) 90 (CC_REGNUM 66) 91 ;; Defined only to make the DWARF description simpler. 92 (VG_REGNUM 67) 93 (P0_REGNUM 68) 94 (P1_REGNUM 69) 95 (P2_REGNUM 70) 96 (P3_REGNUM 71) 97 (P4_REGNUM 72) 98 (P5_REGNUM 73) 99 (P6_REGNUM 74) 100 (P7_REGNUM 75) 101 (P8_REGNUM 76) 102 (P9_REGNUM 77) 103 (P10_REGNUM 78) 104 (P11_REGNUM 79) 105 (P12_REGNUM 80) 106 (P13_REGNUM 81) 107 (P14_REGNUM 82) 108 (P15_REGNUM 83) 109 (LAST_SAVED_REGNUM 83) 110 (FFR_REGNUM 84) 111 ;; "FFR token": a fake register used for representing the scheduling 112 ;; restrictions on FFR-related operations. 113 (FFRT_REGNUM 85) 114 ;; The pair of scratch registers used for stack probing with -fstack-check. 115 ;; Leave R9 alone as a possible choice for the static chain. 116 ;; Note that the use of these registers is mutually exclusive with the use 117 ;; of STACK_CLASH_SVE_CFA_REGNUM, which is for -fstack-clash-protection 118 ;; rather than -fstack-check. 119 (PROBE_STACK_FIRST_REGNUM 10) 120 (PROBE_STACK_SECOND_REGNUM 11) 121 ;; Scratch register used by stack clash protection to calculate 122 ;; SVE CFA offsets during probing. 123 (STACK_CLASH_SVE_CFA_REGNUM 11) 124 ;; Scratch registers for prologue/epilogue use. 125 (EP0_REGNUM 12) 126 (EP1_REGNUM 13) 127 ;; A couple of call-clobbered registers that we need to reserve when 128 ;; tracking speculation this is not ABI, so is subject to change. 129 (SPECULATION_SCRATCH_REGNUM 14) 130 (SPECULATION_TRACKER_REGNUM 15) 131 ;; Scratch registers used in frame layout. 132 (IP0_REGNUM 16) 133 (IP1_REGNUM 17) 134 (FP_REGNUM 29) 135 (LR_REGNUM 30) 136 ] 137) 138 139(define_c_enum "unspec" [ 140 UNSPEC_AUTIA1716 141 UNSPEC_AUTIB1716 142 UNSPEC_AUTIASP 143 UNSPEC_AUTIBSP 144 UNSPEC_CALLEE_ABI 145 UNSPEC_CASESI 146 UNSPEC_CRC32B 147 UNSPEC_CRC32CB 148 UNSPEC_CRC32CH 149 UNSPEC_CRC32CW 150 UNSPEC_CRC32CX 151 UNSPEC_CRC32H 152 UNSPEC_CRC32W 153 UNSPEC_CRC32X 154 UNSPEC_FCVTZS 155 UNSPEC_FCVTZU 156 UNSPEC_FJCVTZS 157 UNSPEC_FRINT32Z 158 UNSPEC_FRINT32X 159 UNSPEC_FRINT64Z 160 UNSPEC_FRINT64X 161 UNSPEC_URECPE 162 UNSPEC_FRECPE 163 UNSPEC_FRECPS 164 UNSPEC_FRECPX 165 UNSPEC_FRINTA 166 UNSPEC_FRINTI 167 UNSPEC_FRINTM 168 UNSPEC_FRINTN 169 UNSPEC_FRINTP 170 UNSPEC_FRINTX 171 UNSPEC_FRINTZ 172 UNSPEC_GOTSMALLPIC 173 UNSPEC_GOTSMALLPIC28K 174 UNSPEC_GOTSMALLTLS 175 UNSPEC_GOTTINYPIC 176 UNSPEC_GOTTINYTLS 177 UNSPEC_LD1 178 UNSPEC_LD2 179 UNSPEC_LD2_DREG 180 UNSPEC_LD2_DUP 181 UNSPEC_LD3 182 UNSPEC_LD3_DREG 183 UNSPEC_LD3_DUP 184 UNSPEC_LD4 185 UNSPEC_LD4_DREG 186 UNSPEC_LD4_DUP 187 UNSPEC_LD2_LANE 188 UNSPEC_LD3_LANE 189 UNSPEC_LD4_LANE 190 UNSPEC_MB 191 UNSPEC_NOP 192 UNSPEC_PACIA1716 193 UNSPEC_PACIB1716 194 UNSPEC_PACIASP 195 UNSPEC_PACIBSP 196 UNSPEC_PRLG_STK 197 UNSPEC_REV 198 UNSPEC_RBIT 199 UNSPEC_SABAL 200 UNSPEC_SABAL2 201 UNSPEC_SABDL 202 UNSPEC_SABDL2 203 UNSPEC_SADALP 204 UNSPEC_SCVTF 205 UNSPEC_SISD_NEG 206 UNSPEC_SISD_SSHL 207 UNSPEC_SISD_USHL 208 UNSPEC_SSHL_2S 209 UNSPEC_ST1 210 UNSPEC_ST2 211 UNSPEC_ST3 212 UNSPEC_ST4 213 UNSPEC_ST2_LANE 214 UNSPEC_ST3_LANE 215 UNSPEC_ST4_LANE 216 UNSPEC_TLS 217 UNSPEC_TLSDESC 218 UNSPEC_TLSLE12 219 UNSPEC_TLSLE24 220 UNSPEC_TLSLE32 221 UNSPEC_TLSLE48 222 UNSPEC_UABAL 223 UNSPEC_UABAL2 224 UNSPEC_UABDL 225 UNSPEC_UABDL2 226 UNSPEC_UADALP 227 UNSPEC_UCVTF 228 UNSPEC_USHL_2S 229 UNSPEC_VSTRUCTDUMMY 230 UNSPEC_SSP_SYSREG 231 UNSPEC_SP_SET 232 UNSPEC_SP_TEST 233 UNSPEC_RSHRN 234 UNSPEC_RSQRT 235 UNSPEC_RSQRTE 236 UNSPEC_RSQRTS 237 UNSPEC_NZCV 238 UNSPEC_XPACLRI 239 UNSPEC_LD1_SVE 240 UNSPEC_ST1_SVE 241 UNSPEC_LDNT1_SVE 242 UNSPEC_STNT1_SVE 243 UNSPEC_LD1RQ 244 UNSPEC_LD1_GATHER 245 UNSPEC_LDFF1_GATHER 246 UNSPEC_LDNT1_GATHER 247 UNSPEC_ST1_SCATTER 248 UNSPEC_STNT1_SCATTER 249 UNSPEC_PRED_X 250 UNSPEC_PRED_Z 251 UNSPEC_PTEST 252 UNSPEC_PTRUE 253 UNSPEC_UNPACKSHI 254 UNSPEC_UNPACKUHI 255 UNSPEC_UNPACKSLO 256 UNSPEC_UNPACKULO 257 UNSPEC_PACK 258 UNSPEC_WHILEGE 259 UNSPEC_WHILEGT 260 UNSPEC_WHILEHI 261 UNSPEC_WHILEHS 262 UNSPEC_WHILELE 263 UNSPEC_WHILELO 264 UNSPEC_WHILELS 265 UNSPEC_WHILELT 266 UNSPEC_WHILERW 267 UNSPEC_WHILEWR 268 UNSPEC_LDN 269 UNSPEC_STN 270 UNSPEC_INSR 271 UNSPEC_CLASTA 272 UNSPEC_CLASTB 273 UNSPEC_FADDA 274 UNSPEC_REV_SUBREG 275 UNSPEC_REINTERPRET 276 UNSPEC_SPECULATION_TRACKER 277 UNSPEC_SPECULATION_TRACKER_REV 278 UNSPEC_COPYSIGN 279 UNSPEC_TTEST ; Represent transaction test. 280 UNSPEC_UPDATE_FFR 281 UNSPEC_UPDATE_FFRT 282 UNSPEC_RDFFR 283 UNSPEC_WRFFR 284 ;; Represents an SVE-style lane index, in which the indexing applies 285 ;; within the containing 128-bit block. 286 UNSPEC_SVE_LANE_SELECT 287 UNSPEC_SVE_CNT_PAT 288 UNSPEC_SVE_PREFETCH 289 UNSPEC_SVE_PREFETCH_GATHER 290 UNSPEC_SVE_COMPACT 291 UNSPEC_SVE_SPLICE 292 UNSPEC_GEN_TAG ; Generate a 4-bit MTE tag. 293 UNSPEC_GEN_TAG_RND ; Generate a random 4-bit MTE tag. 294 UNSPEC_TAG_SPACE ; Translate address to MTE tag address space. 295 UNSPEC_LD1RO 296 UNSPEC_SALT_ADDR 297]) 298 299(define_c_enum "unspecv" [ 300 UNSPECV_EH_RETURN ; Represent EH_RETURN 301 UNSPECV_GET_FPCR ; Represent fetch of FPCR content. 302 UNSPECV_SET_FPCR ; Represent assign of FPCR content. 303 UNSPECV_GET_FPSR ; Represent fetch of FPSR content. 304 UNSPECV_SET_FPSR ; Represent assign of FPSR content. 305 UNSPECV_BLOCKAGE ; Represent a blockage 306 UNSPECV_PROBE_STACK_RANGE ; Represent stack range probing. 307 UNSPECV_SPECULATION_BARRIER ; Represent speculation barrier. 308 UNSPECV_BTI_NOARG ; Represent BTI. 309 UNSPECV_BTI_C ; Represent BTI c. 310 UNSPECV_BTI_J ; Represent BTI j. 311 UNSPECV_BTI_JC ; Represent BTI jc. 312 UNSPECV_TSTART ; Represent transaction start. 313 UNSPECV_TCOMMIT ; Represent transaction commit. 314 UNSPECV_TCANCEL ; Represent transaction cancel. 315 UNSPEC_RNDR ; Represent RNDR 316 UNSPEC_RNDRRS ; Represent RNDRRS 317 ] 318) 319 320;; These constants are used as a const_int in various SVE unspecs 321;; to indicate whether the governing predicate is known to be a PTRUE. 322(define_constants 323 [; Indicates that the predicate might not be a PTRUE. 324 (SVE_MAYBE_NOT_PTRUE 0) 325 326 ; Indicates that the predicate is known to be a PTRUE. 327 (SVE_KNOWN_PTRUE 1)]) 328 329;; These constants are used as a const_int in predicated SVE FP arithmetic 330;; to indicate whether the operation is allowed to make additional lanes 331;; active without worrying about the effect on faulting behavior. 332(define_constants 333 [; Indicates either that all lanes are active or that the instruction may 334 ; operate on inactive inputs even if doing so could induce a fault. 335 (SVE_RELAXED_GP 0) 336 337 ; Indicates that some lanes might be inactive and that the instruction 338 ; must not operate on inactive inputs if doing so could induce a fault. 339 (SVE_STRICT_GP 1)]) 340 341;; If further include files are added the defintion of MD_INCLUDES 342;; must be updated. 343 344(include "constraints.md") 345(include "predicates.md") 346(include "iterators.md") 347 348;; ------------------------------------------------------------------- 349;; Instruction types and attributes 350;; ------------------------------------------------------------------- 351 352; The "type" attribute is included here from AArch32 backend to be able 353; to share pipeline descriptions. 354(include "../arm/types.md") 355 356;; It is important to set the fp or simd attributes to yes when a pattern 357;; alternative uses the FP or SIMD register files, usually signified by use of 358;; the 'w' constraint. This will ensure that the alternative will be 359;; disabled when compiling with -mgeneral-regs-only or with the +nofp/+nosimd 360;; architecture extensions. If all the alternatives in a pattern use the 361;; FP or SIMD registers then the pattern predicate should include TARGET_FLOAT 362;; or TARGET_SIMD. 363 364;; Attributes of the architecture required to support the instruction (or 365;; alternative). This attribute is used to compute attribute "enabled", use type 366;; "any" to enable an alternative in all cases. 367 368(define_enum "arches" [ any rcpc8_4 fp simd sve fp16]) 369 370(define_enum_attr "arch" "arches" (const_string "any")) 371 372;; [For compatibility with Arm in pipeline models] 373;; Attribute that specifies whether or not the instruction touches fp 374;; registers. 375;; Note that this attribute is not used anywhere in either the arm or aarch64 376;; backends except in the scheduling description for xgene1. In that 377;; scheduling description this attribute is used to subclass the load_4 and 378;; load_8 types. 379(define_attr "fp" "no,yes" 380 (if_then_else 381 (eq_attr "arch" "fp") 382 (const_string "yes") 383 (const_string "no"))) 384 385(define_attr "arch_enabled" "no,yes" 386 (if_then_else 387 (ior 388 (eq_attr "arch" "any") 389 390 (and (eq_attr "arch" "rcpc8_4") 391 (match_test "AARCH64_ISA_RCPC8_4")) 392 393 (and (eq_attr "arch" "fp") 394 (match_test "TARGET_FLOAT")) 395 396 (and (eq_attr "arch" "simd") 397 (match_test "TARGET_SIMD")) 398 399 (and (eq_attr "arch" "fp16") 400 (match_test "TARGET_FP_F16INST")) 401 402 (and (eq_attr "arch" "sve") 403 (match_test "TARGET_SVE"))) 404 (const_string "yes") 405 (const_string "no"))) 406 407;; Attribute that controls whether an alternative is enabled or not. 408;; Currently it is only used to disable alternatives which touch fp or simd 409;; registers when -mgeneral-regs-only is specified or to require a special 410;; architecture support. 411(define_attr "enabled" "no,yes" (attr "arch_enabled")) 412 413;; Attribute that specifies whether we are dealing with a branch to a 414;; label that is far away, i.e. further away than the maximum/minimum 415;; representable in a signed 21-bits number. 416;; 0 :=: no 417;; 1 :=: yes 418(define_attr "far_branch" "" (const_int 0)) 419 420;; Attribute that specifies whether the alternative uses MOVPRFX. 421(define_attr "movprfx" "no,yes" (const_string "no")) 422 423;; Attribute to specify that an alternative has the length of a single 424;; instruction plus a speculation barrier. 425(define_attr "sls_length" "none,retbr,casesi" (const_string "none")) 426 427(define_attr "length" "" 428 (cond [(eq_attr "movprfx" "yes") 429 (const_int 8) 430 431 (eq_attr "sls_length" "retbr") 432 (cond [(match_test "!aarch64_harden_sls_retbr_p ()") (const_int 4) 433 (match_test "TARGET_SB") (const_int 8)] 434 (const_int 12)) 435 436 (eq_attr "sls_length" "casesi") 437 (cond [(match_test "!aarch64_harden_sls_retbr_p ()") (const_int 16) 438 (match_test "TARGET_SB") (const_int 20)] 439 (const_int 24)) 440 ] 441 (const_int 4))) 442 443;; Strictly for compatibility with AArch32 in pipeline models, since AArch64 has 444;; no predicated insns. 445(define_attr "predicated" "yes,no" (const_string "no")) 446 447;; Set to true on an insn that requires the speculation tracking state to be 448;; in the tracking register before the insn issues. Otherwise the compiler 449;; may chose to hold the tracking state encoded in SP. 450(define_attr "speculation_barrier" "true,false" (const_string "false")) 451 452;; ------------------------------------------------------------------- 453;; Pipeline descriptions and scheduling 454;; ------------------------------------------------------------------- 455 456;; Processor types. 457(include "aarch64-tune.md") 458 459;; Scheduling 460(include "../arm/cortex-a53.md") 461(include "../arm/cortex-a57.md") 462(include "../arm/exynos-m1.md") 463(include "falkor.md") 464(include "saphira.md") 465(include "thunderx.md") 466(include "../arm/xgene1.md") 467(include "thunderx2t99.md") 468(include "tsv110.md") 469(include "thunderx3t110.md") 470 471;; ------------------------------------------------------------------- 472;; Jumps and other miscellaneous insns 473;; ------------------------------------------------------------------- 474 475(define_insn "indirect_jump" 476 [(set (pc) (match_operand:DI 0 "register_operand" "r"))] 477 "" 478 { 479 output_asm_insn ("br\\t%0", operands); 480 return aarch64_sls_barrier (aarch64_harden_sls_retbr_p ()); 481 } 482 [(set_attr "type" "branch") 483 (set_attr "sls_length" "retbr")] 484) 485 486(define_insn "jump" 487 [(set (pc) (label_ref (match_operand 0 "" "")))] 488 "" 489 "b\\t%l0" 490 [(set_attr "type" "branch")] 491) 492 493(define_expand "cbranch<mode>4" 494 [(set (pc) (if_then_else (match_operator 0 "aarch64_comparison_operator" 495 [(match_operand:GPI 1 "register_operand") 496 (match_operand:GPI 2 "aarch64_plus_operand")]) 497 (label_ref (match_operand 3 "" "")) 498 (pc)))] 499 "" 500 " 501 operands[1] = aarch64_gen_compare_reg (GET_CODE (operands[0]), operands[1], 502 operands[2]); 503 operands[2] = const0_rtx; 504 " 505) 506 507(define_expand "cbranch<mode>4" 508 [(set (pc) (if_then_else (match_operator 0 "aarch64_comparison_operator" 509 [(match_operand:GPF 1 "register_operand") 510 (match_operand:GPF 2 "aarch64_fp_compare_operand")]) 511 (label_ref (match_operand 3 "" "")) 512 (pc)))] 513 "" 514 " 515 operands[1] = aarch64_gen_compare_reg (GET_CODE (operands[0]), operands[1], 516 operands[2]); 517 operands[2] = const0_rtx; 518 " 519) 520 521(define_expand "cbranchcc4" 522 [(set (pc) (if_then_else 523 (match_operator 0 "aarch64_comparison_operator" 524 [(match_operand 1 "cc_register") 525 (match_operand 2 "const0_operand")]) 526 (label_ref (match_operand 3 "" "")) 527 (pc)))] 528 "" 529 "") 530 531(define_insn "@ccmp<CC_ONLY:mode><GPI:mode>" 532 [(set (match_operand:CC_ONLY 1 "cc_register" "") 533 (if_then_else:CC_ONLY 534 (match_operator 4 "aarch64_comparison_operator" 535 [(match_operand 0 "cc_register" "") 536 (const_int 0)]) 537 (compare:CC_ONLY 538 (match_operand:GPI 2 "register_operand" "r,r,r") 539 (match_operand:GPI 3 "aarch64_ccmp_operand" "r,Uss,Usn")) 540 (unspec:CC_ONLY 541 [(match_operand 5 "immediate_operand")] 542 UNSPEC_NZCV)))] 543 "" 544 "@ 545 ccmp\\t%<w>2, %<w>3, %k5, %m4 546 ccmp\\t%<w>2, %3, %k5, %m4 547 ccmn\\t%<w>2, #%n3, %k5, %m4" 548 [(set_attr "type" "alus_sreg,alus_imm,alus_imm")] 549) 550 551(define_insn "@ccmp<CCFP_CCFPE:mode><GPF:mode>" 552 [(set (match_operand:CCFP_CCFPE 1 "cc_register" "") 553 (if_then_else:CCFP_CCFPE 554 (match_operator 4 "aarch64_comparison_operator" 555 [(match_operand 0 "cc_register" "") 556 (const_int 0)]) 557 (compare:CCFP_CCFPE 558 (match_operand:GPF 2 "register_operand" "w") 559 (match_operand:GPF 3 "register_operand" "w")) 560 (unspec:CCFP_CCFPE 561 [(match_operand 5 "immediate_operand")] 562 UNSPEC_NZCV)))] 563 "TARGET_FLOAT" 564 "fccmp<e>\\t%<s>2, %<s>3, %k5, %m4" 565 [(set_attr "type" "fccmp<s>")] 566) 567 568(define_insn "@ccmp<CC_ONLY:mode><GPI:mode>_rev" 569 [(set (match_operand:CC_ONLY 1 "cc_register" "") 570 (if_then_else:CC_ONLY 571 (match_operator 4 "aarch64_comparison_operator" 572 [(match_operand 0 "cc_register" "") 573 (const_int 0)]) 574 (unspec:CC_ONLY 575 [(match_operand 5 "immediate_operand")] 576 UNSPEC_NZCV) 577 (compare:CC_ONLY 578 (match_operand:GPI 2 "register_operand" "r,r,r") 579 (match_operand:GPI 3 "aarch64_ccmp_operand" "r,Uss,Usn"))))] 580 "" 581 "@ 582 ccmp\\t%<w>2, %<w>3, %k5, %M4 583 ccmp\\t%<w>2, %3, %k5, %M4 584 ccmn\\t%<w>2, #%n3, %k5, %M4" 585 [(set_attr "type" "alus_sreg,alus_imm,alus_imm")] 586) 587 588(define_insn "@ccmp<CCFP_CCFPE:mode><GPF:mode>_rev" 589 [(set (match_operand:CCFP_CCFPE 1 "cc_register" "") 590 (if_then_else:CCFP_CCFPE 591 (match_operator 4 "aarch64_comparison_operator" 592 [(match_operand 0 "cc_register" "") 593 (const_int 0)]) 594 (unspec:CCFP_CCFPE 595 [(match_operand 5 "immediate_operand")] 596 UNSPEC_NZCV) 597 (compare:CCFP_CCFPE 598 (match_operand:GPF 2 "register_operand" "w") 599 (match_operand:GPF 3 "register_operand" "w"))))] 600 "TARGET_FLOAT" 601 "fccmp<e>\\t%<s>2, %<s>3, %k5, %M4" 602 [(set_attr "type" "fccmp<s>")] 603) 604 605;; Expansion of signed mod by a power of 2 using CSNEG. 606;; For x0 % n where n is a power of 2 produce: 607;; negs x1, x0 608;; and x0, x0, #(n - 1) 609;; and x1, x1, #(n - 1) 610;; csneg x0, x0, x1, mi 611 612(define_expand "mod<mode>3" 613 [(match_operand:GPI 0 "register_operand") 614 (match_operand:GPI 1 "register_operand") 615 (match_operand:GPI 2 "const_int_operand")] 616 "" 617 { 618 HOST_WIDE_INT val = INTVAL (operands[2]); 619 620 if (val <= 0 621 || exact_log2 (val) <= 0 622 || !aarch64_bitmask_imm (val - 1, <MODE>mode)) 623 FAIL; 624 625 rtx mask = GEN_INT (val - 1); 626 627 /* In the special case of x0 % 2 we can do the even shorter: 628 cmp x0, xzr 629 and x0, x0, 1 630 cneg x0, x0, lt. */ 631 if (val == 2) 632 { 633 rtx masked = gen_reg_rtx (<MODE>mode); 634 rtx ccreg = aarch64_gen_compare_reg (LT, operands[1], const0_rtx); 635 emit_insn (gen_and<mode>3 (masked, operands[1], mask)); 636 rtx x = gen_rtx_LT (VOIDmode, ccreg, const0_rtx); 637 emit_insn (gen_csneg3<mode>_insn (operands[0], x, masked, masked)); 638 DONE; 639 } 640 641 rtx neg_op = gen_reg_rtx (<MODE>mode); 642 rtx_insn *insn = emit_insn (gen_neg<mode>2_compare0 (neg_op, operands[1])); 643 644 /* Extract the condition register and mode. */ 645 rtx cmp = XVECEXP (PATTERN (insn), 0, 0); 646 rtx cc_reg = SET_DEST (cmp); 647 rtx cond = gen_rtx_GE (VOIDmode, cc_reg, const0_rtx); 648 649 rtx masked_pos = gen_reg_rtx (<MODE>mode); 650 emit_insn (gen_and<mode>3 (masked_pos, operands[1], mask)); 651 652 rtx masked_neg = gen_reg_rtx (<MODE>mode); 653 emit_insn (gen_and<mode>3 (masked_neg, neg_op, mask)); 654 655 emit_insn (gen_csneg3<mode>_insn (operands[0], cond, 656 masked_neg, masked_pos)); 657 DONE; 658 } 659) 660 661(define_insn "condjump" 662 [(set (pc) (if_then_else (match_operator 0 "aarch64_comparison_operator" 663 [(match_operand 1 "cc_register" "") (const_int 0)]) 664 (label_ref (match_operand 2 "" "")) 665 (pc)))] 666 "" 667 { 668 /* GCC's traditional style has been to use "beq" instead of "b.eq", etc., 669 but the "." is required for SVE conditions. */ 670 bool use_dot_p = GET_MODE (operands[1]) == CC_NZCmode; 671 if (get_attr_length (insn) == 8) 672 return aarch64_gen_far_branch (operands, 2, "Lbcond", 673 use_dot_p ? "b.%M0\\t" : "b%M0\\t"); 674 else 675 return use_dot_p ? "b.%m0\\t%l2" : "b%m0\\t%l2"; 676 } 677 [(set_attr "type" "branch") 678 (set (attr "length") 679 (if_then_else (and (ge (minus (match_dup 2) (pc)) (const_int -1048576)) 680 (lt (minus (match_dup 2) (pc)) (const_int 1048572))) 681 (const_int 4) 682 (const_int 8))) 683 (set (attr "far_branch") 684 (if_then_else (and (ge (minus (match_dup 2) (pc)) (const_int -1048576)) 685 (lt (minus (match_dup 2) (pc)) (const_int 1048572))) 686 (const_int 0) 687 (const_int 1)))] 688) 689 690;; For a 24-bit immediate CST we can optimize the compare for equality 691;; and branch sequence from: 692;; mov x0, #imm1 693;; movk x0, #imm2, lsl 16 /* x0 contains CST. */ 694;; cmp x1, x0 695;; b<ne,eq> .Label 696;; into the shorter: 697;; sub x0, x1, #(CST & 0xfff000) 698;; subs x0, x0, #(CST & 0x000fff) 699;; b<ne,eq> .Label 700(define_insn_and_split "*compare_condjump<GPI:mode>" 701 [(set (pc) (if_then_else (EQL 702 (match_operand:GPI 0 "register_operand" "r") 703 (match_operand:GPI 1 "aarch64_imm24" "n")) 704 (label_ref:P (match_operand 2 "" "")) 705 (pc)))] 706 "!aarch64_move_imm (INTVAL (operands[1]), <GPI:MODE>mode) 707 && !aarch64_plus_operand (operands[1], <GPI:MODE>mode) 708 && !reload_completed" 709 "#" 710 "&& true" 711 [(const_int 0)] 712 { 713 HOST_WIDE_INT lo_imm = UINTVAL (operands[1]) & 0xfff; 714 HOST_WIDE_INT hi_imm = UINTVAL (operands[1]) & 0xfff000; 715 rtx tmp = gen_reg_rtx (<GPI:MODE>mode); 716 emit_insn (gen_add<GPI:mode>3 (tmp, operands[0], GEN_INT (-hi_imm))); 717 emit_insn (gen_add<GPI:mode>3_compare0 (tmp, tmp, GEN_INT (-lo_imm))); 718 rtx cc_reg = gen_rtx_REG (CC_NZmode, CC_REGNUM); 719 rtx cmp_rtx = gen_rtx_fmt_ee (<EQL:CMP>, <GPI:MODE>mode, 720 cc_reg, const0_rtx); 721 emit_jump_insn (gen_condjump (cmp_rtx, cc_reg, operands[2])); 722 DONE; 723 } 724) 725 726(define_expand "casesi" 727 [(match_operand:SI 0 "register_operand") ; Index 728 (match_operand:SI 1 "const_int_operand") ; Lower bound 729 (match_operand:SI 2 "const_int_operand") ; Total range 730 (match_operand:DI 3 "" "") ; Table label 731 (match_operand:DI 4 "" "")] ; Out of range label 732 "" 733 { 734 if (operands[1] != const0_rtx) 735 { 736 rtx reg = gen_reg_rtx (SImode); 737 738 /* Canonical RTL says that if you have: 739 740 (minus (X) (CONST)) 741 742 then this should be emitted as: 743 744 (plus (X) (-CONST)) 745 746 The use of trunc_int_for_mode ensures that the resulting 747 constant can be represented in SImode, this is important 748 for the corner case where operand[1] is INT_MIN. */ 749 750 operands[1] 751 = GEN_INT (trunc_int_for_mode (-UINTVAL (operands[1]), SImode)); 752 753 if (!(*insn_data[CODE_FOR_addsi3].operand[2].predicate) 754 (operands[1], SImode)) 755 operands[1] = force_reg (SImode, operands[1]); 756 emit_insn (gen_addsi3 (reg, operands[0], operands[1])); 757 operands[0] = reg; 758 } 759 760 if (!aarch64_plus_operand (operands[2], SImode)) 761 operands[2] = force_reg (SImode, operands[2]); 762 emit_jump_insn (gen_cbranchsi4 (gen_rtx_GTU (SImode, const0_rtx, 763 const0_rtx), 764 operands[0], operands[2], operands[4])); 765 766 operands[2] = force_reg (DImode, gen_rtx_LABEL_REF (DImode, operands[3])); 767 operands[2] 768 = gen_rtx_UNSPEC (Pmode, gen_rtvec (2, operands[2], operands[0]), 769 UNSPEC_CASESI); 770 operands[2] = gen_rtx_MEM (DImode, operands[2]); 771 MEM_READONLY_P (operands[2]) = 1; 772 MEM_NOTRAP_P (operands[2]) = 1; 773 emit_jump_insn (gen_casesi_dispatch (operands[2], operands[3])); 774 DONE; 775 } 776) 777 778(define_expand "casesi_dispatch" 779 [(parallel 780 [(set (pc) (match_operand:DI 0 "")) 781 (clobber (reg:CC CC_REGNUM)) 782 (clobber (match_scratch:DI 2)) 783 (clobber (match_scratch:DI 3)) 784 (use (label_ref:DI (match_operand 1 "")))])] 785 "") 786 787(define_insn "*casesi_dispatch" 788 [(parallel 789 [(set (pc) 790 (mem:DI (unspec [(match_operand:DI 0 "register_operand" "r") 791 (match_operand:SI 1 "register_operand" "r")] 792 UNSPEC_CASESI))) 793 (clobber (reg:CC CC_REGNUM)) 794 (clobber (match_scratch:DI 3 "=r")) 795 (clobber (match_scratch:DI 4 "=r")) 796 (use (label_ref:DI (match_operand 2 "" "")))])] 797 "" 798 "* 799 return aarch64_output_casesi (operands); 800 " 801 [(set_attr "sls_length" "casesi") 802 (set_attr "type" "branch")] 803) 804 805(define_insn "nop" 806 [(unspec[(const_int 0)] UNSPEC_NOP)] 807 "" 808 "nop" 809 [(set_attr "type" "no_insn")] 810) 811 812(define_insn "prefetch" 813 [(prefetch (match_operand:DI 0 "aarch64_prefetch_operand" "Dp") 814 (match_operand:QI 1 "const_int_operand" "") 815 (match_operand:QI 2 "const_int_operand" ""))] 816 "" 817 { 818 const char * pftype[2][4] = 819 { 820 {"prfm\\tPLDL1STRM, %0", 821 "prfm\\tPLDL3KEEP, %0", 822 "prfm\\tPLDL2KEEP, %0", 823 "prfm\\tPLDL1KEEP, %0"}, 824 {"prfm\\tPSTL1STRM, %0", 825 "prfm\\tPSTL3KEEP, %0", 826 "prfm\\tPSTL2KEEP, %0", 827 "prfm\\tPSTL1KEEP, %0"}, 828 }; 829 830 int locality = INTVAL (operands[2]); 831 832 gcc_assert (IN_RANGE (locality, 0, 3)); 833 834 /* PRFM accepts the same addresses as a 64-bit LDR so wrap 835 the address into a DImode MEM so that aarch64_print_operand knows 836 how to print it. */ 837 operands[0] = gen_rtx_MEM (DImode, operands[0]); 838 return pftype[INTVAL(operands[1])][locality]; 839 } 840 [(set_attr "type" "load_4")] 841) 842 843(define_insn "trap" 844 [(trap_if (const_int 1) (const_int 8))] 845 "" 846 "brk #1000" 847 [(set_attr "type" "trap")]) 848 849(define_expand "prologue" 850 [(clobber (const_int 0))] 851 "" 852 " 853 aarch64_expand_prologue (); 854 DONE; 855 " 856) 857 858(define_expand "epilogue" 859 [(clobber (const_int 0))] 860 "" 861 " 862 aarch64_expand_epilogue (false); 863 DONE; 864 " 865) 866 867(define_expand "sibcall_epilogue" 868 [(clobber (const_int 0))] 869 "" 870 " 871 aarch64_expand_epilogue (true); 872 DONE; 873 " 874) 875 876(define_insn "*do_return" 877 [(return)] 878 "" 879 { 880 const char *ret = NULL; 881 if (aarch64_return_address_signing_enabled () 882 && (TARGET_PAUTH) 883 && !crtl->calls_eh_return) 884 { 885 if (aarch64_ra_sign_key == AARCH64_KEY_B) 886 ret = "retab"; 887 else 888 ret = "retaa"; 889 } 890 else 891 ret = "ret"; 892 output_asm_insn (ret, operands); 893 return aarch64_sls_barrier (aarch64_harden_sls_retbr_p ()); 894 } 895 [(set_attr "type" "branch") 896 (set_attr "sls_length" "retbr")] 897) 898 899(define_expand "return" 900 [(simple_return)] 901 "aarch64_use_return_insn_p ()" 902 "" 903) 904 905(define_insn "simple_return" 906 [(simple_return)] 907 "" 908 { 909 output_asm_insn ("ret", operands); 910 return aarch64_sls_barrier (aarch64_harden_sls_retbr_p ()); 911 } 912 [(set_attr "type" "branch") 913 (set_attr "sls_length" "retbr")] 914) 915 916(define_insn "*cb<optab><mode>1" 917 [(set (pc) (if_then_else (EQL (match_operand:GPI 0 "register_operand" "r") 918 (const_int 0)) 919 (label_ref (match_operand 1 "" "")) 920 (pc)))] 921 "!aarch64_track_speculation" 922 { 923 if (get_attr_length (insn) == 8) 924 return aarch64_gen_far_branch (operands, 1, "Lcb", "<inv_cb>\\t%<w>0, "); 925 else 926 return "<cbz>\\t%<w>0, %l1"; 927 } 928 [(set_attr "type" "branch") 929 (set (attr "length") 930 (if_then_else (and (ge (minus (match_dup 1) (pc)) (const_int -1048576)) 931 (lt (minus (match_dup 1) (pc)) (const_int 1048572))) 932 (const_int 4) 933 (const_int 8))) 934 (set (attr "far_branch") 935 (if_then_else (and (ge (minus (match_dup 2) (pc)) (const_int -1048576)) 936 (lt (minus (match_dup 2) (pc)) (const_int 1048572))) 937 (const_int 0) 938 (const_int 1)))] 939) 940 941(define_insn "*tb<optab><mode>1" 942 [(set (pc) (if_then_else 943 (EQL (zero_extract:DI (match_operand:GPI 0 "register_operand" "r") 944 (const_int 1) 945 (match_operand 1 946 "aarch64_simd_shift_imm_<mode>" "n")) 947 (const_int 0)) 948 (label_ref (match_operand 2 "" "")) 949 (pc))) 950 (clobber (reg:CC CC_REGNUM))] 951 "!aarch64_track_speculation" 952 { 953 if (get_attr_length (insn) == 8) 954 { 955 if (get_attr_far_branch (insn) == 1) 956 return aarch64_gen_far_branch (operands, 2, "Ltb", 957 "<inv_tb>\\t%<w>0, %1, "); 958 else 959 { 960 operands[1] = GEN_INT (HOST_WIDE_INT_1U << UINTVAL (operands[1])); 961 return "tst\t%<w>0, %1\;<bcond>\t%l2"; 962 } 963 } 964 else 965 return "<tbz>\t%<w>0, %1, %l2"; 966 } 967 [(set_attr "type" "branch") 968 (set (attr "length") 969 (if_then_else (and (ge (minus (match_dup 2) (pc)) (const_int -32768)) 970 (lt (minus (match_dup 2) (pc)) (const_int 32764))) 971 (const_int 4) 972 (const_int 8))) 973 (set (attr "far_branch") 974 (if_then_else (and (ge (minus (match_dup 2) (pc)) (const_int -1048576)) 975 (lt (minus (match_dup 2) (pc)) (const_int 1048572))) 976 (const_int 0) 977 (const_int 1)))] 978 979) 980 981(define_insn "*cb<optab><mode>1" 982 [(set (pc) (if_then_else (LTGE (match_operand:ALLI 0 "register_operand" "r") 983 (const_int 0)) 984 (label_ref (match_operand 1 "" "")) 985 (pc))) 986 (clobber (reg:CC CC_REGNUM))] 987 "!aarch64_track_speculation" 988 { 989 if (get_attr_length (insn) == 8) 990 { 991 if (get_attr_far_branch (insn) == 1) 992 return aarch64_gen_far_branch (operands, 1, "Ltb", 993 "<inv_tb>\\t%<w>0, <sizem1>, "); 994 else 995 { 996 char buf[64]; 997 uint64_t val = ((uint64_t) 1) 998 << (GET_MODE_SIZE (<MODE>mode) * BITS_PER_UNIT - 1); 999 sprintf (buf, "tst\t%%<w>0, %" PRId64, val); 1000 output_asm_insn (buf, operands); 1001 return "<bcond>\t%l1"; 1002 } 1003 } 1004 else 1005 return "<tbz>\t%<w>0, <sizem1>, %l1"; 1006 } 1007 [(set_attr "type" "branch") 1008 (set (attr "length") 1009 (if_then_else (and (ge (minus (match_dup 1) (pc)) (const_int -32768)) 1010 (lt (minus (match_dup 1) (pc)) (const_int 32764))) 1011 (const_int 4) 1012 (const_int 8))) 1013 (set (attr "far_branch") 1014 (if_then_else (and (ge (minus (match_dup 1) (pc)) (const_int -1048576)) 1015 (lt (minus (match_dup 1) (pc)) (const_int 1048572))) 1016 (const_int 0) 1017 (const_int 1)))] 1018) 1019 1020;; ------------------------------------------------------------------- 1021;; Subroutine calls and sibcalls 1022;; ------------------------------------------------------------------- 1023 1024(define_expand "call" 1025 [(parallel 1026 [(call (match_operand 0 "memory_operand") 1027 (match_operand 1 "general_operand")) 1028 (unspec:DI [(match_operand 2 "const_int_operand")] UNSPEC_CALLEE_ABI) 1029 (clobber (reg:DI LR_REGNUM))])] 1030 "" 1031 " 1032 { 1033 aarch64_expand_call (NULL_RTX, operands[0], operands[2], false); 1034 DONE; 1035 }" 1036) 1037 1038(define_insn "*call_insn" 1039 [(call (mem:DI (match_operand:DI 0 "aarch64_call_insn_operand" "Ucr, Usf")) 1040 (match_operand 1 "" "")) 1041 (unspec:DI [(match_operand:DI 2 "const_int_operand")] UNSPEC_CALLEE_ABI) 1042 (clobber (reg:DI LR_REGNUM))] 1043 "" 1044 "@ 1045 * return aarch64_indirect_call_asm (operands[0]); 1046 bl\\t%c0" 1047 [(set_attr "type" "call, call")]) 1048 1049(define_expand "call_value" 1050 [(parallel 1051 [(set (match_operand 0 "") 1052 (call (match_operand 1 "memory_operand") 1053 (match_operand 2 "general_operand"))) 1054 (unspec:DI [(match_operand 3 "const_int_operand")] UNSPEC_CALLEE_ABI) 1055 (clobber (reg:DI LR_REGNUM))])] 1056 "" 1057 " 1058 { 1059 aarch64_expand_call (operands[0], operands[1], operands[3], false); 1060 DONE; 1061 }" 1062) 1063 1064(define_insn "*call_value_insn" 1065 [(set (match_operand 0 "" "") 1066 (call (mem:DI (match_operand:DI 1 "aarch64_call_insn_operand" "Ucr, Usf")) 1067 (match_operand 2 "" ""))) 1068 (unspec:DI [(match_operand:DI 3 "const_int_operand")] UNSPEC_CALLEE_ABI) 1069 (clobber (reg:DI LR_REGNUM))] 1070 "" 1071 "@ 1072 * return aarch64_indirect_call_asm (operands[1]); 1073 bl\\t%c1" 1074 [(set_attr "type" "call, call")] 1075) 1076 1077(define_expand "sibcall" 1078 [(parallel 1079 [(call (match_operand 0 "memory_operand") 1080 (match_operand 1 "general_operand")) 1081 (unspec:DI [(match_operand 2 "const_int_operand")] UNSPEC_CALLEE_ABI) 1082 (return)])] 1083 "" 1084 { 1085 aarch64_expand_call (NULL_RTX, operands[0], operands[2], true); 1086 DONE; 1087 } 1088) 1089 1090(define_expand "sibcall_value" 1091 [(parallel 1092 [(set (match_operand 0 "") 1093 (call (match_operand 1 "memory_operand") 1094 (match_operand 2 "general_operand"))) 1095 (unspec:DI [(match_operand 3 "const_int_operand")] UNSPEC_CALLEE_ABI) 1096 (return)])] 1097 "" 1098 { 1099 aarch64_expand_call (operands[0], operands[1], operands[3], true); 1100 DONE; 1101 } 1102) 1103 1104(define_insn "*sibcall_insn" 1105 [(call (mem:DI (match_operand:DI 0 "aarch64_call_insn_operand" "Ucs, Usf")) 1106 (match_operand 1 "")) 1107 (unspec:DI [(match_operand:DI 2 "const_int_operand")] UNSPEC_CALLEE_ABI) 1108 (return)] 1109 "SIBLING_CALL_P (insn)" 1110 { 1111 if (which_alternative == 0) 1112 { 1113 output_asm_insn ("br\\t%0", operands); 1114 return aarch64_sls_barrier (aarch64_harden_sls_retbr_p ()); 1115 } 1116 return "b\\t%c0"; 1117 } 1118 [(set_attr "type" "branch, branch") 1119 (set_attr "sls_length" "retbr,none")] 1120) 1121 1122(define_insn "*sibcall_value_insn" 1123 [(set (match_operand 0 "") 1124 (call (mem:DI 1125 (match_operand:DI 1 "aarch64_call_insn_operand" "Ucs, Usf")) 1126 (match_operand 2 ""))) 1127 (unspec:DI [(match_operand:DI 3 "const_int_operand")] UNSPEC_CALLEE_ABI) 1128 (return)] 1129 "SIBLING_CALL_P (insn)" 1130 { 1131 if (which_alternative == 0) 1132 { 1133 output_asm_insn ("br\\t%1", operands); 1134 return aarch64_sls_barrier (aarch64_harden_sls_retbr_p ()); 1135 } 1136 return "b\\t%c1"; 1137 } 1138 [(set_attr "type" "branch, branch") 1139 (set_attr "sls_length" "retbr,none")] 1140) 1141 1142;; Call subroutine returning any type. 1143 1144(define_expand "untyped_call" 1145 [(parallel [(call (match_operand 0 "") 1146 (const_int 0)) 1147 (match_operand 1 "") 1148 (match_operand 2 "")])] 1149 "" 1150{ 1151 int i; 1152 1153 /* Untyped calls always use the default ABI. It's only possible to use 1154 ABI variants if we know the type of the target function. */ 1155 emit_call_insn (gen_call (operands[0], const0_rtx, const0_rtx)); 1156 1157 for (i = 0; i < XVECLEN (operands[2], 0); i++) 1158 { 1159 rtx set = XVECEXP (operands[2], 0, i); 1160 emit_move_insn (SET_DEST (set), SET_SRC (set)); 1161 } 1162 1163 /* The optimizer does not know that the call sets the function value 1164 registers we stored in the result block. We avoid problems by 1165 claiming that all hard registers are used and clobbered at this 1166 point. */ 1167 emit_insn (gen_blockage ()); 1168 DONE; 1169}) 1170 1171;; ------------------------------------------------------------------- 1172;; Moves 1173;; ------------------------------------------------------------------- 1174 1175(define_expand "mov<mode>" 1176 [(set (match_operand:SHORT 0 "nonimmediate_operand") 1177 (match_operand:SHORT 1 "general_operand"))] 1178 "" 1179 " 1180 if (GET_CODE (operands[0]) == MEM && operands[1] != const0_rtx) 1181 operands[1] = force_reg (<MODE>mode, operands[1]); 1182 1183 if (GET_CODE (operands[1]) == CONST_POLY_INT) 1184 { 1185 aarch64_expand_mov_immediate (operands[0], operands[1]); 1186 DONE; 1187 } 1188 " 1189) 1190 1191(define_insn "*mov<mode>_aarch64" 1192 [(set (match_operand:SHORT 0 "nonimmediate_operand" "=r,r, w,r ,r,w, m,m,r,w,w") 1193 (match_operand:SHORT 1 "aarch64_mov_operand" " r,M,D<hq>,Usv,m,m,rZ,w,w,r,w"))] 1194 "(register_operand (operands[0], <MODE>mode) 1195 || aarch64_reg_or_zero (operands[1], <MODE>mode))" 1196{ 1197 switch (which_alternative) 1198 { 1199 case 0: 1200 return "mov\t%w0, %w1"; 1201 case 1: 1202 return "mov\t%w0, %1"; 1203 case 2: 1204 return aarch64_output_scalar_simd_mov_immediate (operands[1], 1205 <MODE>mode); 1206 case 3: 1207 return aarch64_output_sve_cnt_immediate (\"cnt\", \"%x0\", operands[1]); 1208 case 4: 1209 return "ldr<size>\t%w0, %1"; 1210 case 5: 1211 return "ldr\t%<size>0, %1"; 1212 case 6: 1213 return "str<size>\t%w1, %0"; 1214 case 7: 1215 return "str\t%<size>1, %0"; 1216 case 8: 1217 return "umov\t%w0, %1.<v>[0]"; 1218 case 9: 1219 return "dup\t%0.<Vallxd>, %w1"; 1220 case 10: 1221 return "dup\t%<Vetype>0, %1.<v>[0]"; 1222 default: 1223 gcc_unreachable (); 1224 } 1225} 1226 ;; The "mov_imm" type for CNT is just a placeholder. 1227 [(set_attr "type" "mov_reg,mov_imm,neon_move,mov_imm,load_4,load_4,store_4, 1228 store_4,neon_to_gp<q>,neon_from_gp<q>,neon_dup") 1229 (set_attr "arch" "*,*,simd,sve,*,*,*,*,simd,simd,simd")] 1230) 1231 1232(define_expand "mov<mode>" 1233 [(set (match_operand:GPI 0 "nonimmediate_operand") 1234 (match_operand:GPI 1 "general_operand"))] 1235 "" 1236 " 1237 if (MEM_P (operands[0]) && !MEM_VOLATILE_P (operands[0]) 1238 && CONST_INT_P (operands[1]) && <MODE>mode == DImode 1239 && aarch64_split_dimode_const_store (operands[0], operands[1])) 1240 DONE; 1241 1242 if (GET_CODE (operands[0]) == MEM && operands[1] != const0_rtx) 1243 operands[1] = force_reg (<MODE>mode, operands[1]); 1244 1245 /* Lower moves of symbolic constants into individual instructions. 1246 Doing this now is sometimes necessary for correctness, since some 1247 sequences require temporary pseudo registers. Lowering now is also 1248 often better for optimization, since more RTL passes get the 1249 chance to optimize the individual instructions. 1250 1251 When called after RA, also split multi-instruction moves into 1252 smaller pieces now, since we can't be sure that sure that there 1253 will be a following split pass. */ 1254 if (CONST_INT_P (operands[1]) 1255 ? (reload_completed 1256 && !aarch64_mov_imm_operand (operands[1], <MODE>mode)) 1257 : CONSTANT_P (operands[1])) 1258 { 1259 aarch64_expand_mov_immediate (operands[0], operands[1]); 1260 DONE; 1261 } 1262 " 1263) 1264 1265(define_insn_and_split "*movsi_aarch64" 1266 [(set (match_operand:SI 0 "nonimmediate_operand" "=r,k,r,r,r,r, r,w, m, m, r, r, w,r,w, w") 1267 (match_operand:SI 1 "aarch64_mov_operand" " r,r,k,M,n,Usv,m,m,rZ,w,Usa,Ush,rZ,w,w,Ds"))] 1268 "(register_operand (operands[0], SImode) 1269 || aarch64_reg_or_zero (operands[1], SImode))" 1270 "@ 1271 mov\\t%w0, %w1 1272 mov\\t%w0, %w1 1273 mov\\t%w0, %w1 1274 mov\\t%w0, %1 1275 # 1276 * return aarch64_output_sve_cnt_immediate (\"cnt\", \"%x0\", operands[1]); 1277 ldr\\t%w0, %1 1278 ldr\\t%s0, %1 1279 str\\t%w1, %0 1280 str\\t%s1, %0 1281 adr\\t%x0, %c1 1282 adrp\\t%x0, %A1 1283 fmov\\t%s0, %w1 1284 fmov\\t%w0, %s1 1285 fmov\\t%s0, %s1 1286 * return aarch64_output_scalar_simd_mov_immediate (operands[1], SImode);" 1287 "CONST_INT_P (operands[1]) && !aarch64_move_imm (INTVAL (operands[1]), SImode) 1288 && REG_P (operands[0]) && GP_REGNUM_P (REGNO (operands[0]))" 1289 [(const_int 0)] 1290 "{ 1291 aarch64_expand_mov_immediate (operands[0], operands[1]); 1292 DONE; 1293 }" 1294 ;; The "mov_imm" type for CNT is just a placeholder. 1295 [(set_attr "type" "mov_reg,mov_reg,mov_reg,mov_imm,mov_imm,mov_imm,load_4, 1296 load_4,store_4,store_4,adr,adr,f_mcr,f_mrc,fmov,neon_move") 1297 (set_attr "arch" "*,*,*,*,*,sve,*,fp,*,fp,*,*,fp,fp,fp,simd")] 1298) 1299 1300(define_insn_and_split "*movdi_aarch64" 1301 [(set (match_operand:DI 0 "nonimmediate_operand" "=r,k,r,r,r,r,r, r,w, m,m, r, r, w,r,w, w") 1302 (match_operand:DI 1 "aarch64_mov_operand" " r,r,k,N,M,n,Usv,m,m,rZ,w,Usa,Ush,rZ,w,w,Dd"))] 1303 "(register_operand (operands[0], DImode) 1304 || aarch64_reg_or_zero (operands[1], DImode))" 1305 "@ 1306 mov\\t%x0, %x1 1307 mov\\t%0, %x1 1308 mov\\t%x0, %1 1309 mov\\t%x0, %1 1310 mov\\t%w0, %1 1311 # 1312 * return aarch64_output_sve_cnt_immediate (\"cnt\", \"%x0\", operands[1]); 1313 ldr\\t%x0, %1 1314 ldr\\t%d0, %1 1315 str\\t%x1, %0 1316 str\\t%d1, %0 1317 adr\\t%x0, %c1 1318 adrp\\t%x0, %A1 1319 fmov\\t%d0, %x1 1320 fmov\\t%x0, %d1 1321 fmov\\t%d0, %d1 1322 * return aarch64_output_scalar_simd_mov_immediate (operands[1], DImode);" 1323 "(CONST_INT_P (operands[1]) && !aarch64_move_imm (INTVAL (operands[1]), DImode)) 1324 && REG_P (operands[0]) && GP_REGNUM_P (REGNO (operands[0]))" 1325 [(const_int 0)] 1326 "{ 1327 aarch64_expand_mov_immediate (operands[0], operands[1]); 1328 DONE; 1329 }" 1330 ;; The "mov_imm" type for CNTD is just a placeholder. 1331 [(set_attr "type" "mov_reg,mov_reg,mov_reg,mov_imm,mov_imm,mov_imm,mov_imm, 1332 load_8,load_8,store_8,store_8,adr,adr,f_mcr,f_mrc,fmov, 1333 neon_move") 1334 (set_attr "arch" "*,*,*,*,*,*,sve,*,fp,*,fp,*,*,fp,fp,fp,simd")] 1335) 1336 1337(define_insn "insv_imm<mode>" 1338 [(set (zero_extract:GPI (match_operand:GPI 0 "register_operand" "+r") 1339 (const_int 16) 1340 (match_operand:GPI 1 "const_int_operand" "n")) 1341 (match_operand:GPI 2 "const_int_operand" "n"))] 1342 "UINTVAL (operands[1]) < GET_MODE_BITSIZE (<MODE>mode) 1343 && UINTVAL (operands[1]) % 16 == 0" 1344 "movk\\t%<w>0, %X2, lsl %1" 1345 [(set_attr "type" "mov_imm")] 1346) 1347 1348;; Match MOVK as a normal AND and IOR operation. 1349(define_insn "aarch64_movk<mode>" 1350 [(set (match_operand:GPI 0 "register_operand" "=r") 1351 (ior:GPI (and:GPI (match_operand:GPI 1 "register_operand" "0") 1352 (match_operand:GPI 2 "const_int_operand")) 1353 (match_operand:GPI 3 "const_int_operand")))] 1354 "aarch64_movk_shift (rtx_mode_t (operands[2], <MODE>mode), 1355 rtx_mode_t (operands[3], <MODE>mode)) >= 0" 1356 { 1357 int shift = aarch64_movk_shift (rtx_mode_t (operands[2], <MODE>mode), 1358 rtx_mode_t (operands[3], <MODE>mode)); 1359 operands[2] = gen_int_mode (UINTVAL (operands[3]) >> shift, SImode); 1360 operands[3] = gen_int_mode (shift, SImode); 1361 return "movk\\t%<w>0, #%X2, lsl %3"; 1362 } 1363 [(set_attr "type" "mov_imm")] 1364) 1365 1366(define_expand "movti" 1367 [(set (match_operand:TI 0 "nonimmediate_operand") 1368 (match_operand:TI 1 "general_operand"))] 1369 "" 1370 " 1371 if (GET_CODE (operands[0]) == MEM && operands[1] != const0_rtx) 1372 operands[1] = force_reg (TImode, operands[1]); 1373 1374 if (GET_CODE (operands[1]) == CONST_POLY_INT) 1375 { 1376 emit_move_insn (gen_lowpart (DImode, operands[0]), 1377 gen_lowpart (DImode, operands[1])); 1378 emit_move_insn (gen_highpart (DImode, operands[0]), const0_rtx); 1379 DONE; 1380 } 1381 " 1382) 1383 1384(define_insn "*movti_aarch64" 1385 [(set (match_operand:TI 0 1386 "nonimmediate_operand" "= r,w,w, r,w,r,m,m,w,m") 1387 (match_operand:TI 1 1388 "aarch64_movti_operand" " rUti,Z,r, w,w,m,r,Z,m,w"))] 1389 "(register_operand (operands[0], TImode) 1390 || aarch64_reg_or_zero (operands[1], TImode))" 1391 "@ 1392 # 1393 movi\\t%0.2d, #0 1394 # 1395 # 1396 mov\\t%0.16b, %1.16b 1397 ldp\\t%0, %H0, %1 1398 stp\\t%1, %H1, %0 1399 stp\\txzr, xzr, %0 1400 ldr\\t%q0, %1 1401 str\\t%q1, %0" 1402 [(set_attr "type" "multiple,neon_move,f_mcr,f_mrc,neon_logic_q, \ 1403 load_16,store_16,store_16,\ 1404 load_16,store_16") 1405 (set_attr "length" "8,4,8,8,4,4,4,4,4,4") 1406 (set_attr "arch" "*,simd,*,*,simd,*,*,*,fp,fp")] 1407) 1408 1409;; Split a TImode register-register or register-immediate move into 1410;; its component DImode pieces, taking care to handle overlapping 1411;; source and dest registers. 1412(define_split 1413 [(set (match_operand:TI 0 "register_operand" "") 1414 (match_operand:TI 1 "aarch64_reg_or_imm" ""))] 1415 "reload_completed && aarch64_split_128bit_move_p (operands[0], operands[1])" 1416 [(const_int 0)] 1417{ 1418 aarch64_split_128bit_move (operands[0], operands[1]); 1419 DONE; 1420}) 1421 1422(define_expand "mov<mode>" 1423 [(set (match_operand:GPF_TF_F16_MOV 0 "nonimmediate_operand") 1424 (match_operand:GPF_TF_F16_MOV 1 "general_operand"))] 1425 "" 1426 { 1427 if (!TARGET_FLOAT) 1428 { 1429 aarch64_err_no_fpadvsimd (<MODE>mode); 1430 machine_mode intmode 1431 = int_mode_for_size (GET_MODE_BITSIZE (<MODE>mode), 0).require (); 1432 emit_move_insn (gen_lowpart (intmode, operands[0]), 1433 gen_lowpart (intmode, operands[1])); 1434 DONE; 1435 } 1436 1437 if (GET_CODE (operands[0]) == MEM 1438 && ! (GET_CODE (operands[1]) == CONST_DOUBLE 1439 && aarch64_float_const_zero_rtx_p (operands[1]))) 1440 operands[1] = force_reg (<MODE>mode, operands[1]); 1441 } 1442) 1443 1444(define_insn "*mov<mode>_aarch64" 1445 [(set (match_operand:HFBF 0 "nonimmediate_operand" "=w,w , w,?r,w,w ,w ,w,m,r,m ,r") 1446 (match_operand:HFBF 1 "general_operand" "Y ,?rY,?r, w,w,Ufc,Uvi,m,w,m,rY,r"))] 1447 "TARGET_FLOAT && (register_operand (operands[0], <MODE>mode) 1448 || aarch64_reg_or_fp_zero (operands[1], <MODE>mode))" 1449 "@ 1450 movi\\t%0.4h, #0 1451 fmov\\t%h0, %w1 1452 dup\\t%w0.4h, %w1 1453 umov\\t%w0, %1.h[0] 1454 mov\\t%0.h[0], %1.h[0] 1455 fmov\\t%h0, %1 1456 * return aarch64_output_scalar_simd_mov_immediate (operands[1], HImode); 1457 ldr\\t%h0, %1 1458 str\\t%h1, %0 1459 ldrh\\t%w0, %1 1460 strh\\t%w1, %0 1461 mov\\t%w0, %w1" 1462 [(set_attr "type" "neon_move,f_mcr,neon_move,neon_to_gp, neon_move,fconsts, \ 1463 neon_move,f_loads,f_stores,load_4,store_4,mov_reg") 1464 (set_attr "arch" "simd,fp16,simd,simd,simd,fp16,simd,*,*,*,*,*")] 1465) 1466 1467(define_insn "*movsf_aarch64" 1468 [(set (match_operand:SF 0 "nonimmediate_operand" "=w,w ,?r,w,w ,w ,w,m,r,m ,r,r") 1469 (match_operand:SF 1 "general_operand" "Y ,?rY, w,w,Ufc,Uvi,m,w,m,rY,r,M"))] 1470 "TARGET_FLOAT && (register_operand (operands[0], SFmode) 1471 || aarch64_reg_or_fp_zero (operands[1], SFmode))" 1472 "@ 1473 movi\\t%0.2s, #0 1474 fmov\\t%s0, %w1 1475 fmov\\t%w0, %s1 1476 fmov\\t%s0, %s1 1477 fmov\\t%s0, %1 1478 * return aarch64_output_scalar_simd_mov_immediate (operands[1], SImode); 1479 ldr\\t%s0, %1 1480 str\\t%s1, %0 1481 ldr\\t%w0, %1 1482 str\\t%w1, %0 1483 mov\\t%w0, %w1 1484 mov\\t%w0, %1" 1485 [(set_attr "type" "neon_move,f_mcr,f_mrc,fmov,fconsts,neon_move,\ 1486 f_loads,f_stores,load_4,store_4,mov_reg,\ 1487 fconsts") 1488 (set_attr "arch" "simd,*,*,*,*,simd,*,*,*,*,*,*")] 1489) 1490 1491(define_insn "*movdf_aarch64" 1492 [(set (match_operand:DF 0 "nonimmediate_operand" "=w, w ,?r,w,w ,w ,w,m,r,m ,r,r") 1493 (match_operand:DF 1 "general_operand" "Y , ?rY, w,w,Ufc,Uvi,m,w,m,rY,r,N"))] 1494 "TARGET_FLOAT && (register_operand (operands[0], DFmode) 1495 || aarch64_reg_or_fp_zero (operands[1], DFmode))" 1496 "@ 1497 movi\\t%d0, #0 1498 fmov\\t%d0, %x1 1499 fmov\\t%x0, %d1 1500 fmov\\t%d0, %d1 1501 fmov\\t%d0, %1 1502 * return aarch64_output_scalar_simd_mov_immediate (operands[1], DImode); 1503 ldr\\t%d0, %1 1504 str\\t%d1, %0 1505 ldr\\t%x0, %1 1506 str\\t%x1, %0 1507 mov\\t%x0, %x1 1508 mov\\t%x0, %1" 1509 [(set_attr "type" "neon_move,f_mcr,f_mrc,fmov,fconstd,neon_move,\ 1510 f_loadd,f_stored,load_8,store_8,mov_reg,\ 1511 fconstd") 1512 (set_attr "arch" "simd,*,*,*,*,simd,*,*,*,*,*,*")] 1513) 1514 1515(define_split 1516 [(set (match_operand:GPF_HF 0 "nonimmediate_operand") 1517 (match_operand:GPF_HF 1 "general_operand"))] 1518 "can_create_pseudo_p () 1519 && !aarch64_can_const_movi_rtx_p (operands[1], <MODE>mode) 1520 && !aarch64_float_const_representable_p (operands[1]) 1521 && aarch64_float_const_rtx_p (operands[1])" 1522 [(const_int 0)] 1523 { 1524 unsigned HOST_WIDE_INT ival; 1525 if (!aarch64_reinterpret_float_as_int (operands[1], &ival)) 1526 FAIL; 1527 1528 rtx tmp = gen_reg_rtx (<FCVT_TARGET>mode); 1529 emit_move_insn (tmp, gen_int_mode (ival, <FCVT_TARGET>mode)); 1530 emit_move_insn (operands[0], gen_lowpart (<MODE>mode, tmp)); 1531 DONE; 1532 } 1533) 1534 1535(define_insn "*movtf_aarch64" 1536 [(set (match_operand:TF 0 1537 "nonimmediate_operand" "=w,?r ,w ,?r,w,?w,w,m,?r,m ,m") 1538 (match_operand:TF 1 1539 "general_operand" " w,?rY,?r,w ,Y,Y ,m,w,m ,?r,Y"))] 1540 "TARGET_FLOAT && (register_operand (operands[0], TFmode) 1541 || aarch64_reg_or_fp_zero (operands[1], TFmode))" 1542 "@ 1543 mov\\t%0.16b, %1.16b 1544 # 1545 # 1546 # 1547 movi\\t%0.2d, #0 1548 fmov\\t%s0, wzr 1549 ldr\\t%q0, %1 1550 str\\t%q1, %0 1551 ldp\\t%0, %H0, %1 1552 stp\\t%1, %H1, %0 1553 stp\\txzr, xzr, %0" 1554 [(set_attr "type" "logic_reg,multiple,f_mcr,f_mrc,neon_move_q,f_mcr,\ 1555 f_loadd,f_stored,load_16,store_16,store_16") 1556 (set_attr "length" "4,8,8,8,4,4,4,4,4,4,4") 1557 (set_attr "arch" "simd,*,*,*,simd,*,*,*,*,*,*")] 1558) 1559 1560(define_split 1561 [(set (match_operand:TF 0 "register_operand" "") 1562 (match_operand:TF 1 "nonmemory_operand" ""))] 1563 "reload_completed && aarch64_split_128bit_move_p (operands[0], operands[1])" 1564 [(const_int 0)] 1565 { 1566 aarch64_split_128bit_move (operands[0], operands[1]); 1567 DONE; 1568 } 1569) 1570 1571;; 0 is dst 1572;; 1 is src 1573;; 2 is size of copy in bytes 1574;; 3 is alignment 1575 1576(define_expand "cpymemdi" 1577 [(match_operand:BLK 0 "memory_operand") 1578 (match_operand:BLK 1 "memory_operand") 1579 (match_operand:DI 2 "immediate_operand") 1580 (match_operand:DI 3 "immediate_operand")] 1581 "!STRICT_ALIGNMENT" 1582{ 1583 if (aarch64_expand_cpymem (operands)) 1584 DONE; 1585 FAIL; 1586} 1587) 1588 1589;; 0 is dst 1590;; 1 is val 1591;; 2 is size of copy in bytes 1592;; 3 is alignment 1593 1594(define_expand "setmemdi" 1595 [(set (match_operand:BLK 0 "memory_operand") ;; Dest 1596 (match_operand:QI 2 "nonmemory_operand")) ;; Value 1597 (use (match_operand:DI 1 "immediate_operand")) ;; Length 1598 (match_operand 3 "immediate_operand")] ;; Align 1599 "TARGET_SIMD" 1600{ 1601 if (aarch64_expand_setmem (operands)) 1602 DONE; 1603 1604 FAIL; 1605}) 1606 1607;; Operands 1 and 3 are tied together by the final condition; so we allow 1608;; fairly lax checking on the second memory operation. 1609(define_insn "load_pair_sw_<SX:mode><SX2:mode>" 1610 [(set (match_operand:SX 0 "register_operand" "=r,w") 1611 (match_operand:SX 1 "aarch64_mem_pair_operand" "Ump,Ump")) 1612 (set (match_operand:SX2 2 "register_operand" "=r,w") 1613 (match_operand:SX2 3 "memory_operand" "m,m"))] 1614 "rtx_equal_p (XEXP (operands[3], 0), 1615 plus_constant (Pmode, 1616 XEXP (operands[1], 0), 1617 GET_MODE_SIZE (<SX:MODE>mode)))" 1618 "@ 1619 ldp\\t%w0, %w2, %z1 1620 ldp\\t%s0, %s2, %z1" 1621 [(set_attr "type" "load_8,neon_load1_2reg") 1622 (set_attr "arch" "*,fp")] 1623) 1624 1625;; Storing different modes that can still be merged 1626(define_insn "load_pair_dw_<DX:mode><DX2:mode>" 1627 [(set (match_operand:DX 0 "register_operand" "=r,w") 1628 (match_operand:DX 1 "aarch64_mem_pair_operand" "Ump,Ump")) 1629 (set (match_operand:DX2 2 "register_operand" "=r,w") 1630 (match_operand:DX2 3 "memory_operand" "m,m"))] 1631 "rtx_equal_p (XEXP (operands[3], 0), 1632 plus_constant (Pmode, 1633 XEXP (operands[1], 0), 1634 GET_MODE_SIZE (<DX:MODE>mode)))" 1635 "@ 1636 ldp\\t%x0, %x2, %z1 1637 ldp\\t%d0, %d2, %z1" 1638 [(set_attr "type" "load_16,neon_load1_2reg") 1639 (set_attr "arch" "*,fp")] 1640) 1641 1642(define_insn "load_pair_dw_tftf" 1643 [(set (match_operand:TF 0 "register_operand" "=w") 1644 (match_operand:TF 1 "aarch64_mem_pair_operand" "Ump")) 1645 (set (match_operand:TF 2 "register_operand" "=w") 1646 (match_operand:TF 3 "memory_operand" "m"))] 1647 "TARGET_SIMD 1648 && rtx_equal_p (XEXP (operands[3], 0), 1649 plus_constant (Pmode, 1650 XEXP (operands[1], 0), 1651 GET_MODE_SIZE (TFmode)))" 1652 "ldp\\t%q0, %q2, %z1" 1653 [(set_attr "type" "neon_ldp_q") 1654 (set_attr "fp" "yes")] 1655) 1656 1657;; Operands 0 and 2 are tied together by the final condition; so we allow 1658;; fairly lax checking on the second memory operation. 1659(define_insn "store_pair_sw_<SX:mode><SX2:mode>" 1660 [(set (match_operand:SX 0 "aarch64_mem_pair_operand" "=Ump,Ump") 1661 (match_operand:SX 1 "aarch64_reg_zero_or_fp_zero" "rYZ,w")) 1662 (set (match_operand:SX2 2 "memory_operand" "=m,m") 1663 (match_operand:SX2 3 "aarch64_reg_zero_or_fp_zero" "rYZ,w"))] 1664 "rtx_equal_p (XEXP (operands[2], 0), 1665 plus_constant (Pmode, 1666 XEXP (operands[0], 0), 1667 GET_MODE_SIZE (<SX:MODE>mode)))" 1668 "@ 1669 stp\\t%w1, %w3, %z0 1670 stp\\t%s1, %s3, %z0" 1671 [(set_attr "type" "store_8,neon_store1_2reg") 1672 (set_attr "arch" "*,fp")] 1673) 1674 1675;; Storing different modes that can still be merged 1676(define_insn "store_pair_dw_<DX:mode><DX2:mode>" 1677 [(set (match_operand:DX 0 "aarch64_mem_pair_operand" "=Ump,Ump") 1678 (match_operand:DX 1 "aarch64_reg_zero_or_fp_zero" "rYZ,w")) 1679 (set (match_operand:DX2 2 "memory_operand" "=m,m") 1680 (match_operand:DX2 3 "aarch64_reg_zero_or_fp_zero" "rYZ,w"))] 1681 "rtx_equal_p (XEXP (operands[2], 0), 1682 plus_constant (Pmode, 1683 XEXP (operands[0], 0), 1684 GET_MODE_SIZE (<DX:MODE>mode)))" 1685 "@ 1686 stp\\t%x1, %x3, %z0 1687 stp\\t%d1, %d3, %z0" 1688 [(set_attr "type" "store_16,neon_store1_2reg") 1689 (set_attr "arch" "*,fp")] 1690) 1691 1692(define_insn "store_pair_dw_tftf" 1693 [(set (match_operand:TF 0 "aarch64_mem_pair_operand" "=Ump") 1694 (match_operand:TF 1 "register_operand" "w")) 1695 (set (match_operand:TF 2 "memory_operand" "=m") 1696 (match_operand:TF 3 "register_operand" "w"))] 1697 "TARGET_SIMD && 1698 rtx_equal_p (XEXP (operands[2], 0), 1699 plus_constant (Pmode, 1700 XEXP (operands[0], 0), 1701 GET_MODE_SIZE (TFmode)))" 1702 "stp\\t%q1, %q3, %z0" 1703 [(set_attr "type" "neon_stp_q") 1704 (set_attr "fp" "yes")] 1705) 1706 1707;; Load pair with post-index writeback. This is primarily used in function 1708;; epilogues. 1709(define_insn "loadwb_pair<GPI:mode>_<P:mode>" 1710 [(parallel 1711 [(set (match_operand:P 0 "register_operand" "=k") 1712 (plus:P (match_operand:P 1 "register_operand" "0") 1713 (match_operand:P 4 "aarch64_mem_pair_offset" "n"))) 1714 (set (match_operand:GPI 2 "register_operand" "=r") 1715 (mem:GPI (match_dup 1))) 1716 (set (match_operand:GPI 3 "register_operand" "=r") 1717 (mem:GPI (plus:P (match_dup 1) 1718 (match_operand:P 5 "const_int_operand" "n"))))])] 1719 "INTVAL (operands[5]) == GET_MODE_SIZE (<GPI:MODE>mode)" 1720 "ldp\\t%<GPI:w>2, %<GPI:w>3, [%1], %4" 1721 [(set_attr "type" "load_<GPI:ldpstp_sz>")] 1722) 1723 1724(define_insn "loadwb_pair<GPF:mode>_<P:mode>" 1725 [(parallel 1726 [(set (match_operand:P 0 "register_operand" "=k") 1727 (plus:P (match_operand:P 1 "register_operand" "0") 1728 (match_operand:P 4 "aarch64_mem_pair_offset" "n"))) 1729 (set (match_operand:GPF 2 "register_operand" "=w") 1730 (mem:GPF (match_dup 1))) 1731 (set (match_operand:GPF 3 "register_operand" "=w") 1732 (mem:GPF (plus:P (match_dup 1) 1733 (match_operand:P 5 "const_int_operand" "n"))))])] 1734 "INTVAL (operands[5]) == GET_MODE_SIZE (<GPF:MODE>mode)" 1735 "ldp\\t%<GPF:w>2, %<GPF:w>3, [%1], %4" 1736 [(set_attr "type" "neon_load1_2reg")] 1737) 1738 1739(define_insn "loadwb_pair<TX:mode>_<P:mode>" 1740 [(parallel 1741 [(set (match_operand:P 0 "register_operand" "=k") 1742 (plus:P (match_operand:P 1 "register_operand" "0") 1743 (match_operand:P 4 "aarch64_mem_pair_offset" "n"))) 1744 (set (match_operand:TX 2 "register_operand" "=w") 1745 (mem:TX (match_dup 1))) 1746 (set (match_operand:TX 3 "register_operand" "=w") 1747 (mem:TX (plus:P (match_dup 1) 1748 (match_operand:P 5 "const_int_operand" "n"))))])] 1749 "TARGET_SIMD && INTVAL (operands[5]) == GET_MODE_SIZE (<TX:MODE>mode)" 1750 "ldp\\t%q2, %q3, [%1], %4" 1751 [(set_attr "type" "neon_ldp_q")] 1752) 1753 1754;; Store pair with pre-index writeback. This is primarily used in function 1755;; prologues. 1756(define_insn "storewb_pair<GPI:mode>_<P:mode>" 1757 [(parallel 1758 [(set (match_operand:P 0 "register_operand" "=&k") 1759 (plus:P (match_operand:P 1 "register_operand" "0") 1760 (match_operand:P 4 "aarch64_mem_pair_offset" "n"))) 1761 (set (mem:GPI (plus:P (match_dup 0) 1762 (match_dup 4))) 1763 (match_operand:GPI 2 "register_operand" "r")) 1764 (set (mem:GPI (plus:P (match_dup 0) 1765 (match_operand:P 5 "const_int_operand" "n"))) 1766 (match_operand:GPI 3 "register_operand" "r"))])] 1767 "INTVAL (operands[5]) == INTVAL (operands[4]) + GET_MODE_SIZE (<GPI:MODE>mode)" 1768 "stp\\t%<GPI:w>2, %<GPI:w>3, [%0, %4]!" 1769 [(set_attr "type" "store_<GPI:ldpstp_sz>")] 1770) 1771 1772(define_insn "storewb_pair<GPF:mode>_<P:mode>" 1773 [(parallel 1774 [(set (match_operand:P 0 "register_operand" "=&k") 1775 (plus:P (match_operand:P 1 "register_operand" "0") 1776 (match_operand:P 4 "aarch64_mem_pair_offset" "n"))) 1777 (set (mem:GPF (plus:P (match_dup 0) 1778 (match_dup 4))) 1779 (match_operand:GPF 2 "register_operand" "w")) 1780 (set (mem:GPF (plus:P (match_dup 0) 1781 (match_operand:P 5 "const_int_operand" "n"))) 1782 (match_operand:GPF 3 "register_operand" "w"))])] 1783 "INTVAL (operands[5]) == INTVAL (operands[4]) + GET_MODE_SIZE (<GPF:MODE>mode)" 1784 "stp\\t%<GPF:w>2, %<GPF:w>3, [%0, %4]!" 1785 [(set_attr "type" "neon_store1_2reg<q>")] 1786) 1787 1788(define_insn "storewb_pair<TX:mode>_<P:mode>" 1789 [(parallel 1790 [(set (match_operand:P 0 "register_operand" "=&k") 1791 (plus:P (match_operand:P 1 "register_operand" "0") 1792 (match_operand:P 4 "aarch64_mem_pair_offset" "n"))) 1793 (set (mem:TX (plus:P (match_dup 0) 1794 (match_dup 4))) 1795 (match_operand:TX 2 "register_operand" "w")) 1796 (set (mem:TX (plus:P (match_dup 0) 1797 (match_operand:P 5 "const_int_operand" "n"))) 1798 (match_operand:TX 3 "register_operand" "w"))])] 1799 "TARGET_SIMD 1800 && INTVAL (operands[5]) 1801 == INTVAL (operands[4]) + GET_MODE_SIZE (<TX:MODE>mode)" 1802 "stp\\t%q2, %q3, [%0, %4]!" 1803 [(set_attr "type" "neon_stp_q")] 1804) 1805 1806;; ------------------------------------------------------------------- 1807;; Sign/Zero extension 1808;; ------------------------------------------------------------------- 1809 1810(define_expand "<optab>sidi2" 1811 [(set (match_operand:DI 0 "register_operand") 1812 (ANY_EXTEND:DI (match_operand:SI 1 "nonimmediate_operand")))] 1813 "" 1814) 1815 1816(define_insn "*extendsidi2_aarch64" 1817 [(set (match_operand:DI 0 "register_operand" "=r,r") 1818 (sign_extend:DI (match_operand:SI 1 "nonimmediate_operand" "r,m")))] 1819 "" 1820 "@ 1821 sxtw\t%0, %w1 1822 ldrsw\t%0, %1" 1823 [(set_attr "type" "extend,load_4")] 1824) 1825 1826(define_insn "*load_pair_extendsidi2_aarch64" 1827 [(set (match_operand:DI 0 "register_operand" "=r") 1828 (sign_extend:DI (match_operand:SI 1 "aarch64_mem_pair_operand" "Ump"))) 1829 (set (match_operand:DI 2 "register_operand" "=r") 1830 (sign_extend:DI (match_operand:SI 3 "memory_operand" "m")))] 1831 "rtx_equal_p (XEXP (operands[3], 0), 1832 plus_constant (Pmode, 1833 XEXP (operands[1], 0), 1834 GET_MODE_SIZE (SImode)))" 1835 "ldpsw\\t%0, %2, %z1" 1836 [(set_attr "type" "load_8")] 1837) 1838 1839(define_insn "*zero_extendsidi2_aarch64" 1840 [(set (match_operand:DI 0 "register_operand" "=r,r,w,w,r,w") 1841 (zero_extend:DI (match_operand:SI 1 "nonimmediate_operand" "r,m,r,m,w,w")))] 1842 "" 1843 "@ 1844 uxtw\t%0, %w1 1845 ldr\t%w0, %1 1846 fmov\t%s0, %w1 1847 ldr\t%s0, %1 1848 fmov\t%w0, %s1 1849 fmov\t%s0, %s1" 1850 [(set_attr "type" "mov_reg,load_4,f_mcr,f_loads,f_mrc,fmov") 1851 (set_attr "arch" "*,*,fp,fp,fp,fp")] 1852) 1853 1854(define_insn "*load_pair_zero_extendsidi2_aarch64" 1855 [(set (match_operand:DI 0 "register_operand" "=r,w") 1856 (zero_extend:DI (match_operand:SI 1 "aarch64_mem_pair_operand" "Ump,Ump"))) 1857 (set (match_operand:DI 2 "register_operand" "=r,w") 1858 (zero_extend:DI (match_operand:SI 3 "memory_operand" "m,m")))] 1859 "rtx_equal_p (XEXP (operands[3], 0), 1860 plus_constant (Pmode, 1861 XEXP (operands[1], 0), 1862 GET_MODE_SIZE (SImode)))" 1863 "@ 1864 ldp\t%w0, %w2, %z1 1865 ldp\t%s0, %s2, %z1" 1866 [(set_attr "type" "load_8,neon_load1_2reg") 1867 (set_attr "arch" "*,fp")] 1868) 1869 1870(define_expand "<ANY_EXTEND:optab><SHORT:mode><GPI:mode>2" 1871 [(set (match_operand:GPI 0 "register_operand") 1872 (ANY_EXTEND:GPI (match_operand:SHORT 1 "nonimmediate_operand")))] 1873 "" 1874) 1875 1876(define_insn "*extend<SHORT:mode><GPI:mode>2_aarch64" 1877 [(set (match_operand:GPI 0 "register_operand" "=r,r") 1878 (sign_extend:GPI (match_operand:SHORT 1 "nonimmediate_operand" "r,m")))] 1879 "" 1880 "@ 1881 sxt<SHORT:size>\t%<GPI:w>0, %w1 1882 ldrs<SHORT:size>\t%<GPI:w>0, %1" 1883 [(set_attr "type" "extend,load_4")] 1884) 1885 1886(define_insn "*zero_extend<SHORT:mode><GPI:mode>2_aarch64" 1887 [(set (match_operand:GPI 0 "register_operand" "=r,r,w") 1888 (zero_extend:GPI (match_operand:SHORT 1 "nonimmediate_operand" "r,m,m")))] 1889 "" 1890 "@ 1891 and\t%<GPI:w>0, %<GPI:w>1, <SHORT:short_mask> 1892 ldr<SHORT:size>\t%w0, %1 1893 ldr\t%<SHORT:size>0, %1" 1894 [(set_attr "type" "logic_imm,load_4,f_loads") 1895 (set_attr "arch" "*,*,fp")] 1896) 1897 1898(define_expand "<optab>qihi2" 1899 [(set (match_operand:HI 0 "register_operand") 1900 (ANY_EXTEND:HI (match_operand:QI 1 "nonimmediate_operand")))] 1901 "" 1902) 1903 1904(define_insn "*extendqihi2_aarch64" 1905 [(set (match_operand:HI 0 "register_operand" "=r,r") 1906 (sign_extend:HI (match_operand:QI 1 "nonimmediate_operand" "r,m")))] 1907 "" 1908 "@ 1909 sxtb\t%w0, %w1 1910 ldrsb\t%w0, %1" 1911 [(set_attr "type" "extend,load_4")] 1912) 1913 1914(define_insn "*zero_extendqihi2_aarch64" 1915 [(set (match_operand:HI 0 "register_operand" "=r,r") 1916 (zero_extend:HI (match_operand:QI 1 "nonimmediate_operand" "r,m")))] 1917 "" 1918 "@ 1919 and\t%w0, %w1, 255 1920 ldrb\t%w0, %1" 1921 [(set_attr "type" "logic_imm,load_4")] 1922) 1923 1924;; ------------------------------------------------------------------- 1925;; Simple arithmetic 1926;; ------------------------------------------------------------------- 1927 1928(define_expand "add<mode>3" 1929 [(set 1930 (match_operand:GPI 0 "register_operand") 1931 (plus:GPI (match_operand:GPI 1 "register_operand") 1932 (match_operand:GPI 2 "aarch64_pluslong_or_poly_operand")))] 1933 "" 1934{ 1935 /* If operands[1] is a subreg extract the inner RTX. */ 1936 rtx op1 = REG_P (operands[1]) ? operands[1] : SUBREG_REG (operands[1]); 1937 1938 /* If the constant is too large for a single instruction and isn't frame 1939 based, split off the immediate so it is available for CSE. */ 1940 if (!aarch64_plus_immediate (operands[2], <MODE>mode) 1941 && !(TARGET_SVE && aarch64_sve_plus_immediate (operands[2], <MODE>mode)) 1942 && can_create_pseudo_p () 1943 && (!REG_P (op1) 1944 || !REGNO_PTR_FRAME_P (REGNO (op1)))) 1945 operands[2] = force_reg (<MODE>mode, operands[2]); 1946 /* Some tunings prefer to avoid VL-based operations. 1947 Split off the poly immediate here. The rtx costs hook will reject attempts 1948 to combine them back. */ 1949 else if (GET_CODE (operands[2]) == CONST_POLY_INT 1950 && can_create_pseudo_p () 1951 && (aarch64_tune_params.extra_tuning_flags 1952 & AARCH64_EXTRA_TUNE_CSE_SVE_VL_CONSTANTS)) 1953 operands[2] = force_reg (<MODE>mode, operands[2]); 1954 /* Expand polynomial additions now if the destination is the stack 1955 pointer, since we don't want to use that as a temporary. */ 1956 else if (operands[0] == stack_pointer_rtx 1957 && aarch64_split_add_offset_immediate (operands[2], <MODE>mode)) 1958 { 1959 aarch64_split_add_offset (<MODE>mode, operands[0], operands[1], 1960 operands[2], NULL_RTX, NULL_RTX); 1961 DONE; 1962 } 1963}) 1964 1965(define_insn "*add<mode>3_aarch64" 1966 [(set 1967 (match_operand:GPI 0 "register_operand" "=rk,rk,w,rk,r,r,rk") 1968 (plus:GPI 1969 (match_operand:GPI 1 "register_operand" "%rk,rk,w,rk,rk,0,rk") 1970 (match_operand:GPI 2 "aarch64_pluslong_operand" "I,r,w,J,Uaa,Uai,Uav")))] 1971 "" 1972 "@ 1973 add\\t%<w>0, %<w>1, %2 1974 add\\t%<w>0, %<w>1, %<w>2 1975 add\\t%<rtn>0<vas>, %<rtn>1<vas>, %<rtn>2<vas> 1976 sub\\t%<w>0, %<w>1, #%n2 1977 # 1978 * return aarch64_output_sve_scalar_inc_dec (operands[2]); 1979 * return aarch64_output_sve_addvl_addpl (operands[2]);" 1980 ;; The "alu_imm" types for INC/DEC and ADDVL/ADDPL are just placeholders. 1981 [(set_attr "type" "alu_imm,alu_sreg,neon_add,alu_imm,multiple,alu_imm,alu_imm") 1982 (set_attr "arch" "*,*,simd,*,*,sve,sve")] 1983) 1984 1985;; zero_extend version of above 1986(define_insn "*addsi3_aarch64_uxtw" 1987 [(set 1988 (match_operand:DI 0 "register_operand" "=rk,rk,rk,r") 1989 (zero_extend:DI 1990 (plus:SI (match_operand:SI 1 "register_operand" "%rk,rk,rk,rk") 1991 (match_operand:SI 2 "aarch64_pluslong_operand" "I,r,J,Uaa"))))] 1992 "" 1993 "@ 1994 add\\t%w0, %w1, %2 1995 add\\t%w0, %w1, %w2 1996 sub\\t%w0, %w1, #%n2 1997 #" 1998 [(set_attr "type" "alu_imm,alu_sreg,alu_imm,multiple")] 1999) 2000 2001;; If there's a free register, and we can load the constant with a 2002;; single instruction, do so. This has a chance to improve scheduling. 2003(define_peephole2 2004 [(match_scratch:GPI 3 "r") 2005 (set (match_operand:GPI 0 "register_operand") 2006 (plus:GPI 2007 (match_operand:GPI 1 "register_operand") 2008 (match_operand:GPI 2 "aarch64_pluslong_strict_immedate")))] 2009 "aarch64_move_imm (INTVAL (operands[2]), <MODE>mode)" 2010 [(set (match_dup 3) (match_dup 2)) 2011 (set (match_dup 0) (plus:GPI (match_dup 1) (match_dup 3)))] 2012) 2013 2014(define_peephole2 2015 [(match_scratch:SI 3 "r") 2016 (set (match_operand:DI 0 "register_operand") 2017 (zero_extend:DI 2018 (plus:SI 2019 (match_operand:SI 1 "register_operand") 2020 (match_operand:SI 2 "aarch64_pluslong_strict_immedate"))))] 2021 "aarch64_move_imm (INTVAL (operands[2]), SImode)" 2022 [(set (match_dup 3) (match_dup 2)) 2023 (set (match_dup 0) (zero_extend:DI (plus:SI (match_dup 1) (match_dup 3))))] 2024) 2025 2026;; After peephole2 has had a chance to run, split any remaining long 2027;; additions into two add immediates. 2028(define_split 2029 [(set (match_operand:GPI 0 "register_operand") 2030 (plus:GPI 2031 (match_operand:GPI 1 "register_operand") 2032 (match_operand:GPI 2 "aarch64_pluslong_strict_immedate")))] 2033 "epilogue_completed" 2034 [(set (match_dup 0) (plus:GPI (match_dup 1) (match_dup 3))) 2035 (set (match_dup 0) (plus:GPI (match_dup 0) (match_dup 4)))] 2036 { 2037 HOST_WIDE_INT i = INTVAL (operands[2]); 2038 HOST_WIDE_INT s = (i >= 0 ? i & 0xfff : -(-i & 0xfff)); 2039 operands[3] = GEN_INT (i - s); 2040 operands[4] = GEN_INT (s); 2041 } 2042) 2043 2044;; Match addition of polynomial offsets that require one temporary, for which 2045;; we can use the early-clobbered destination register. This is a separate 2046;; pattern so that the early clobber doesn't affect register allocation 2047;; for other forms of addition. However, we still need to provide an 2048;; all-register alternative, in case the offset goes out of range after 2049;; elimination. For completeness we might as well provide all GPR-based 2050;; alternatives from the main pattern. 2051;; 2052;; We don't have a pattern for additions requiring two temporaries since at 2053;; present LRA doesn't allow new scratches to be added during elimination. 2054;; Such offsets should be rare anyway. 2055;; 2056;; ??? But if we added LRA support for new scratches, much of the ugliness 2057;; here would go away. We could just handle all polynomial constants in 2058;; this pattern. 2059(define_insn_and_split "*add<mode>3_poly_1" 2060 [(set 2061 (match_operand:GPI 0 "register_operand" "=r,r,r,r,r,r,&r") 2062 (plus:GPI 2063 (match_operand:GPI 1 "register_operand" "%rk,rk,rk,rk,0,rk,rk") 2064 (match_operand:GPI 2 "aarch64_pluslong_or_poly_operand" "I,r,J,Uaa,Uai,Uav,Uat")))] 2065 "TARGET_SVE && operands[0] != stack_pointer_rtx" 2066 "@ 2067 add\\t%<w>0, %<w>1, %2 2068 add\\t%<w>0, %<w>1, %<w>2 2069 sub\\t%<w>0, %<w>1, #%n2 2070 # 2071 * return aarch64_output_sve_scalar_inc_dec (operands[2]); 2072 * return aarch64_output_sve_addvl_addpl (operands[2]); 2073 #" 2074 "&& epilogue_completed 2075 && !reg_overlap_mentioned_p (operands[0], operands[1]) 2076 && aarch64_split_add_offset_immediate (operands[2], <MODE>mode)" 2077 [(const_int 0)] 2078 { 2079 aarch64_split_add_offset (<MODE>mode, operands[0], operands[1], 2080 operands[2], operands[0], NULL_RTX); 2081 DONE; 2082 } 2083 ;; The "alu_imm" types for INC/DEC and ADDVL/ADDPL are just placeholders. 2084 [(set_attr "type" "alu_imm,alu_sreg,alu_imm,multiple,alu_imm,alu_imm,multiple")] 2085) 2086 2087(define_split 2088 [(set (match_operand:DI 0 "register_operand") 2089 (zero_extend:DI 2090 (plus:SI 2091 (match_operand:SI 1 "register_operand") 2092 (match_operand:SI 2 "aarch64_pluslong_strict_immedate"))))] 2093 "epilogue_completed" 2094 [(set (match_dup 5) (plus:SI (match_dup 1) (match_dup 3))) 2095 (set (match_dup 0) (zero_extend:DI (plus:SI (match_dup 5) (match_dup 4))))] 2096 { 2097 HOST_WIDE_INT i = INTVAL (operands[2]); 2098 HOST_WIDE_INT s = (i >= 0 ? i & 0xfff : -(-i & 0xfff)); 2099 operands[3] = GEN_INT (i - s); 2100 operands[4] = GEN_INT (s); 2101 operands[5] = gen_lowpart (SImode, operands[0]); 2102 } 2103) 2104 2105(define_expand "addv<mode>4" 2106 [(match_operand:GPI 0 "register_operand") 2107 (match_operand:GPI 1 "register_operand") 2108 (match_operand:GPI 2 "aarch64_plus_operand") 2109 (label_ref (match_operand 3 "" ""))] 2110 "" 2111{ 2112 if (CONST_INT_P (operands[2])) 2113 emit_insn (gen_add<mode>3_compareV_imm (operands[0], operands[1], 2114 operands[2])); 2115 else 2116 emit_insn (gen_add<mode>3_compareV (operands[0], operands[1], operands[2])); 2117 aarch64_gen_unlikely_cbranch (NE, CC_Vmode, operands[3]); 2118 2119 DONE; 2120}) 2121 2122(define_expand "uaddv<mode>4" 2123 [(match_operand:GPI 0 "register_operand") 2124 (match_operand:GPI 1 "register_operand") 2125 (match_operand:GPI 2 "register_operand") 2126 (label_ref (match_operand 3 "" ""))] 2127 "" 2128{ 2129 emit_insn (gen_add<mode>3_compareC (operands[0], operands[1], operands[2])); 2130 aarch64_gen_unlikely_cbranch (LTU, CC_Cmode, operands[3]); 2131 2132 DONE; 2133}) 2134 2135(define_expand "addti3" 2136 [(set (match_operand:TI 0 "register_operand") 2137 (plus:TI (match_operand:TI 1 "register_operand") 2138 (match_operand:TI 2 "aarch64_reg_or_imm")))] 2139 "" 2140{ 2141 rtx low_dest, op1_low, op2_low, high_dest, op1_high, op2_high; 2142 2143 aarch64_addti_scratch_regs (operands[1], operands[2], 2144 &low_dest, &op1_low, &op2_low, 2145 &high_dest, &op1_high, &op2_high); 2146 2147 if (op2_low == const0_rtx) 2148 { 2149 low_dest = op1_low; 2150 if (!aarch64_pluslong_operand (op2_high, DImode)) 2151 op2_high = force_reg (DImode, op2_high); 2152 emit_insn (gen_adddi3 (high_dest, op1_high, op2_high)); 2153 } 2154 else 2155 { 2156 emit_insn (gen_adddi3_compareC (low_dest, op1_low, 2157 force_reg (DImode, op2_low))); 2158 emit_insn (gen_adddi3_carryin (high_dest, op1_high, 2159 force_reg (DImode, op2_high))); 2160 } 2161 2162 emit_move_insn (gen_lowpart (DImode, operands[0]), low_dest); 2163 emit_move_insn (gen_highpart (DImode, operands[0]), high_dest); 2164 2165 DONE; 2166}) 2167 2168(define_expand "addvti4" 2169 [(match_operand:TI 0 "register_operand") 2170 (match_operand:TI 1 "register_operand") 2171 (match_operand:TI 2 "aarch64_reg_or_imm") 2172 (label_ref (match_operand 3 "" ""))] 2173 "" 2174{ 2175 rtx low_dest, op1_low, op2_low, high_dest, op1_high, op2_high; 2176 2177 aarch64_addti_scratch_regs (operands[1], operands[2], 2178 &low_dest, &op1_low, &op2_low, 2179 &high_dest, &op1_high, &op2_high); 2180 2181 if (op2_low == const0_rtx) 2182 { 2183 low_dest = op1_low; 2184 emit_insn (gen_adddi3_compareV (high_dest, op1_high, 2185 force_reg (DImode, op2_high))); 2186 } 2187 else 2188 { 2189 emit_insn (gen_adddi3_compareC (low_dest, op1_low, 2190 force_reg (DImode, op2_low))); 2191 emit_insn (gen_adddi3_carryinV (high_dest, op1_high, 2192 force_reg (DImode, op2_high))); 2193 } 2194 2195 emit_move_insn (gen_lowpart (DImode, operands[0]), low_dest); 2196 emit_move_insn (gen_highpart (DImode, operands[0]), high_dest); 2197 2198 aarch64_gen_unlikely_cbranch (NE, CC_Vmode, operands[3]); 2199 DONE; 2200}) 2201 2202(define_expand "uaddvti4" 2203 [(match_operand:TI 0 "register_operand") 2204 (match_operand:TI 1 "register_operand") 2205 (match_operand:TI 2 "aarch64_reg_or_imm") 2206 (label_ref (match_operand 3 "" ""))] 2207 "" 2208{ 2209 rtx low_dest, op1_low, op2_low, high_dest, op1_high, op2_high; 2210 2211 aarch64_addti_scratch_regs (operands[1], operands[2], 2212 &low_dest, &op1_low, &op2_low, 2213 &high_dest, &op1_high, &op2_high); 2214 2215 if (op2_low == const0_rtx) 2216 { 2217 low_dest = op1_low; 2218 emit_insn (gen_adddi3_compareC (high_dest, op1_high, 2219 force_reg (DImode, op2_high))); 2220 } 2221 else 2222 { 2223 emit_insn (gen_adddi3_compareC (low_dest, op1_low, 2224 force_reg (DImode, op2_low))); 2225 emit_insn (gen_adddi3_carryinC (high_dest, op1_high, 2226 force_reg (DImode, op2_high))); 2227 } 2228 2229 emit_move_insn (gen_lowpart (DImode, operands[0]), low_dest); 2230 emit_move_insn (gen_highpart (DImode, operands[0]), high_dest); 2231 2232 aarch64_gen_unlikely_cbranch (GEU, CC_ADCmode, operands[3]); 2233 DONE; 2234 }) 2235 2236(define_insn "add<mode>3_compare0" 2237 [(set (reg:CC_NZ CC_REGNUM) 2238 (compare:CC_NZ 2239 (plus:GPI (match_operand:GPI 1 "register_operand" "%rk,rk,rk") 2240 (match_operand:GPI 2 "aarch64_plus_operand" "r,I,J")) 2241 (const_int 0))) 2242 (set (match_operand:GPI 0 "register_operand" "=r,r,r") 2243 (plus:GPI (match_dup 1) (match_dup 2)))] 2244 "" 2245 "@ 2246 adds\\t%<w>0, %<w>1, %<w>2 2247 adds\\t%<w>0, %<w>1, %2 2248 subs\\t%<w>0, %<w>1, #%n2" 2249 [(set_attr "type" "alus_sreg,alus_imm,alus_imm")] 2250) 2251 2252;; zero_extend version of above 2253(define_insn "*addsi3_compare0_uxtw" 2254 [(set (reg:CC_NZ CC_REGNUM) 2255 (compare:CC_NZ 2256 (plus:SI (match_operand:SI 1 "register_operand" "%rk,rk,rk") 2257 (match_operand:SI 2 "aarch64_plus_operand" "r,I,J")) 2258 (const_int 0))) 2259 (set (match_operand:DI 0 "register_operand" "=r,r,r") 2260 (zero_extend:DI (plus:SI (match_dup 1) (match_dup 2))))] 2261 "" 2262 "@ 2263 adds\\t%w0, %w1, %w2 2264 adds\\t%w0, %w1, %2 2265 subs\\t%w0, %w1, #%n2" 2266 [(set_attr "type" "alus_sreg,alus_imm,alus_imm")] 2267) 2268 2269(define_insn "*add<mode>3_compareC_cconly" 2270 [(set (reg:CC_C CC_REGNUM) 2271 (compare:CC_C 2272 (plus:GPI 2273 (match_operand:GPI 0 "register_operand" "r,r,r") 2274 (match_operand:GPI 1 "aarch64_plus_operand" "r,I,J")) 2275 (match_dup 0)))] 2276 "" 2277 "@ 2278 cmn\\t%<w>0, %<w>1 2279 cmn\\t%<w>0, %1 2280 cmp\\t%<w>0, #%n1" 2281 [(set_attr "type" "alus_sreg,alus_imm,alus_imm")] 2282) 2283 2284(define_insn "add<mode>3_compareC" 2285 [(set (reg:CC_C CC_REGNUM) 2286 (compare:CC_C 2287 (plus:GPI 2288 (match_operand:GPI 1 "register_operand" "rk,rk,rk") 2289 (match_operand:GPI 2 "aarch64_plus_operand" "r,I,J")) 2290 (match_dup 1))) 2291 (set (match_operand:GPI 0 "register_operand" "=r,r,r") 2292 (plus:GPI (match_dup 1) (match_dup 2)))] 2293 "" 2294 "@ 2295 adds\\t%<w>0, %<w>1, %<w>2 2296 adds\\t%<w>0, %<w>1, %2 2297 subs\\t%<w>0, %<w>1, #%n2" 2298 [(set_attr "type" "alus_sreg,alus_imm,alus_imm")] 2299) 2300 2301(define_insn "*add<mode>3_compareV_cconly_imm" 2302 [(set (reg:CC_V CC_REGNUM) 2303 (compare:CC_V 2304 (plus:<DWI> 2305 (sign_extend:<DWI> (match_operand:GPI 0 "register_operand" "r,r")) 2306 (match_operand:<DWI> 1 "const_scalar_int_operand" "")) 2307 (sign_extend:<DWI> 2308 (plus:GPI 2309 (match_dup 0) 2310 (match_operand:GPI 2 "aarch64_plus_immediate" "I,J")))))] 2311 "INTVAL (operands[1]) == INTVAL (operands[2])" 2312 "@ 2313 cmn\\t%<w>0, %<w>1 2314 cmp\\t%<w>0, #%n1" 2315 [(set_attr "type" "alus_imm")] 2316) 2317 2318(define_insn "*add<mode>3_compareV_cconly" 2319 [(set (reg:CC_V CC_REGNUM) 2320 (compare:CC_V 2321 (plus:<DWI> 2322 (sign_extend:<DWI> (match_operand:GPI 0 "register_operand" "r")) 2323 (sign_extend:<DWI> (match_operand:GPI 1 "register_operand" "r"))) 2324 (sign_extend:<DWI> (plus:GPI (match_dup 0) (match_dup 1)))))] 2325 "" 2326 "cmn\\t%<w>0, %<w>1" 2327 [(set_attr "type" "alus_sreg")] 2328) 2329 2330(define_insn "add<mode>3_compareV_imm" 2331 [(set (reg:CC_V CC_REGNUM) 2332 (compare:CC_V 2333 (plus:<DWI> 2334 (sign_extend:<DWI> 2335 (match_operand:GPI 1 "register_operand" "rk,rk")) 2336 (match_operand:GPI 2 "aarch64_plus_immediate" "I,J")) 2337 (sign_extend:<DWI> 2338 (plus:GPI (match_dup 1) (match_dup 2))))) 2339 (set (match_operand:GPI 0 "register_operand" "=r,r") 2340 (plus:GPI (match_dup 1) (match_dup 2)))] 2341 "" 2342 "@ 2343 adds\\t%<w>0, %<w>1, %<w>2 2344 subs\\t%<w>0, %<w>1, #%n2" 2345 [(set_attr "type" "alus_imm,alus_imm")] 2346) 2347 2348(define_insn "add<mode>3_compareV" 2349 [(set (reg:CC_V CC_REGNUM) 2350 (compare:CC_V 2351 (plus:<DWI> 2352 (sign_extend:<DWI> (match_operand:GPI 1 "register_operand" "rk")) 2353 (sign_extend:<DWI> (match_operand:GPI 2 "register_operand" "r"))) 2354 (sign_extend:<DWI> (plus:GPI (match_dup 1) (match_dup 2))))) 2355 (set (match_operand:GPI 0 "register_operand" "=r") 2356 (plus:GPI (match_dup 1) (match_dup 2)))] 2357 "" 2358 "adds\\t%<w>0, %<w>1, %<w>2" 2359 [(set_attr "type" "alus_sreg")] 2360) 2361 2362(define_insn "*adds_shift_imm_<mode>" 2363 [(set (reg:CC_NZ CC_REGNUM) 2364 (compare:CC_NZ 2365 (plus:GPI (ASHIFT:GPI 2366 (match_operand:GPI 1 "register_operand" "r") 2367 (match_operand:QI 2 "aarch64_shift_imm_<mode>" "n")) 2368 (match_operand:GPI 3 "register_operand" "r")) 2369 (const_int 0))) 2370 (set (match_operand:GPI 0 "register_operand" "=r") 2371 (plus:GPI (ASHIFT:GPI (match_dup 1) (match_dup 2)) 2372 (match_dup 3)))] 2373 "" 2374 "adds\\t%<w>0, %<w>3, %<w>1, <shift> %2" 2375 [(set_attr "type" "alus_shift_imm")] 2376) 2377 2378(define_insn "*subs_shift_imm_<mode>" 2379 [(set (reg:CC_NZ CC_REGNUM) 2380 (compare:CC_NZ 2381 (minus:GPI (match_operand:GPI 1 "register_operand" "r") 2382 (ASHIFT:GPI 2383 (match_operand:GPI 2 "register_operand" "r") 2384 (match_operand:QI 3 "aarch64_shift_imm_<mode>" "n"))) 2385 (const_int 0))) 2386 (set (match_operand:GPI 0 "register_operand" "=r") 2387 (minus:GPI (match_dup 1) 2388 (ASHIFT:GPI (match_dup 2) (match_dup 3))))] 2389 "" 2390 "subs\\t%<w>0, %<w>1, %<w>2, <shift> %3" 2391 [(set_attr "type" "alus_shift_imm")] 2392) 2393 2394(define_insn "*adds_<optab><ALLX:mode>_<GPI:mode>" 2395 [(set (reg:CC_NZ CC_REGNUM) 2396 (compare:CC_NZ 2397 (plus:GPI 2398 (ANY_EXTEND:GPI (match_operand:ALLX 1 "register_operand" "r")) 2399 (match_operand:GPI 2 "register_operand" "rk")) 2400 (const_int 0))) 2401 (set (match_operand:GPI 0 "register_operand" "=r") 2402 (plus:GPI (ANY_EXTEND:GPI (match_dup 1)) (match_dup 2)))] 2403 "" 2404 "adds\\t%<GPI:w>0, %<GPI:w>2, %w1, <su>xt<ALLX:size>" 2405 [(set_attr "type" "alus_ext")] 2406) 2407 2408(define_insn "*subs_<optab><ALLX:mode>_<GPI:mode>" 2409 [(set (reg:CC_NZ CC_REGNUM) 2410 (compare:CC_NZ 2411 (minus:GPI (match_operand:GPI 1 "register_operand" "rk") 2412 (ANY_EXTEND:GPI 2413 (match_operand:ALLX 2 "register_operand" "r"))) 2414 (const_int 0))) 2415 (set (match_operand:GPI 0 "register_operand" "=r") 2416 (minus:GPI (match_dup 1) (ANY_EXTEND:GPI (match_dup 2))))] 2417 "" 2418 "subs\\t%<GPI:w>0, %<GPI:w>1, %w2, <su>xt<ALLX:size>" 2419 [(set_attr "type" "alus_ext")] 2420) 2421 2422(define_insn "*adds_<optab><ALLX:mode>_shift_<GPI:mode>" 2423 [(set (reg:CC_NZ CC_REGNUM) 2424 (compare:CC_NZ 2425 (plus:GPI (ashift:GPI 2426 (ANY_EXTEND:GPI 2427 (match_operand:ALLX 1 "register_operand" "r")) 2428 (match_operand 2 "aarch64_imm3" "Ui3")) 2429 (match_operand:GPI 3 "register_operand" "rk")) 2430 (const_int 0))) 2431 (set (match_operand:GPI 0 "register_operand" "=rk") 2432 (plus:GPI (ashift:GPI (ANY_EXTEND:GPI (match_dup 1)) 2433 (match_dup 2)) 2434 (match_dup 3)))] 2435 "" 2436 "adds\\t%<GPI:w>0, %<GPI:w>3, %w1, <su>xt<ALLX:size> %2" 2437 [(set_attr "type" "alus_ext")] 2438) 2439 2440(define_insn "*subs_<optab><ALLX:mode>_shift_<GPI:mode>" 2441 [(set (reg:CC_NZ CC_REGNUM) 2442 (compare:CC_NZ 2443 (minus:GPI (match_operand:GPI 1 "register_operand" "rk") 2444 (ashift:GPI 2445 (ANY_EXTEND:GPI 2446 (match_operand:ALLX 2 "register_operand" "r")) 2447 (match_operand 3 "aarch64_imm3" "Ui3"))) 2448 (const_int 0))) 2449 (set (match_operand:GPI 0 "register_operand" "=rk") 2450 (minus:GPI (match_dup 1) 2451 (ashift:GPI (ANY_EXTEND:GPI (match_dup 2)) 2452 (match_dup 3))))] 2453 "" 2454 "subs\\t%<GPI:w>0, %<GPI:w>1, %w2, <su>xt<ALLX:size> %3" 2455 [(set_attr "type" "alus_ext")] 2456) 2457 2458(define_insn "*add<mode>3nr_compare0" 2459 [(set (reg:CC_NZ CC_REGNUM) 2460 (compare:CC_NZ 2461 (plus:GPI (match_operand:GPI 0 "register_operand" "%r,r,r") 2462 (match_operand:GPI 1 "aarch64_plus_operand" "r,I,J")) 2463 (const_int 0)))] 2464 "" 2465 "@ 2466 cmn\\t%<w>0, %<w>1 2467 cmn\\t%<w>0, %1 2468 cmp\\t%<w>0, #%n1" 2469 [(set_attr "type" "alus_sreg,alus_imm,alus_imm")] 2470) 2471 2472(define_insn "aarch64_sub<mode>_compare0" 2473 [(set (reg:CC_NZ CC_REGNUM) 2474 (compare:CC_NZ 2475 (minus:GPI (match_operand:GPI 0 "register_operand" "r") 2476 (match_operand:GPI 1 "aarch64_plus_operand" "r")) 2477 (const_int 0)))] 2478 "" 2479 "cmp\\t%<w>0, %<w>1" 2480 [(set_attr "type" "alus_sreg")] 2481) 2482 2483(define_insn "*compare_neg<mode>" 2484 [(set (reg:CC_Z CC_REGNUM) 2485 (compare:CC_Z 2486 (neg:GPI (match_operand:GPI 0 "register_operand" "r")) 2487 (match_operand:GPI 1 "register_operand" "r")))] 2488 "" 2489 "cmn\\t%<w>1, %<w>0" 2490 [(set_attr "type" "alus_sreg")] 2491) 2492 2493(define_insn "*add_<shift>_<mode>" 2494 [(set (match_operand:GPI 0 "register_operand" "=r") 2495 (plus:GPI (ASHIFT:GPI (match_operand:GPI 1 "register_operand" "r") 2496 (match_operand:QI 2 "aarch64_shift_imm_<mode>" "n")) 2497 (match_operand:GPI 3 "register_operand" "r")))] 2498 "" 2499 "add\\t%<w>0, %<w>3, %<w>1, <shift> %2" 2500 [(set_attr "autodetect_type" "alu_shift_<shift>_op2")] 2501) 2502 2503;; zero_extend version of above 2504(define_insn "*add_<shift>_si_uxtw" 2505 [(set (match_operand:DI 0 "register_operand" "=r") 2506 (zero_extend:DI 2507 (plus:SI (ASHIFT:SI (match_operand:SI 1 "register_operand" "r") 2508 (match_operand:QI 2 "aarch64_shift_imm_si" "n")) 2509 (match_operand:SI 3 "register_operand" "r"))))] 2510 "" 2511 "add\\t%w0, %w3, %w1, <shift> %2" 2512 [(set_attr "autodetect_type" "alu_shift_<shift>_op2")] 2513) 2514 2515(define_insn "*add_<optab><ALLX:mode>_<GPI:mode>" 2516 [(set (match_operand:GPI 0 "register_operand" "=rk") 2517 (plus:GPI (ANY_EXTEND:GPI (match_operand:ALLX 1 "register_operand" "r")) 2518 (match_operand:GPI 2 "register_operand" "r")))] 2519 "" 2520 "add\\t%<GPI:w>0, %<GPI:w>2, %w1, <su>xt<ALLX:size>" 2521 [(set_attr "type" "alu_ext")] 2522) 2523 2524;; zero_extend version of above 2525(define_insn "*add_<optab><SHORT:mode>_si_uxtw" 2526 [(set (match_operand:DI 0 "register_operand" "=rk") 2527 (zero_extend:DI 2528 (plus:SI (ANY_EXTEND:SI (match_operand:SHORT 1 "register_operand" "r")) 2529 (match_operand:GPI 2 "register_operand" "r"))))] 2530 "" 2531 "add\\t%w0, %w2, %w1, <su>xt<SHORT:size>" 2532 [(set_attr "type" "alu_ext")] 2533) 2534 2535(define_insn "*add_<optab><ALLX:mode>_shft_<GPI:mode>" 2536 [(set (match_operand:GPI 0 "register_operand" "=rk") 2537 (plus:GPI (ashift:GPI (ANY_EXTEND:GPI 2538 (match_operand:ALLX 1 "register_operand" "r")) 2539 (match_operand 2 "aarch64_imm3" "Ui3")) 2540 (match_operand:GPI 3 "register_operand" "r")))] 2541 "" 2542 "add\\t%<GPI:w>0, %<GPI:w>3, %w1, <su>xt<ALLX:size> %2" 2543 [(set_attr "type" "alu_ext")] 2544) 2545 2546;; zero_extend version of above 2547(define_insn "*add_<optab><SHORT:mode>_shft_si_uxtw" 2548 [(set (match_operand:DI 0 "register_operand" "=rk") 2549 (zero_extend:DI 2550 (plus:SI (ashift:SI (ANY_EXTEND:SI 2551 (match_operand:SHORT 1 "register_operand" "r")) 2552 (match_operand 2 "aarch64_imm3" "Ui3")) 2553 (match_operand:SI 3 "register_operand" "r"))))] 2554 "" 2555 "add\\t%w0, %w3, %w1, <su>xt<SHORT:size> %2" 2556 [(set_attr "type" "alu_ext")] 2557) 2558 2559(define_expand "add<mode>3_carryin" 2560 [(set (match_operand:GPI 0 "register_operand") 2561 (plus:GPI 2562 (plus:GPI 2563 (ltu:GPI (reg:CC_C CC_REGNUM) (const_int 0)) 2564 (match_operand:GPI 1 "aarch64_reg_or_zero")) 2565 (match_operand:GPI 2 "aarch64_reg_or_zero")))] 2566 "" 2567 "" 2568) 2569 2570;; Note that add with carry with two zero inputs is matched by cset, 2571;; and that add with carry with one zero input is matched by cinc. 2572 2573(define_insn "*add<mode>3_carryin" 2574 [(set (match_operand:GPI 0 "register_operand" "=r") 2575 (plus:GPI 2576 (plus:GPI 2577 (match_operand:GPI 3 "aarch64_carry_operation" "") 2578 (match_operand:GPI 1 "register_operand" "r")) 2579 (match_operand:GPI 2 "register_operand" "r")))] 2580 "" 2581 "adc\\t%<w>0, %<w>1, %<w>2" 2582 [(set_attr "type" "adc_reg")] 2583) 2584 2585;; zero_extend version of above 2586(define_insn "*addsi3_carryin_uxtw" 2587 [(set (match_operand:DI 0 "register_operand" "=r") 2588 (zero_extend:DI 2589 (plus:SI 2590 (plus:SI 2591 (match_operand:SI 3 "aarch64_carry_operation" "") 2592 (match_operand:SI 1 "register_operand" "r")) 2593 (match_operand:SI 2 "register_operand" "r"))))] 2594 "" 2595 "adc\\t%w0, %w1, %w2" 2596 [(set_attr "type" "adc_reg")] 2597) 2598 2599(define_expand "add<mode>3_carryinC" 2600 [(parallel 2601 [(set (match_dup 3) 2602 (compare:CC_ADC 2603 (plus:<DWI> 2604 (plus:<DWI> 2605 (match_dup 4) 2606 (zero_extend:<DWI> 2607 (match_operand:GPI 1 "register_operand"))) 2608 (zero_extend:<DWI> 2609 (match_operand:GPI 2 "register_operand"))) 2610 (match_dup 6))) 2611 (set (match_operand:GPI 0 "register_operand") 2612 (plus:GPI 2613 (plus:GPI (match_dup 5) (match_dup 1)) 2614 (match_dup 2)))])] 2615 "" 2616{ 2617 operands[3] = gen_rtx_REG (CC_ADCmode, CC_REGNUM); 2618 rtx ccin = gen_rtx_REG (CC_Cmode, CC_REGNUM); 2619 operands[4] = gen_rtx_LTU (<DWI>mode, ccin, const0_rtx); 2620 operands[5] = gen_rtx_LTU (<MODE>mode, ccin, const0_rtx); 2621 operands[6] = immed_wide_int_const (wi::shwi (1, <DWI>mode) 2622 << GET_MODE_BITSIZE (<MODE>mode), 2623 TImode); 2624}) 2625 2626(define_insn "*add<mode>3_carryinC_zero" 2627 [(set (reg:CC_ADC CC_REGNUM) 2628 (compare:CC_ADC 2629 (plus:<DWI> 2630 (match_operand:<DWI> 2 "aarch64_carry_operation" "") 2631 (zero_extend:<DWI> (match_operand:GPI 1 "register_operand" "r"))) 2632 (match_operand 4 "const_scalar_int_operand" ""))) 2633 (set (match_operand:GPI 0 "register_operand" "=r") 2634 (plus:GPI (match_operand:GPI 3 "aarch64_carry_operation" "") 2635 (match_dup 1)))] 2636 "rtx_mode_t (operands[4], <DWI>mode) 2637 == (wi::shwi (1, <DWI>mode) << (unsigned) GET_MODE_BITSIZE (<MODE>mode))" 2638 "adcs\\t%<w>0, %<w>1, <w>zr" 2639 [(set_attr "type" "adc_reg")] 2640) 2641 2642(define_insn "*add<mode>3_carryinC" 2643 [(set (reg:CC_ADC CC_REGNUM) 2644 (compare:CC_ADC 2645 (plus:<DWI> 2646 (plus:<DWI> 2647 (match_operand:<DWI> 3 "aarch64_carry_operation" "") 2648 (zero_extend:<DWI> (match_operand:GPI 1 "register_operand" "r"))) 2649 (zero_extend:<DWI> (match_operand:GPI 2 "register_operand" "r"))) 2650 (match_operand 5 "const_scalar_int_operand" ""))) 2651 (set (match_operand:GPI 0 "register_operand" "=r") 2652 (plus:GPI 2653 (plus:GPI (match_operand:GPI 4 "aarch64_carry_operation" "") 2654 (match_dup 1)) 2655 (match_dup 2)))] 2656 "rtx_mode_t (operands[5], <DWI>mode) 2657 == (wi::shwi (1, <DWI>mode) << (unsigned) GET_MODE_BITSIZE (<MODE>mode))" 2658 "adcs\\t%<w>0, %<w>1, %<w>2" 2659 [(set_attr "type" "adc_reg")] 2660) 2661 2662(define_expand "add<mode>3_carryinV" 2663 [(parallel 2664 [(set (reg:CC_V CC_REGNUM) 2665 (compare:CC_V 2666 (plus:<DWI> 2667 (plus:<DWI> 2668 (match_dup 3) 2669 (sign_extend:<DWI> 2670 (match_operand:GPI 1 "register_operand"))) 2671 (sign_extend:<DWI> 2672 (match_operand:GPI 2 "register_operand"))) 2673 (sign_extend:<DWI> 2674 (plus:GPI 2675 (plus:GPI (match_dup 4) (match_dup 1)) 2676 (match_dup 2))))) 2677 (set (match_operand:GPI 0 "register_operand") 2678 (plus:GPI 2679 (plus:GPI (match_dup 4) (match_dup 1)) 2680 (match_dup 2)))])] 2681 "" 2682{ 2683 rtx cc = gen_rtx_REG (CC_Cmode, CC_REGNUM); 2684 operands[3] = gen_rtx_LTU (<DWI>mode, cc, const0_rtx); 2685 operands[4] = gen_rtx_LTU (<MODE>mode, cc, const0_rtx); 2686}) 2687 2688(define_insn "*add<mode>3_carryinV_zero" 2689 [(set (reg:CC_V CC_REGNUM) 2690 (compare:CC_V 2691 (plus:<DWI> 2692 (match_operand:<DWI> 2 "aarch64_carry_operation" "") 2693 (sign_extend:<DWI> (match_operand:GPI 1 "register_operand" "r"))) 2694 (sign_extend:<DWI> 2695 (plus:GPI 2696 (match_operand:GPI 3 "aarch64_carry_operation" "") 2697 (match_dup 1))))) 2698 (set (match_operand:GPI 0 "register_operand" "=r") 2699 (plus:GPI (match_dup 3) (match_dup 1)))] 2700 "" 2701 "adcs\\t%<w>0, %<w>1, <w>zr" 2702 [(set_attr "type" "adc_reg")] 2703) 2704 2705(define_insn "*add<mode>3_carryinV" 2706 [(set (reg:CC_V CC_REGNUM) 2707 (compare:CC_V 2708 (plus:<DWI> 2709 (plus:<DWI> 2710 (match_operand:<DWI> 3 "aarch64_carry_operation" "") 2711 (sign_extend:<DWI> (match_operand:GPI 1 "register_operand" "r"))) 2712 (sign_extend:<DWI> (match_operand:GPI 2 "register_operand" "r"))) 2713 (sign_extend:<DWI> 2714 (plus:GPI 2715 (plus:GPI 2716 (match_operand:GPI 4 "aarch64_carry_operation" "") 2717 (match_dup 1)) 2718 (match_dup 2))))) 2719 (set (match_operand:GPI 0 "register_operand" "=r") 2720 (plus:GPI 2721 (plus:GPI (match_dup 4) (match_dup 1)) 2722 (match_dup 2)))] 2723 "" 2724 "adcs\\t%<w>0, %<w>1, %<w>2" 2725 [(set_attr "type" "adc_reg")] 2726) 2727 2728(define_insn "*add_uxt<mode>_shift2" 2729 [(set (match_operand:GPI 0 "register_operand" "=rk") 2730 (plus:GPI (and:GPI 2731 (ashift:GPI (match_operand:GPI 1 "register_operand" "r") 2732 (match_operand 2 "aarch64_imm3" "Ui3")) 2733 (match_operand 3 "const_int_operand" "n")) 2734 (match_operand:GPI 4 "register_operand" "r")))] 2735 "aarch64_uxt_size (INTVAL (operands[2]), INTVAL (operands[3])) != 0" 2736 "* 2737 operands[3] = GEN_INT (aarch64_uxt_size (INTVAL(operands[2]), 2738 INTVAL (operands[3]))); 2739 return \"add\t%<w>0, %<w>4, %w1, uxt%e3 %2\";" 2740 [(set_attr "type" "alu_ext")] 2741) 2742 2743;; zero_extend version of above 2744(define_insn "*add_uxtsi_shift2_uxtw" 2745 [(set (match_operand:DI 0 "register_operand" "=rk") 2746 (zero_extend:DI 2747 (plus:SI (and:SI 2748 (ashift:SI (match_operand:SI 1 "register_operand" "r") 2749 (match_operand 2 "aarch64_imm3" "Ui3")) 2750 (match_operand 3 "const_int_operand" "n")) 2751 (match_operand:SI 4 "register_operand" "r"))))] 2752 "aarch64_uxt_size (INTVAL (operands[2]), INTVAL (operands[3])) != 0" 2753 "* 2754 operands[3] = GEN_INT (aarch64_uxt_size (INTVAL (operands[2]), 2755 INTVAL (operands[3]))); 2756 return \"add\t%w0, %w4, %w1, uxt%e3 %2\";" 2757 [(set_attr "type" "alu_ext")] 2758) 2759 2760(define_insn "subsi3" 2761 [(set (match_operand:SI 0 "register_operand" "=rk") 2762 (minus:SI (match_operand:SI 1 "register_operand" "rk") 2763 (match_operand:SI 2 "register_operand" "r")))] 2764 "" 2765 "sub\\t%w0, %w1, %w2" 2766 [(set_attr "type" "alu_sreg")] 2767) 2768 2769;; zero_extend version of above 2770(define_insn "*subsi3_uxtw" 2771 [(set (match_operand:DI 0 "register_operand" "=rk") 2772 (zero_extend:DI 2773 (minus:SI (match_operand:SI 1 "register_operand" "rk") 2774 (match_operand:SI 2 "register_operand" "r"))))] 2775 "" 2776 "sub\\t%w0, %w1, %w2" 2777 [(set_attr "type" "alu_sreg")] 2778) 2779 2780(define_insn "subdi3" 2781 [(set (match_operand:DI 0 "register_operand" "=rk,w") 2782 (minus:DI (match_operand:DI 1 "register_operand" "rk,w") 2783 (match_operand:DI 2 "register_operand" "r,w")))] 2784 "" 2785 "@ 2786 sub\\t%x0, %x1, %x2 2787 sub\\t%d0, %d1, %d2" 2788 [(set_attr "type" "alu_sreg, neon_sub") 2789 (set_attr "arch" "*,simd")] 2790) 2791 2792(define_expand "subv<GPI:mode>4" 2793 [(match_operand:GPI 0 "register_operand") 2794 (match_operand:GPI 1 "register_operand") 2795 (match_operand:GPI 2 "aarch64_plus_operand") 2796 (label_ref (match_operand 3 "" ""))] 2797 "" 2798{ 2799 if (CONST_INT_P (operands[2])) 2800 emit_insn (gen_subv<mode>_imm (operands[0], operands[1], operands[2])); 2801 else 2802 emit_insn (gen_subv<mode>_insn (operands[0], operands[1], operands[2])); 2803 aarch64_gen_unlikely_cbranch (NE, CC_Vmode, operands[3]); 2804 2805 DONE; 2806}) 2807 2808(define_insn "subv<GPI:mode>_insn" 2809 [(set (reg:CC_V CC_REGNUM) 2810 (compare:CC_V 2811 (sign_extend:<DWI> 2812 (minus:GPI 2813 (match_operand:GPI 1 "register_operand" "rk") 2814 (match_operand:GPI 2 "register_operand" "r"))) 2815 (minus:<DWI> (sign_extend:<DWI> (match_dup 1)) 2816 (sign_extend:<DWI> (match_dup 2))))) 2817 (set (match_operand:GPI 0 "register_operand" "=r") 2818 (minus:GPI (match_dup 1) (match_dup 2)))] 2819 "" 2820 "subs\\t%<w>0, %<w>1, %<w>2" 2821 [(set_attr "type" "alus_sreg")] 2822) 2823 2824(define_insn "subv<GPI:mode>_imm" 2825 [(set (reg:CC_V CC_REGNUM) 2826 (compare:CC_V 2827 (sign_extend:<DWI> 2828 (minus:GPI 2829 (match_operand:GPI 1 "register_operand" "rk,rk") 2830 (match_operand:GPI 2 "aarch64_plus_immediate" "I,J"))) 2831 (minus:<DWI> (sign_extend:<DWI> (match_dup 1)) 2832 (match_dup 2)))) 2833 (set (match_operand:GPI 0 "register_operand" "=r,r") 2834 (minus:GPI (match_dup 1) (match_dup 2)))] 2835 "" 2836 "@ 2837 subs\\t%<w>0, %<w>1, %2 2838 adds\\t%<w>0, %<w>1, #%n2" 2839 [(set_attr "type" "alus_sreg")] 2840) 2841 2842(define_expand "negv<GPI:mode>3" 2843 [(match_operand:GPI 0 "register_operand") 2844 (match_operand:GPI 1 "register_operand") 2845 (label_ref (match_operand 2 "" ""))] 2846 "" 2847 { 2848 emit_insn (gen_negv<mode>_insn (operands[0], operands[1])); 2849 aarch64_gen_unlikely_cbranch (NE, CC_Vmode, operands[2]); 2850 2851 DONE; 2852 } 2853) 2854 2855(define_insn "negv<GPI:mode>_insn" 2856 [(set (reg:CC_V CC_REGNUM) 2857 (compare:CC_V 2858 (sign_extend:<DWI> 2859 (neg:GPI (match_operand:GPI 1 "register_operand" "r"))) 2860 (neg:<DWI> (sign_extend:<DWI> (match_dup 1))))) 2861 (set (match_operand:GPI 0 "register_operand" "=r") 2862 (neg:GPI (match_dup 1)))] 2863 "" 2864 "negs\\t%<w>0, %<w>1" 2865 [(set_attr "type" "alus_sreg")] 2866) 2867 2868(define_insn "negv<GPI:mode>_cmp_only" 2869 [(set (reg:CC_V CC_REGNUM) 2870 (compare:CC_V 2871 (sign_extend:<DWI> 2872 (neg:GPI (match_operand:GPI 0 "register_operand" "r"))) 2873 (neg:<DWI> (sign_extend:<DWI> (match_dup 0)))))] 2874 "" 2875 "negs\\t%<w>zr, %<w>0" 2876 [(set_attr "type" "alus_sreg")] 2877) 2878 2879(define_insn "*cmpv<GPI:mode>_insn" 2880 [(set (reg:CC_V CC_REGNUM) 2881 (compare:CC_V 2882 (sign_extend:<DWI> 2883 (minus:GPI (match_operand:GPI 0 "register_operand" "r,r,r") 2884 (match_operand:GPI 1 "aarch64_plus_operand" "r,I,J"))) 2885 (minus:<DWI> (sign_extend:<DWI> (match_dup 0)) 2886 (sign_extend:<DWI> (match_dup 1)))))] 2887 "" 2888 "@ 2889 cmp\\t%<w>0, %<w>1 2890 cmp\\t%<w>0, %1 2891 cmp\\t%<w>0, #%n1" 2892 [(set_attr "type" "alus_sreg")] 2893) 2894 2895(define_expand "usubv<mode>4" 2896 [(match_operand:GPI 0 "register_operand") 2897 (match_operand:GPI 1 "aarch64_reg_or_zero") 2898 (match_operand:GPI 2 "aarch64_reg_or_zero") 2899 (label_ref (match_operand 3 "" ""))] 2900 "" 2901{ 2902 emit_insn (gen_sub<mode>3_compare1 (operands[0], operands[1], operands[2])); 2903 aarch64_gen_unlikely_cbranch (LTU, CCmode, operands[3]); 2904 2905 DONE; 2906}) 2907 2908(define_expand "subti3" 2909 [(set (match_operand:TI 0 "register_operand") 2910 (minus:TI (match_operand:TI 1 "aarch64_reg_or_zero") 2911 (match_operand:TI 2 "register_operand")))] 2912 "" 2913{ 2914 rtx low_dest, op1_low, op2_low, high_dest, op1_high, op2_high; 2915 2916 aarch64_subvti_scratch_regs (operands[1], operands[2], 2917 &low_dest, &op1_low, &op2_low, 2918 &high_dest, &op1_high, &op2_high); 2919 2920 emit_insn (gen_subdi3_compare1 (low_dest, op1_low, op2_low)); 2921 emit_insn (gen_subdi3_carryin (high_dest, op1_high, op2_high)); 2922 2923 emit_move_insn (gen_lowpart (DImode, operands[0]), low_dest); 2924 emit_move_insn (gen_highpart (DImode, operands[0]), high_dest); 2925 DONE; 2926}) 2927 2928(define_expand "subvti4" 2929 [(match_operand:TI 0 "register_operand") 2930 (match_operand:TI 1 "register_operand") 2931 (match_operand:TI 2 "aarch64_reg_or_imm") 2932 (label_ref (match_operand 3 "" ""))] 2933 "" 2934{ 2935 rtx low_dest, op1_low, op2_low, high_dest, op1_high, op2_high; 2936 2937 aarch64_subvti_scratch_regs (operands[1], operands[2], 2938 &low_dest, &op1_low, &op2_low, 2939 &high_dest, &op1_high, &op2_high); 2940 aarch64_expand_subvti (operands[0], low_dest, op1_low, op2_low, 2941 high_dest, op1_high, op2_high, false); 2942 2943 aarch64_gen_unlikely_cbranch (NE, CC_Vmode, operands[3]); 2944 DONE; 2945}) 2946 2947(define_expand "usubvti4" 2948 [(match_operand:TI 0 "register_operand") 2949 (match_operand:TI 1 "register_operand") 2950 (match_operand:TI 2 "aarch64_reg_or_imm") 2951 (label_ref (match_operand 3 "" ""))] 2952 "" 2953{ 2954 rtx low_dest, op1_low, op2_low, high_dest, op1_high, op2_high; 2955 2956 aarch64_subvti_scratch_regs (operands[1], operands[2], 2957 &low_dest, &op1_low, &op2_low, 2958 &high_dest, &op1_high, &op2_high); 2959 aarch64_expand_subvti (operands[0], low_dest, op1_low, op2_low, 2960 high_dest, op1_high, op2_high, true); 2961 2962 aarch64_gen_unlikely_cbranch (LTU, CCmode, operands[3]); 2963 DONE; 2964}) 2965 2966(define_expand "negvti3" 2967 [(match_operand:TI 0 "register_operand") 2968 (match_operand:TI 1 "register_operand") 2969 (label_ref (match_operand 2 "" ""))] 2970 "" 2971 { 2972 emit_insn (gen_negdi_carryout (gen_lowpart (DImode, operands[0]), 2973 gen_lowpart (DImode, operands[1]))); 2974 emit_insn (gen_negvdi_carryinV (gen_highpart (DImode, operands[0]), 2975 gen_highpart (DImode, operands[1]))); 2976 aarch64_gen_unlikely_cbranch (NE, CC_Vmode, operands[2]); 2977 2978 DONE; 2979 } 2980) 2981 2982(define_insn "negdi_carryout" 2983 [(set (reg:CC CC_REGNUM) 2984 (compare:CC 2985 (const_int 0) (match_operand:DI 1 "register_operand" "r"))) 2986 (set (match_operand:DI 0 "register_operand" "=r") 2987 (neg:DI (match_dup 1)))] 2988 "" 2989 "negs\\t%0, %1" 2990 [(set_attr "type" "alus_sreg")] 2991) 2992 2993(define_insn "negvdi_carryinV" 2994 [(set (reg:CC_V CC_REGNUM) 2995 (compare:CC_V 2996 (neg:TI (plus:TI 2997 (ltu:TI (reg:CC CC_REGNUM) (const_int 0)) 2998 (sign_extend:TI (match_operand:DI 1 "register_operand" "r")))) 2999 (sign_extend:TI 3000 (neg:DI (plus:DI (ltu:DI (reg:CC CC_REGNUM) (const_int 0)) 3001 (match_dup 1)))))) 3002 (set (match_operand:DI 0 "register_operand" "=r") 3003 (neg:DI (plus:DI (ltu:DI (reg:CC CC_REGNUM) (const_int 0)) 3004 (match_dup 1))))] 3005 "" 3006 "ngcs\\t%0, %1" 3007 [(set_attr "type" "alus_sreg")] 3008) 3009 3010(define_insn "*sub<mode>3_compare0" 3011 [(set (reg:CC_NZ CC_REGNUM) 3012 (compare:CC_NZ (minus:GPI (match_operand:GPI 1 "register_operand" "rk") 3013 (match_operand:GPI 2 "register_operand" "r")) 3014 (const_int 0))) 3015 (set (match_operand:GPI 0 "register_operand" "=r") 3016 (minus:GPI (match_dup 1) (match_dup 2)))] 3017 "" 3018 "subs\\t%<w>0, %<w>1, %<w>2" 3019 [(set_attr "type" "alus_sreg")] 3020) 3021 3022;; zero_extend version of above 3023(define_insn "*subsi3_compare0_uxtw" 3024 [(set (reg:CC_NZ CC_REGNUM) 3025 (compare:CC_NZ (minus:SI (match_operand:SI 1 "register_operand" "rk") 3026 (match_operand:SI 2 "register_operand" "r")) 3027 (const_int 0))) 3028 (set (match_operand:DI 0 "register_operand" "=r") 3029 (zero_extend:DI (minus:SI (match_dup 1) (match_dup 2))))] 3030 "" 3031 "subs\\t%w0, %w1, %w2" 3032 [(set_attr "type" "alus_sreg")] 3033) 3034 3035(define_insn "sub<mode>3_compare1_imm" 3036 [(set (reg:CC CC_REGNUM) 3037 (compare:CC 3038 (match_operand:GPI 1 "register_operand" "rk,rk") 3039 (match_operand:GPI 2 "aarch64_plus_immediate" "I,J"))) 3040 (set (match_operand:GPI 0 "register_operand" "=r,r") 3041 (plus:GPI 3042 (match_dup 1) 3043 (match_operand:GPI 3 "aarch64_plus_immediate" "J,I")))] 3044 "UINTVAL (operands[2]) == -UINTVAL (operands[3])" 3045 "@ 3046 subs\\t%<w>0, %<w>1, %2 3047 adds\\t%<w>0, %<w>1, #%n2" 3048 [(set_attr "type" "alus_imm")] 3049) 3050 3051(define_insn "sub<mode>3_compare1" 3052 [(set (reg:CC CC_REGNUM) 3053 (compare:CC 3054 (match_operand:GPI 1 "aarch64_reg_or_zero" "rkZ") 3055 (match_operand:GPI 2 "aarch64_reg_or_zero" "rZ"))) 3056 (set (match_operand:GPI 0 "register_operand" "=r") 3057 (minus:GPI (match_dup 1) (match_dup 2)))] 3058 "" 3059 "subs\\t%<w>0, %<w>1, %<w>2" 3060 [(set_attr "type" "alus_sreg")] 3061) 3062 3063(define_peephole2 3064 [(set (match_operand:GPI 0 "aarch64_general_reg") 3065 (minus:GPI (match_operand:GPI 1 "aarch64_reg_or_zero") 3066 (match_operand:GPI 2 "aarch64_reg_or_zero"))) 3067 (set (reg:CC CC_REGNUM) 3068 (compare:CC 3069 (match_dup 1) 3070 (match_dup 2)))] 3071 "!reg_overlap_mentioned_p (operands[0], operands[1]) 3072 && !reg_overlap_mentioned_p (operands[0], operands[2])" 3073 [(const_int 0)] 3074 { 3075 emit_insn (gen_sub<mode>3_compare1 (operands[0], operands[1], 3076 operands[2])); 3077 DONE; 3078 } 3079) 3080 3081;; Same as the above peephole but with the compare and minus in 3082;; swapped order. The restriction on overlap between operand 0 3083;; and operands 1 and 2 doesn't apply here. 3084(define_peephole2 3085 [(set (reg:CC CC_REGNUM) 3086 (compare:CC 3087 (match_operand:GPI 1 "aarch64_reg_or_zero") 3088 (match_operand:GPI 2 "aarch64_reg_or_zero"))) 3089 (set (match_operand:GPI 0 "aarch64_general_reg") 3090 (minus:GPI (match_dup 1) 3091 (match_dup 2)))] 3092 "" 3093 [(const_int 0)] 3094 { 3095 emit_insn (gen_sub<mode>3_compare1 (operands[0], operands[1], 3096 operands[2])); 3097 DONE; 3098 } 3099) 3100 3101(define_peephole2 3102 [(set (match_operand:GPI 0 "aarch64_general_reg") 3103 (plus:GPI (match_operand:GPI 1 "register_operand") 3104 (match_operand:GPI 2 "aarch64_plus_immediate"))) 3105 (set (reg:CC CC_REGNUM) 3106 (compare:CC 3107 (match_dup 1) 3108 (match_operand:GPI 3 "const_int_operand")))] 3109 "!reg_overlap_mentioned_p (operands[0], operands[1]) 3110 && INTVAL (operands[3]) == -INTVAL (operands[2])" 3111 [(const_int 0)] 3112 { 3113 emit_insn (gen_sub<mode>3_compare1_imm (operands[0], operands[1], 3114 operands[3], operands[2])); 3115 DONE; 3116 } 3117) 3118 3119;; Same as the above peephole but with the compare and minus in 3120;; swapped order. The restriction on overlap between operand 0 3121;; and operands 1 doesn't apply here. 3122(define_peephole2 3123 [(set (reg:CC CC_REGNUM) 3124 (compare:CC 3125 (match_operand:GPI 1 "register_operand") 3126 (match_operand:GPI 3 "const_int_operand"))) 3127 (set (match_operand:GPI 0 "aarch64_general_reg") 3128 (plus:GPI (match_dup 1) 3129 (match_operand:GPI 2 "aarch64_plus_immediate")))] 3130 "INTVAL (operands[3]) == -INTVAL (operands[2])" 3131 [(const_int 0)] 3132 { 3133 emit_insn (gen_sub<mode>3_compare1_imm (operands[0], operands[1], 3134 operands[3], operands[2])); 3135 DONE; 3136 } 3137) 3138 3139(define_insn "*sub_<shift>_<mode>" 3140 [(set (match_operand:GPI 0 "register_operand" "=r") 3141 (minus:GPI (match_operand:GPI 3 "register_operand" "r") 3142 (ASHIFT:GPI 3143 (match_operand:GPI 1 "register_operand" "r") 3144 (match_operand:QI 2 "aarch64_shift_imm_<mode>" "n"))))] 3145 "" 3146 "sub\\t%<w>0, %<w>3, %<w>1, <shift> %2" 3147 [(set_attr "autodetect_type" "alu_shift_<shift>_op2")] 3148) 3149 3150;; zero_extend version of above 3151(define_insn "*sub_<shift>_si_uxtw" 3152 [(set (match_operand:DI 0 "register_operand" "=r") 3153 (zero_extend:DI 3154 (minus:SI (match_operand:SI 3 "register_operand" "r") 3155 (ASHIFT:SI 3156 (match_operand:SI 1 "register_operand" "r") 3157 (match_operand:QI 2 "aarch64_shift_imm_si" "n")))))] 3158 "" 3159 "sub\\t%w0, %w3, %w1, <shift> %2" 3160 [(set_attr "autodetect_type" "alu_shift_<shift>_op2")] 3161) 3162 3163(define_insn "*sub_<optab><ALLX:mode>_<GPI:mode>" 3164 [(set (match_operand:GPI 0 "register_operand" "=rk") 3165 (minus:GPI (match_operand:GPI 1 "register_operand" "rk") 3166 (ANY_EXTEND:GPI 3167 (match_operand:ALLX 2 "register_operand" "r"))))] 3168 "" 3169 "sub\\t%<GPI:w>0, %<GPI:w>1, %w2, <su>xt<ALLX:size>" 3170 [(set_attr "type" "alu_ext")] 3171) 3172 3173;; zero_extend version of above 3174(define_insn "*sub_<optab><SHORT:mode>_si_uxtw" 3175 [(set (match_operand:DI 0 "register_operand" "=rk") 3176 (zero_extend:DI 3177 (minus:SI (match_operand:SI 1 "register_operand" "rk") 3178 (ANY_EXTEND:SI 3179 (match_operand:SHORT 2 "register_operand" "r")))))] 3180 "" 3181 "sub\\t%w0, %w1, %w2, <su>xt<SHORT:size>" 3182 [(set_attr "type" "alu_ext")] 3183) 3184 3185(define_insn "*sub_<optab><ALLX:mode>_shft_<GPI:mode>" 3186 [(set (match_operand:GPI 0 "register_operand" "=rk") 3187 (minus:GPI (match_operand:GPI 1 "register_operand" "rk") 3188 (ashift:GPI (ANY_EXTEND:GPI 3189 (match_operand:ALLX 2 "register_operand" "r")) 3190 (match_operand 3 "aarch64_imm3" "Ui3"))))] 3191 "" 3192 "sub\\t%<GPI:w>0, %<GPI:w>1, %w2, <su>xt<ALLX:size> %3" 3193 [(set_attr "type" "alu_ext")] 3194) 3195 3196;; zero_extend version of above 3197(define_insn "*sub_<optab><SHORT:mode>_shft_si_uxtw" 3198 [(set (match_operand:DI 0 "register_operand" "=rk") 3199 (zero_extend:DI 3200 (minus:SI (match_operand:SI 1 "register_operand" "rk") 3201 (ashift:SI (ANY_EXTEND:SI 3202 (match_operand:SHORT 2 "register_operand" "r")) 3203 (match_operand 3 "aarch64_imm3" "Ui3")))))] 3204 "" 3205 "sub\\t%w0, %w1, %w2, <su>xt<SHORT:size> %3" 3206 [(set_attr "type" "alu_ext")] 3207) 3208 3209;; The hardware description is op1 + ~op2 + C. 3210;; = op1 + (-op2 + 1) + (1 - !C) 3211;; = op1 - op2 - 1 + 1 - !C 3212;; = op1 - op2 - !C. 3213;; We describe the latter. 3214 3215(define_insn "*sub<mode>3_carryin0" 3216 [(set (match_operand:GPI 0 "register_operand" "=r") 3217 (minus:GPI 3218 (match_operand:GPI 1 "aarch64_reg_or_zero" "rZ") 3219 (match_operand:GPI 2 "aarch64_borrow_operation" "")))] 3220 "" 3221 "sbc\\t%<w>0, %<w>1, <w>zr" 3222 [(set_attr "type" "adc_reg")] 3223) 3224 3225;; zero_extend version of the above 3226(define_insn "*subsi3_carryin_uxtw" 3227 [(set (match_operand:DI 0 "register_operand" "=r") 3228 (zero_extend:DI 3229 (minus:SI 3230 (match_operand:SI 1 "aarch64_reg_or_zero" "rZ") 3231 (match_operand:SI 2 "aarch64_borrow_operation" ""))))] 3232 "" 3233 "sbc\\t%w0, %w1, wzr" 3234 [(set_attr "type" "adc_reg")] 3235) 3236 3237(define_expand "sub<mode>3_carryin" 3238 [(set (match_operand:GPI 0 "register_operand") 3239 (minus:GPI 3240 (minus:GPI 3241 (match_operand:GPI 1 "aarch64_reg_or_zero") 3242 (match_operand:GPI 2 "register_operand")) 3243 (ltu:GPI (reg:CC CC_REGNUM) (const_int 0))))] 3244 "" 3245 "" 3246) 3247 3248(define_insn "*sub<mode>3_carryin" 3249 [(set (match_operand:GPI 0 "register_operand" "=r") 3250 (minus:GPI 3251 (minus:GPI 3252 (match_operand:GPI 1 "aarch64_reg_or_zero" "rZ") 3253 (match_operand:GPI 2 "register_operand" "r")) 3254 (match_operand:GPI 3 "aarch64_borrow_operation" "")))] 3255 3256 "" 3257 "sbc\\t%<w>0, %<w>1, %<w>2" 3258 [(set_attr "type" "adc_reg")] 3259) 3260 3261;; zero_extend version of the above 3262(define_insn "*subsi3_carryin_uxtw" 3263 [(set (match_operand:DI 0 "register_operand" "=r") 3264 (zero_extend:DI 3265 (minus:SI 3266 (minus:SI 3267 (match_operand:SI 1 "aarch64_reg_or_zero" "rZ") 3268 (match_operand:SI 2 "register_operand" "r")) 3269 (match_operand:SI 3 "aarch64_borrow_operation" ""))))] 3270 3271 "" 3272 "sbc\\t%w0, %w1, %w2" 3273 [(set_attr "type" "adc_reg")] 3274) 3275 3276(define_insn "*sub<mode>3_carryin_alt" 3277 [(set (match_operand:GPI 0 "register_operand" "=r") 3278 (minus:GPI 3279 (minus:GPI 3280 (match_operand:GPI 1 "aarch64_reg_or_zero" "rZ") 3281 (match_operand:GPI 3 "aarch64_borrow_operation" "")) 3282 (match_operand:GPI 2 "register_operand" "r")))] 3283 "" 3284 "sbc\\t%<w>0, %<w>1, %<w>2" 3285 [(set_attr "type" "adc_reg")] 3286) 3287 3288;; zero_extend version of the above 3289(define_insn "*subsi3_carryin_alt_uxtw" 3290 [(set (match_operand:DI 0 "register_operand" "=r") 3291 (zero_extend:DI 3292 (minus:SI 3293 (minus:SI 3294 (match_operand:SI 1 "aarch64_reg_or_zero" "rZ") 3295 (match_operand:SI 3 "aarch64_borrow_operation" "")) 3296 (match_operand:SI 2 "register_operand" "r"))))] 3297 "" 3298 "sbc\\t%w0, %w1, %w2" 3299 [(set_attr "type" "adc_reg")] 3300) 3301 3302(define_expand "usub<GPI:mode>3_carryinC" 3303 [(parallel 3304 [(set (reg:CC CC_REGNUM) 3305 (compare:CC 3306 (zero_extend:<DWI> 3307 (match_operand:GPI 1 "aarch64_reg_or_zero")) 3308 (plus:<DWI> 3309 (zero_extend:<DWI> 3310 (match_operand:GPI 2 "register_operand")) 3311 (ltu:<DWI> (reg:CC CC_REGNUM) (const_int 0))))) 3312 (set (match_operand:GPI 0 "register_operand") 3313 (minus:GPI 3314 (minus:GPI (match_dup 1) (match_dup 2)) 3315 (ltu:GPI (reg:CC CC_REGNUM) (const_int 0))))])] 3316 "" 3317) 3318 3319(define_insn "*usub<GPI:mode>3_carryinC_z1" 3320 [(set (reg:CC CC_REGNUM) 3321 (compare:CC 3322 (const_int 0) 3323 (plus:<DWI> 3324 (zero_extend:<DWI> 3325 (match_operand:GPI 1 "register_operand" "r")) 3326 (match_operand:<DWI> 2 "aarch64_borrow_operation" "")))) 3327 (set (match_operand:GPI 0 "register_operand" "=r") 3328 (minus:GPI 3329 (neg:GPI (match_dup 1)) 3330 (match_operand:GPI 3 "aarch64_borrow_operation" "")))] 3331 "" 3332 "sbcs\\t%<w>0, <w>zr, %<w>1" 3333 [(set_attr "type" "adc_reg")] 3334) 3335 3336(define_insn "*usub<GPI:mode>3_carryinC_z2" 3337 [(set (reg:CC CC_REGNUM) 3338 (compare:CC 3339 (zero_extend:<DWI> 3340 (match_operand:GPI 1 "register_operand" "r")) 3341 (match_operand:<DWI> 2 "aarch64_borrow_operation" ""))) 3342 (set (match_operand:GPI 0 "register_operand" "=r") 3343 (minus:GPI 3344 (match_dup 1) 3345 (match_operand:GPI 3 "aarch64_borrow_operation" "")))] 3346 "" 3347 "sbcs\\t%<w>0, %<w>1, <w>zr" 3348 [(set_attr "type" "adc_reg")] 3349) 3350 3351(define_insn "*usub<GPI:mode>3_carryinC" 3352 [(set (reg:CC CC_REGNUM) 3353 (compare:CC 3354 (zero_extend:<DWI> 3355 (match_operand:GPI 1 "register_operand" "r")) 3356 (plus:<DWI> 3357 (zero_extend:<DWI> 3358 (match_operand:GPI 2 "register_operand" "r")) 3359 (match_operand:<DWI> 3 "aarch64_borrow_operation" "")))) 3360 (set (match_operand:GPI 0 "register_operand" "=r") 3361 (minus:GPI 3362 (minus:GPI (match_dup 1) (match_dup 2)) 3363 (match_operand:GPI 4 "aarch64_borrow_operation" "")))] 3364 "" 3365 "sbcs\\t%<w>0, %<w>1, %<w>2" 3366 [(set_attr "type" "adc_reg")] 3367) 3368 3369(define_expand "sub<GPI:mode>3_carryinV" 3370 [(parallel 3371 [(set (reg:CC_V CC_REGNUM) 3372 (compare:CC_V 3373 (minus:<DWI> 3374 (sign_extend:<DWI> 3375 (match_operand:GPI 1 "aarch64_reg_or_zero")) 3376 (plus:<DWI> 3377 (sign_extend:<DWI> 3378 (match_operand:GPI 2 "register_operand")) 3379 (ltu:<DWI> (reg:CC CC_REGNUM) (const_int 0)))) 3380 (sign_extend:<DWI> 3381 (minus:GPI (match_dup 1) 3382 (plus:GPI (ltu:GPI (reg:CC CC_REGNUM) (const_int 0)) 3383 (match_dup 2)))))) 3384 (set (match_operand:GPI 0 "register_operand") 3385 (minus:GPI 3386 (minus:GPI (match_dup 1) (match_dup 2)) 3387 (ltu:GPI (reg:CC CC_REGNUM) (const_int 0))))])] 3388 "" 3389) 3390 3391(define_insn "*sub<mode>3_carryinV_z2" 3392 [(set (reg:CC_V CC_REGNUM) 3393 (compare:CC_V 3394 (minus:<DWI> 3395 (sign_extend:<DWI> (match_operand:GPI 1 "register_operand" "r")) 3396 (match_operand:<DWI> 2 "aarch64_borrow_operation" "")) 3397 (sign_extend:<DWI> 3398 (minus:GPI (match_dup 1) 3399 (match_operand:GPI 3 "aarch64_borrow_operation" ""))))) 3400 (set (match_operand:GPI 0 "register_operand" "=r") 3401 (minus:GPI 3402 (match_dup 1) (match_dup 3)))] 3403 "" 3404 "sbcs\\t%<w>0, %<w>1, <w>zr" 3405 [(set_attr "type" "adc_reg")] 3406) 3407 3408(define_insn "*sub<mode>3_carryinV" 3409 [(set (reg:CC_V CC_REGNUM) 3410 (compare:CC_V 3411 (minus:<DWI> 3412 (sign_extend:<DWI> 3413 (match_operand:GPI 1 "register_operand" "r")) 3414 (plus:<DWI> 3415 (sign_extend:<DWI> 3416 (match_operand:GPI 2 "register_operand" "r")) 3417 (match_operand:<DWI> 3 "aarch64_borrow_operation" ""))) 3418 (sign_extend:<DWI> 3419 (minus:GPI 3420 (match_dup 1) 3421 (plus:GPI (match_operand:GPI 4 "aarch64_borrow_operation" "") 3422 (match_dup 2)))))) 3423 (set (match_operand:GPI 0 "register_operand" "=r") 3424 (minus:GPI 3425 (minus:GPI (match_dup 1) (match_dup 2)) 3426 (match_dup 4)))] 3427 "" 3428 "sbcs\\t%<w>0, %<w>1, %<w>2" 3429 [(set_attr "type" "adc_reg")] 3430) 3431 3432(define_insn "*sub_uxt<mode>_shift2" 3433 [(set (match_operand:GPI 0 "register_operand" "=rk") 3434 (minus:GPI (match_operand:GPI 4 "register_operand" "rk") 3435 (and:GPI 3436 (ashift:GPI (match_operand:GPI 1 "register_operand" "r") 3437 (match_operand 2 "aarch64_imm3" "Ui3")) 3438 (match_operand 3 "const_int_operand" "n"))))] 3439 "aarch64_uxt_size (INTVAL (operands[2]),INTVAL (operands[3])) != 0" 3440 "* 3441 operands[3] = GEN_INT (aarch64_uxt_size (INTVAL (operands[2]), 3442 INTVAL (operands[3]))); 3443 return \"sub\t%<w>0, %<w>4, %w1, uxt%e3 %2\";" 3444 [(set_attr "type" "alu_ext")] 3445) 3446 3447;; zero_extend version of above 3448(define_insn "*sub_uxtsi_shift2_uxtw" 3449 [(set (match_operand:DI 0 "register_operand" "=rk") 3450 (zero_extend:DI 3451 (minus:SI (match_operand:SI 4 "register_operand" "rk") 3452 (and:SI 3453 (ashift:SI (match_operand:SI 1 "register_operand" "r") 3454 (match_operand 2 "aarch64_imm3" "Ui3")) 3455 (match_operand 3 "const_int_operand" "n")))))] 3456 "aarch64_uxt_size (INTVAL (operands[2]),INTVAL (operands[3])) != 0" 3457 "* 3458 operands[3] = GEN_INT (aarch64_uxt_size (INTVAL (operands[2]), 3459 INTVAL (operands[3]))); 3460 return \"sub\t%w0, %w4, %w1, uxt%e3 %2\";" 3461 [(set_attr "type" "alu_ext")] 3462) 3463 3464(define_expand "abs<mode>2" 3465 [(match_operand:GPI 0 "register_operand") 3466 (match_operand:GPI 1 "register_operand")] 3467 "" 3468 { 3469 rtx ccreg = aarch64_gen_compare_reg (LT, operands[1], const0_rtx); 3470 rtx x = gen_rtx_LT (VOIDmode, ccreg, const0_rtx); 3471 emit_insn (gen_csneg3<mode>_insn (operands[0], x, operands[1], operands[1])); 3472 DONE; 3473 } 3474) 3475 3476(define_insn "neg<mode>2" 3477 [(set (match_operand:GPI 0 "register_operand" "=r,w") 3478 (neg:GPI (match_operand:GPI 1 "register_operand" "r,w")))] 3479 "" 3480 "@ 3481 neg\\t%<w>0, %<w>1 3482 neg\\t%<rtn>0<vas>, %<rtn>1<vas>" 3483 [(set_attr "type" "alu_sreg, neon_neg<q>") 3484 (set_attr "arch" "*,simd")] 3485) 3486 3487;; zero_extend version of above 3488(define_insn "*negsi2_uxtw" 3489 [(set (match_operand:DI 0 "register_operand" "=r") 3490 (zero_extend:DI (neg:SI (match_operand:SI 1 "register_operand" "r"))))] 3491 "" 3492 "neg\\t%w0, %w1" 3493 [(set_attr "type" "alu_sreg")] 3494) 3495 3496(define_insn "*ngc<mode>" 3497 [(set (match_operand:GPI 0 "register_operand" "=r") 3498 (minus:GPI 3499 (neg:GPI (match_operand:GPI 2 "aarch64_borrow_operation" "")) 3500 (match_operand:GPI 1 "register_operand" "r")))] 3501 "" 3502 "ngc\\t%<w>0, %<w>1" 3503 [(set_attr "type" "adc_reg")] 3504) 3505 3506(define_insn "*ngcsi_uxtw" 3507 [(set (match_operand:DI 0 "register_operand" "=r") 3508 (zero_extend:DI 3509 (minus:SI 3510 (neg:SI (match_operand:SI 2 "aarch64_borrow_operation" "")) 3511 (match_operand:SI 1 "register_operand" "r"))))] 3512 "" 3513 "ngc\\t%w0, %w1" 3514 [(set_attr "type" "adc_reg")] 3515) 3516 3517(define_insn "neg<mode>2_compare0" 3518 [(set (reg:CC_NZ CC_REGNUM) 3519 (compare:CC_NZ (neg:GPI (match_operand:GPI 1 "register_operand" "r")) 3520 (const_int 0))) 3521 (set (match_operand:GPI 0 "register_operand" "=r") 3522 (neg:GPI (match_dup 1)))] 3523 "" 3524 "negs\\t%<w>0, %<w>1" 3525 [(set_attr "type" "alus_sreg")] 3526) 3527 3528;; zero_extend version of above 3529(define_insn "*negsi2_compare0_uxtw" 3530 [(set (reg:CC_NZ CC_REGNUM) 3531 (compare:CC_NZ (neg:SI (match_operand:SI 1 "register_operand" "r")) 3532 (const_int 0))) 3533 (set (match_operand:DI 0 "register_operand" "=r") 3534 (zero_extend:DI (neg:SI (match_dup 1))))] 3535 "" 3536 "negs\\t%w0, %w1" 3537 [(set_attr "type" "alus_sreg")] 3538) 3539 3540(define_insn "*neg_<shift><mode>3_compare0" 3541 [(set (reg:CC_NZ CC_REGNUM) 3542 (compare:CC_NZ 3543 (neg:GPI (ASHIFT:GPI 3544 (match_operand:GPI 1 "register_operand" "r") 3545 (match_operand:QI 2 "aarch64_shift_imm_<mode>" "n"))) 3546 (const_int 0))) 3547 (set (match_operand:GPI 0 "register_operand" "=r") 3548 (neg:GPI (ASHIFT:GPI (match_dup 1) (match_dup 2))))] 3549 "" 3550 "negs\\t%<w>0, %<w>1, <shift> %2" 3551 [(set_attr "type" "alus_shift_imm")] 3552) 3553 3554(define_insn "*neg_<shift>_<mode>2" 3555 [(set (match_operand:GPI 0 "register_operand" "=r") 3556 (neg:GPI (ASHIFT:GPI 3557 (match_operand:GPI 1 "register_operand" "r") 3558 (match_operand:QI 2 "aarch64_shift_imm_<mode>" "n"))))] 3559 "" 3560 "neg\\t%<w>0, %<w>1, <shift> %2" 3561 [(set_attr "autodetect_type" "alu_shift_<shift>_op2")] 3562) 3563 3564;; zero_extend version of above 3565(define_insn "*neg_<shift>_si2_uxtw" 3566 [(set (match_operand:DI 0 "register_operand" "=r") 3567 (zero_extend:DI 3568 (neg:SI (ASHIFT:SI 3569 (match_operand:SI 1 "register_operand" "r") 3570 (match_operand:QI 2 "aarch64_shift_imm_si" "n")))))] 3571 "" 3572 "neg\\t%w0, %w1, <shift> %2" 3573 [(set_attr "autodetect_type" "alu_shift_<shift>_op2")] 3574) 3575 3576(define_insn "*neg_asr_si2_extr" 3577 [(set (match_operand:SI 0 "register_operand" "=r") 3578 (neg:SI (match_operator:SI 4 "subreg_lowpart_operator" 3579 [(sign_extract:DI 3580 (match_operand:DI 1 "register_operand" "r") 3581 (match_operand 3 "aarch64_simd_shift_imm_offset_si" "n") 3582 (match_operand 2 "aarch64_simd_shift_imm_offset_si" "n"))])))] 3583 "INTVAL (operands[2]) + INTVAL (operands[3]) == 32" 3584 "neg\\t%w0, %w1, asr %2" 3585 [(set_attr "autodetect_type" "alu_shift_asr_op2")] 3586) 3587 3588(define_insn "mul<mode>3" 3589 [(set (match_operand:GPI 0 "register_operand" "=r") 3590 (mult:GPI (match_operand:GPI 1 "register_operand" "r") 3591 (match_operand:GPI 2 "register_operand" "r")))] 3592 "" 3593 "mul\\t%<w>0, %<w>1, %<w>2" 3594 [(set_attr "type" "mul")] 3595) 3596 3597;; zero_extend version of above 3598(define_insn "*mulsi3_uxtw" 3599 [(set (match_operand:DI 0 "register_operand" "=r") 3600 (zero_extend:DI 3601 (mult:SI (match_operand:SI 1 "register_operand" "r") 3602 (match_operand:SI 2 "register_operand" "r"))))] 3603 "" 3604 "mul\\t%w0, %w1, %w2" 3605 [(set_attr "type" "mul")] 3606) 3607 3608(define_insn "madd<mode>" 3609 [(set (match_operand:GPI 0 "register_operand" "=r") 3610 (plus:GPI (mult:GPI (match_operand:GPI 1 "register_operand" "r") 3611 (match_operand:GPI 2 "register_operand" "r")) 3612 (match_operand:GPI 3 "register_operand" "r")))] 3613 "" 3614 "madd\\t%<w>0, %<w>1, %<w>2, %<w>3" 3615 [(set_attr "type" "mla")] 3616) 3617 3618;; zero_extend version of above 3619(define_insn "*maddsi_uxtw" 3620 [(set (match_operand:DI 0 "register_operand" "=r") 3621 (zero_extend:DI 3622 (plus:SI (mult:SI (match_operand:SI 1 "register_operand" "r") 3623 (match_operand:SI 2 "register_operand" "r")) 3624 (match_operand:SI 3 "register_operand" "r"))))] 3625 "" 3626 "madd\\t%w0, %w1, %w2, %w3" 3627 [(set_attr "type" "mla")] 3628) 3629 3630(define_insn "*msub<mode>" 3631 [(set (match_operand:GPI 0 "register_operand" "=r") 3632 (minus:GPI (match_operand:GPI 3 "register_operand" "r") 3633 (mult:GPI (match_operand:GPI 1 "register_operand" "r") 3634 (match_operand:GPI 2 "register_operand" "r"))))] 3635 3636 "" 3637 "msub\\t%<w>0, %<w>1, %<w>2, %<w>3" 3638 [(set_attr "type" "mla")] 3639) 3640 3641;; zero_extend version of above 3642(define_insn "*msubsi_uxtw" 3643 [(set (match_operand:DI 0 "register_operand" "=r") 3644 (zero_extend:DI 3645 (minus:SI (match_operand:SI 3 "register_operand" "r") 3646 (mult:SI (match_operand:SI 1 "register_operand" "r") 3647 (match_operand:SI 2 "register_operand" "r")))))] 3648 3649 "" 3650 "msub\\t%w0, %w1, %w2, %w3" 3651 [(set_attr "type" "mla")] 3652) 3653 3654(define_insn "*mul<mode>_neg" 3655 [(set (match_operand:GPI 0 "register_operand" "=r") 3656 (mult:GPI (neg:GPI (match_operand:GPI 1 "register_operand" "r")) 3657 (match_operand:GPI 2 "register_operand" "r")))] 3658 3659 "" 3660 "mneg\\t%<w>0, %<w>1, %<w>2" 3661 [(set_attr "type" "mul")] 3662) 3663 3664;; zero_extend version of above 3665(define_insn "*mulsi_neg_uxtw" 3666 [(set (match_operand:DI 0 "register_operand" "=r") 3667 (zero_extend:DI 3668 (mult:SI (neg:SI (match_operand:SI 1 "register_operand" "r")) 3669 (match_operand:SI 2 "register_operand" "r"))))] 3670 3671 "" 3672 "mneg\\t%w0, %w1, %w2" 3673 [(set_attr "type" "mul")] 3674) 3675 3676(define_insn "<su_optab>mulsidi3" 3677 [(set (match_operand:DI 0 "register_operand" "=r") 3678 (mult:DI (ANY_EXTEND:DI (match_operand:SI 1 "register_operand" "r")) 3679 (ANY_EXTEND:DI (match_operand:SI 2 "register_operand" "r"))))] 3680 "" 3681 "<su>mull\\t%0, %w1, %w2" 3682 [(set_attr "type" "<su>mull")] 3683) 3684 3685(define_insn "<su_optab>maddsidi4" 3686 [(set (match_operand:DI 0 "register_operand" "=r") 3687 (plus:DI (mult:DI 3688 (ANY_EXTEND:DI (match_operand:SI 1 "register_operand" "r")) 3689 (ANY_EXTEND:DI (match_operand:SI 2 "register_operand" "r"))) 3690 (match_operand:DI 3 "register_operand" "r")))] 3691 "" 3692 "<su>maddl\\t%0, %w1, %w2, %3" 3693 [(set_attr "type" "<su>mlal")] 3694) 3695 3696(define_insn "<su_optab>msubsidi4" 3697 [(set (match_operand:DI 0 "register_operand" "=r") 3698 (minus:DI 3699 (match_operand:DI 3 "register_operand" "r") 3700 (mult:DI (ANY_EXTEND:DI (match_operand:SI 1 "register_operand" "r")) 3701 (ANY_EXTEND:DI 3702 (match_operand:SI 2 "register_operand" "r")))))] 3703 "" 3704 "<su>msubl\\t%0, %w1, %w2, %3" 3705 [(set_attr "type" "<su>mlal")] 3706) 3707 3708(define_insn "*<su_optab>mulsidi_neg" 3709 [(set (match_operand:DI 0 "register_operand" "=r") 3710 (mult:DI (neg:DI 3711 (ANY_EXTEND:DI (match_operand:SI 1 "register_operand" "r"))) 3712 (ANY_EXTEND:DI (match_operand:SI 2 "register_operand" "r"))))] 3713 "" 3714 "<su>mnegl\\t%0, %w1, %w2" 3715 [(set_attr "type" "<su>mull")] 3716) 3717 3718(define_expand "<su_optab>mulditi3" 3719 [(set (match_operand:TI 0 "register_operand") 3720 (mult:TI (ANY_EXTEND:TI (match_operand:DI 1 "register_operand")) 3721 (ANY_EXTEND:TI (match_operand:DI 2 "register_operand"))))] 3722 "" 3723{ 3724 rtx low = gen_reg_rtx (DImode); 3725 emit_insn (gen_muldi3 (low, operands[1], operands[2])); 3726 3727 rtx high = gen_reg_rtx (DImode); 3728 emit_insn (gen_<su>muldi3_highpart (high, operands[1], operands[2])); 3729 3730 emit_move_insn (gen_lowpart (DImode, operands[0]), low); 3731 emit_move_insn (gen_highpart (DImode, operands[0]), high); 3732 DONE; 3733}) 3734 3735;; The default expansion of multi3 using umuldi3_highpart will perform 3736;; the additions in an order that fails to combine into two madd insns. 3737(define_expand "multi3" 3738 [(set (match_operand:TI 0 "register_operand") 3739 (mult:TI (match_operand:TI 1 "register_operand") 3740 (match_operand:TI 2 "register_operand")))] 3741 "" 3742{ 3743 rtx l0 = gen_reg_rtx (DImode); 3744 rtx l1 = gen_lowpart (DImode, operands[1]); 3745 rtx l2 = gen_lowpart (DImode, operands[2]); 3746 rtx h0 = gen_reg_rtx (DImode); 3747 rtx h1 = gen_highpart (DImode, operands[1]); 3748 rtx h2 = gen_highpart (DImode, operands[2]); 3749 3750 emit_insn (gen_muldi3 (l0, l1, l2)); 3751 emit_insn (gen_umuldi3_highpart (h0, l1, l2)); 3752 emit_insn (gen_madddi (h0, h1, l2, h0)); 3753 emit_insn (gen_madddi (h0, l1, h2, h0)); 3754 3755 emit_move_insn (gen_lowpart (DImode, operands[0]), l0); 3756 emit_move_insn (gen_highpart (DImode, operands[0]), h0); 3757 DONE; 3758}) 3759 3760(define_insn "<su>muldi3_highpart" 3761 [(set (match_operand:DI 0 "register_operand" "=r") 3762 (truncate:DI 3763 (lshiftrt:TI 3764 (mult:TI 3765 (ANY_EXTEND:TI (match_operand:DI 1 "register_operand" "r")) 3766 (ANY_EXTEND:TI (match_operand:DI 2 "register_operand" "r"))) 3767 (const_int 64))))] 3768 "" 3769 "<su>mulh\\t%0, %1, %2" 3770 [(set_attr "type" "<su>mull")] 3771) 3772 3773(define_insn "<su_optab>div<mode>3" 3774 [(set (match_operand:GPI 0 "register_operand" "=r") 3775 (ANY_DIV:GPI (match_operand:GPI 1 "register_operand" "r") 3776 (match_operand:GPI 2 "register_operand" "r")))] 3777 "" 3778 "<su>div\\t%<w>0, %<w>1, %<w>2" 3779 [(set_attr "type" "<su>div")] 3780) 3781 3782;; zero_extend version of above 3783(define_insn "*<su_optab>divsi3_uxtw" 3784 [(set (match_operand:DI 0 "register_operand" "=r") 3785 (zero_extend:DI 3786 (ANY_DIV:SI (match_operand:SI 1 "register_operand" "r") 3787 (match_operand:SI 2 "register_operand" "r"))))] 3788 "" 3789 "<su>div\\t%w0, %w1, %w2" 3790 [(set_attr "type" "<su>div")] 3791) 3792 3793;; ------------------------------------------------------------------- 3794;; Comparison insns 3795;; ------------------------------------------------------------------- 3796 3797(define_insn "cmp<mode>" 3798 [(set (reg:CC CC_REGNUM) 3799 (compare:CC (match_operand:GPI 0 "register_operand" "rk,rk,rk") 3800 (match_operand:GPI 1 "aarch64_plus_operand" "r,I,J")))] 3801 "" 3802 "@ 3803 cmp\\t%<w>0, %<w>1 3804 cmp\\t%<w>0, %1 3805 cmn\\t%<w>0, #%n1" 3806 [(set_attr "type" "alus_sreg,alus_imm,alus_imm")] 3807) 3808 3809(define_insn "fcmp<mode>" 3810 [(set (reg:CCFP CC_REGNUM) 3811 (compare:CCFP (match_operand:GPF 0 "register_operand" "w,w") 3812 (match_operand:GPF 1 "aarch64_fp_compare_operand" "Y,w")))] 3813 "TARGET_FLOAT" 3814 "@ 3815 fcmp\\t%<s>0, #0.0 3816 fcmp\\t%<s>0, %<s>1" 3817 [(set_attr "type" "fcmp<s>")] 3818) 3819 3820(define_insn "fcmpe<mode>" 3821 [(set (reg:CCFPE CC_REGNUM) 3822 (compare:CCFPE (match_operand:GPF 0 "register_operand" "w,w") 3823 (match_operand:GPF 1 "aarch64_fp_compare_operand" "Y,w")))] 3824 "TARGET_FLOAT" 3825 "@ 3826 fcmpe\\t%<s>0, #0.0 3827 fcmpe\\t%<s>0, %<s>1" 3828 [(set_attr "type" "fcmp<s>")] 3829) 3830 3831(define_insn "*cmp_swp_<shift>_reg<mode>" 3832 [(set (reg:CC_SWP CC_REGNUM) 3833 (compare:CC_SWP (ASHIFT:GPI 3834 (match_operand:GPI 0 "register_operand" "r") 3835 (match_operand:QI 1 "aarch64_shift_imm_<mode>" "n")) 3836 (match_operand:GPI 2 "aarch64_reg_or_zero" "rZ")))] 3837 "" 3838 "cmp\\t%<w>2, %<w>0, <shift> %1" 3839 [(set_attr "type" "alus_shift_imm")] 3840) 3841 3842(define_insn "*cmp_swp_<optab><ALLX:mode>_reg<GPI:mode>" 3843 [(set (reg:CC_SWP CC_REGNUM) 3844 (compare:CC_SWP (ANY_EXTEND:GPI 3845 (match_operand:ALLX 0 "register_operand" "r")) 3846 (match_operand:GPI 1 "register_operand" "r")))] 3847 "" 3848 "cmp\\t%<GPI:w>1, %w0, <su>xt<ALLX:size>" 3849 [(set_attr "type" "alus_ext")] 3850) 3851 3852(define_insn "*cmp_swp_<optab><ALLX:mode>_shft_<GPI:mode>" 3853 [(set (reg:CC_SWP CC_REGNUM) 3854 (compare:CC_SWP (ashift:GPI 3855 (ANY_EXTEND:GPI 3856 (match_operand:ALLX 0 "register_operand" "r")) 3857 (match_operand 1 "aarch64_imm3" "Ui3")) 3858 (match_operand:GPI 2 "register_operand" "r")))] 3859 "" 3860 "cmp\\t%<GPI:w>2, %w0, <su>xt<ALLX:size> %1" 3861 [(set_attr "type" "alus_ext")] 3862) 3863 3864;; ------------------------------------------------------------------- 3865;; Store-flag and conditional select insns 3866;; ------------------------------------------------------------------- 3867 3868(define_expand "cstore<mode>4" 3869 [(set (match_operand:SI 0 "register_operand") 3870 (match_operator:SI 1 "aarch64_comparison_operator" 3871 [(match_operand:GPI 2 "register_operand") 3872 (match_operand:GPI 3 "aarch64_plus_operand")]))] 3873 "" 3874 " 3875 operands[2] = aarch64_gen_compare_reg (GET_CODE (operands[1]), operands[2], 3876 operands[3]); 3877 operands[3] = const0_rtx; 3878 " 3879) 3880 3881(define_expand "cstorecc4" 3882 [(set (match_operand:SI 0 "register_operand") 3883 (match_operator 1 "aarch64_comparison_operator_mode" 3884 [(match_operand 2 "cc_register") 3885 (match_operand 3 "const0_operand")]))] 3886 "" 3887"{ 3888 emit_insn (gen_rtx_SET (operands[0], operands[1])); 3889 DONE; 3890}") 3891 3892 3893(define_expand "cstore<mode>4" 3894 [(set (match_operand:SI 0 "register_operand") 3895 (match_operator:SI 1 "aarch64_comparison_operator_mode" 3896 [(match_operand:GPF 2 "register_operand") 3897 (match_operand:GPF 3 "aarch64_fp_compare_operand")]))] 3898 "" 3899 " 3900 operands[2] = aarch64_gen_compare_reg (GET_CODE (operands[1]), operands[2], 3901 operands[3]); 3902 operands[3] = const0_rtx; 3903 " 3904) 3905 3906(define_insn "aarch64_cstore<mode>" 3907 [(set (match_operand:ALLI 0 "register_operand" "=r") 3908 (match_operator:ALLI 1 "aarch64_comparison_operator_mode" 3909 [(match_operand 2 "cc_register" "") (const_int 0)]))] 3910 "" 3911 "cset\\t%<w>0, %m1" 3912 [(set_attr "type" "csel")] 3913) 3914 3915;; For a 24-bit immediate CST we can optimize the compare for equality 3916;; and branch sequence from: 3917;; mov x0, #imm1 3918;; movk x0, #imm2, lsl 16 /* x0 contains CST. */ 3919;; cmp x1, x0 3920;; cset x2, <ne,eq> 3921;; into the shorter: 3922;; sub x0, x1, #(CST & 0xfff000) 3923;; subs x0, x0, #(CST & 0x000fff) 3924;; cset x2, <ne, eq>. 3925(define_insn_and_split "*compare_cstore<mode>_insn" 3926 [(set (match_operand:GPI 0 "register_operand" "=r") 3927 (EQL:GPI (match_operand:GPI 1 "register_operand" "r") 3928 (match_operand:GPI 2 "aarch64_imm24" "n"))) 3929 (clobber (reg:CC CC_REGNUM))] 3930 "!aarch64_move_imm (INTVAL (operands[2]), <MODE>mode) 3931 && !aarch64_plus_operand (operands[2], <MODE>mode) 3932 && !reload_completed" 3933 "#" 3934 "&& true" 3935 [(const_int 0)] 3936 { 3937 HOST_WIDE_INT lo_imm = UINTVAL (operands[2]) & 0xfff; 3938 HOST_WIDE_INT hi_imm = UINTVAL (operands[2]) & 0xfff000; 3939 rtx tmp = gen_reg_rtx (<MODE>mode); 3940 emit_insn (gen_add<mode>3 (tmp, operands[1], GEN_INT (-hi_imm))); 3941 emit_insn (gen_add<mode>3_compare0 (tmp, tmp, GEN_INT (-lo_imm))); 3942 rtx cc_reg = gen_rtx_REG (CC_NZmode, CC_REGNUM); 3943 rtx cmp_rtx = gen_rtx_fmt_ee (<EQL:CMP>, <MODE>mode, cc_reg, const0_rtx); 3944 emit_insn (gen_aarch64_cstore<mode> (operands[0], cmp_rtx, cc_reg)); 3945 DONE; 3946 } 3947 [(set_attr "type" "csel")] 3948) 3949 3950;; zero_extend version of the above 3951(define_insn "*cstoresi_insn_uxtw" 3952 [(set (match_operand:DI 0 "register_operand" "=r") 3953 (zero_extend:DI 3954 (match_operator:SI 1 "aarch64_comparison_operator_mode" 3955 [(match_operand 2 "cc_register" "") (const_int 0)])))] 3956 "" 3957 "cset\\t%w0, %m1" 3958 [(set_attr "type" "csel")] 3959) 3960 3961(define_insn "cstore<mode>_neg" 3962 [(set (match_operand:ALLI 0 "register_operand" "=r") 3963 (neg:ALLI (match_operator:ALLI 1 "aarch64_comparison_operator_mode" 3964 [(match_operand 2 "cc_register" "") (const_int 0)])))] 3965 "" 3966 "csetm\\t%<w>0, %m1" 3967 [(set_attr "type" "csel")] 3968) 3969 3970;; zero_extend version of the above 3971(define_insn "*cstoresi_neg_uxtw" 3972 [(set (match_operand:DI 0 "register_operand" "=r") 3973 (zero_extend:DI 3974 (neg:SI (match_operator:SI 1 "aarch64_comparison_operator_mode" 3975 [(match_operand 2 "cc_register" "") (const_int 0)]))))] 3976 "" 3977 "csetm\\t%w0, %m1" 3978 [(set_attr "type" "csel")] 3979) 3980 3981(define_expand "cmov<mode>6" 3982 [(set (match_operand:GPI 0 "register_operand") 3983 (if_then_else:GPI 3984 (match_operator 1 "aarch64_comparison_operator" 3985 [(match_operand:GPI 2 "register_operand") 3986 (match_operand:GPI 3 "aarch64_plus_operand")]) 3987 (match_operand:GPI 4 "register_operand") 3988 (match_operand:GPI 5 "register_operand")))] 3989 "" 3990 " 3991 operands[2] = aarch64_gen_compare_reg (GET_CODE (operands[1]), operands[2], 3992 operands[3]); 3993 operands[3] = const0_rtx; 3994 " 3995) 3996 3997(define_expand "cmov<mode>6" 3998 [(set (match_operand:GPF 0 "register_operand") 3999 (if_then_else:GPF 4000 (match_operator 1 "aarch64_comparison_operator" 4001 [(match_operand:GPF 2 "register_operand") 4002 (match_operand:GPF 3 "aarch64_fp_compare_operand")]) 4003 (match_operand:GPF 4 "register_operand") 4004 (match_operand:GPF 5 "register_operand")))] 4005 "" 4006 " 4007 operands[2] = aarch64_gen_compare_reg (GET_CODE (operands[1]), operands[2], 4008 operands[3]); 4009 operands[3] = const0_rtx; 4010 " 4011) 4012 4013(define_insn "*cmov<mode>_insn" 4014 [(set (match_operand:ALLI 0 "register_operand" "=r,r,r,r,r,r,r") 4015 (if_then_else:ALLI 4016 (match_operator 1 "aarch64_comparison_operator" 4017 [(match_operand 2 "cc_register" "") (const_int 0)]) 4018 (match_operand:ALLI 3 "aarch64_reg_zero_or_m1_or_1" "rZ,rZ,UsM,rZ,Ui1,UsM,Ui1") 4019 (match_operand:ALLI 4 "aarch64_reg_zero_or_m1_or_1" "rZ,UsM,rZ,Ui1,rZ,UsM,Ui1")))] 4020 "!((operands[3] == const1_rtx && operands[4] == constm1_rtx) 4021 || (operands[3] == constm1_rtx && operands[4] == const1_rtx))" 4022 ;; Final two alternatives should be unreachable, but included for completeness 4023 "@ 4024 csel\\t%<w>0, %<w>3, %<w>4, %m1 4025 csinv\\t%<w>0, %<w>3, <w>zr, %m1 4026 csinv\\t%<w>0, %<w>4, <w>zr, %M1 4027 csinc\\t%<w>0, %<w>3, <w>zr, %m1 4028 csinc\\t%<w>0, %<w>4, <w>zr, %M1 4029 mov\\t%<w>0, -1 4030 mov\\t%<w>0, 1" 4031 [(set_attr "type" "csel, csel, csel, csel, csel, mov_imm, mov_imm")] 4032) 4033 4034;; zero_extend version of above 4035(define_insn "*cmovsi_insn_uxtw" 4036 [(set (match_operand:DI 0 "register_operand" "=r,r,r,r,r,r,r") 4037 (zero_extend:DI 4038 (if_then_else:SI 4039 (match_operator 1 "aarch64_comparison_operator" 4040 [(match_operand 2 "cc_register" "") (const_int 0)]) 4041 (match_operand:SI 3 "aarch64_reg_zero_or_m1_or_1" "rZ,rZ,UsM,rZ,Ui1,UsM,Ui1") 4042 (match_operand:SI 4 "aarch64_reg_zero_or_m1_or_1" "rZ,UsM,rZ,Ui1,rZ,UsM,Ui1"))))] 4043 "!((operands[3] == const1_rtx && operands[4] == constm1_rtx) 4044 || (operands[3] == constm1_rtx && operands[4] == const1_rtx))" 4045 ;; Final two alternatives should be unreachable, but included for completeness 4046 "@ 4047 csel\\t%w0, %w3, %w4, %m1 4048 csinv\\t%w0, %w3, wzr, %m1 4049 csinv\\t%w0, %w4, wzr, %M1 4050 csinc\\t%w0, %w3, wzr, %m1 4051 csinc\\t%w0, %w4, wzr, %M1 4052 mov\\t%w0, -1 4053 mov\\t%w0, 1" 4054 [(set_attr "type" "csel, csel, csel, csel, csel, mov_imm, mov_imm")] 4055) 4056 4057(define_insn "*cmovdi_insn_uxtw" 4058 [(set (match_operand:DI 0 "register_operand" "=r") 4059 (if_then_else:DI 4060 (match_operator 1 "aarch64_comparison_operator" 4061 [(match_operand 2 "cc_register" "") (const_int 0)]) 4062 (zero_extend:DI (match_operand:SI 3 "register_operand" "r")) 4063 (zero_extend:DI (match_operand:SI 4 "register_operand" "r"))))] 4064 "" 4065 "csel\\t%w0, %w3, %w4, %m1" 4066 [(set_attr "type" "csel")] 4067) 4068 4069(define_insn "*cmov<mode>_insn" 4070 [(set (match_operand:GPF 0 "register_operand" "=w") 4071 (if_then_else:GPF 4072 (match_operator 1 "aarch64_comparison_operator" 4073 [(match_operand 2 "cc_register" "") (const_int 0)]) 4074 (match_operand:GPF 3 "register_operand" "w") 4075 (match_operand:GPF 4 "register_operand" "w")))] 4076 "TARGET_FLOAT" 4077 "fcsel\\t%<s>0, %<s>3, %<s>4, %m1" 4078 [(set_attr "type" "fcsel")] 4079) 4080 4081(define_expand "mov<mode>cc" 4082 [(set (match_operand:ALLI 0 "register_operand") 4083 (if_then_else:ALLI (match_operand 1 "aarch64_comparison_operator") 4084 (match_operand:ALLI 2 "register_operand") 4085 (match_operand:ALLI 3 "register_operand")))] 4086 "" 4087 { 4088 rtx ccreg; 4089 enum rtx_code code = GET_CODE (operands[1]); 4090 4091 if (code == UNEQ || code == LTGT) 4092 FAIL; 4093 4094 ccreg = aarch64_gen_compare_reg (code, XEXP (operands[1], 0), 4095 XEXP (operands[1], 1)); 4096 operands[1] = gen_rtx_fmt_ee (code, VOIDmode, ccreg, const0_rtx); 4097 } 4098) 4099 4100(define_expand "mov<GPF:mode><GPI:mode>cc" 4101 [(set (match_operand:GPI 0 "register_operand") 4102 (if_then_else:GPI (match_operand 1 "aarch64_comparison_operator") 4103 (match_operand:GPF 2 "register_operand") 4104 (match_operand:GPF 3 "register_operand")))] 4105 "" 4106 { 4107 rtx ccreg; 4108 enum rtx_code code = GET_CODE (operands[1]); 4109 4110 if (code == UNEQ || code == LTGT) 4111 FAIL; 4112 4113 ccreg = aarch64_gen_compare_reg (code, XEXP (operands[1], 0), 4114 XEXP (operands[1], 1)); 4115 operands[1] = gen_rtx_fmt_ee (code, VOIDmode, ccreg, const0_rtx); 4116 } 4117) 4118 4119(define_expand "mov<mode>cc" 4120 [(set (match_operand:GPF 0 "register_operand") 4121 (if_then_else:GPF (match_operand 1 "aarch64_comparison_operator") 4122 (match_operand:GPF 2 "register_operand") 4123 (match_operand:GPF 3 "register_operand")))] 4124 "" 4125 { 4126 rtx ccreg; 4127 enum rtx_code code = GET_CODE (operands[1]); 4128 4129 if (code == UNEQ || code == LTGT) 4130 FAIL; 4131 4132 ccreg = aarch64_gen_compare_reg (code, XEXP (operands[1], 0), 4133 XEXP (operands[1], 1)); 4134 operands[1] = gen_rtx_fmt_ee (code, VOIDmode, ccreg, const0_rtx); 4135 } 4136) 4137 4138(define_expand "<neg_not_op><mode>cc" 4139 [(set (match_operand:GPI 0 "register_operand") 4140 (if_then_else:GPI (match_operand 1 "aarch64_comparison_operator") 4141 (NEG_NOT:GPI (match_operand:GPI 2 "register_operand")) 4142 (match_operand:GPI 3 "register_operand")))] 4143 "" 4144 { 4145 rtx ccreg; 4146 enum rtx_code code = GET_CODE (operands[1]); 4147 4148 if (code == UNEQ || code == LTGT) 4149 FAIL; 4150 4151 ccreg = aarch64_gen_compare_reg (code, XEXP (operands[1], 0), 4152 XEXP (operands[1], 1)); 4153 operands[1] = gen_rtx_fmt_ee (code, VOIDmode, ccreg, const0_rtx); 4154 } 4155) 4156 4157;; CRC32 instructions. 4158(define_insn "aarch64_<crc_variant>" 4159 [(set (match_operand:SI 0 "register_operand" "=r") 4160 (unspec:SI [(match_operand:SI 1 "register_operand" "r") 4161 (match_operand:<crc_mode> 2 "register_operand" "r")] 4162 CRC))] 4163 "TARGET_CRC32" 4164 { 4165 if (GET_MODE_BITSIZE (<crc_mode>mode) >= 64) 4166 return "<crc_variant>\\t%w0, %w1, %x2"; 4167 else 4168 return "<crc_variant>\\t%w0, %w1, %w2"; 4169 } 4170 [(set_attr "type" "crc")] 4171) 4172 4173(define_insn "*csinc2<mode>_insn" 4174 [(set (match_operand:GPI 0 "register_operand" "=r") 4175 (plus:GPI (match_operand 2 "aarch64_comparison_operation" "") 4176 (match_operand:GPI 1 "register_operand" "r")))] 4177 "" 4178 "cinc\\t%<w>0, %<w>1, %m2" 4179 [(set_attr "type" "csel")] 4180) 4181 4182(define_insn "csinc3<mode>_insn" 4183 [(set (match_operand:GPI 0 "register_operand" "=r") 4184 (if_then_else:GPI 4185 (match_operand 1 "aarch64_comparison_operation" "") 4186 (plus:GPI (match_operand:GPI 2 "register_operand" "r") 4187 (const_int 1)) 4188 (match_operand:GPI 3 "aarch64_reg_or_zero" "rZ")))] 4189 "" 4190 "csinc\\t%<w>0, %<w>3, %<w>2, %M1" 4191 [(set_attr "type" "csel")] 4192) 4193 4194(define_insn "*csinv3<mode>_insn" 4195 [(set (match_operand:GPI 0 "register_operand" "=r") 4196 (if_then_else:GPI 4197 (match_operand 1 "aarch64_comparison_operation" "") 4198 (not:GPI (match_operand:GPI 2 "register_operand" "r")) 4199 (match_operand:GPI 3 "aarch64_reg_or_zero" "rZ")))] 4200 "" 4201 "csinv\\t%<w>0, %<w>3, %<w>2, %M1" 4202 [(set_attr "type" "csel")] 4203) 4204 4205(define_insn "csneg3_uxtw_insn" 4206 [(set (match_operand:DI 0 "register_operand" "=r") 4207 (zero_extend:DI 4208 (if_then_else:SI 4209 (match_operand 1 "aarch64_comparison_operation" "") 4210 (neg:SI (match_operand:SI 2 "register_operand" "r")) 4211 (match_operand:SI 3 "aarch64_reg_or_zero" "rZ"))))] 4212 "" 4213 "csneg\\t%w0, %w3, %w2, %M1" 4214 [(set_attr "type" "csel")] 4215) 4216 4217(define_insn "csneg3<mode>_insn" 4218 [(set (match_operand:GPI 0 "register_operand" "=r") 4219 (if_then_else:GPI 4220 (match_operand 1 "aarch64_comparison_operation" "") 4221 (neg:GPI (match_operand:GPI 2 "register_operand" "r")) 4222 (match_operand:GPI 3 "aarch64_reg_or_zero" "rZ")))] 4223 "" 4224 "csneg\\t%<w>0, %<w>3, %<w>2, %M1" 4225 [(set_attr "type" "csel")] 4226) 4227 4228(define_insn "*csinv3_uxtw_insn1" 4229 [(set (match_operand:DI 0 "register_operand" "=r") 4230 (if_then_else:DI 4231 (match_operand 1 "aarch64_comparison_operation" "") 4232 (zero_extend:DI 4233 (match_operand:SI 2 "register_operand" "r")) 4234 (zero_extend:DI 4235 (NEG_NOT:SI (match_operand:SI 3 "register_operand" "r")))))] 4236 "" 4237 "cs<neg_not_cs>\\t%w0, %w2, %w3, %m1" 4238 [(set_attr "type" "csel")] 4239) 4240 4241(define_insn "*csinv3_uxtw_insn2" 4242 [(set (match_operand:DI 0 "register_operand" "=r") 4243 (if_then_else:DI 4244 (match_operand 1 "aarch64_comparison_operation" "") 4245 (zero_extend:DI 4246 (NEG_NOT:SI (match_operand:SI 2 "register_operand" "r"))) 4247 (zero_extend:DI 4248 (match_operand:SI 3 "register_operand" "r"))))] 4249 "" 4250 "cs<neg_not_cs>\\t%w0, %w3, %w2, %M1" 4251 [(set_attr "type" "csel")] 4252) 4253 4254(define_insn "*csinv3_uxtw_insn3" 4255 [(set (match_operand:DI 0 "register_operand" "=r") 4256 (if_then_else:DI 4257 (match_operand 1 "aarch64_comparison_operation" "") 4258 (zero_extend:DI 4259 (NEG_NOT:SI (match_operand:SI 2 "register_operand" "r"))) 4260 (const_int 0)))] 4261 "" 4262 "cs<neg_not_cs>\\t%w0, wzr, %w2, %M1" 4263 [(set_attr "type" "csel")] 4264) 4265 4266;; If X can be loaded by a single CNT[BHWD] instruction, 4267;; 4268;; A = UMAX (B, X) 4269;; 4270;; is equivalent to: 4271;; 4272;; TMP = UQDEC[BHWD] (B, X) 4273;; A = TMP + X 4274;; 4275;; Defining the pattern this way means that: 4276;; 4277;; A = UMAX (B, X) - X 4278;; 4279;; becomes: 4280;; 4281;; TMP1 = UQDEC[BHWD] (B, X) 4282;; TMP2 = TMP1 + X 4283;; A = TMP2 - X 4284;; 4285;; which combine can optimize to: 4286;; 4287;; A = UQDEC[BHWD] (B, X) 4288;; 4289;; We don't use match_operand predicates because the order of the operands 4290;; can vary: the CNT[BHWD] constant will come first if the other operand is 4291;; a simpler constant (such as a CONST_INT), otherwise it will come second. 4292(define_expand "umax<mode>3" 4293 [(set (match_operand:GPI 0 "register_operand") 4294 (umax:GPI (match_operand:GPI 1 "") 4295 (match_operand:GPI 2 "")))] 4296 "TARGET_SVE" 4297 { 4298 if (aarch64_sve_cnt_immediate (operands[1], <MODE>mode)) 4299 std::swap (operands[1], operands[2]); 4300 else if (!aarch64_sve_cnt_immediate (operands[2], <MODE>mode)) 4301 FAIL; 4302 rtx temp = gen_reg_rtx (<MODE>mode); 4303 operands[1] = force_reg (<MODE>mode, operands[1]); 4304 emit_insn (gen_aarch64_uqdec<mode> (temp, operands[1], operands[2])); 4305 emit_insn (gen_add<mode>3 (operands[0], temp, operands[2])); 4306 DONE; 4307 } 4308) 4309 4310;; Saturating unsigned subtraction of a CNT[BHWD] immediate. 4311(define_insn "aarch64_uqdec<mode>" 4312 [(set (match_operand:GPI 0 "register_operand" "=r") 4313 (minus:GPI 4314 (umax:GPI (match_operand:GPI 1 "register_operand" "0") 4315 (match_operand:GPI 2 "aarch64_sve_cnt_immediate" "Usv")) 4316 (match_dup 2)))] 4317 "TARGET_SVE" 4318 { 4319 return aarch64_output_sve_cnt_immediate ("uqdec", "%<w>0", operands[2]); 4320 } 4321) 4322 4323;; ------------------------------------------------------------------- 4324;; Logical operations 4325;; ------------------------------------------------------------------- 4326 4327 4328(define_insn_and_split "*aarch64_and<mode>_imm2" 4329 [(set (match_operand:GPI 0 "register_operand" "=rk") 4330 (and:GPI (match_operand:GPI 1 "register_operand" "%r") 4331 (match_operand:GPI 2 "aarch64_logical_and_immediate" "<lconst2>")))] 4332 "" 4333 "#" 4334 "true" 4335 [(const_int 0)] 4336 { 4337 HOST_WIDE_INT val = INTVAL (operands[2]); 4338 rtx imm1 = GEN_INT (aarch64_and_split_imm1 (val)); 4339 rtx imm2 = GEN_INT (aarch64_and_split_imm2 (val)); 4340 4341 emit_insn (gen_and<mode>3 (operands[0], operands[1], imm1)); 4342 emit_insn (gen_and<mode>3 (operands[0], operands[0], imm2)); 4343 DONE; 4344 } 4345) 4346 4347(define_insn "<optab><mode>3" 4348 [(set (match_operand:GPI 0 "register_operand" "=r,rk,w") 4349 (LOGICAL:GPI (match_operand:GPI 1 "register_operand" "%r,r,w") 4350 (match_operand:GPI 2 "aarch64_logical_operand" "r,<lconst>,w")))] 4351 "" 4352 "@ 4353 <logical>\\t%<w>0, %<w>1, %<w>2 4354 <logical>\\t%<w>0, %<w>1, %2 4355 <logical>\\t%0.<Vbtype>, %1.<Vbtype>, %2.<Vbtype>" 4356 [(set_attr "type" "logic_reg,logic_imm,neon_logic") 4357 (set_attr "arch" "*,*,simd")] 4358) 4359 4360;; zero_extend version of above 4361(define_insn "*<optab>si3_uxtw" 4362 [(set (match_operand:DI 0 "register_operand" "=r,rk") 4363 (zero_extend:DI 4364 (LOGICAL:SI (match_operand:SI 1 "register_operand" "%r,r") 4365 (match_operand:SI 2 "aarch64_logical_operand" "r,K"))))] 4366 "" 4367 "@ 4368 <logical>\\t%w0, %w1, %w2 4369 <logical>\\t%w0, %w1, %2" 4370 [(set_attr "type" "logic_reg,logic_imm")] 4371) 4372 4373(define_insn "*and<mode>3_compare0" 4374 [(set (reg:CC_NZ CC_REGNUM) 4375 (compare:CC_NZ 4376 (and:GPI (match_operand:GPI 1 "register_operand" "%r,r") 4377 (match_operand:GPI 2 "aarch64_logical_operand" "r,<lconst>")) 4378 (const_int 0))) 4379 (set (match_operand:GPI 0 "register_operand" "=r,r") 4380 (and:GPI (match_dup 1) (match_dup 2)))] 4381 "" 4382 "@ 4383 ands\\t%<w>0, %<w>1, %<w>2 4384 ands\\t%<w>0, %<w>1, %2" 4385 [(set_attr "type" "logics_reg,logics_imm")] 4386) 4387 4388;; zero_extend version of above 4389(define_insn "*andsi3_compare0_uxtw" 4390 [(set (reg:CC_NZ CC_REGNUM) 4391 (compare:CC_NZ 4392 (and:SI (match_operand:SI 1 "register_operand" "%r,r") 4393 (match_operand:SI 2 "aarch64_logical_operand" "r,K")) 4394 (const_int 0))) 4395 (set (match_operand:DI 0 "register_operand" "=r,r") 4396 (zero_extend:DI (and:SI (match_dup 1) (match_dup 2))))] 4397 "" 4398 "@ 4399 ands\\t%w0, %w1, %w2 4400 ands\\t%w0, %w1, %2" 4401 [(set_attr "type" "logics_reg,logics_imm")] 4402) 4403 4404(define_insn "*and_<SHIFT:optab><mode>3_compare0" 4405 [(set (reg:CC_NZ CC_REGNUM) 4406 (compare:CC_NZ 4407 (and:GPI (SHIFT:GPI 4408 (match_operand:GPI 1 "register_operand" "r") 4409 (match_operand:QI 2 "aarch64_shift_imm_<mode>" "n")) 4410 (match_operand:GPI 3 "register_operand" "r")) 4411 (const_int 0))) 4412 (set (match_operand:GPI 0 "register_operand" "=r") 4413 (and:GPI (SHIFT:GPI (match_dup 1) (match_dup 2)) (match_dup 3)))] 4414 "" 4415 "ands\\t%<w>0, %<w>3, %<w>1, <SHIFT:shift> %2" 4416 [(set_attr "type" "logics_shift_imm")] 4417) 4418 4419;; zero_extend version of above 4420(define_insn "*and_<SHIFT:optab>si3_compare0_uxtw" 4421 [(set (reg:CC_NZ CC_REGNUM) 4422 (compare:CC_NZ 4423 (and:SI (SHIFT:SI 4424 (match_operand:SI 1 "register_operand" "r") 4425 (match_operand:QI 2 "aarch64_shift_imm_si" "n")) 4426 (match_operand:SI 3 "register_operand" "r")) 4427 (const_int 0))) 4428 (set (match_operand:DI 0 "register_operand" "=r") 4429 (zero_extend:DI (and:SI (SHIFT:SI (match_dup 1) (match_dup 2)) 4430 (match_dup 3))))] 4431 "" 4432 "ands\\t%w0, %w3, %w1, <SHIFT:shift> %2" 4433 [(set_attr "type" "logics_shift_imm")] 4434) 4435 4436(define_insn "*<LOGICAL:optab>_<SHIFT:optab><mode>3" 4437 [(set (match_operand:GPI 0 "register_operand" "=r") 4438 (LOGICAL:GPI (SHIFT:GPI 4439 (match_operand:GPI 1 "register_operand" "r") 4440 (match_operand:QI 2 "aarch64_shift_imm_<mode>" "n")) 4441 (match_operand:GPI 3 "register_operand" "r")))] 4442 "" 4443 "<LOGICAL:logical>\\t%<w>0, %<w>3, %<w>1, <SHIFT:shift> %2" 4444 [(set_attr "type" "logic_shift_imm")] 4445) 4446 4447(define_split 4448 [(set (match_operand:GPI 0 "register_operand") 4449 (LOGICAL:GPI 4450 (and:GPI (ashift:GPI (match_operand:GPI 1 "register_operand") 4451 (match_operand:QI 2 "aarch64_shift_imm_<mode>")) 4452 (match_operand:GPI 3 "const_int_operand")) 4453 (zero_extend:GPI (match_operand 4 "register_operand"))))] 4454 "can_create_pseudo_p () 4455 && ((paradoxical_subreg_p (operands[1]) 4456 && rtx_equal_p (SUBREG_REG (operands[1]), operands[4])) 4457 || (REG_P (operands[1]) 4458 && REG_P (operands[4]) 4459 && REGNO (operands[1]) == REGNO (operands[4]))) 4460 && (trunc_int_for_mode (GET_MODE_MASK (GET_MODE (operands[4])) 4461 << INTVAL (operands[2]), <MODE>mode) 4462 == INTVAL (operands[3]))" 4463 [(set (match_dup 5) (zero_extend:GPI (match_dup 4))) 4464 (set (match_dup 0) (LOGICAL:GPI (ashift:GPI (match_dup 5) (match_dup 2)) 4465 (match_dup 5)))] 4466 "operands[5] = gen_reg_rtx (<MODE>mode);" 4467) 4468 4469(define_split 4470 [(set (match_operand:GPI 0 "register_operand") 4471 (LOGICAL:GPI 4472 (and:GPI (ashift:GPI (match_operand:GPI 1 "register_operand") 4473 (match_operand:QI 2 "aarch64_shift_imm_<mode>")) 4474 (match_operand:GPI 4 "const_int_operand")) 4475 (and:GPI (match_dup 1) (match_operand:GPI 3 "const_int_operand"))))] 4476 "can_create_pseudo_p () 4477 && pow2_or_zerop (UINTVAL (operands[3]) + 1) 4478 && (trunc_int_for_mode (UINTVAL (operands[3]) 4479 << INTVAL (operands[2]), <MODE>mode) 4480 == INTVAL (operands[4]))" 4481 [(set (match_dup 5) (and:GPI (match_dup 1) (match_dup 3))) 4482 (set (match_dup 0) (LOGICAL:GPI (ashift:GPI (match_dup 5) (match_dup 2)) 4483 (match_dup 5)))] 4484 "operands[5] = gen_reg_rtx (<MODE>mode);" 4485) 4486 4487(define_split 4488 [(set (match_operand:GPI 0 "register_operand") 4489 (LOGICAL:GPI 4490 (ashift:GPI (sign_extend:GPI (match_operand 1 "register_operand")) 4491 (match_operand:QI 2 "aarch64_shift_imm_<mode>")) 4492 (sign_extend:GPI (match_dup 1))))] 4493 "can_create_pseudo_p ()" 4494 [(set (match_dup 3) (sign_extend:GPI (match_dup 1))) 4495 (set (match_dup 0) (LOGICAL:GPI (ashift:GPI (match_dup 3) (match_dup 2)) 4496 (match_dup 3)))] 4497 "operands[3] = gen_reg_rtx (<MODE>mode);" 4498) 4499 4500(define_insn "*<optab>_rol<mode>3" 4501 [(set (match_operand:GPI 0 "register_operand" "=r") 4502 (LOGICAL:GPI (rotate:GPI 4503 (match_operand:GPI 1 "register_operand" "r") 4504 (match_operand:QI 2 "aarch64_shift_imm_<mode>" "n")) 4505 (match_operand:GPI 3 "register_operand" "r")))] 4506 "" 4507 "<logical>\\t%<w>0, %<w>3, %<w>1, ror #(<sizen> - %2)" 4508 [(set_attr "type" "logic_shift_imm")] 4509) 4510 4511;; zero_extend versions of above 4512(define_insn "*<LOGICAL:optab>_<SHIFT:optab>si3_uxtw" 4513 [(set (match_operand:DI 0 "register_operand" "=r") 4514 (zero_extend:DI 4515 (LOGICAL:SI (SHIFT:SI 4516 (match_operand:SI 1 "register_operand" "r") 4517 (match_operand:QI 2 "aarch64_shift_imm_si" "n")) 4518 (match_operand:SI 3 "register_operand" "r"))))] 4519 "" 4520 "<LOGICAL:logical>\\t%w0, %w3, %w1, <SHIFT:shift> %2" 4521 [(set_attr "type" "logic_shift_imm")] 4522) 4523 4524(define_insn "*<optab>_rolsi3_uxtw" 4525 [(set (match_operand:DI 0 "register_operand" "=r") 4526 (zero_extend:DI 4527 (LOGICAL:SI (rotate:SI 4528 (match_operand:SI 1 "register_operand" "r") 4529 (match_operand:QI 2 "aarch64_shift_imm_si" "n")) 4530 (match_operand:SI 3 "register_operand" "r"))))] 4531 "" 4532 "<logical>\\t%w0, %w3, %w1, ror #(32 - %2)" 4533 [(set_attr "type" "logic_shift_imm")] 4534) 4535 4536(define_insn "one_cmpl<mode>2" 4537 [(set (match_operand:GPI 0 "register_operand" "=r,w") 4538 (not:GPI (match_operand:GPI 1 "register_operand" "r,w")))] 4539 "" 4540 "@ 4541 mvn\\t%<w>0, %<w>1 4542 mvn\\t%0.8b, %1.8b" 4543 [(set_attr "type" "logic_reg,neon_logic") 4544 (set_attr "arch" "*,simd")] 4545) 4546 4547(define_insn "*one_cmpl_zero_extend" 4548 [(set (match_operand:DI 0 "register_operand" "=r") 4549 (zero_extend:DI 4550 (not:SI (match_operand:SI 1 "register_operand" "r"))))] 4551 "" 4552 "mvn\\t%w0, %w1" 4553 [(set_attr "type" "logic_reg")] 4554) 4555 4556(define_insn "*one_cmpl_<optab><mode>2" 4557 [(set (match_operand:GPI 0 "register_operand" "=r") 4558 (not:GPI (SHIFT:GPI (match_operand:GPI 1 "register_operand" "r") 4559 (match_operand:QI 2 "aarch64_shift_imm_<mode>" "n"))))] 4560 "" 4561 "mvn\\t%<w>0, %<w>1, <shift> %2" 4562 [(set_attr "type" "logic_shift_imm")] 4563) 4564 4565;; Binary logical operators negating one operand, i.e. (a & !b), (a | !b). 4566 4567(define_insn "*<NLOGICAL:optab>_one_cmpl<mode>3" 4568 [(set (match_operand:GPI 0 "register_operand" "=r,w") 4569 (NLOGICAL:GPI (not:GPI (match_operand:GPI 1 "register_operand" "r,w")) 4570 (match_operand:GPI 2 "register_operand" "r,w")))] 4571 "" 4572 "@ 4573 <NLOGICAL:nlogical>\\t%<w>0, %<w>2, %<w>1 4574 <NLOGICAL:nlogical>\\t%0.<Vbtype>, %2.<Vbtype>, %1.<Vbtype>" 4575 [(set_attr "type" "logic_reg,neon_logic") 4576 (set_attr "arch" "*,simd")] 4577) 4578 4579(define_insn "*<NLOGICAL:optab>_one_cmplsidi3_ze" 4580 [(set (match_operand:DI 0 "register_operand" "=r") 4581 (zero_extend:DI 4582 (NLOGICAL:SI (not:SI (match_operand:SI 1 "register_operand" "r")) 4583 (match_operand:SI 2 "register_operand" "r"))))] 4584 "" 4585 "<NLOGICAL:nlogical>\\t%w0, %w2, %w1" 4586 [(set_attr "type" "logic_reg")] 4587) 4588 4589(define_insn "*xor_one_cmplsidi3_ze" 4590 [(set (match_operand:DI 0 "register_operand" "=r") 4591 (zero_extend:DI 4592 (not:SI (xor:SI (match_operand:SI 1 "register_operand" "r") 4593 (match_operand:SI 2 "register_operand" "r")))))] 4594 "" 4595 "eon\\t%w0, %w1, %w2" 4596 [(set_attr "type" "logic_reg")] 4597) 4598 4599;; (xor (not a) b) is simplify_rtx-ed down to (not (xor a b)). 4600;; eon does not operate on SIMD registers so the vector variant must be split. 4601(define_insn_and_split "*xor_one_cmpl<mode>3" 4602 [(set (match_operand:GPI 0 "register_operand" "=r,w") 4603 (not:GPI (xor:GPI (match_operand:GPI 1 "register_operand" "r,?w") 4604 (match_operand:GPI 2 "register_operand" "r,w"))))] 4605 "" 4606 "@ 4607 eon\\t%<w>0, %<w>1, %<w>2 4608 #" 4609 "reload_completed && FP_REGNUM_P (REGNO (operands[0]))" ;; For SIMD registers. 4610 [(set (match_operand:GPI 0 "register_operand" "=w") 4611 (xor:GPI (match_operand:GPI 1 "register_operand" "w") 4612 (match_operand:GPI 2 "register_operand" "w"))) 4613 (set (match_dup 0) (not:GPI (match_dup 0)))] 4614 "" 4615 [(set_attr "type" "logic_reg,multiple") 4616 (set_attr "arch" "*,simd")] 4617) 4618 4619(define_insn "*and_one_cmpl<mode>3_compare0" 4620 [(set (reg:CC_NZ CC_REGNUM) 4621 (compare:CC_NZ 4622 (and:GPI (not:GPI 4623 (match_operand:GPI 1 "register_operand" "r")) 4624 (match_operand:GPI 2 "register_operand" "r")) 4625 (const_int 0))) 4626 (set (match_operand:GPI 0 "register_operand" "=r") 4627 (and:GPI (not:GPI (match_dup 1)) (match_dup 2)))] 4628 "" 4629 "bics\\t%<w>0, %<w>2, %<w>1" 4630 [(set_attr "type" "logics_reg")] 4631) 4632 4633;; zero_extend version of above 4634(define_insn "*and_one_cmplsi3_compare0_uxtw" 4635 [(set (reg:CC_NZ CC_REGNUM) 4636 (compare:CC_NZ 4637 (and:SI (not:SI 4638 (match_operand:SI 1 "register_operand" "r")) 4639 (match_operand:SI 2 "register_operand" "r")) 4640 (const_int 0))) 4641 (set (match_operand:DI 0 "register_operand" "=r") 4642 (zero_extend:DI (and:SI (not:SI (match_dup 1)) (match_dup 2))))] 4643 "" 4644 "bics\\t%w0, %w2, %w1" 4645 [(set_attr "type" "logics_reg")] 4646) 4647 4648(define_insn "*and_one_cmpl<mode>3_compare0_no_reuse" 4649 [(set (reg:CC_NZ CC_REGNUM) 4650 (compare:CC_NZ 4651 (and:GPI (not:GPI 4652 (match_operand:GPI 0 "register_operand" "r")) 4653 (match_operand:GPI 1 "register_operand" "r")) 4654 (const_int 0)))] 4655 "" 4656 "bics\\t<w>zr, %<w>1, %<w>0" 4657 [(set_attr "type" "logics_reg")] 4658) 4659 4660(define_insn "<LOGICAL:optab>_one_cmpl_<SHIFT:optab><mode>3" 4661 [(set (match_operand:GPI 0 "register_operand" "=r") 4662 (LOGICAL:GPI (not:GPI 4663 (SHIFT:GPI 4664 (match_operand:GPI 1 "register_operand" "r") 4665 (match_operand:QI 2 "aarch64_shift_imm_<mode>" "n"))) 4666 (match_operand:GPI 3 "register_operand" "r")))] 4667 "" 4668 "<LOGICAL:nlogical>\\t%<w>0, %<w>3, %<w>1, <SHIFT:shift> %2" 4669 [(set_attr "type" "logic_shift_imm")] 4670) 4671 4672(define_insn "*eor_one_cmpl_<SHIFT:optab><mode>3_alt" 4673 [(set (match_operand:GPI 0 "register_operand" "=r") 4674 (not:GPI (xor:GPI 4675 (SHIFT:GPI 4676 (match_operand:GPI 1 "register_operand" "r") 4677 (match_operand:QI 2 "aarch64_shift_imm_<mode>" "n")) 4678 (match_operand:GPI 3 "register_operand" "r"))))] 4679 "" 4680 "eon\\t%<w>0, %<w>3, %<w>1, <SHIFT:shift> %2" 4681 [(set_attr "type" "logic_shift_imm")] 4682) 4683 4684;; Zero-extend version of the above. 4685(define_insn "*eor_one_cmpl_<SHIFT:optab>sidi3_alt_ze" 4686 [(set (match_operand:DI 0 "register_operand" "=r") 4687 (zero_extend:DI 4688 (not:SI (xor:SI 4689 (SHIFT:SI 4690 (match_operand:SI 1 "register_operand" "r") 4691 (match_operand:QI 2 "aarch64_shift_imm_si" "n")) 4692 (match_operand:SI 3 "register_operand" "r")))))] 4693 "" 4694 "eon\\t%w0, %w3, %w1, <SHIFT:shift> %2" 4695 [(set_attr "type" "logic_shift_imm")] 4696) 4697 4698(define_insn "*and_one_cmpl_<SHIFT:optab><mode>3_compare0" 4699 [(set (reg:CC_NZ CC_REGNUM) 4700 (compare:CC_NZ 4701 (and:GPI (not:GPI 4702 (SHIFT:GPI 4703 (match_operand:GPI 1 "register_operand" "r") 4704 (match_operand:QI 2 "aarch64_shift_imm_<mode>" "n"))) 4705 (match_operand:GPI 3 "register_operand" "r")) 4706 (const_int 0))) 4707 (set (match_operand:GPI 0 "register_operand" "=r") 4708 (and:GPI (not:GPI 4709 (SHIFT:GPI 4710 (match_dup 1) (match_dup 2))) (match_dup 3)))] 4711 "" 4712 "bics\\t%<w>0, %<w>3, %<w>1, <SHIFT:shift> %2" 4713 [(set_attr "type" "logics_shift_imm")] 4714) 4715 4716;; zero_extend version of above 4717(define_insn "*and_one_cmpl_<SHIFT:optab>si3_compare0_uxtw" 4718 [(set (reg:CC_NZ CC_REGNUM) 4719 (compare:CC_NZ 4720 (and:SI (not:SI 4721 (SHIFT:SI 4722 (match_operand:SI 1 "register_operand" "r") 4723 (match_operand:QI 2 "aarch64_shift_imm_si" "n"))) 4724 (match_operand:SI 3 "register_operand" "r")) 4725 (const_int 0))) 4726 (set (match_operand:DI 0 "register_operand" "=r") 4727 (zero_extend:DI (and:SI 4728 (not:SI 4729 (SHIFT:SI (match_dup 1) (match_dup 2))) (match_dup 3))))] 4730 "" 4731 "bics\\t%w0, %w3, %w1, <SHIFT:shift> %2" 4732 [(set_attr "type" "logics_shift_imm")] 4733) 4734 4735(define_insn "*and_one_cmpl_<SHIFT:optab><mode>3_compare0_no_reuse" 4736 [(set (reg:CC_NZ CC_REGNUM) 4737 (compare:CC_NZ 4738 (and:GPI (not:GPI 4739 (SHIFT:GPI 4740 (match_operand:GPI 0 "register_operand" "r") 4741 (match_operand:QI 1 "aarch64_shift_imm_<mode>" "n"))) 4742 (match_operand:GPI 2 "register_operand" "r")) 4743 (const_int 0)))] 4744 "" 4745 "bics\\t<w>zr, %<w>2, %<w>0, <SHIFT:shift> %1" 4746 [(set_attr "type" "logics_shift_imm")] 4747) 4748 4749(define_insn "clz<mode>2" 4750 [(set (match_operand:GPI 0 "register_operand" "=r") 4751 (clz:GPI (match_operand:GPI 1 "register_operand" "r")))] 4752 "" 4753 "clz\\t%<w>0, %<w>1" 4754 [(set_attr "type" "clz")] 4755) 4756 4757(define_expand "ffs<mode>2" 4758 [(match_operand:GPI 0 "register_operand") 4759 (match_operand:GPI 1 "register_operand")] 4760 "" 4761 { 4762 rtx ccreg = aarch64_gen_compare_reg (EQ, operands[1], const0_rtx); 4763 rtx x = gen_rtx_NE (VOIDmode, ccreg, const0_rtx); 4764 4765 emit_insn (gen_rbit<mode>2 (operands[0], operands[1])); 4766 emit_insn (gen_clz<mode>2 (operands[0], operands[0])); 4767 emit_insn (gen_csinc3<mode>_insn (operands[0], x, operands[0], const0_rtx)); 4768 DONE; 4769 } 4770) 4771 4772;; Pop count be done via the "CNT" instruction in AdvSIMD. 4773;; 4774;; MOV v.1d, x0 4775;; CNT v1.8b, v.8b 4776;; ADDV b2, v1.8b 4777;; MOV w0, v2.b[0] 4778 4779(define_expand "popcount<mode>2" 4780 [(match_operand:GPI 0 "register_operand") 4781 (match_operand:GPI 1 "register_operand")] 4782 "TARGET_SIMD" 4783{ 4784 rtx v = gen_reg_rtx (V8QImode); 4785 rtx v1 = gen_reg_rtx (V8QImode); 4786 rtx in = operands[1]; 4787 rtx out = operands[0]; 4788 if(<MODE>mode == SImode) 4789 { 4790 rtx tmp; 4791 tmp = gen_reg_rtx (DImode); 4792 /* If we have SImode, zero extend to DImode, pop count does 4793 not change if we have extra zeros. */ 4794 emit_insn (gen_zero_extendsidi2 (tmp, in)); 4795 in = tmp; 4796 } 4797 emit_move_insn (v, gen_lowpart (V8QImode, in)); 4798 emit_insn (gen_popcountv8qi2 (v1, v)); 4799 emit_insn (gen_aarch64_zero_extend<mode>_reduc_plus_v8qi (out, v1)); 4800 DONE; 4801}) 4802 4803(define_insn "clrsb<mode>2" 4804 [(set (match_operand:GPI 0 "register_operand" "=r") 4805 (clrsb:GPI (match_operand:GPI 1 "register_operand" "r")))] 4806 "" 4807 "cls\\t%<w>0, %<w>1" 4808 [(set_attr "type" "clz")] 4809) 4810 4811(define_insn "rbit<mode>2" 4812 [(set (match_operand:GPI 0 "register_operand" "=r") 4813 (unspec:GPI [(match_operand:GPI 1 "register_operand" "r")] UNSPEC_RBIT))] 4814 "" 4815 "rbit\\t%<w>0, %<w>1" 4816 [(set_attr "type" "rbit")] 4817) 4818 4819;; Split after reload into RBIT + CLZ. Since RBIT is represented as an UNSPEC 4820;; it is unlikely to fold with any other operation, so keep this as a CTZ 4821;; expression and split after reload to enable scheduling them apart if 4822;; needed. 4823 4824(define_insn_and_split "ctz<mode>2" 4825 [(set (match_operand:GPI 0 "register_operand" "=r") 4826 (ctz:GPI (match_operand:GPI 1 "register_operand" "r")))] 4827 "" 4828 "#" 4829 "reload_completed" 4830 [(const_int 0)] 4831 " 4832 emit_insn (gen_rbit<mode>2 (operands[0], operands[1])); 4833 emit_insn (gen_clz<mode>2 (operands[0], operands[0])); 4834 DONE; 4835") 4836 4837(define_insn "*and<mode>_compare0" 4838 [(set (reg:CC_NZ CC_REGNUM) 4839 (compare:CC_NZ 4840 (match_operand:SHORT 0 "register_operand" "r") 4841 (const_int 0)))] 4842 "" 4843 "tst\\t%<w>0, <short_mask>" 4844 [(set_attr "type" "alus_imm")] 4845) 4846 4847(define_insn "*ands<GPI:mode>_compare0" 4848 [(set (reg:CC_NZ CC_REGNUM) 4849 (compare:CC_NZ 4850 (zero_extend:GPI (match_operand:SHORT 1 "register_operand" "r")) 4851 (const_int 0))) 4852 (set (match_operand:GPI 0 "register_operand" "=r") 4853 (zero_extend:GPI (match_dup 1)))] 4854 "" 4855 "ands\\t%<GPI:w>0, %<GPI:w>1, <short_mask>" 4856 [(set_attr "type" "alus_imm")] 4857) 4858 4859(define_insn "*and<mode>3nr_compare0" 4860 [(set (reg:CC_NZ CC_REGNUM) 4861 (compare:CC_NZ 4862 (and:GPI (match_operand:GPI 0 "register_operand" "%r,r") 4863 (match_operand:GPI 1 "aarch64_logical_operand" "r,<lconst>")) 4864 (const_int 0)))] 4865 "" 4866 "@ 4867 tst\\t%<w>0, %<w>1 4868 tst\\t%<w>0, %1" 4869 [(set_attr "type" "logics_reg,logics_imm")] 4870) 4871 4872(define_split 4873 [(set (reg:CC_NZ CC_REGNUM) 4874 (compare:CC_NZ 4875 (and:GPI (match_operand:GPI 0 "register_operand") 4876 (match_operand:GPI 1 "aarch64_mov_imm_operand")) 4877 (const_int 0))) 4878 (clobber (match_operand:SI 2 "register_operand"))] 4879 "" 4880 [(set (match_dup 2) (match_dup 1)) 4881 (set (reg:CC_NZ CC_REGNUM) 4882 (compare:CC_NZ 4883 (and:GPI (match_dup 0) 4884 (match_dup 2)) 4885 (const_int 0)))] 4886) 4887 4888(define_insn "*and<mode>3nr_compare0_zextract" 4889 [(set (reg:CC_NZ CC_REGNUM) 4890 (compare:CC_NZ 4891 (zero_extract:GPI (match_operand:GPI 0 "register_operand" "r") 4892 (match_operand:GPI 1 "const_int_operand" "n") 4893 (match_operand:GPI 2 "const_int_operand" "n")) 4894 (const_int 0)))] 4895 "INTVAL (operands[1]) > 0 4896 && ((INTVAL (operands[1]) + INTVAL (operands[2])) 4897 <= GET_MODE_BITSIZE (<MODE>mode)) 4898 && aarch64_bitmask_imm ( 4899 UINTVAL (aarch64_mask_from_zextract_ops (operands[1], 4900 operands[2])), 4901 <MODE>mode)" 4902 { 4903 operands[1] 4904 = aarch64_mask_from_zextract_ops (operands[1], operands[2]); 4905 return "tst\\t%<w>0, %1"; 4906 } 4907 [(set_attr "type" "logics_shift_imm")] 4908) 4909 4910(define_insn "*and_<SHIFT:optab><mode>3nr_compare0" 4911 [(set (reg:CC_NZ CC_REGNUM) 4912 (compare:CC_NZ 4913 (and:GPI (SHIFT:GPI 4914 (match_operand:GPI 0 "register_operand" "r") 4915 (match_operand:QI 1 "aarch64_shift_imm_<mode>" "n")) 4916 (match_operand:GPI 2 "register_operand" "r")) 4917 (const_int 0)))] 4918 "" 4919 "tst\\t%<w>2, %<w>0, <SHIFT:shift> %1" 4920 [(set_attr "type" "logics_shift_imm")] 4921) 4922 4923(define_split 4924 [(set (reg:CC_NZ CC_REGNUM) 4925 (compare:CC_NZ 4926 (and:GPI (SHIFT:GPI 4927 (match_operand:GPI 0 "register_operand") 4928 (match_operand:QI 1 "aarch64_shift_imm_<mode>")) 4929 (match_operand:GPI 2 "aarch64_mov_imm_operand")) 4930 (const_int 0))) 4931 (clobber (match_operand:SI 3 "register_operand"))] 4932 "" 4933 [(set (match_dup 3) (match_dup 2)) 4934 (set (reg:CC_NZ CC_REGNUM) 4935 (compare:CC_NZ 4936 (and:GPI (SHIFT:GPI 4937 (match_dup 0) 4938 (match_dup 1)) 4939 (match_dup 3)) 4940 (const_int 0)))] 4941) 4942 4943;; ------------------------------------------------------------------- 4944;; Shifts 4945;; ------------------------------------------------------------------- 4946 4947(define_expand "<optab><mode>3" 4948 [(set (match_operand:GPI 0 "register_operand") 4949 (ASHIFT:GPI (match_operand:GPI 1 "register_operand") 4950 (match_operand:QI 2 "aarch64_reg_or_imm")))] 4951 "" 4952 { 4953 if (CONST_INT_P (operands[2])) 4954 { 4955 operands[2] = GEN_INT (INTVAL (operands[2]) 4956 & (GET_MODE_BITSIZE (<MODE>mode) - 1)); 4957 4958 if (operands[2] == const0_rtx) 4959 { 4960 emit_insn (gen_mov<mode> (operands[0], operands[1])); 4961 DONE; 4962 } 4963 } 4964 } 4965) 4966 4967(define_expand "ashl<mode>3" 4968 [(set (match_operand:SHORT 0 "register_operand") 4969 (ashift:SHORT (match_operand:SHORT 1 "register_operand") 4970 (match_operand:QI 2 "const_int_operand")))] 4971 "" 4972 { 4973 operands[2] = GEN_INT (INTVAL (operands[2]) & GET_MODE_MASK (<MODE>mode)); 4974 4975 if (operands[2] == const0_rtx) 4976 { 4977 emit_insn (gen_mov<mode> (operands[0], operands[1])); 4978 DONE; 4979 } 4980 } 4981) 4982 4983(define_expand "rotr<mode>3" 4984 [(set (match_operand:GPI 0 "register_operand") 4985 (rotatert:GPI (match_operand:GPI 1 "register_operand") 4986 (match_operand:QI 2 "aarch64_reg_or_imm")))] 4987 "" 4988 { 4989 if (CONST_INT_P (operands[2])) 4990 { 4991 operands[2] = GEN_INT (INTVAL (operands[2]) 4992 & (GET_MODE_BITSIZE (<MODE>mode) - 1)); 4993 4994 if (operands[2] == const0_rtx) 4995 { 4996 emit_insn (gen_mov<mode> (operands[0], operands[1])); 4997 DONE; 4998 } 4999 } 5000 } 5001) 5002 5003(define_expand "rotl<mode>3" 5004 [(set (match_operand:GPI 0 "register_operand") 5005 (rotatert:GPI (match_operand:GPI 1 "register_operand") 5006 (match_operand:QI 2 "aarch64_reg_or_imm")))] 5007 "" 5008 { 5009 /* (SZ - cnt) % SZ == -cnt % SZ */ 5010 if (CONST_INT_P (operands[2])) 5011 { 5012 operands[2] = GEN_INT ((-UINTVAL (operands[2])) 5013 & (GET_MODE_BITSIZE (<MODE>mode) - 1)); 5014 if (operands[2] == const0_rtx) 5015 { 5016 emit_insn (gen_mov<mode> (operands[0], operands[1])); 5017 DONE; 5018 } 5019 } 5020 else 5021 operands[2] = expand_simple_unop (QImode, NEG, operands[2], 5022 NULL_RTX, 1); 5023 } 5024) 5025 5026;; When the LSL, LSR, ASR, ROR instructions operate on all register arguments 5027;; they truncate the shift/rotate amount by the size of the registers they 5028;; operate on: 32 for W-regs, 64 for X-regs. This allows us to optimise away 5029;; such redundant masking instructions. GCC can do that automatically when 5030;; SHIFT_COUNT_TRUNCATED is true, but we can't enable it for TARGET_SIMD 5031;; because some of the SISD shift alternatives don't perform this truncations. 5032;; So this pattern exists to catch such cases. 5033 5034(define_insn "*aarch64_<optab>_reg_<mode>3_mask1" 5035 [(set (match_operand:GPI 0 "register_operand" "=r") 5036 (SHIFT:GPI 5037 (match_operand:GPI 1 "register_operand" "r") 5038 (match_operator 4 "subreg_lowpart_operator" 5039 [(and:GPI (match_operand:GPI 2 "register_operand" "r") 5040 (match_operand 3 "const_int_operand" "n"))])))] 5041 "(~INTVAL (operands[3]) & (GET_MODE_BITSIZE (<MODE>mode) - 1)) == 0" 5042 "<shift>\t%<w>0, %<w>1, %<w>2" 5043 [(set_attr "type" "shift_reg")] 5044) 5045 5046(define_insn_and_split "*aarch64_<optab>_reg_<mode>3_neg_mask2" 5047 [(set (match_operand:GPI 0 "register_operand" "=&r") 5048 (SHIFT:GPI 5049 (match_operand:GPI 1 "register_operand" "r") 5050 (match_operator 4 "subreg_lowpart_operator" 5051 [(neg:SI (and:SI (match_operand:SI 2 "register_operand" "r") 5052 (match_operand 3 "const_int_operand" "n")))])))] 5053 "((~INTVAL (operands[3]) & (GET_MODE_BITSIZE (<MODE>mode) - 1)) == 0)" 5054 "#" 5055 "&& true" 5056 [(const_int 0)] 5057 { 5058 rtx tmp = (can_create_pseudo_p () ? gen_reg_rtx (SImode) 5059 : lowpart_subreg (SImode, operands[0], <MODE>mode)); 5060 emit_insn (gen_negsi2 (tmp, operands[2])); 5061 5062 rtx and_op = gen_rtx_AND (SImode, tmp, operands[3]); 5063 rtx subreg_tmp = gen_rtx_SUBREG (GET_MODE (operands[4]), and_op, 5064 SUBREG_BYTE (operands[4])); 5065 emit_insn (gen_<optab><mode>3 (operands[0], operands[1], subreg_tmp)); 5066 DONE; 5067 } 5068) 5069 5070(define_insn_and_split "*aarch64_ashl_reg_<mode>3_minus_mask" 5071 [(set (match_operand:GPI 0 "register_operand" "=&r") 5072 (ashift:GPI 5073 (match_operand:GPI 1 "register_operand" "r") 5074 (minus:QI (match_operand 2 "const_int_operand" "n") 5075 (match_operator 5 "subreg_lowpart_operator" 5076 [(and:SI (match_operand:SI 3 "register_operand" "r") 5077 (match_operand 4 "const_int_operand" "n"))]))))] 5078 "((~INTVAL (operands[4]) & (GET_MODE_BITSIZE (<MODE>mode) - 1)) == 0) 5079 && INTVAL (operands[2]) == GET_MODE_BITSIZE (<MODE>mode)" 5080 "#" 5081 "&& true" 5082 [(const_int 0)] 5083 { 5084 rtx tmp = (can_create_pseudo_p () ? gen_reg_rtx (SImode) 5085 : operands[0]); 5086 5087 emit_insn (gen_negsi2 (tmp, operands[3])); 5088 5089 rtx and_op = gen_rtx_AND (SImode, tmp, operands[4]); 5090 rtx subreg_tmp = gen_rtx_SUBREG (GET_MODE (operands[5]), and_op, 5091 SUBREG_BYTE (operands[5])); 5092 5093 emit_insn (gen_ashl<mode>3 (operands[0], operands[1], subreg_tmp)); 5094 DONE; 5095 } 5096) 5097 5098(define_insn "*aarch64_<optab>_reg_di3_mask2" 5099 [(set (match_operand:DI 0 "register_operand" "=r") 5100 (SHIFT:DI 5101 (match_operand:DI 1 "register_operand" "r") 5102 (match_operator 4 "subreg_lowpart_operator" 5103 [(and:SI (match_operand:SI 2 "register_operand" "r") 5104 (match_operand 3 "const_int_operand" "n"))])))] 5105 "((~INTVAL (operands[3]) & (GET_MODE_BITSIZE (DImode) - 1)) == 0)" 5106{ 5107 rtx xop[3]; 5108 xop[0] = operands[0]; 5109 xop[1] = operands[1]; 5110 xop[2] = gen_lowpart (GET_MODE (operands[4]), operands[2]); 5111 output_asm_insn ("<shift>\t%x0, %x1, %x2", xop); 5112 return ""; 5113} 5114 [(set_attr "type" "shift_reg")] 5115) 5116 5117(define_insn_and_split "*aarch64_<optab>_reg_minus<mode>3" 5118 [(set (match_operand:GPI 0 "register_operand" "=&r") 5119 (ASHIFT:GPI 5120 (match_operand:GPI 1 "register_operand" "r") 5121 (minus:QI (match_operand 2 "const_int_operand" "n") 5122 (match_operand:QI 3 "register_operand" "r"))))] 5123 "INTVAL (operands[2]) == GET_MODE_BITSIZE (<MODE>mode)" 5124 "#" 5125 "&& true" 5126 [(const_int 0)] 5127 { 5128 rtx subreg_tmp = gen_lowpart (SImode, operands[3]); 5129 5130 rtx tmp = (can_create_pseudo_p () ? gen_reg_rtx (SImode) 5131 : gen_lowpart (SImode, operands[0])); 5132 5133 emit_insn (gen_negsi2 (tmp, subreg_tmp)); 5134 5135 rtx and_op = gen_rtx_AND (SImode, tmp, 5136 GEN_INT (GET_MODE_BITSIZE (<MODE>mode) - 1)); 5137 5138 rtx subreg_tmp2 = gen_lowpart_SUBREG (QImode, and_op); 5139 5140 emit_insn (gen_<optab><mode>3 (operands[0], operands[1], subreg_tmp2)); 5141 DONE; 5142 } 5143 [(set_attr "length" "8")] 5144) 5145 5146;; Logical left shift using SISD or Integer instruction 5147(define_insn "*aarch64_ashl_sisd_or_int_<mode>3" 5148 [(set (match_operand:GPI 0 "register_operand" "=r,r,w,w") 5149 (ashift:GPI 5150 (match_operand:GPI 1 "register_operand" "r,r,w,w") 5151 (match_operand:QI 2 "aarch64_reg_or_shift_imm_<mode>" "Us<cmode>,r,Us<cmode>,w")))] 5152 "" 5153 "@ 5154 lsl\t%<w>0, %<w>1, %2 5155 lsl\t%<w>0, %<w>1, %<w>2 5156 shl\t%<rtn>0<vas>, %<rtn>1<vas>, %2 5157 ushl\t%<rtn>0<vas>, %<rtn>1<vas>, %<rtn>2<vas>" 5158 [(set_attr "type" "bfx,shift_reg,neon_shift_imm<q>, neon_shift_reg<q>") 5159 (set_attr "arch" "*,*,simd,simd")] 5160) 5161 5162;; Logical right shift using SISD or Integer instruction 5163(define_insn "*aarch64_lshr_sisd_or_int_<mode>3" 5164 [(set (match_operand:GPI 0 "register_operand" "=r,r,w,&w,&w") 5165 (lshiftrt:GPI 5166 (match_operand:GPI 1 "register_operand" "r,r,w,w,w") 5167 (match_operand:QI 2 "aarch64_reg_or_shift_imm_<mode>" 5168 "Us<cmode>,r,Us<cmode_simd>,w,0")))] 5169 "" 5170 "@ 5171 lsr\t%<w>0, %<w>1, %2 5172 lsr\t%<w>0, %<w>1, %<w>2 5173 ushr\t%<rtn>0<vas>, %<rtn>1<vas>, %2 5174 # 5175 #" 5176 [(set_attr "type" "bfx,shift_reg,neon_shift_imm<q>,neon_shift_reg<q>,neon_shift_reg<q>") 5177 (set_attr "arch" "*,*,simd,simd,simd")] 5178) 5179 5180(define_split 5181 [(set (match_operand:DI 0 "aarch64_simd_register") 5182 (lshiftrt:DI 5183 (match_operand:DI 1 "aarch64_simd_register") 5184 (match_operand:QI 2 "aarch64_simd_register")))] 5185 "TARGET_SIMD && reload_completed" 5186 [(set (match_dup 3) 5187 (unspec:QI [(match_dup 2)] UNSPEC_SISD_NEG)) 5188 (set (match_dup 0) 5189 (unspec:DI [(match_dup 1) (match_dup 3)] UNSPEC_SISD_USHL))] 5190 { 5191 operands[3] = gen_lowpart (QImode, operands[0]); 5192 } 5193) 5194 5195(define_split 5196 [(set (match_operand:SI 0 "aarch64_simd_register") 5197 (lshiftrt:SI 5198 (match_operand:SI 1 "aarch64_simd_register") 5199 (match_operand:QI 2 "aarch64_simd_register")))] 5200 "TARGET_SIMD && reload_completed" 5201 [(set (match_dup 3) 5202 (unspec:QI [(match_dup 2)] UNSPEC_SISD_NEG)) 5203 (set (match_dup 0) 5204 (unspec:SI [(match_dup 1) (match_dup 3)] UNSPEC_USHL_2S))] 5205 { 5206 operands[3] = gen_lowpart (QImode, operands[0]); 5207 } 5208) 5209 5210;; Arithmetic right shift using SISD or Integer instruction 5211(define_insn "*aarch64_ashr_sisd_or_int_<mode>3" 5212 [(set (match_operand:GPI 0 "register_operand" "=r,r,w,&w,&w") 5213 (ashiftrt:GPI 5214 (match_operand:GPI 1 "register_operand" "r,r,w,w,w") 5215 (match_operand:QI 2 "aarch64_reg_or_shift_imm_di" 5216 "Us<cmode>,r,Us<cmode_simd>,w,0")))] 5217 "" 5218 "@ 5219 asr\t%<w>0, %<w>1, %2 5220 asr\t%<w>0, %<w>1, %<w>2 5221 sshr\t%<rtn>0<vas>, %<rtn>1<vas>, %2 5222 # 5223 #" 5224 [(set_attr "type" "bfx,shift_reg,neon_shift_imm<q>,neon_shift_reg<q>,neon_shift_reg<q>") 5225 (set_attr "arch" "*,*,simd,simd,simd")] 5226) 5227 5228(define_split 5229 [(set (match_operand:DI 0 "aarch64_simd_register") 5230 (ashiftrt:DI 5231 (match_operand:DI 1 "aarch64_simd_register") 5232 (match_operand:QI 2 "aarch64_simd_register")))] 5233 "TARGET_SIMD && reload_completed" 5234 [(set (match_dup 3) 5235 (unspec:QI [(match_dup 2)] UNSPEC_SISD_NEG)) 5236 (set (match_dup 0) 5237 (unspec:DI [(match_dup 1) (match_dup 3)] UNSPEC_SISD_SSHL))] 5238{ 5239 operands[3] = gen_lowpart (QImode, operands[0]); 5240} 5241) 5242 5243(define_split 5244 [(set (match_operand:SI 0 "aarch64_simd_register") 5245 (ashiftrt:SI 5246 (match_operand:SI 1 "aarch64_simd_register") 5247 (match_operand:QI 2 "aarch64_simd_register")))] 5248 "TARGET_SIMD && reload_completed" 5249 [(set (match_dup 3) 5250 (unspec:QI [(match_dup 2)] UNSPEC_SISD_NEG)) 5251 (set (match_dup 0) 5252 (unspec:SI [(match_dup 1) (match_dup 3)] UNSPEC_SSHL_2S))] 5253{ 5254 operands[3] = gen_lowpart (QImode, operands[0]); 5255} 5256) 5257 5258(define_insn "*aarch64_sisd_ushl" 5259 [(set (match_operand:DI 0 "register_operand" "=w") 5260 (unspec:DI [(match_operand:DI 1 "register_operand" "w") 5261 (match_operand:QI 2 "register_operand" "w")] 5262 UNSPEC_SISD_USHL))] 5263 "TARGET_SIMD" 5264 "ushl\t%d0, %d1, %d2" 5265 [(set_attr "type" "neon_shift_reg")] 5266) 5267 5268(define_insn "*aarch64_ushl_2s" 5269 [(set (match_operand:SI 0 "register_operand" "=w") 5270 (unspec:SI [(match_operand:SI 1 "register_operand" "w") 5271 (match_operand:QI 2 "register_operand" "w")] 5272 UNSPEC_USHL_2S))] 5273 "TARGET_SIMD" 5274 "ushl\t%0.2s, %1.2s, %2.2s" 5275 [(set_attr "type" "neon_shift_reg")] 5276) 5277 5278(define_insn "*aarch64_sisd_sshl" 5279 [(set (match_operand:DI 0 "register_operand" "=w") 5280 (unspec:DI [(match_operand:DI 1 "register_operand" "w") 5281 (match_operand:QI 2 "register_operand" "w")] 5282 UNSPEC_SISD_SSHL))] 5283 "TARGET_SIMD" 5284 "sshl\t%d0, %d1, %d2" 5285 [(set_attr "type" "neon_shift_reg")] 5286) 5287 5288(define_insn "*aarch64_sshl_2s" 5289 [(set (match_operand:SI 0 "register_operand" "=w") 5290 (unspec:SI [(match_operand:SI 1 "register_operand" "w") 5291 (match_operand:QI 2 "register_operand" "w")] 5292 UNSPEC_SSHL_2S))] 5293 "TARGET_SIMD" 5294 "sshl\t%0.2s, %1.2s, %2.2s" 5295 [(set_attr "type" "neon_shift_reg")] 5296) 5297 5298(define_insn "*aarch64_sisd_neg_qi" 5299 [(set (match_operand:QI 0 "register_operand" "=w") 5300 (unspec:QI [(match_operand:QI 1 "register_operand" "w")] 5301 UNSPEC_SISD_NEG))] 5302 "TARGET_SIMD" 5303 "neg\t%d0, %d1" 5304 [(set_attr "type" "neon_neg")] 5305) 5306 5307;; Rotate right 5308(define_insn "*ror<mode>3_insn" 5309 [(set (match_operand:GPI 0 "register_operand" "=r,r") 5310 (rotatert:GPI 5311 (match_operand:GPI 1 "register_operand" "r,r") 5312 (match_operand:QI 2 "aarch64_reg_or_shift_imm_<mode>" "Us<cmode>,r")))] 5313 "" 5314 "@ 5315 ror\\t%<w>0, %<w>1, %2 5316 ror\\t%<w>0, %<w>1, %<w>2" 5317 [(set_attr "type" "rotate_imm,shift_reg")] 5318) 5319 5320;; zero_extend version of above 5321(define_insn "*<optab>si3_insn_uxtw" 5322 [(set (match_operand:DI 0 "register_operand" "=r,r") 5323 (zero_extend:DI (SHIFT:SI 5324 (match_operand:SI 1 "register_operand" "r,r") 5325 (match_operand:QI 2 "aarch64_reg_or_shift_imm_si" "Uss,r"))))] 5326 "" 5327 "@ 5328 <shift>\\t%w0, %w1, %2 5329 <shift>\\t%w0, %w1, %w2" 5330 [(set_attr "type" "bfx,shift_reg")] 5331) 5332 5333(define_insn "*<optab><mode>3_insn" 5334 [(set (match_operand:SHORT 0 "register_operand" "=r") 5335 (ASHIFT:SHORT (match_operand:SHORT 1 "register_operand" "r") 5336 (match_operand 2 "const_int_operand" "n")))] 5337 "UINTVAL (operands[2]) < GET_MODE_BITSIZE (<MODE>mode)" 5338{ 5339 operands[3] = GEN_INT (<sizen> - UINTVAL (operands[2])); 5340 return "<bfshift>\t%w0, %w1, %2, %3"; 5341} 5342 [(set_attr "type" "bfx")] 5343) 5344 5345(define_insn "*extr<mode>5_insn" 5346 [(set (match_operand:GPI 0 "register_operand" "=r") 5347 (ior:GPI (ashift:GPI (match_operand:GPI 1 "register_operand" "r") 5348 (match_operand 3 "const_int_operand" "n")) 5349 (lshiftrt:GPI (match_operand:GPI 2 "register_operand" "r") 5350 (match_operand 4 "const_int_operand" "n"))))] 5351 "UINTVAL (operands[3]) < GET_MODE_BITSIZE (<MODE>mode) && 5352 (UINTVAL (operands[3]) + UINTVAL (operands[4]) == GET_MODE_BITSIZE (<MODE>mode))" 5353 "extr\\t%<w>0, %<w>1, %<w>2, %4" 5354 [(set_attr "type" "rotate_imm")] 5355) 5356 5357;; There are no canonicalisation rules for ashift and lshiftrt inside an ior 5358;; so we have to match both orderings. 5359(define_insn "*extr<mode>5_insn_alt" 5360 [(set (match_operand:GPI 0 "register_operand" "=r") 5361 (ior:GPI (lshiftrt:GPI (match_operand:GPI 2 "register_operand" "r") 5362 (match_operand 4 "const_int_operand" "n")) 5363 (ashift:GPI (match_operand:GPI 1 "register_operand" "r") 5364 (match_operand 3 "const_int_operand" "n"))))] 5365 "UINTVAL (operands[3]) < GET_MODE_BITSIZE (<MODE>mode) 5366 && (UINTVAL (operands[3]) + UINTVAL (operands[4]) 5367 == GET_MODE_BITSIZE (<MODE>mode))" 5368 "extr\\t%<w>0, %<w>1, %<w>2, %4" 5369 [(set_attr "type" "rotate_imm")] 5370) 5371 5372;; zero_extend version of the above 5373(define_insn "*extrsi5_insn_uxtw" 5374 [(set (match_operand:DI 0 "register_operand" "=r") 5375 (zero_extend:DI 5376 (ior:SI (ashift:SI (match_operand:SI 1 "register_operand" "r") 5377 (match_operand 3 "const_int_operand" "n")) 5378 (lshiftrt:SI (match_operand:SI 2 "register_operand" "r") 5379 (match_operand 4 "const_int_operand" "n")))))] 5380 "UINTVAL (operands[3]) < 32 && 5381 (UINTVAL (operands[3]) + UINTVAL (operands[4]) == 32)" 5382 "extr\\t%w0, %w1, %w2, %4" 5383 [(set_attr "type" "rotate_imm")] 5384) 5385 5386(define_insn "*extrsi5_insn_uxtw_alt" 5387 [(set (match_operand:DI 0 "register_operand" "=r") 5388 (zero_extend:DI 5389 (ior:SI (lshiftrt:SI (match_operand:SI 2 "register_operand" "r") 5390 (match_operand 4 "const_int_operand" "n")) 5391 (ashift:SI (match_operand:SI 1 "register_operand" "r") 5392 (match_operand 3 "const_int_operand" "n")))))] 5393 "UINTVAL (operands[3]) < 32 && 5394 (UINTVAL (operands[3]) + UINTVAL (operands[4]) == 32)" 5395 "extr\\t%w0, %w1, %w2, %4" 5396 [(set_attr "type" "rotate_imm")] 5397) 5398 5399(define_insn "*extrsi5_insn_di" 5400 [(set (match_operand:SI 0 "register_operand" "=r") 5401 (ior:SI (ashift:SI (match_operand:SI 1 "register_operand" "r") 5402 (match_operand 3 "const_int_operand" "n")) 5403 (match_operator:SI 6 "subreg_lowpart_operator" 5404 [(zero_extract:DI 5405 (match_operand:DI 2 "register_operand" "r") 5406 (match_operand 5 "const_int_operand" "n") 5407 (match_operand 4 "const_int_operand" "n"))])))] 5408 "UINTVAL (operands[3]) < 32 5409 && UINTVAL (operands[3]) + UINTVAL (operands[4]) == 32 5410 && INTVAL (operands[3]) == INTVAL (operands[5])" 5411 "extr\\t%w0, %w1, %w2, %4" 5412 [(set_attr "type" "rotate_imm")] 5413) 5414 5415(define_insn "*ror<mode>3_insn" 5416 [(set (match_operand:GPI 0 "register_operand" "=r") 5417 (rotate:GPI (match_operand:GPI 1 "register_operand" "r") 5418 (match_operand 2 "const_int_operand" "n")))] 5419 "UINTVAL (operands[2]) < GET_MODE_BITSIZE (<MODE>mode)" 5420{ 5421 operands[3] = GEN_INT (<sizen> - UINTVAL (operands[2])); 5422 return "ror\\t%<w>0, %<w>1, %3"; 5423} 5424 [(set_attr "type" "rotate_imm")] 5425) 5426 5427;; zero_extend version of the above 5428(define_insn "*rorsi3_insn_uxtw" 5429 [(set (match_operand:DI 0 "register_operand" "=r") 5430 (zero_extend:DI 5431 (rotate:SI (match_operand:SI 1 "register_operand" "r") 5432 (match_operand 2 "const_int_operand" "n"))))] 5433 "UINTVAL (operands[2]) < 32" 5434{ 5435 operands[3] = GEN_INT (32 - UINTVAL (operands[2])); 5436 return "ror\\t%w0, %w1, %3"; 5437} 5438 [(set_attr "type" "rotate_imm")] 5439) 5440 5441(define_insn "*<ANY_EXTEND:optab><GPI:mode>_ashl<SHORT:mode>" 5442 [(set (match_operand:GPI 0 "register_operand" "=r") 5443 (ANY_EXTEND:GPI 5444 (ashift:SHORT (match_operand:SHORT 1 "register_operand" "r") 5445 (match_operand 2 "const_int_operand" "n"))))] 5446 "UINTVAL (operands[2]) < GET_MODE_BITSIZE (<SHORT:MODE>mode)" 5447{ 5448 operands[3] = GEN_INT (<SHORT:sizen> - UINTVAL (operands[2])); 5449 return "<su>bfiz\t%<GPI:w>0, %<GPI:w>1, %2, %3"; 5450} 5451 [(set_attr "type" "bfx")] 5452) 5453 5454(define_insn "*zero_extend<GPI:mode>_lshr<SHORT:mode>" 5455 [(set (match_operand:GPI 0 "register_operand" "=r") 5456 (zero_extend:GPI 5457 (lshiftrt:SHORT (match_operand:SHORT 1 "register_operand" "r") 5458 (match_operand 2 "const_int_operand" "n"))))] 5459 "UINTVAL (operands[2]) < GET_MODE_BITSIZE (<SHORT:MODE>mode)" 5460{ 5461 operands[3] = GEN_INT (<SHORT:sizen> - UINTVAL (operands[2])); 5462 return "ubfx\t%<GPI:w>0, %<GPI:w>1, %2, %3"; 5463} 5464 [(set_attr "type" "bfx")] 5465) 5466 5467(define_insn "*extend<GPI:mode>_ashr<SHORT:mode>" 5468 [(set (match_operand:GPI 0 "register_operand" "=r") 5469 (sign_extend:GPI 5470 (ashiftrt:SHORT (match_operand:SHORT 1 "register_operand" "r") 5471 (match_operand 2 "const_int_operand" "n"))))] 5472 "UINTVAL (operands[2]) < GET_MODE_BITSIZE (<SHORT:MODE>mode)" 5473{ 5474 operands[3] = GEN_INT (<SHORT:sizen> - UINTVAL (operands[2])); 5475 return "sbfx\\t%<GPI:w>0, %<GPI:w>1, %2, %3"; 5476} 5477 [(set_attr "type" "bfx")] 5478) 5479 5480;; ------------------------------------------------------------------- 5481;; Bitfields 5482;; ------------------------------------------------------------------- 5483 5484(define_expand "<optab>" 5485 [(set (match_operand:DI 0 "register_operand") 5486 (ANY_EXTRACT:DI (match_operand:DI 1 "register_operand") 5487 (match_operand 2 5488 "aarch64_simd_shift_imm_offset_di") 5489 (match_operand 3 "aarch64_simd_shift_imm_di")))] 5490 "" 5491 { 5492 if (!IN_RANGE (INTVAL (operands[2]) + INTVAL (operands[3]), 5493 1, GET_MODE_BITSIZE (DImode) - 1)) 5494 FAIL; 5495 } 5496) 5497 5498 5499(define_insn "*<optab><mode>" 5500 [(set (match_operand:GPI 0 "register_operand" "=r") 5501 (ANY_EXTRACT:GPI (match_operand:GPI 1 "register_operand" "r") 5502 (match_operand 2 5503 "aarch64_simd_shift_imm_offset_<mode>" "n") 5504 (match_operand 3 5505 "aarch64_simd_shift_imm_<mode>" "n")))] 5506 "IN_RANGE (INTVAL (operands[2]) + INTVAL (operands[3]), 5507 1, GET_MODE_BITSIZE (<MODE>mode) - 1)" 5508 "<su>bfx\\t%<w>0, %<w>1, %3, %2" 5509 [(set_attr "type" "bfx")] 5510) 5511 5512;; When the bit position and width add up to 32 we can use a W-reg LSR 5513;; instruction taking advantage of the implicit zero-extension of the X-reg. 5514(define_split 5515 [(set (match_operand:DI 0 "register_operand") 5516 (zero_extract:DI (match_operand:DI 1 "register_operand") 5517 (match_operand 2 5518 "aarch64_simd_shift_imm_offset_di") 5519 (match_operand 3 5520 "aarch64_simd_shift_imm_di")))] 5521 "IN_RANGE (INTVAL (operands[2]) + INTVAL (operands[3]), 1, 5522 GET_MODE_BITSIZE (DImode) - 1) 5523 && (INTVAL (operands[2]) + INTVAL (operands[3])) 5524 == GET_MODE_BITSIZE (SImode)" 5525 [(set (match_dup 0) 5526 (zero_extend:DI (lshiftrt:SI (match_dup 4) (match_dup 3))))] 5527 { 5528 operands[4] = gen_lowpart (SImode, operands[1]); 5529 } 5530) 5531 5532;; Bitfield Insert (insv) 5533(define_expand "insv<mode>" 5534 [(set (zero_extract:GPI (match_operand:GPI 0 "register_operand") 5535 (match_operand 1 "const_int_operand") 5536 (match_operand 2 "const_int_operand")) 5537 (match_operand:GPI 3 "general_operand"))] 5538 "" 5539{ 5540 unsigned HOST_WIDE_INT width = UINTVAL (operands[1]); 5541 unsigned HOST_WIDE_INT pos = UINTVAL (operands[2]); 5542 rtx value = operands[3]; 5543 5544 if (width == 0 || (pos + width) > GET_MODE_BITSIZE (<MODE>mode)) 5545 FAIL; 5546 5547 if (CONST_INT_P (value)) 5548 { 5549 unsigned HOST_WIDE_INT mask = ((unsigned HOST_WIDE_INT)1 << width) - 1; 5550 5551 /* Prefer AND/OR for inserting all zeros or all ones. */ 5552 if ((UINTVAL (value) & mask) == 0 5553 || (UINTVAL (value) & mask) == mask) 5554 FAIL; 5555 5556 /* 16-bit aligned 16-bit wide insert is handled by insv_imm. */ 5557 if (width == 16 && (pos % 16) == 0) 5558 DONE; 5559 } 5560 operands[3] = force_reg (<MODE>mode, value); 5561}) 5562 5563(define_insn "*insv_reg<mode>" 5564 [(set (zero_extract:GPI (match_operand:GPI 0 "register_operand" "+r") 5565 (match_operand 1 "const_int_operand" "n") 5566 (match_operand 2 "const_int_operand" "n")) 5567 (match_operand:GPI 3 "register_operand" "r"))] 5568 "!(UINTVAL (operands[1]) == 0 5569 || (UINTVAL (operands[2]) + UINTVAL (operands[1]) 5570 > GET_MODE_BITSIZE (<MODE>mode)))" 5571 "bfi\\t%<w>0, %<w>3, %2, %1" 5572 [(set_attr "type" "bfm")] 5573) 5574 5575(define_insn "*aarch64_bfi<GPI:mode><ALLX:mode>4" 5576 [(set (zero_extract:GPI (match_operand:GPI 0 "register_operand" "+r") 5577 (match_operand 1 "const_int_operand" "n") 5578 (match_operand 2 "const_int_operand" "n")) 5579 (zero_extend:GPI (match_operand:ALLX 3 "register_operand" "r")))] 5580 "UINTVAL (operands[1]) <= <ALLX:sizen>" 5581 "bfi\\t%<GPI:w>0, %<GPI:w>3, %2, %1" 5582 [(set_attr "type" "bfm")] 5583) 5584 5585;; Match a bfi instruction where the shift of OP3 means that we are 5586;; actually copying the least significant bits of OP3 into OP0 by way 5587;; of the AND masks and the IOR instruction. A similar instruction 5588;; with the two parts of the IOR swapped around was never triggered 5589;; in a bootstrap build and test of GCC so it was not included. 5590 5591(define_insn "*aarch64_bfi<GPI:mode>5_shift" 5592 [(set (match_operand:GPI 0 "register_operand" "=r") 5593 (ior:GPI (and:GPI (match_operand:GPI 1 "register_operand" "0") 5594 (match_operand:GPI 2 "const_int_operand" "n")) 5595 (and:GPI (ashift:GPI 5596 (match_operand:GPI 3 "register_operand" "r") 5597 (match_operand:GPI 4 "aarch64_simd_shift_imm_<mode>" "n")) 5598 (match_operand:GPI 5 "const_int_operand" "n"))))] 5599 "aarch64_masks_and_shift_for_bfi_p (<MODE>mode, UINTVAL (operands[2]), 5600 UINTVAL (operands[4]), 5601 UINTVAL(operands[5]))" 5602 "bfi\t%<GPI:w>0, %<GPI:w>3, %4, %P5" 5603 [(set_attr "type" "bfm")] 5604) 5605 5606(define_insn "*aarch64_bfi<GPI:mode>5_shift_alt" 5607 [(set (match_operand:GPI 0 "register_operand" "=r") 5608 (ior:GPI (and:GPI (ashift:GPI 5609 (match_operand:GPI 1 "register_operand" "r") 5610 (match_operand:GPI 2 "aarch64_simd_shift_imm_<mode>" "n")) 5611 (match_operand:GPI 3 "const_int_operand" "n")) 5612 (and:GPI (match_operand:GPI 4 "register_operand" "0") 5613 (match_operand:GPI 5 "const_int_operand" "n"))))] 5614 "aarch64_masks_and_shift_for_bfi_p (<MODE>mode, UINTVAL (operands[5]), 5615 UINTVAL (operands[2]), 5616 UINTVAL(operands[3]))" 5617 "bfi\t%<GPI:w>0, %<GPI:w>1, %2, %P3" 5618 [(set_attr "type" "bfm")] 5619) 5620 5621;; Like *aarch64_bfi<GPI:mode>5_shift but with no and of the ashift because 5622;; the shift is large enough to remove the need for an AND instruction. 5623 5624(define_insn "*aarch64_bfi<GPI:mode>4_noand" 5625 [(set (match_operand:GPI 0 "register_operand" "=r") 5626 (ior:GPI (and:GPI (match_operand:GPI 1 "register_operand" "0") 5627 (match_operand:GPI 2 "const_int_operand" "n")) 5628 (ashift:GPI 5629 (match_operand:GPI 3 "register_operand" "r") 5630 (match_operand:GPI 4 "aarch64_simd_shift_imm_<mode>" "n"))))] 5631 "aarch64_masks_and_shift_for_bfi_p (<MODE>mode, UINTVAL (operands[2]), 5632 UINTVAL (operands[4]), 5633 HOST_WIDE_INT_M1U << UINTVAL (operands[4]) )" 5634{ 5635 operands[5] = GEN_INT (GET_MODE_BITSIZE (<MODE>mode) - UINTVAL (operands[4])); 5636 return "bfi\t%<GPI:w>0, %<GPI:w>3, %4, %5"; 5637} 5638 [(set_attr "type" "bfm")] 5639) 5640 5641(define_insn "*aarch64_bfi<GPI:mode>4_noand_alt" 5642 [(set (match_operand:GPI 0 "register_operand" "=r") 5643 (ior:GPI (ashift:GPI 5644 (match_operand:GPI 1 "register_operand" "r") 5645 (match_operand:GPI 2 "aarch64_simd_shift_imm_<mode>" "n")) 5646 (and:GPI (match_operand:GPI 3 "register_operand" "0") 5647 (match_operand:GPI 4 "const_int_operand" "n"))))] 5648 "aarch64_masks_and_shift_for_bfi_p (<MODE>mode, UINTVAL (operands[4]), 5649 UINTVAL (operands[2]), 5650 HOST_WIDE_INT_M1U << UINTVAL (operands[2]) )" 5651{ 5652 operands[5] = GEN_INT (GET_MODE_BITSIZE (<MODE>mode) - UINTVAL (operands[2])); 5653 return "bfi\t%<GPI:w>0, %<GPI:w>1, %2, %5"; 5654} 5655 [(set_attr "type" "bfm")] 5656) 5657 5658;; Like *aarch64_bfi<GPI:mode>5_shift but with no shifting, we are just 5659;; copying the least significant bits of OP3 to OP0. We need two versions 5660;; of the instruction to handle different checks on the constant values. 5661 5662(define_insn "*aarch64_bfi<GPI:mode>4_noshift" 5663 [(set (match_operand:GPI 0 "register_operand" "=r") 5664 (ior:GPI (and:GPI (match_operand:GPI 1 "register_operand" "0") 5665 (match_operand:GPI 2 "const_int_operand" "n")) 5666 (and:GPI (match_operand:GPI 3 "register_operand" "r") 5667 (match_operand:GPI 4 "const_int_operand" "n"))))] 5668 "aarch64_masks_and_shift_for_bfi_p (<MODE>mode, UINTVAL (operands[2]), 0, 5669 UINTVAL (operands[4]))" 5670 "bfi\t%<GPI:w>0, %<GPI:w>3, 0, %P4" 5671 [(set_attr "type" "bfm")] 5672) 5673 5674(define_insn "*aarch64_bfi<GPI:mode>4_noshift_alt" 5675 [(set (match_operand:GPI 0 "register_operand" "=r") 5676 (ior:GPI (and:GPI (match_operand:GPI 3 "register_operand" "r") 5677 (match_operand:GPI 4 "const_int_operand" "n")) 5678 (and:GPI (match_operand:GPI 1 "register_operand" "0") 5679 (match_operand:GPI 2 "const_int_operand" "n"))))] 5680 "aarch64_masks_and_shift_for_bfi_p (<MODE>mode, UINTVAL (operands[2]), 0, 5681 UINTVAL (operands[4]))" 5682 "bfi\t%<GPI:w>0, %<GPI:w>3, 0, %P4" 5683 [(set_attr "type" "bfm")] 5684) 5685 5686(define_insn "*aarch64_bfxil<mode>_extr" 5687 [(set (match_operand:GPI 0 "register_operand" "=r") 5688 (ior:GPI (and:GPI (match_operand:GPI 1 "register_operand" "0") 5689 (match_operand:GPI 2 "const_int_operand" "n")) 5690 (zero_extract:GPI 5691 (match_operand:GPI 3 "register_operand" "r") 5692 (match_operand:GPI 4 "aarch64_simd_shift_imm_<mode>" "n") 5693 (match_operand:GPI 5 "aarch64_simd_shift_imm_<mode>" "n"))))] 5694 "UINTVAL (operands[2]) == HOST_WIDE_INT_M1U << INTVAL (operands[4]) 5695 && INTVAL (operands[4]) 5696 && (UINTVAL (operands[4]) + UINTVAL (operands[5]) 5697 <= GET_MODE_BITSIZE (<MODE>mode))" 5698 "bfxil\t%<GPI:w>0, %<GPI:w>3, %5, %4" 5699 [(set_attr "type" "bfm")] 5700) 5701 5702(define_insn "*aarch64_bfxilsi_extrdi" 5703 [(set (match_operand:SI 0 "register_operand" "=r") 5704 (ior:SI (and:SI (match_operand:SI 1 "register_operand" "0") 5705 (match_operand:SI 2 "const_int_operand" "n")) 5706 (match_operator:SI 6 "subreg_lowpart_operator" 5707 [(zero_extract:DI 5708 (match_operand:DI 3 "register_operand" "r") 5709 (match_operand:SI 4 "aarch64_simd_shift_imm_si" "n") 5710 (match_operand:SI 5 "aarch64_simd_shift_imm_si" "n"))])))] 5711 "UINTVAL (operands[2]) == HOST_WIDE_INT_M1U << INTVAL (operands[4]) 5712 && INTVAL (operands[4]) 5713 && UINTVAL (operands[4]) + UINTVAL (operands[5]) <= 32" 5714 "bfxil\t%w0, %w3, %5, %4" 5715 [(set_attr "type" "bfm")] 5716) 5717 5718(define_insn "*extr_insv_lower_reg<mode>" 5719 [(set (zero_extract:GPI (match_operand:GPI 0 "register_operand" "+r") 5720 (match_operand 1 "const_int_operand" "n") 5721 (const_int 0)) 5722 (zero_extract:GPI (match_operand:GPI 2 "register_operand" "r") 5723 (match_dup 1) 5724 (match_operand 3 "const_int_operand" "n")))] 5725 "!(UINTVAL (operands[1]) == 0 5726 || (UINTVAL (operands[3]) + UINTVAL (operands[1]) 5727 > GET_MODE_BITSIZE (<MODE>mode)))" 5728 "bfxil\\t%<w>0, %<w>2, %3, %1" 5729 [(set_attr "type" "bfm")] 5730) 5731 5732(define_insn "*<optab><ALLX:mode>_shft_<GPI:mode>" 5733 [(set (match_operand:GPI 0 "register_operand" "=r") 5734 (ashift:GPI (ANY_EXTEND:GPI 5735 (match_operand:ALLX 1 "register_operand" "r")) 5736 (match_operand 2 "const_int_operand" "n")))] 5737 "UINTVAL (operands[2]) < <GPI:sizen>" 5738{ 5739 operands[3] = (<ALLX:sizen> <= (<GPI:sizen> - UINTVAL (operands[2]))) 5740 ? GEN_INT (<ALLX:sizen>) 5741 : GEN_INT (<GPI:sizen> - UINTVAL (operands[2])); 5742 return "<su>bfiz\t%<GPI:w>0, %<GPI:w>1, %2, %3"; 5743} 5744 [(set_attr "type" "bfx")] 5745) 5746 5747;; XXX We should match (any_extend (ashift)) here, like (and (ashift)) below 5748 5749(define_insn "*andim_ashift<mode>_bfiz" 5750 [(set (match_operand:GPI 0 "register_operand" "=r") 5751 (and:GPI (ashift:GPI (match_operand:GPI 1 "register_operand" "r") 5752 (match_operand 2 "const_int_operand" "n")) 5753 (match_operand 3 "const_int_operand" "n")))] 5754 "aarch64_mask_and_shift_for_ubfiz_p (<MODE>mode, operands[3], operands[2])" 5755 "ubfiz\\t%<w>0, %<w>1, %2, %P3" 5756 [(set_attr "type" "bfx")] 5757) 5758 5759;; Match sbfiz pattern in a shift left + shift right operation. 5760 5761(define_insn "*ashift<mode>_extv_bfiz" 5762 [(set (match_operand:GPI 0 "register_operand" "=r") 5763 (ashift:GPI (sign_extract:GPI (match_operand:GPI 1 "register_operand" "r") 5764 (match_operand 2 "aarch64_simd_shift_imm_offset_<mode>" "n") 5765 (const_int 0)) 5766 (match_operand 3 "aarch64_simd_shift_imm_<mode>" "n")))] 5767 "IN_RANGE (INTVAL (operands[2]) + INTVAL (operands[3]), 5768 1, GET_MODE_BITSIZE (<MODE>mode) - 1)" 5769 "sbfiz\\t%<w>0, %<w>1, %3, %2" 5770 [(set_attr "type" "bfx")] 5771) 5772 5773(define_insn "*ashiftsi_extvdi_bfiz" 5774 [(set (match_operand:SI 0 "register_operand" "=r") 5775 (ashift:SI 5776 (match_operator:SI 4 "subreg_lowpart_operator" 5777 [(sign_extract:DI 5778 (match_operand:DI 1 "register_operand" "r") 5779 (match_operand 2 "aarch64_simd_shift_imm_offset_si") 5780 (const_int 0))]) 5781 (match_operand 3 "aarch64_simd_shift_imm_si")))] 5782 "IN_RANGE (INTVAL (operands[2]) + INTVAL (operands[3]), 5783 1, GET_MODE_BITSIZE (SImode) - 1)" 5784 "sbfiz\\t%w0, %w1, %3, %2" 5785 [(set_attr "type" "bfx")] 5786) 5787 5788;; When the bit position and width of the equivalent extraction add up to 32 5789;; we can use a W-reg LSL instruction taking advantage of the implicit 5790;; zero-extension of the X-reg. 5791(define_split 5792 [(set (match_operand:DI 0 "register_operand") 5793 (and:DI (ashift:DI (match_operand:DI 1 "register_operand") 5794 (match_operand 2 "const_int_operand")) 5795 (match_operand 3 "const_int_operand")))] 5796 "aarch64_mask_and_shift_for_ubfiz_p (DImode, operands[3], operands[2]) 5797 && (INTVAL (operands[2]) + popcount_hwi (INTVAL (operands[3]))) 5798 == GET_MODE_BITSIZE (SImode)" 5799 [(set (match_dup 0) 5800 (zero_extend:DI (ashift:SI (match_dup 4) (match_dup 2))))] 5801 { 5802 operands[4] = gen_lowpart (SImode, operands[1]); 5803 } 5804) 5805 5806(define_insn "bswap<mode>2" 5807 [(set (match_operand:GPI 0 "register_operand" "=r") 5808 (bswap:GPI (match_operand:GPI 1 "register_operand" "r")))] 5809 "" 5810 "rev\\t%<w>0, %<w>1" 5811 [(set_attr "type" "rev")] 5812) 5813 5814(define_insn "bswaphi2" 5815 [(set (match_operand:HI 0 "register_operand" "=r") 5816 (bswap:HI (match_operand:HI 1 "register_operand" "r")))] 5817 "" 5818 "rev16\\t%w0, %w1" 5819 [(set_attr "type" "rev")] 5820) 5821 5822(define_insn "*aarch64_bfxil<mode>" 5823 [(set (match_operand:GPI 0 "register_operand" "=r,r") 5824 (ior:GPI (and:GPI (match_operand:GPI 1 "register_operand" "r,0") 5825 (match_operand:GPI 3 "const_int_operand" "n, Ulc")) 5826 (and:GPI (match_operand:GPI 2 "register_operand" "0,r") 5827 (match_operand:GPI 4 "const_int_operand" "Ulc, n"))))] 5828 "(INTVAL (operands[3]) == ~INTVAL (operands[4])) 5829 && (aarch64_high_bits_all_ones_p (INTVAL (operands[3])) 5830 || aarch64_high_bits_all_ones_p (INTVAL (operands[4])))" 5831 { 5832 switch (which_alternative) 5833 { 5834 case 0: 5835 operands[3] = GEN_INT (ctz_hwi (~INTVAL (operands[3]))); 5836 return "bfxil\\t%<w>0, %<w>1, 0, %3"; 5837 case 1: 5838 operands[3] = GEN_INT (ctz_hwi (~INTVAL (operands[4]))); 5839 return "bfxil\\t%<w>0, %<w>2, 0, %3"; 5840 default: 5841 gcc_unreachable (); 5842 } 5843 } 5844 [(set_attr "type" "bfm")] 5845) 5846 5847; Zero-extended version of above (aarch64_bfxil) 5848(define_insn "*aarch64_bfxilsi_uxtw" 5849 [(set (match_operand:DI 0 "register_operand" "=r,r") 5850 (zero_extend:DI (ior:SI (and:SI (match_operand:SI 1 "register_operand" 5851 "r,0") 5852 (match_operand:SI 3 "const_int_operand" "n, Ulc")) 5853 (and:SI (match_operand:SI 2 "register_operand" "0,r") 5854 (match_operand:SI 4 "const_int_operand" "Ulc, n")))))] 5855 "(INTVAL (operands[3]) == ~INTVAL (operands[4])) 5856 && (aarch64_high_bits_all_ones_p (INTVAL (operands[3])) 5857 || aarch64_high_bits_all_ones_p (INTVAL (operands[4])))" 5858 { 5859 switch (which_alternative) 5860 { 5861 case 0: 5862 operands[3] = GEN_INT (ctz_hwi (~INTVAL (operands[3]))); 5863 return "bfxil\\t%w0, %w1, 0, %3"; 5864 case 1: 5865 operands[3] = GEN_INT (ctz_hwi (~INTVAL (operands[4]))); 5866 return "bfxil\\t%w0, %w2, 0, %3"; 5867 default: 5868 gcc_unreachable (); 5869 } 5870 } 5871 [(set_attr "type" "bfm")] 5872) 5873 5874;; There are no canonicalisation rules for the position of the lshiftrt, ashift 5875;; operations within an IOR/AND RTX, therefore we have two patterns matching 5876;; each valid permutation. 5877 5878(define_insn "rev16<mode>2" 5879 [(set (match_operand:GPI 0 "register_operand" "=r") 5880 (ior:GPI (and:GPI (ashift:GPI (match_operand:GPI 1 "register_operand" "r") 5881 (const_int 8)) 5882 (match_operand:GPI 3 "const_int_operand" "n")) 5883 (and:GPI (lshiftrt:GPI (match_dup 1) 5884 (const_int 8)) 5885 (match_operand:GPI 2 "const_int_operand" "n"))))] 5886 "aarch_rev16_shleft_mask_imm_p (operands[3], <MODE>mode) 5887 && aarch_rev16_shright_mask_imm_p (operands[2], <MODE>mode)" 5888 "rev16\\t%<w>0, %<w>1" 5889 [(set_attr "type" "rev")] 5890) 5891 5892(define_insn "rev16<mode>2_alt" 5893 [(set (match_operand:GPI 0 "register_operand" "=r") 5894 (ior:GPI (and:GPI (lshiftrt:GPI (match_operand:GPI 1 "register_operand" "r") 5895 (const_int 8)) 5896 (match_operand:GPI 2 "const_int_operand" "n")) 5897 (and:GPI (ashift:GPI (match_dup 1) 5898 (const_int 8)) 5899 (match_operand:GPI 3 "const_int_operand" "n"))))] 5900 "aarch_rev16_shleft_mask_imm_p (operands[3], <MODE>mode) 5901 && aarch_rev16_shright_mask_imm_p (operands[2], <MODE>mode)" 5902 "rev16\\t%<w>0, %<w>1" 5903 [(set_attr "type" "rev")] 5904) 5905 5906;; zero_extend version of above 5907(define_insn "*bswapsi2_uxtw" 5908 [(set (match_operand:DI 0 "register_operand" "=r") 5909 (zero_extend:DI (bswap:SI (match_operand:SI 1 "register_operand" "r"))))] 5910 "" 5911 "rev\\t%w0, %w1" 5912 [(set_attr "type" "rev")] 5913) 5914 5915;; ------------------------------------------------------------------- 5916;; Floating-point intrinsics 5917;; ------------------------------------------------------------------- 5918 5919;; frint floating-point round to integral standard patterns. 5920;; Expands to btrunc, ceil, floor, nearbyint, rint, round, frintn. 5921 5922(define_insn "<frint_pattern><mode>2" 5923 [(set (match_operand:GPF_F16 0 "register_operand" "=w") 5924 (unspec:GPF_F16 [(match_operand:GPF_F16 1 "register_operand" "w")] 5925 FRINT))] 5926 "TARGET_FLOAT" 5927 "frint<frint_suffix>\\t%<s>0, %<s>1" 5928 [(set_attr "type" "f_rint<stype>")] 5929) 5930 5931;; frcvt floating-point round to integer and convert standard patterns. 5932;; Expands to lbtrunc, lceil, lfloor, lround. 5933(define_insn "l<fcvt_pattern><su_optab><GPF_F16:mode><GPI:mode>2" 5934 [(set (match_operand:GPI 0 "register_operand" "=r") 5935 (FIXUORS:GPI 5936 (unspec:GPF_F16 [(match_operand:GPF_F16 1 "register_operand" "w")] 5937 FCVT)))] 5938 "TARGET_FLOAT" 5939 "fcvt<frint_suffix><su>\\t%<GPI:w>0, %<GPF_F16:s>1" 5940 [(set_attr "type" "f_cvtf2i")] 5941) 5942 5943(define_insn "*aarch64_fcvt<su_optab><GPF:mode><GPI:mode>2_mult" 5944 [(set (match_operand:GPI 0 "register_operand" "=r") 5945 (FIXUORS:GPI 5946 (mult:GPF 5947 (match_operand:GPF 1 "register_operand" "w") 5948 (match_operand:GPF 2 "aarch64_fp_pow2" "F"))))] 5949 "TARGET_FLOAT 5950 && IN_RANGE (aarch64_fpconst_pow_of_2 (operands[2]), 1, 5951 GET_MODE_BITSIZE (<GPI:MODE>mode))" 5952 { 5953 int fbits = aarch64_fpconst_pow_of_2 (operands[2]); 5954 char buf[64]; 5955 snprintf (buf, 64, "fcvtz<su>\\t%%<GPI:w>0, %%<GPF:s>1, #%d", fbits); 5956 output_asm_insn (buf, operands); 5957 return ""; 5958 } 5959 [(set_attr "type" "f_cvtf2i")] 5960) 5961 5962;; fma - expand fma into patterns with the accumulator operand first since 5963;; reusing the accumulator results in better register allocation. 5964;; The register allocator considers copy preferences in operand order, 5965;; so this prefers fmadd s0, s1, s2, s0 over fmadd s1, s1, s2, s0. 5966 5967(define_expand "fma<mode>4" 5968 [(set (match_operand:GPF_F16 0 "register_operand") 5969 (fma:GPF_F16 (match_operand:GPF_F16 1 "register_operand") 5970 (match_operand:GPF_F16 2 "register_operand") 5971 (match_operand:GPF_F16 3 "register_operand")))] 5972 "TARGET_FLOAT" 5973) 5974 5975(define_insn "*aarch64_fma<mode>4" 5976 [(set (match_operand:GPF_F16 0 "register_operand" "=w") 5977 (fma:GPF_F16 (match_operand:GPF_F16 2 "register_operand" "w") 5978 (match_operand:GPF_F16 3 "register_operand" "w") 5979 (match_operand:GPF_F16 1 "register_operand" "w")))] 5980 "TARGET_FLOAT" 5981 "fmadd\\t%<s>0, %<s>2, %<s>3, %<s>1" 5982 [(set_attr "type" "fmac<stype>")] 5983) 5984 5985(define_expand "fnma<mode>4" 5986 [(set (match_operand:GPF_F16 0 "register_operand") 5987 (fma:GPF_F16 5988 (neg:GPF_F16 (match_operand:GPF_F16 1 "register_operand")) 5989 (match_operand:GPF_F16 2 "register_operand") 5990 (match_operand:GPF_F16 3 "register_operand")))] 5991 "TARGET_FLOAT" 5992) 5993 5994(define_insn "*aarch64_fnma<mode>4" 5995 [(set (match_operand:GPF_F16 0 "register_operand" "=w") 5996 (fma:GPF_F16 5997 (neg:GPF_F16 (match_operand:GPF_F16 2 "register_operand" "w")) 5998 (match_operand:GPF_F16 3 "register_operand" "w") 5999 (match_operand:GPF_F16 1 "register_operand" "w")))] 6000 "TARGET_FLOAT" 6001 "fmsub\\t%<s>0, %<s>2, %<s>3, %<s>1" 6002 [(set_attr "type" "fmac<stype>")] 6003) 6004 6005 6006(define_expand "fms<mode>4" 6007 [(set (match_operand:GPF 0 "register_operand") 6008 (fma:GPF (match_operand:GPF 1 "register_operand") 6009 (match_operand:GPF 2 "register_operand") 6010 (neg:GPF (match_operand:GPF 3 "register_operand"))))] 6011 "TARGET_FLOAT" 6012) 6013 6014(define_insn "*aarch64_fms<mode>4" 6015 [(set (match_operand:GPF 0 "register_operand" "=w") 6016 (fma:GPF (match_operand:GPF 2 "register_operand" "w") 6017 (match_operand:GPF 3 "register_operand" "w") 6018 (neg:GPF (match_operand:GPF 1 "register_operand" "w"))))] 6019 "TARGET_FLOAT" 6020 "fnmsub\\t%<s>0, %<s>2, %<s>3, %<s>1" 6021 [(set_attr "type" "fmac<s>")] 6022) 6023 6024(define_expand "fnms<mode>4" 6025 [(set (match_operand:GPF 0 "register_operand") 6026 (fma:GPF (neg:GPF (match_operand:GPF 1 "register_operand")) 6027 (match_operand:GPF 2 "register_operand") 6028 (neg:GPF (match_operand:GPF 3 "register_operand"))))] 6029 "TARGET_FLOAT" 6030) 6031 6032(define_insn "*aarch64_fnms<mode>4" 6033 [(set (match_operand:GPF 0 "register_operand" "=w") 6034 (fma:GPF (neg:GPF (match_operand:GPF 2 "register_operand" "w")) 6035 (match_operand:GPF 3 "register_operand" "w") 6036 (neg:GPF (match_operand:GPF 1 "register_operand" "w"))))] 6037 "TARGET_FLOAT" 6038 "fnmadd\\t%<s>0, %<s>2, %<s>3, %<s>1" 6039 [(set_attr "type" "fmac<s>")] 6040) 6041 6042;; If signed zeros are ignored, -(a * b + c) = -a * b - c. 6043(define_insn "*aarch64_fnmadd<mode>4" 6044 [(set (match_operand:GPF 0 "register_operand" "=w") 6045 (neg:GPF (fma:GPF (match_operand:GPF 2 "register_operand" "w") 6046 (match_operand:GPF 3 "register_operand" "w") 6047 (match_operand:GPF 1 "register_operand" "w"))))] 6048 "!HONOR_SIGNED_ZEROS (<MODE>mode) && TARGET_FLOAT" 6049 "fnmadd\\t%<s>0, %<s>2, %<s>3, %<s>1" 6050 [(set_attr "type" "fmac<s>")] 6051) 6052 6053;; ------------------------------------------------------------------- 6054;; Floating-point conversions 6055;; ------------------------------------------------------------------- 6056 6057(define_insn "extendsfdf2" 6058 [(set (match_operand:DF 0 "register_operand" "=w") 6059 (float_extend:DF (match_operand:SF 1 "register_operand" "w")))] 6060 "TARGET_FLOAT" 6061 "fcvt\\t%d0, %s1" 6062 [(set_attr "type" "f_cvt")] 6063) 6064 6065(define_insn "extendhfsf2" 6066 [(set (match_operand:SF 0 "register_operand" "=w") 6067 (float_extend:SF (match_operand:HF 1 "register_operand" "w")))] 6068 "TARGET_FLOAT" 6069 "fcvt\\t%s0, %h1" 6070 [(set_attr "type" "f_cvt")] 6071) 6072 6073(define_insn "extendhfdf2" 6074 [(set (match_operand:DF 0 "register_operand" "=w") 6075 (float_extend:DF (match_operand:HF 1 "register_operand" "w")))] 6076 "TARGET_FLOAT" 6077 "fcvt\\t%d0, %h1" 6078 [(set_attr "type" "f_cvt")] 6079) 6080 6081(define_insn "truncdfsf2" 6082 [(set (match_operand:SF 0 "register_operand" "=w") 6083 (float_truncate:SF (match_operand:DF 1 "register_operand" "w")))] 6084 "TARGET_FLOAT" 6085 "fcvt\\t%s0, %d1" 6086 [(set_attr "type" "f_cvt")] 6087) 6088 6089(define_insn "truncsfhf2" 6090 [(set (match_operand:HF 0 "register_operand" "=w") 6091 (float_truncate:HF (match_operand:SF 1 "register_operand" "w")))] 6092 "TARGET_FLOAT" 6093 "fcvt\\t%h0, %s1" 6094 [(set_attr "type" "f_cvt")] 6095) 6096 6097(define_insn "truncdfhf2" 6098 [(set (match_operand:HF 0 "register_operand" "=w") 6099 (float_truncate:HF (match_operand:DF 1 "register_operand" "w")))] 6100 "TARGET_FLOAT" 6101 "fcvt\\t%h0, %d1" 6102 [(set_attr "type" "f_cvt")] 6103) 6104 6105;; Convert SF -> SI or DF -> DI while preferring w = w register constraints 6106;; and making r = w more expensive 6107 6108(define_insn "<optab>_trunc<fcvt_target><GPI:mode>2" 6109 [(set (match_operand:GPI 0 "register_operand" "=w,?r") 6110 (FIXUORS:GPI (match_operand:<FCVT_TARGET> 1 "register_operand" "w,w")))] 6111 "TARGET_FLOAT" 6112 "@ 6113 fcvtz<su>\t%<s>0, %<s>1 6114 fcvtz<su>\t%<w>0, %<s>1" 6115 [(set_attr "type" "neon_fp_to_int_s,f_cvtf2i") 6116 (set_attr "arch" "simd,fp")] 6117) 6118 6119;; Convert HF -> SI or DI 6120 6121(define_insn "<optab>_trunchf<GPI:mode>2" 6122 [(set (match_operand:GPI 0 "register_operand" "=r") 6123 (FIXUORS:GPI (match_operand:HF 1 "register_operand" "w")))] 6124 "TARGET_FP_F16INST" 6125 "fcvtz<su>\t%<w>0, %h1" 6126 [(set_attr "type" "f_cvtf2i")] 6127) 6128 6129;; Convert DF -> SI or SF -> DI which can only be accomplished with 6130;; input in a fp register and output in a integer register 6131 6132(define_insn "<optab>_trunc<fcvt_change_mode><GPI:mode>2" 6133 [(set (match_operand:GPI 0 "register_operand" "=r") 6134 (FIXUORS:GPI (match_operand:<FCVT_CHANGE_MODE> 1 "register_operand" "w")))] 6135 "TARGET_FLOAT" 6136 "fcvtz<su>\t%<w>0, %<fpw>1" 6137 [(set_attr "type" "f_cvtf2i")] 6138) 6139 6140(define_insn "*fix_to_zero_extend<mode>di2" 6141 [(set (match_operand:DI 0 "register_operand" "=r") 6142 (zero_extend:DI 6143 (unsigned_fix:SI 6144 (match_operand:GPF 1 "register_operand" "w"))))] 6145 "TARGET_FLOAT" 6146 "fcvtzu\t%w0, %<s>1" 6147 [(set_attr "type" "f_cvtf2i")] 6148) 6149 6150;; Equal width integer to fp and multiply combine. 6151(define_insn "*aarch64_<su_optab>cvtf<fcvt_target><GPF:mode>2_mult" 6152 [(set (match_operand:GPF 0 "register_operand" "=w,w") 6153 (mult:GPF (FLOATUORS:GPF 6154 (match_operand:<FCVT_TARGET> 1 "register_operand" "w,?r")) 6155 (match_operand:GPF 2 "aarch64_fp_pow2_recip" "Dt,Dt")))] 6156 "TARGET_FLOAT" 6157 { 6158 operands[2] = GEN_INT (aarch64_fpconst_pow2_recip (operands[2])); 6159 switch (which_alternative) 6160 { 6161 case 0: 6162 return "<su_optab>cvtf\t%<GPF:s>0, %<s>1, #%2"; 6163 case 1: 6164 return "<su_optab>cvtf\t%<GPF:s>0, %<w1>1, #%2"; 6165 default: 6166 gcc_unreachable (); 6167 } 6168 } 6169 [(set_attr "type" "neon_int_to_fp_<Vetype>,f_cvti2f") 6170 (set_attr "arch" "simd,fp")] 6171) 6172 6173;; Unequal width integer to fp and multiply combine. 6174(define_insn "*aarch64_<su_optab>cvtf<fcvt_iesize><GPF:mode>2_mult" 6175 [(set (match_operand:GPF 0 "register_operand" "=w") 6176 (mult:GPF (FLOATUORS:GPF 6177 (match_operand:<FCVT_IESIZE> 1 "register_operand" "r")) 6178 (match_operand:GPF 2 "aarch64_fp_pow2_recip" "Dt")))] 6179 "TARGET_FLOAT" 6180 { 6181 operands[2] = GEN_INT (aarch64_fpconst_pow2_recip (operands[2])); 6182 return "<su_optab>cvtf\t%<GPF:s>0, %<w2>1, #%2"; 6183 } 6184 [(set_attr "type" "f_cvti2f")] 6185) 6186 6187;; Equal width integer to fp conversion. 6188(define_insn "<optab><fcvt_target><GPF:mode>2" 6189 [(set (match_operand:GPF 0 "register_operand" "=w,w") 6190 (FLOATUORS:GPF (match_operand:<FCVT_TARGET> 1 "register_operand" "w,?r")))] 6191 "TARGET_FLOAT" 6192 "@ 6193 <su_optab>cvtf\t%<GPF:s>0, %<s>1 6194 <su_optab>cvtf\t%<GPF:s>0, %<w1>1" 6195 [(set_attr "type" "neon_int_to_fp_<Vetype>,f_cvti2f") 6196 (set_attr "arch" "simd,fp")] 6197) 6198 6199;; Unequal width integer to fp conversions. 6200(define_insn "<optab><fcvt_iesize><GPF:mode>2" 6201 [(set (match_operand:GPF 0 "register_operand" "=w") 6202 (FLOATUORS:GPF (match_operand:<FCVT_IESIZE> 1 "register_operand" "r")))] 6203 "TARGET_FLOAT" 6204 "<su_optab>cvtf\t%<GPF:s>0, %<w2>1" 6205 [(set_attr "type" "f_cvti2f")] 6206) 6207 6208;; If we do not have ARMv8.2-A 16-bit floating point extensions, the 6209;; midend will arrange for an SImode conversion to HFmode to first go 6210;; through DFmode, then to HFmode. But first it will try converting 6211;; to DImode then down, which would match our DImode pattern below and 6212;; give very poor code-generation. So, we must provide our own emulation 6213;; of the mid-end logic. 6214 6215(define_insn "aarch64_fp16_<optab><mode>hf2" 6216 [(set (match_operand:HF 0 "register_operand" "=w") 6217 (FLOATUORS:HF (match_operand:GPI 1 "register_operand" "r")))] 6218 "TARGET_FP_F16INST" 6219 "<su_optab>cvtf\t%h0, %<w>1" 6220 [(set_attr "type" "f_cvti2f")] 6221) 6222 6223(define_expand "<optab>sihf2" 6224 [(set (match_operand:HF 0 "register_operand") 6225 (FLOATUORS:HF (match_operand:SI 1 "register_operand")))] 6226 "TARGET_FLOAT" 6227{ 6228 if (TARGET_FP_F16INST) 6229 emit_insn (gen_aarch64_fp16_<optab>sihf2 (operands[0], operands[1])); 6230 else 6231 { 6232 rtx convert_target = gen_reg_rtx (DFmode); 6233 emit_insn (gen_<optab>sidf2 (convert_target, operands[1])); 6234 emit_insn (gen_truncdfhf2 (operands[0], convert_target)); 6235 } 6236 DONE; 6237} 6238) 6239 6240;; For DImode there is no wide enough floating-point mode that we 6241;; can convert through natively (TFmode would work, but requires a library 6242;; call). However, we know that any value >= 65504 will be rounded 6243;; to infinity on conversion. This is well within the range of SImode, so 6244;; we can: 6245;; Saturate to SImode. 6246;; Convert from that to DFmode 6247;; Convert from that to HFmode (phew!). 6248;; Note that the saturation to SImode requires the SIMD extensions. If 6249;; we ever need to provide this pattern where the SIMD extensions are not 6250;; available, we would need a different approach. 6251 6252(define_expand "<optab>dihf2" 6253 [(set (match_operand:HF 0 "register_operand") 6254 (FLOATUORS:HF (match_operand:DI 1 "register_operand")))] 6255 "TARGET_FLOAT && (TARGET_FP_F16INST || TARGET_SIMD)" 6256{ 6257 if (TARGET_FP_F16INST) 6258 emit_insn (gen_aarch64_fp16_<optab>dihf2 (operands[0], operands[1])); 6259 else 6260 { 6261 rtx sat_target = gen_reg_rtx (SImode); 6262 emit_insn (gen_aarch64_<su_optab>qmovndi (sat_target, operands[1])); 6263 emit_insn (gen_<optab>sihf2 (operands[0], sat_target)); 6264 } 6265 6266 DONE; 6267} 6268) 6269 6270;; Convert between fixed-point and floating-point (scalar modes) 6271 6272(define_insn "<FCVT_F2FIXED:fcvt_fixed_insn><GPF:mode>3" 6273 [(set (match_operand:<GPF:FCVT_TARGET> 0 "register_operand" "=r, w") 6274 (unspec:<GPF:FCVT_TARGET> [(match_operand:GPF 1 "register_operand" "w, w") 6275 (match_operand:SI 2 "immediate_operand" "i, i")] 6276 FCVT_F2FIXED))] 6277 "" 6278 "@ 6279 <FCVT_F2FIXED:fcvt_fixed_insn>\t%<GPF:w1>0, %<GPF:s>1, #%2 6280 <FCVT_F2FIXED:fcvt_fixed_insn>\t%<GPF:s>0, %<GPF:s>1, #%2" 6281 [(set_attr "type" "f_cvtf2i, neon_fp_to_int_<GPF:Vetype>") 6282 (set_attr "arch" "fp,simd")] 6283) 6284 6285(define_insn "<FCVT_FIXED2F:fcvt_fixed_insn><GPI:mode>3" 6286 [(set (match_operand:<GPI:FCVT_TARGET> 0 "register_operand" "=w, w") 6287 (unspec:<GPI:FCVT_TARGET> [(match_operand:GPI 1 "register_operand" "r, w") 6288 (match_operand:SI 2 "immediate_operand" "i, i")] 6289 FCVT_FIXED2F))] 6290 "" 6291 "@ 6292 <FCVT_FIXED2F:fcvt_fixed_insn>\t%<GPI:v>0, %<GPI:w>1, #%2 6293 <FCVT_FIXED2F:fcvt_fixed_insn>\t%<GPI:v>0, %<GPI:v>1, #%2" 6294 [(set_attr "type" "f_cvti2f, neon_int_to_fp_<GPI:Vetype>") 6295 (set_attr "arch" "fp,simd")] 6296) 6297 6298(define_insn "<FCVT_F2FIXED:fcvt_fixed_insn>hf<mode>3" 6299 [(set (match_operand:GPI 0 "register_operand" "=r") 6300 (unspec:GPI [(match_operand:HF 1 "register_operand" "w") 6301 (match_operand:SI 2 "immediate_operand" "i")] 6302 FCVT_F2FIXED))] 6303 "TARGET_FP_F16INST" 6304 "<FCVT_F2FIXED:fcvt_fixed_insn>\t%<GPI:w>0, %h1, #%2" 6305 [(set_attr "type" "f_cvtf2i")] 6306) 6307 6308(define_insn "<FCVT_FIXED2F:fcvt_fixed_insn><mode>hf3" 6309 [(set (match_operand:HF 0 "register_operand" "=w") 6310 (unspec:HF [(match_operand:GPI 1 "register_operand" "r") 6311 (match_operand:SI 2 "immediate_operand" "i")] 6312 FCVT_FIXED2F))] 6313 "TARGET_FP_F16INST" 6314 "<FCVT_FIXED2F:fcvt_fixed_insn>\t%h0, %<GPI:w>1, #%2" 6315 [(set_attr "type" "f_cvti2f")] 6316) 6317 6318(define_insn "<FCVT_F2FIXED:fcvt_fixed_insn>hf3" 6319 [(set (match_operand:HI 0 "register_operand" "=w") 6320 (unspec:HI [(match_operand:HF 1 "register_operand" "w") 6321 (match_operand:SI 2 "immediate_operand" "i")] 6322 FCVT_F2FIXED))] 6323 "TARGET_SIMD" 6324 "<FCVT_F2FIXED:fcvt_fixed_insn>\t%h0, %h1, #%2" 6325 [(set_attr "type" "neon_fp_to_int_s")] 6326) 6327 6328(define_insn "<FCVT_FIXED2F:fcvt_fixed_insn>hi3" 6329 [(set (match_operand:HF 0 "register_operand" "=w") 6330 (unspec:HF [(match_operand:HI 1 "register_operand" "w") 6331 (match_operand:SI 2 "immediate_operand" "i")] 6332 FCVT_FIXED2F))] 6333 "TARGET_SIMD" 6334 "<FCVT_FIXED2F:fcvt_fixed_insn>\t%h0, %h1, #%2" 6335 [(set_attr "type" "neon_int_to_fp_s")] 6336) 6337 6338;; ------------------------------------------------------------------- 6339;; Floating-point arithmetic 6340;; ------------------------------------------------------------------- 6341 6342(define_insn "add<mode>3" 6343 [(set (match_operand:GPF_F16 0 "register_operand" "=w") 6344 (plus:GPF_F16 6345 (match_operand:GPF_F16 1 "register_operand" "w") 6346 (match_operand:GPF_F16 2 "register_operand" "w")))] 6347 "TARGET_FLOAT" 6348 "fadd\\t%<s>0, %<s>1, %<s>2" 6349 [(set_attr "type" "fadd<stype>")] 6350) 6351 6352(define_insn "sub<mode>3" 6353 [(set (match_operand:GPF_F16 0 "register_operand" "=w") 6354 (minus:GPF_F16 6355 (match_operand:GPF_F16 1 "register_operand" "w") 6356 (match_operand:GPF_F16 2 "register_operand" "w")))] 6357 "TARGET_FLOAT" 6358 "fsub\\t%<s>0, %<s>1, %<s>2" 6359 [(set_attr "type" "fadd<stype>")] 6360) 6361 6362(define_insn "mul<mode>3" 6363 [(set (match_operand:GPF_F16 0 "register_operand" "=w") 6364 (mult:GPF_F16 6365 (match_operand:GPF_F16 1 "register_operand" "w") 6366 (match_operand:GPF_F16 2 "register_operand" "w")))] 6367 "TARGET_FLOAT" 6368 "fmul\\t%<s>0, %<s>1, %<s>2" 6369 [(set_attr "type" "fmul<stype>")] 6370) 6371 6372(define_insn "*fnmul<mode>3" 6373 [(set (match_operand:GPF 0 "register_operand" "=w") 6374 (mult:GPF 6375 (neg:GPF (match_operand:GPF 1 "register_operand" "w")) 6376 (match_operand:GPF 2 "register_operand" "w")))] 6377 "TARGET_FLOAT && !flag_rounding_math" 6378 "fnmul\\t%<s>0, %<s>1, %<s>2" 6379 [(set_attr "type" "fmul<s>")] 6380) 6381 6382(define_insn "*fnmul<mode>3" 6383 [(set (match_operand:GPF 0 "register_operand" "=w") 6384 (neg:GPF (mult:GPF 6385 (match_operand:GPF 1 "register_operand" "w") 6386 (match_operand:GPF 2 "register_operand" "w"))))] 6387 "TARGET_FLOAT" 6388 "fnmul\\t%<s>0, %<s>1, %<s>2" 6389 [(set_attr "type" "fmul<s>")] 6390) 6391 6392(define_expand "div<mode>3" 6393 [(set (match_operand:GPF_F16 0 "register_operand") 6394 (div:GPF_F16 (match_operand:GPF_F16 1 "general_operand") 6395 (match_operand:GPF_F16 2 "register_operand")))] 6396 "TARGET_FLOAT" 6397{ 6398 if (aarch64_emit_approx_div (operands[0], operands[1], operands[2])) 6399 DONE; 6400 6401 operands[1] = force_reg (<MODE>mode, operands[1]); 6402}) 6403 6404(define_insn "*div<mode>3" 6405 [(set (match_operand:GPF_F16 0 "register_operand" "=w") 6406 (div:GPF_F16 (match_operand:GPF_F16 1 "register_operand" "w") 6407 (match_operand:GPF_F16 2 "register_operand" "w")))] 6408 "TARGET_FLOAT" 6409 "fdiv\\t%<s>0, %<s>1, %<s>2" 6410 [(set_attr "type" "fdiv<stype>")] 6411) 6412 6413(define_insn "neg<mode>2" 6414 [(set (match_operand:GPF_F16 0 "register_operand" "=w") 6415 (neg:GPF_F16 (match_operand:GPF_F16 1 "register_operand" "w")))] 6416 "TARGET_FLOAT" 6417 "fneg\\t%<s>0, %<s>1" 6418 [(set_attr "type" "ffarith<stype>")] 6419) 6420 6421(define_expand "sqrt<mode>2" 6422 [(set (match_operand:GPF_F16 0 "register_operand") 6423 (sqrt:GPF_F16 (match_operand:GPF_F16 1 "register_operand")))] 6424 "TARGET_FLOAT" 6425{ 6426 if (aarch64_emit_approx_sqrt (operands[0], operands[1], false)) 6427 DONE; 6428}) 6429 6430(define_insn "*sqrt<mode>2" 6431 [(set (match_operand:GPF_F16 0 "register_operand" "=w") 6432 (sqrt:GPF_F16 (match_operand:GPF_F16 1 "register_operand" "w")))] 6433 "TARGET_FLOAT" 6434 "fsqrt\\t%<s>0, %<s>1" 6435 [(set_attr "type" "fsqrt<stype>")] 6436) 6437 6438(define_insn "abs<mode>2" 6439 [(set (match_operand:GPF_F16 0 "register_operand" "=w") 6440 (abs:GPF_F16 (match_operand:GPF_F16 1 "register_operand" "w")))] 6441 "TARGET_FLOAT" 6442 "fabs\\t%<s>0, %<s>1" 6443 [(set_attr "type" "ffarith<stype>")] 6444) 6445 6446;; Given that smax/smin do not specify the result when either input is NaN, 6447;; we could use either FMAXNM or FMAX for smax, and either FMINNM or FMIN 6448;; for smin. 6449 6450(define_insn "smax<mode>3" 6451 [(set (match_operand:GPF 0 "register_operand" "=w") 6452 (smax:GPF (match_operand:GPF 1 "register_operand" "w") 6453 (match_operand:GPF 2 "register_operand" "w")))] 6454 "TARGET_FLOAT" 6455 "fmaxnm\\t%<s>0, %<s>1, %<s>2" 6456 [(set_attr "type" "f_minmax<s>")] 6457) 6458 6459(define_insn "smin<mode>3" 6460 [(set (match_operand:GPF 0 "register_operand" "=w") 6461 (smin:GPF (match_operand:GPF 1 "register_operand" "w") 6462 (match_operand:GPF 2 "register_operand" "w")))] 6463 "TARGET_FLOAT" 6464 "fminnm\\t%<s>0, %<s>1, %<s>2" 6465 [(set_attr "type" "f_minmax<s>")] 6466) 6467 6468;; Scalar forms for fmax, fmin, fmaxnm, fminnm. 6469;; fmaxnm and fminnm are used for the fmax<mode>3 standard pattern names, 6470;; which implement the IEEE fmax ()/fmin () functions. 6471(define_insn "<maxmin_uns><mode>3" 6472 [(set (match_operand:GPF_F16 0 "register_operand" "=w") 6473 (unspec:GPF_F16 [(match_operand:GPF_F16 1 "register_operand" "w") 6474 (match_operand:GPF_F16 2 "register_operand" "w")] 6475 FMAXMIN_UNS))] 6476 "TARGET_FLOAT" 6477 "<maxmin_uns_op>\\t%<s>0, %<s>1, %<s>2" 6478 [(set_attr "type" "f_minmax<stype>")] 6479) 6480 6481(define_expand "lrint<GPF:mode><GPI:mode>2" 6482 [(match_operand:GPI 0 "register_operand") 6483 (match_operand:GPF 1 "register_operand")] 6484 "TARGET_FLOAT 6485 && ((GET_MODE_BITSIZE (<GPF:MODE>mode) <= LONG_TYPE_SIZE) 6486 || !flag_trapping_math || flag_fp_int_builtin_inexact)" 6487{ 6488 rtx cvt = gen_reg_rtx (<GPF:MODE>mode); 6489 emit_insn (gen_rint<GPF:mode>2 (cvt, operands[1])); 6490 emit_insn (gen_lbtrunc<GPF:mode><GPI:mode>2 (operands[0], cvt)); 6491 DONE; 6492} 6493) 6494 6495;; For copysign (x, y), we want to generate: 6496;; 6497;; LDR d2, #(1 << 63) 6498;; BSL v2.8b, [y], [x] 6499;; 6500;; or another, equivalent, sequence using one of BSL/BIT/BIF. Because 6501;; we expect these operations to nearly always operate on 6502;; floating-point values, we do not want the operation to be 6503;; simplified into a bit-field insert operation that operates on the 6504;; integer side, since typically that would involve three inter-bank 6505;; register copies. As we do not expect copysign to be followed by 6506;; other logical operations on the result, it seems preferable to keep 6507;; this as an unspec operation, rather than exposing the underlying 6508;; logic to the compiler. 6509 6510(define_expand "copysign<GPF:mode>3" 6511 [(match_operand:GPF 0 "register_operand") 6512 (match_operand:GPF 1 "register_operand") 6513 (match_operand:GPF 2 "register_operand")] 6514 "TARGET_FLOAT && TARGET_SIMD" 6515{ 6516 rtx bitmask = gen_reg_rtx (<V_INT_EQUIV>mode); 6517 emit_move_insn (bitmask, GEN_INT (HOST_WIDE_INT_M1U 6518 << (GET_MODE_BITSIZE (<MODE>mode) - 1))); 6519 emit_insn (gen_copysign<mode>3_insn (operands[0], operands[1], operands[2], 6520 bitmask)); 6521 DONE; 6522} 6523) 6524 6525(define_insn "copysign<GPF:mode>3_insn" 6526 [(set (match_operand:GPF 0 "register_operand" "=w,w,w,r") 6527 (unspec:GPF [(match_operand:GPF 1 "register_operand" "w,0,w,r") 6528 (match_operand:GPF 2 "register_operand" "w,w,0,0") 6529 (match_operand:<V_INT_EQUIV> 3 "register_operand" "0,w,w,X")] 6530 UNSPEC_COPYSIGN))] 6531 "TARGET_FLOAT && TARGET_SIMD" 6532 "@ 6533 bsl\\t%0.<Vbtype>, %2.<Vbtype>, %1.<Vbtype> 6534 bit\\t%0.<Vbtype>, %2.<Vbtype>, %3.<Vbtype> 6535 bif\\t%0.<Vbtype>, %1.<Vbtype>, %3.<Vbtype> 6536 bfxil\\t%<w1>0, %<w1>1, #0, <sizem1>" 6537 [(set_attr "type" "neon_bsl<q>,neon_bsl<q>,neon_bsl<q>,bfm")] 6538) 6539 6540 6541;; For xorsign (x, y), we want to generate: 6542;; 6543;; LDR d2, #1<<63 6544;; AND v3.8B, v1.8B, v2.8B 6545;; EOR v0.8B, v0.8B, v3.8B 6546;; 6547 6548(define_expand "xorsign<mode>3" 6549 [(match_operand:GPF 0 "register_operand") 6550 (match_operand:GPF 1 "register_operand") 6551 (match_operand:GPF 2 "register_operand")] 6552 "TARGET_FLOAT && TARGET_SIMD" 6553{ 6554 6555 machine_mode imode = <V_INT_EQUIV>mode; 6556 rtx mask = gen_reg_rtx (imode); 6557 rtx op1x = gen_reg_rtx (imode); 6558 rtx op2x = gen_reg_rtx (imode); 6559 6560 int bits = GET_MODE_BITSIZE (<MODE>mode) - 1; 6561 emit_move_insn (mask, GEN_INT (trunc_int_for_mode (HOST_WIDE_INT_M1U << bits, 6562 imode))); 6563 6564 emit_insn (gen_and<v_int_equiv>3 (op2x, mask, 6565 lowpart_subreg (imode, operands[2], 6566 <MODE>mode))); 6567 emit_insn (gen_xor<v_int_equiv>3 (op1x, 6568 lowpart_subreg (imode, operands[1], 6569 <MODE>mode), 6570 op2x)); 6571 emit_move_insn (operands[0], 6572 lowpart_subreg (<MODE>mode, op1x, imode)); 6573 DONE; 6574} 6575) 6576 6577;; ------------------------------------------------------------------- 6578;; Reload support 6579;; ------------------------------------------------------------------- 6580;; Reload Scalar Floating point modes from constant pool. 6581;; The AArch64 port doesn't have __int128 constant move support. 6582;; The patterns need constraints due to TARGET_SECONDARY_RELOAD hook. 6583(define_expand "@aarch64_reload_movcp<GPF_TF:mode><P:mode>" 6584 [(set (match_operand:GPF_TF 0 "register_operand" "=w") 6585 (mem:GPF_TF (match_operand 1 "aarch64_constant_pool_symref" "S"))) 6586 (clobber (match_operand:P 2 "register_operand" "=&r"))] 6587 "TARGET_FLOAT" 6588 { 6589 aarch64_expand_mov_immediate (operands[2], XEXP (operands[1], 0)); 6590 emit_move_insn (operands[0], gen_rtx_MEM (<GPF_TF:MODE>mode, operands[2])); 6591 DONE; 6592 } 6593) 6594 6595;; Reload Vector modes from constant pool. 6596(define_expand "@aarch64_reload_movcp<VALL:mode><P:mode>" 6597 [(set (match_operand:VALL 0 "register_operand" "=w") 6598 (mem:VALL (match_operand 1 "aarch64_constant_pool_symref" "S"))) 6599 (clobber (match_operand:P 2 "register_operand" "=&r"))] 6600 "TARGET_FLOAT" 6601 { 6602 aarch64_expand_mov_immediate (operands[2], XEXP (operands[1], 0)); 6603 emit_move_insn (operands[0], gen_rtx_MEM (<VALL:MODE>mode, operands[2])); 6604 DONE; 6605 } 6606) 6607 6608(define_expand "@aarch64_reload_mov<mode>" 6609 [(set (match_operand:TX 0 "register_operand" "=w") 6610 (match_operand:TX 1 "register_operand" "w")) 6611 (clobber (match_operand:DI 2 "register_operand" "=&r")) 6612 ] 6613 "TARGET_FLOAT" 6614 { 6615 rtx op0 = simplify_gen_subreg (TImode, operands[0], <MODE>mode, 0); 6616 rtx op1 = simplify_gen_subreg (TImode, operands[1], <MODE>mode, 0); 6617 gen_aarch64_movtilow_tilow (op0, op1); 6618 gen_aarch64_movdi_tihigh (operands[2], op1); 6619 gen_aarch64_movtihigh_di (op0, operands[2]); 6620 DONE; 6621 } 6622) 6623 6624;; The following secondary reload helpers patterns are invoked 6625;; after or during reload as we don't want these patterns to start 6626;; kicking in during the combiner. 6627 6628(define_insn "@aarch64_movdi_<mode>low" 6629 [(set (match_operand:DI 0 "register_operand" "=r") 6630 (zero_extract:DI (match_operand:TX 1 "register_operand" "w") 6631 (const_int 64) (const_int 0)))] 6632 "TARGET_FLOAT && (reload_completed || reload_in_progress)" 6633 "fmov\\t%x0, %d1" 6634 [(set_attr "type" "f_mrc") 6635 (set_attr "length" "4") 6636 ]) 6637 6638(define_insn "@aarch64_movdi_<mode>high" 6639 [(set (match_operand:DI 0 "register_operand" "=r") 6640 (zero_extract:DI (match_operand:TX 1 "register_operand" "w") 6641 (const_int 64) (const_int 64)))] 6642 "TARGET_FLOAT && (reload_completed || reload_in_progress)" 6643 "fmov\\t%x0, %1.d[1]" 6644 [(set_attr "type" "f_mrc") 6645 (set_attr "length" "4") 6646 ]) 6647 6648(define_insn "@aarch64_mov<mode>high_di" 6649 [(set (zero_extract:TX (match_operand:TX 0 "register_operand" "+w") 6650 (const_int 64) (const_int 64)) 6651 (zero_extend:TX (match_operand:DI 1 "register_operand" "r")))] 6652 "TARGET_FLOAT && (reload_completed || reload_in_progress)" 6653 "fmov\\t%0.d[1], %x1" 6654 [(set_attr "type" "f_mcr") 6655 (set_attr "length" "4") 6656 ]) 6657 6658(define_insn "@aarch64_mov<mode>low_di" 6659 [(set (match_operand:TX 0 "register_operand" "=w") 6660 (zero_extend:TX (match_operand:DI 1 "register_operand" "r")))] 6661 "TARGET_FLOAT && (reload_completed || reload_in_progress)" 6662 "fmov\\t%d0, %x1" 6663 [(set_attr "type" "f_mcr") 6664 (set_attr "length" "4") 6665 ]) 6666 6667(define_insn "aarch64_movtilow_tilow" 6668 [(set (match_operand:TI 0 "register_operand" "=w") 6669 (zero_extend:TI 6670 (truncate:DI (match_operand:TI 1 "register_operand" "w"))))] 6671 "TARGET_FLOAT && (reload_completed || reload_in_progress)" 6672 "fmov\\t%d0, %d1" 6673 [(set_attr "type" "fmov") 6674 (set_attr "length" "4") 6675 ]) 6676 6677;; There is a deliberate reason why the parameters of high and lo_sum's 6678;; don't have modes for ADRP and ADD instructions. This is to allow high 6679;; and lo_sum's to be used with the labels defining the jump tables in 6680;; rodata section. 6681 6682(define_expand "add_losym" 6683 [(set (match_operand 0 "register_operand") 6684 (lo_sum (match_operand 1 "register_operand") 6685 (match_operand 2 "aarch64_valid_symref")))] 6686 "" 6687{ 6688 machine_mode mode = GET_MODE (operands[0]); 6689 6690 emit_insn ((mode == DImode 6691 ? gen_add_losym_di 6692 : gen_add_losym_si) (operands[0], 6693 operands[1], 6694 operands[2])); 6695 DONE; 6696}) 6697 6698(define_insn "add_losym_<mode>" 6699 [(set (match_operand:P 0 "register_operand" "=r") 6700 (lo_sum:P (match_operand:P 1 "register_operand" "r") 6701 (match_operand 2 "aarch64_valid_symref" "S")))] 6702 "" 6703 "add\\t%<w>0, %<w>1, :lo12:%c2" 6704 [(set_attr "type" "alu_imm")] 6705) 6706 6707(define_insn "ldr_got_small_<mode>" 6708 [(set (match_operand:PTR 0 "register_operand" "=r") 6709 (unspec:PTR [(mem:PTR (lo_sum:PTR 6710 (match_operand:PTR 1 "register_operand" "r") 6711 (match_operand:PTR 2 "aarch64_valid_symref" "S")))] 6712 UNSPEC_GOTSMALLPIC))] 6713 "" 6714 "ldr\\t%<w>0, [%1, #:got_lo12:%c2]" 6715 [(set_attr "type" "load_<ldst_sz>")] 6716) 6717 6718(define_insn "ldr_got_small_sidi" 6719 [(set (match_operand:DI 0 "register_operand" "=r") 6720 (zero_extend:DI 6721 (unspec:SI [(mem:SI (lo_sum:DI 6722 (match_operand:DI 1 "register_operand" "r") 6723 (match_operand:DI 2 "aarch64_valid_symref" "S")))] 6724 UNSPEC_GOTSMALLPIC)))] 6725 "TARGET_ILP32" 6726 "ldr\\t%w0, [%1, #:got_lo12:%c2]" 6727 [(set_attr "type" "load_4")] 6728) 6729 6730(define_insn "ldr_got_small_28k_<mode>" 6731 [(set (match_operand:PTR 0 "register_operand" "=r") 6732 (unspec:PTR [(mem:PTR (lo_sum:PTR 6733 (match_operand:PTR 1 "register_operand" "r") 6734 (match_operand:PTR 2 "aarch64_valid_symref" "S")))] 6735 UNSPEC_GOTSMALLPIC28K))] 6736 "" 6737 "ldr\\t%<w>0, [%1, #:<got_modifier>:%c2]" 6738 [(set_attr "type" "load_<ldst_sz>")] 6739) 6740 6741(define_insn "ldr_got_small_28k_sidi" 6742 [(set (match_operand:DI 0 "register_operand" "=r") 6743 (zero_extend:DI 6744 (unspec:SI [(mem:SI (lo_sum:DI 6745 (match_operand:DI 1 "register_operand" "r") 6746 (match_operand:DI 2 "aarch64_valid_symref" "S")))] 6747 UNSPEC_GOTSMALLPIC28K)))] 6748 "TARGET_ILP32" 6749 "ldr\\t%w0, [%1, #:gotpage_lo14:%c2]" 6750 [(set_attr "type" "load_4")] 6751) 6752 6753(define_insn "@ldr_got_tiny_<mode>" 6754 [(set (match_operand:PTR 0 "register_operand" "=r") 6755 (unspec:PTR [(match_operand:PTR 1 "aarch64_valid_symref" "S")] 6756 UNSPEC_GOTTINYPIC))] 6757 "" 6758 "ldr\t%<w>0, %L1" 6759 [(set_attr "type" "load_<ldst_sz>")] 6760) 6761 6762(define_insn "ldr_got_tiny_sidi" 6763 [(set (match_operand:DI 0 "register_operand" "=r") 6764 (zero_extend:DI 6765 (unspec:SI [(match_operand:DI 1 "aarch64_valid_symref" "S")] 6766 UNSPEC_GOTTINYPIC)))] 6767 "TARGET_ILP32" 6768 "ldr\t%w0, %L1" 6769 [(set_attr "type" "load_4")] 6770) 6771 6772(define_insn "aarch64_load_tp_hard" 6773 [(set (match_operand:DI 0 "register_operand" "=r") 6774 (unspec:DI [(const_int 0)] UNSPEC_TLS))] 6775 "" 6776 "mrs\\t%0, tpidr_el0" 6777 [(set_attr "type" "mrs")] 6778) 6779 6780;; The TLS ABI specifically requires that the compiler does not schedule 6781;; instructions in the TLS stubs, in order to enable linker relaxation. 6782;; Therefore we treat the stubs as an atomic sequence. 6783(define_expand "tlsgd_small_<mode>" 6784 [(parallel [(set (match_operand:PTR 0 "register_operand") 6785 (call (mem:DI (match_dup 2)) (const_int 1))) 6786 (unspec:DI [(const_int 0)] UNSPEC_CALLEE_ABI) 6787 (unspec:DI [(match_operand 1 "aarch64_valid_symref")] UNSPEC_GOTSMALLTLS) 6788 (clobber (reg:DI LR_REGNUM))])] 6789 "" 6790{ 6791 operands[2] = aarch64_tls_get_addr (); 6792}) 6793 6794(define_insn "*tlsgd_small_<mode>" 6795 [(set (match_operand:PTR 0 "register_operand" "") 6796 (call (mem:DI (match_operand:DI 2 "" "")) (const_int 1))) 6797 (unspec:DI [(const_int 0)] UNSPEC_CALLEE_ABI) 6798 (unspec:DI [(match_operand 1 "aarch64_valid_symref" "S")] UNSPEC_GOTSMALLTLS) 6799 (clobber (reg:DI LR_REGNUM)) 6800 ] 6801 "" 6802 "adrp\\tx0, %A1\;add\\tx0, x0, %L1\;bl\\t%2\;nop" 6803 [(set_attr "type" "call") 6804 (set_attr "length" "16")]) 6805 6806(define_insn "tlsie_small_<mode>" 6807 [(set (match_operand:PTR 0 "register_operand" "=r") 6808 (unspec:PTR [(match_operand 1 "aarch64_tls_ie_symref" "S")] 6809 UNSPEC_GOTSMALLTLS))] 6810 "" 6811 "adrp\\t%0, %A1\;ldr\\t%<w>0, [%0, #%L1]" 6812 [(set_attr "type" "load_4") 6813 (set_attr "length" "8")] 6814) 6815 6816(define_insn "tlsie_small_sidi" 6817 [(set (match_operand:DI 0 "register_operand" "=r") 6818 (zero_extend:DI 6819 (unspec:SI [(match_operand 1 "aarch64_tls_ie_symref" "S")] 6820 UNSPEC_GOTSMALLTLS)))] 6821 "" 6822 "adrp\\t%0, %A1\;ldr\\t%w0, [%0, #%L1]" 6823 [(set_attr "type" "load_4") 6824 (set_attr "length" "8")] 6825) 6826 6827(define_insn "tlsie_tiny_<mode>" 6828 [(set (match_operand:PTR 0 "register_operand" "=&r") 6829 (unspec:PTR [(match_operand 1 "aarch64_tls_ie_symref" "S") 6830 (match_operand:PTR 2 "register_operand" "r")] 6831 UNSPEC_GOTTINYTLS))] 6832 "" 6833 "ldr\\t%<w>0, %L1\;add\\t%<w>0, %<w>0, %<w>2" 6834 [(set_attr "type" "multiple") 6835 (set_attr "length" "8")] 6836) 6837 6838(define_insn "tlsie_tiny_sidi" 6839 [(set (match_operand:DI 0 "register_operand" "=&r") 6840 (zero_extend:DI 6841 (unspec:SI [(match_operand 1 "aarch64_tls_ie_symref" "S") 6842 (match_operand:DI 2 "register_operand" "r") 6843 ] 6844 UNSPEC_GOTTINYTLS)))] 6845 "" 6846 "ldr\\t%w0, %L1\;add\\t%w0, %w0, %w2" 6847 [(set_attr "type" "multiple") 6848 (set_attr "length" "8")] 6849) 6850 6851(define_insn "tlsle12_<mode>" 6852 [(set (match_operand:P 0 "register_operand" "=r") 6853 (unspec:P [(match_operand:P 1 "register_operand" "r") 6854 (match_operand 2 "aarch64_tls_le_symref" "S")] 6855 UNSPEC_TLSLE12))] 6856 "" 6857 "add\\t%<w>0, %<w>1, #%L2"; 6858 [(set_attr "type" "alu_sreg") 6859 (set_attr "length" "4")] 6860) 6861 6862(define_insn "tlsle24_<mode>" 6863 [(set (match_operand:P 0 "register_operand" "=r") 6864 (unspec:P [(match_operand:P 1 "register_operand" "r") 6865 (match_operand 2 "aarch64_tls_le_symref" "S")] 6866 UNSPEC_TLSLE24))] 6867 "" 6868 "add\\t%<w>0, %<w>1, #%G2, lsl #12\;add\\t%<w>0, %<w>0, #%L2" 6869 [(set_attr "type" "multiple") 6870 (set_attr "length" "8")] 6871) 6872 6873(define_insn "tlsle32_<mode>" 6874 [(set (match_operand:P 0 "register_operand" "=r") 6875 (unspec:P [(match_operand 1 "aarch64_tls_le_symref" "S")] 6876 UNSPEC_TLSLE32))] 6877 "" 6878 "movz\\t%<w>0, #:tprel_g1:%1\;movk\\t%<w>0, #:tprel_g0_nc:%1" 6879 [(set_attr "type" "multiple") 6880 (set_attr "length" "8")] 6881) 6882 6883(define_insn "tlsle48_<mode>" 6884 [(set (match_operand:P 0 "register_operand" "=r") 6885 (unspec:P [(match_operand 1 "aarch64_tls_le_symref" "S")] 6886 UNSPEC_TLSLE48))] 6887 "" 6888 "movz\\t%<w>0, #:tprel_g2:%1\;movk\\t%<w>0, #:tprel_g1_nc:%1\;movk\\t%<w>0, #:tprel_g0_nc:%1" 6889 [(set_attr "type" "multiple") 6890 (set_attr "length" "12")] 6891) 6892 6893(define_expand "tlsdesc_small_<mode>" 6894 [(unspec:PTR [(match_operand 0 "aarch64_valid_symref")] UNSPEC_TLSDESC)] 6895 "TARGET_TLS_DESC" 6896 { 6897 if (TARGET_SVE) 6898 { 6899 rtx abi = gen_int_mode (aarch64_tlsdesc_abi_id (), DImode); 6900 rtx_insn *call 6901 = emit_call_insn (gen_tlsdesc_small_sve_<mode> (operands[0], abi)); 6902 RTL_CONST_CALL_P (call) = 1; 6903 } 6904 else 6905 emit_insn (gen_tlsdesc_small_advsimd_<mode> (operands[0])); 6906 DONE; 6907 } 6908) 6909 6910;; tlsdesc calls preserve all core and Advanced SIMD registers except 6911;; R0 and LR. 6912(define_insn "tlsdesc_small_advsimd_<mode>" 6913 [(set (reg:PTR R0_REGNUM) 6914 (unspec:PTR [(match_operand 0 "aarch64_valid_symref" "S")] 6915 UNSPEC_TLSDESC)) 6916 (clobber (reg:DI LR_REGNUM)) 6917 (clobber (reg:CC CC_REGNUM)) 6918 (clobber (match_scratch:DI 1 "=r")) 6919 (use (reg:DI FP_REGNUM))] 6920 "TARGET_TLS_DESC && !TARGET_SVE" 6921 "adrp\\tx0, %A0\;ldr\\t%<w>1, [x0, #%L0]\;add\\t<w>0, <w>0, %L0\;.tlsdesccall\\t%0\;blr\\t%1" 6922 [(set_attr "type" "call") 6923 (set_attr "length" "16")]) 6924 6925;; For SVE, model tlsdesc calls as normal calls, with the callee ABI 6926;; describing the extra call-preserved guarantees. This would work 6927;; for non-SVE too, but avoiding a call is probably better if we can. 6928(define_insn "tlsdesc_small_sve_<mode>" 6929 [(set (reg:PTR R0_REGNUM) 6930 (call (mem:DI (unspec:PTR 6931 [(match_operand 0 "aarch64_valid_symref")] 6932 UNSPEC_TLSDESC)) 6933 (const_int 0))) 6934 (unspec:DI [(match_operand:DI 1 "const_int_operand")] UNSPEC_CALLEE_ABI) 6935 (clobber (reg:DI LR_REGNUM))] 6936 "TARGET_TLS_DESC && TARGET_SVE" 6937 "adrp\\tx0, %A0\;ldr\\t<w>30, [x0, #%L0]\;add\\t<w>0, <w>0, %L0\;.tlsdesccall\\t%0\;blr\\tx30" 6938 [(set_attr "type" "call") 6939 (set_attr "length" "16")]) 6940 6941(define_insn "stack_tie" 6942 [(set (mem:BLK (scratch)) 6943 (unspec:BLK [(match_operand:DI 0 "register_operand" "rk") 6944 (match_operand:DI 1 "register_operand" "rk")] 6945 UNSPEC_PRLG_STK))] 6946 "" 6947 "" 6948 [(set_attr "length" "0")] 6949) 6950 6951(define_insn "aarch64_fjcvtzs" 6952 [(set (match_operand:SI 0 "register_operand" "=r") 6953 (unspec:SI [(match_operand:DF 1 "register_operand" "w")] 6954 UNSPEC_FJCVTZS)) 6955 (clobber (reg:CC CC_REGNUM))] 6956 "TARGET_JSCVT" 6957 "fjcvtzs\\t%w0, %d1" 6958 [(set_attr "type" "f_cvtf2i")] 6959) 6960 6961;; Pointer authentication patterns are always provided. In architecture 6962;; revisions prior to ARMv8.3-A these HINT instructions operate as NOPs. 6963;; This lets the user write portable software which authenticates pointers 6964;; when run on something which implements ARMv8.3-A, and which runs 6965;; correctly, but does not authenticate pointers, where ARMv8.3-A is not 6966;; implemented. 6967 6968;; Signing/Authenticating R30 using SP as the salt. 6969 6970(define_insn "<pauth_mnem_prefix>sp" 6971 [(set (reg:DI R30_REGNUM) 6972 (unspec:DI [(reg:DI R30_REGNUM) (reg:DI SP_REGNUM)] PAUTH_LR_SP))] 6973 "" 6974 "hint\t<pauth_hint_num> // <pauth_mnem_prefix>sp"; 6975) 6976 6977;; Signing/Authenticating X17 using X16 as the salt. 6978 6979(define_insn "<pauth_mnem_prefix>1716" 6980 [(set (reg:DI R17_REGNUM) 6981 (unspec:DI [(reg:DI R17_REGNUM) (reg:DI R16_REGNUM)] PAUTH_17_16))] 6982 "" 6983 "hint\t<pauth_hint_num> // <pauth_mnem_prefix>1716"; 6984) 6985 6986;; Stripping the signature in R30. 6987 6988(define_insn "xpaclri" 6989 [(set (reg:DI R30_REGNUM) (unspec:DI [(reg:DI R30_REGNUM)] UNSPEC_XPACLRI))] 6990 "" 6991 "hint\t7 // xpaclri" 6992) 6993 6994;; UNSPEC_VOLATILE is considered to use and clobber all hard registers and 6995;; all of memory. This blocks insns from being moved across this point. 6996 6997(define_insn "blockage" 6998 [(unspec_volatile [(const_int 0)] UNSPECV_BLOCKAGE)] 6999 "" 7000 "" 7001 [(set_attr "length" "0") 7002 (set_attr "type" "block")] 7003) 7004 7005(define_insn "probe_stack_range" 7006 [(set (match_operand:DI 0 "register_operand" "=rk") 7007 (unspec_volatile:DI [(match_operand:DI 1 "register_operand" "0") 7008 (match_operand:DI 2 "register_operand" "r")] 7009 UNSPECV_PROBE_STACK_RANGE))] 7010 "" 7011{ 7012 return aarch64_output_probe_stack_range (operands[0], operands[2]); 7013} 7014 [(set_attr "length" "32")] 7015) 7016 7017;; This instruction is used to generate the stack clash stack adjustment and 7018;; probing loop. We can't change the control flow during prologue and epilogue 7019;; code generation. So we must emit a volatile unspec and expand it later on. 7020 7021(define_insn "@probe_sve_stack_clash_<mode>" 7022 [(set (match_operand:P 0 "register_operand" "=rk") 7023 (unspec_volatile:P [(match_operand:P 1 "register_operand" "0") 7024 (match_operand:P 2 "register_operand" "r") 7025 (match_operand:P 3 "const_int_operand" "n") 7026 (match_operand:P 4 "aarch64_plus_immediate" "L")] 7027 UNSPECV_PROBE_STACK_RANGE))] 7028 "TARGET_SVE" 7029{ 7030 return aarch64_output_probe_sve_stack_clash (operands[0], operands[2], 7031 operands[3], operands[4]); 7032} 7033 [(set_attr "length" "28")] 7034) 7035 7036;; Named pattern for expanding thread pointer reference. 7037(define_expand "get_thread_pointerdi" 7038 [(match_operand:DI 0 "register_operand")] 7039 "" 7040{ 7041 rtx tmp = aarch64_load_tp (operands[0]); 7042 if (tmp != operands[0]) 7043 emit_move_insn (operands[0], tmp); 7044 DONE; 7045}) 7046 7047;; Defined for -mstack-protector-guard=sysreg, which goes through this 7048;; pattern rather than stack_protect_combined_set. Our implementation 7049;; of the latter can handle both. 7050(define_expand "stack_protect_set" 7051 [(match_operand 0 "memory_operand") 7052 (match_operand 1 "")] 7053 "" 7054{ 7055 emit_insn (gen_stack_protect_combined_set (operands[0], operands[1])); 7056 DONE; 7057}) 7058 7059(define_expand "stack_protect_combined_set" 7060 [(match_operand 0 "memory_operand") 7061 (match_operand 1 "")] 7062 "" 7063{ 7064 machine_mode mode = GET_MODE (operands[0]); 7065 operands[1] = aarch64_stack_protect_canary_mem (mode, operands[1], 7066 AARCH64_SALT_SSP_SET); 7067 emit_insn ((mode == DImode 7068 ? gen_stack_protect_set_di 7069 : gen_stack_protect_set_si) (operands[0], operands[1])); 7070 DONE; 7071}) 7072 7073;; Operand 1 is either AARCH64_SALT_SSP_SET or AARCH64_SALT_SSP_TEST. 7074(define_insn "reg_stack_protect_address_<mode>" 7075 [(set (match_operand:PTR 0 "register_operand" "=r") 7076 (unspec:PTR [(match_operand 1 "const_int_operand")] 7077 UNSPEC_SSP_SYSREG))] 7078 "aarch64_stack_protector_guard != SSP_GLOBAL" 7079 { 7080 char buf[150]; 7081 snprintf (buf, 150, "mrs\\t%%<w>0, %s", 7082 aarch64_stack_protector_guard_reg_str); 7083 output_asm_insn (buf, operands); 7084 return ""; 7085 } 7086 [(set_attr "type" "mrs")]) 7087 7088;; DO NOT SPLIT THIS PATTERN. It is important for security reasons that the 7089;; canary value does not live beyond the life of this sequence. 7090(define_insn "stack_protect_set_<mode>" 7091 [(set (match_operand:PTR 0 "memory_operand" "=m") 7092 (unspec:PTR [(match_operand:PTR 1 "memory_operand" "m")] 7093 UNSPEC_SP_SET)) 7094 (set (match_scratch:PTR 2 "=&r") (const_int 0))] 7095 "" 7096 "ldr\\t%<w>2, %1\;str\\t%<w>2, %0\;mov\t%<w>2, 0" 7097 [(set_attr "length" "12") 7098 (set_attr "type" "multiple")]) 7099 7100;; Defined for -mstack-protector-guard=sysreg, which goes through this 7101;; pattern rather than stack_protect_combined_test. Our implementation 7102;; of the latter can handle both. 7103(define_expand "stack_protect_test" 7104 [(match_operand 0 "memory_operand") 7105 (match_operand 1 "") 7106 (match_operand 2)] 7107 "" 7108{ 7109 emit_insn (gen_stack_protect_combined_test (operands[0], operands[1], 7110 operands[2])); 7111 DONE; 7112}) 7113 7114(define_expand "stack_protect_combined_test" 7115 [(match_operand 0 "memory_operand") 7116 (match_operand 1 "") 7117 (match_operand 2)] 7118 "" 7119{ 7120 machine_mode mode = GET_MODE (operands[0]); 7121 operands[1] = aarch64_stack_protect_canary_mem (mode, operands[1], 7122 AARCH64_SALT_SSP_TEST); 7123 emit_insn ((mode == DImode 7124 ? gen_stack_protect_test_di 7125 : gen_stack_protect_test_si) (operands[0], operands[1])); 7126 7127 rtx cc_reg = gen_rtx_REG (CCmode, CC_REGNUM); 7128 emit_jump_insn (gen_condjump (gen_rtx_EQ (VOIDmode, cc_reg, const0_rtx), 7129 cc_reg, operands[2])); 7130 DONE; 7131}) 7132 7133;; DO NOT SPLIT THIS PATTERN. It is important for security reasons that the 7134;; canary value does not live beyond the end of this sequence. 7135(define_insn "stack_protect_test_<mode>" 7136 [(set (reg:CC CC_REGNUM) 7137 (unspec:CC [(match_operand:PTR 0 "memory_operand" "m") 7138 (match_operand:PTR 1 "memory_operand" "m")] 7139 UNSPEC_SP_TEST)) 7140 (clobber (match_scratch:PTR 2 "=&r")) 7141 (clobber (match_scratch:PTR 3 "=&r"))] 7142 "" 7143 "ldr\t%<w>2, %0\;ldr\t%<w>3, %1\;subs\t%<w>2, %<w>2, %<w>3\;mov\t%3, 0" 7144 [(set_attr "length" "16") 7145 (set_attr "type" "multiple")]) 7146 7147;; Write into the Floating-point Status or Control Register. 7148(define_insn "@aarch64_set_<fpscr_name><GPI:mode>" 7149 [(unspec_volatile [(match_operand:GPI 0 "register_operand" "r")] SET_FPSCR)] 7150 "" 7151 "msr\\t<fpscr_name>, %0" 7152 [(set_attr "type" "mrs")]) 7153 7154;; Read into the Floating-point Status or Control Register. 7155(define_insn "@aarch64_get_<fpscr_name><GPI:mode>" 7156 [(set (match_operand:GPI 0 "register_operand" "=r") 7157 (unspec_volatile:GPI [(const_int 0)] GET_FPSCR))] 7158 "" 7159 "mrs\\t%0, <fpscr_name>" 7160 [(set_attr "type" "mrs")]) 7161 7162;; Define the subtract-one-and-jump insns so loop.c 7163;; knows what to generate. 7164(define_expand "doloop_end" 7165 [(use (match_operand 0 "" "")) ; loop pseudo 7166 (use (match_operand 1 "" ""))] ; label 7167 "optimize > 0 && flag_modulo_sched" 7168{ 7169 rtx s0; 7170 rtx bcomp; 7171 rtx loc_ref; 7172 rtx cc_reg; 7173 rtx insn; 7174 rtx cmp; 7175 7176 /* Currently SMS relies on the do-loop pattern to recognize loops 7177 where (1) the control part consists of all insns defining and/or 7178 using a certain 'count' register and (2) the loop count can be 7179 adjusted by modifying this register prior to the loop. 7180 ??? The possible introduction of a new block to initialize the 7181 new IV can potentially affect branch optimizations. */ 7182 7183 if (GET_MODE (operands[0]) != DImode) 7184 FAIL; 7185 7186 s0 = operands [0]; 7187 insn = emit_insn (gen_adddi3_compare0 (s0, s0, GEN_INT (-1))); 7188 7189 cmp = XVECEXP (PATTERN (insn), 0, 0); 7190 cc_reg = SET_DEST (cmp); 7191 bcomp = gen_rtx_NE (VOIDmode, cc_reg, const0_rtx); 7192 loc_ref = gen_rtx_LABEL_REF (VOIDmode, operands [1]); 7193 emit_jump_insn (gen_rtx_SET (pc_rtx, 7194 gen_rtx_IF_THEN_ELSE (VOIDmode, bcomp, 7195 loc_ref, pc_rtx))); 7196 DONE; 7197}) 7198 7199;; Track speculation through conditional branches. We assume that 7200;; SPECULATION_TRACKER_REGNUM is reserved for this purpose when necessary. 7201(define_insn "speculation_tracker" 7202 [(set (reg:DI SPECULATION_TRACKER_REGNUM) 7203 (unspec:DI [(reg:DI SPECULATION_TRACKER_REGNUM) (match_operand 0)] 7204 UNSPEC_SPECULATION_TRACKER))] 7205 "" 7206 { 7207 operands[1] = gen_rtx_REG (DImode, SPECULATION_TRACKER_REGNUM); 7208 output_asm_insn ("csel\\t%1, %1, xzr, %m0", operands); 7209 return ""; 7210 } 7211 [(set_attr "type" "csel")] 7212) 7213 7214;; Like speculation_tracker, but track the inverse condition. 7215(define_insn "speculation_tracker_rev" 7216 [(set (reg:DI SPECULATION_TRACKER_REGNUM) 7217 (unspec:DI [(reg:DI SPECULATION_TRACKER_REGNUM) (match_operand 0)] 7218 UNSPEC_SPECULATION_TRACKER_REV))] 7219 "" 7220 { 7221 operands[1] = gen_rtx_REG (DImode, SPECULATION_TRACKER_REGNUM); 7222 output_asm_insn ("csel\\t%1, %1, xzr, %M0", operands); 7223 return ""; 7224 } 7225 [(set_attr "type" "csel")] 7226) 7227 7228;; BTI <target> instructions 7229(define_insn "bti_noarg" 7230 [(unspec_volatile [(const_int 0)] UNSPECV_BTI_NOARG)] 7231 "" 7232 "hint\t32 // bti" 7233 [(set_attr "type" "no_insn")] 7234) 7235 7236(define_insn "bti_c" 7237 [(unspec_volatile [(const_int 0)] UNSPECV_BTI_C)] 7238 "" 7239 "hint\t34 // bti c" 7240 [(set_attr "type" "no_insn")] 7241) 7242 7243(define_insn "bti_j" 7244 [(unspec_volatile [(const_int 0)] UNSPECV_BTI_J)] 7245 "" 7246 "hint\t36 // bti j" 7247 [(set_attr "type" "no_insn")] 7248) 7249 7250(define_insn "bti_jc" 7251 [(unspec_volatile [(const_int 0)] UNSPECV_BTI_JC)] 7252 "" 7253 "hint\t38 // bti jc" 7254 [(set_attr "type" "no_insn")] 7255) 7256 7257;; Hard speculation barrier. 7258(define_insn "speculation_barrier" 7259 [(unspec_volatile [(const_int 0)] UNSPECV_SPECULATION_BARRIER)] 7260 "" 7261 "isb\;dsb\\tsy" 7262 [(set_attr "length" "8") 7263 (set_attr "type" "block") 7264 (set_attr "speculation_barrier" "true")] 7265) 7266 7267;; Support for __builtin_speculation_safe_value when we have speculation 7268;; tracking enabled. Use the speculation tracker to decide whether to 7269;; copy operand 1 to the target, or to copy the fail value (operand 2). 7270(define_expand "@despeculate_copy<ALLI_TI:mode>" 7271 [(set (match_operand:ALLI_TI 0 "register_operand") 7272 (unspec_volatile:ALLI_TI 7273 [(match_operand:ALLI_TI 1 "register_operand") 7274 (match_operand:ALLI_TI 2 "aarch64_reg_or_zero") 7275 (use (reg:DI SPECULATION_TRACKER_REGNUM)) 7276 (clobber (reg:CC CC_REGNUM))] UNSPECV_SPECULATION_BARRIER))] 7277 "" 7278 " 7279 { 7280 if (operands[2] == const0_rtx) 7281 { 7282 rtx tracker; 7283 if (<MODE>mode == TImode) 7284 tracker = gen_rtx_REG (DImode, SPECULATION_TRACKER_REGNUM); 7285 else 7286 tracker = gen_rtx_REG (<MODE>mode, SPECULATION_TRACKER_REGNUM); 7287 7288 emit_insn (gen_despeculate_simple<mode> (operands[0], operands[1], 7289 tracker)); 7290 DONE; 7291 } 7292 } 7293 " 7294) 7295 7296;; Patterns to match despeculate_copy<mode>. Note that "hint 0x14" is the 7297;; encoding for CSDB, but will work in older versions of the assembler. 7298(define_insn "*despeculate_copy<ALLI:mode>_insn" 7299 [(set (match_operand:ALLI 0 "register_operand" "=r") 7300 (unspec_volatile:ALLI 7301 [(match_operand:ALLI 1 "register_operand" "r") 7302 (match_operand:ALLI 2 "aarch64_reg_or_zero" "rZ") 7303 (use (reg:DI SPECULATION_TRACKER_REGNUM)) 7304 (clobber (reg:CC CC_REGNUM))] UNSPECV_SPECULATION_BARRIER))] 7305 "" 7306 { 7307 operands[3] = gen_rtx_REG (DImode, SPECULATION_TRACKER_REGNUM); 7308 output_asm_insn ("cmp\\t%3, #0\;csel\\t%<w>0, %<w>1, %<w>2, ne\;hint\t0x14 // csdb", 7309 operands); 7310 return ""; 7311 } 7312 [(set_attr "length" "12") 7313 (set_attr "type" "block") 7314 (set_attr "speculation_barrier" "true")] 7315) 7316 7317;; Pattern to match despeculate_copyti 7318(define_insn "*despeculate_copyti_insn" 7319 [(set (match_operand:TI 0 "register_operand" "=r") 7320 (unspec_volatile:TI 7321 [(match_operand:TI 1 "register_operand" "r") 7322 (match_operand:TI 2 "aarch64_reg_or_zero" "rZ") 7323 (use (reg:DI SPECULATION_TRACKER_REGNUM)) 7324 (clobber (reg:CC CC_REGNUM))] UNSPECV_SPECULATION_BARRIER))] 7325 "" 7326 { 7327 operands[3] = gen_rtx_REG (DImode, SPECULATION_TRACKER_REGNUM); 7328 output_asm_insn 7329 ("cmp\\t%3, #0\;csel\\t%0, %1, %2, ne\;csel\\t%H0, %H1, %H2, ne\;hint\t0x14 // csdb", 7330 operands); 7331 return ""; 7332 } 7333 [(set_attr "length" "16") 7334 (set_attr "type" "block") 7335 (set_attr "speculation_barrier" "true")] 7336) 7337 7338(define_insn "despeculate_simple<ALLI:mode>" 7339 [(set (match_operand:ALLI 0 "register_operand" "=r") 7340 (unspec_volatile:ALLI 7341 [(match_operand:ALLI 1 "register_operand" "r") 7342 (use (match_operand:ALLI 2 "register_operand" ""))] 7343 UNSPECV_SPECULATION_BARRIER))] 7344 "" 7345 "and\\t%<w>0, %<w>1, %<w>2\;hint\t0x14 // csdb" 7346 [(set_attr "type" "block") 7347 (set_attr "length" "8") 7348 (set_attr "speculation_barrier" "true")] 7349) 7350 7351(define_insn "despeculate_simpleti" 7352 [(set (match_operand:TI 0 "register_operand" "=r") 7353 (unspec_volatile:TI 7354 [(match_operand:TI 1 "register_operand" "r") 7355 (use (match_operand:DI 2 "register_operand" ""))] 7356 UNSPECV_SPECULATION_BARRIER))] 7357 "" 7358 "and\\t%0, %1, %2\;and\\t%H0, %H1, %2\;hint\t0x14 // csdb" 7359 [(set_attr "type" "block") 7360 (set_attr "length" "12") 7361 (set_attr "speculation_barrier" "true")] 7362) 7363 7364(define_insn "aarch64_<frintnzs_op><mode>" 7365 [(set (match_operand:VSFDF 0 "register_operand" "=w") 7366 (unspec:VSFDF [(match_operand:VSFDF 1 "register_operand" "w")] 7367 FRINTNZX))] 7368 "TARGET_FRINT && TARGET_FLOAT 7369 && !(VECTOR_MODE_P (<MODE>mode) && !TARGET_SIMD)" 7370 "<frintnzs_op>\\t%<v>0<Vmtype>, %<v>1<Vmtype>" 7371 [(set_attr "type" "f_rint<stype>")] 7372) 7373 7374;; Transactional Memory Extension (TME) instructions. 7375 7376(define_insn "tstart" 7377 [(set (match_operand:DI 0 "register_operand" "=r") 7378 (unspec_volatile:DI [(const_int 0)] UNSPECV_TSTART)) 7379 (clobber (mem:BLK (scratch)))] 7380 "TARGET_TME" 7381 "tstart\\t%0" 7382 [(set_attr "type" "tme")] 7383) 7384 7385(define_insn "ttest" 7386 [(set (match_operand:DI 0 "register_operand" "=r") 7387 (unspec_volatile:DI [(const_int 0)] UNSPEC_TTEST)) 7388 (clobber (mem:BLK (scratch)))] 7389 "TARGET_TME" 7390 "ttest\\t%0" 7391 [(set_attr "type" "tme")] 7392) 7393 7394(define_insn "tcommit" 7395 [(unspec_volatile:BLK [(const_int 0)] UNSPECV_TCOMMIT) 7396 (clobber (mem:BLK (scratch)))] 7397 "TARGET_TME" 7398 "tcommit" 7399 [(set_attr "type" "tme")] 7400) 7401 7402(define_insn "tcancel" 7403 [(unspec_volatile:BLK 7404 [(match_operand 0 "const_int_operand" "n")] UNSPECV_TCANCEL) 7405 (clobber (mem:BLK (scratch)))] 7406 "TARGET_TME && (UINTVAL (operands[0]) <= 65535)" 7407 "tcancel\\t#%0" 7408 [(set_attr "type" "tme")] 7409) 7410 7411(define_insn "aarch64_rndr" 7412 [(set (match_operand:DI 0 "register_operand" "=r") 7413 (unspec_volatile:DI [(const_int 0)] UNSPEC_RNDR)) 7414 (set (reg:CC_Z CC_REGNUM) 7415 (unspec_volatile:CC_Z [(const_int 0)] UNSPEC_RNDR))] 7416 "TARGET_RNG" 7417 "mrs\t%0, RNDR" 7418 [(set_attr "type" "mrs")] 7419) 7420 7421(define_insn "aarch64_rndrrs" 7422 [(set (match_operand:DI 0 "register_operand" "=r") 7423 (unspec_volatile:DI [(const_int 0)] UNSPEC_RNDRRS)) 7424 (set (reg:CC_Z CC_REGNUM) 7425 (unspec_volatile:CC_Z [(const_int 0)] UNSPEC_RNDRRS))] 7426 "TARGET_RNG" 7427 "mrs\t%0, RNDRRS" 7428 [(set_attr "type" "mrs")] 7429) 7430 7431;; Memory Tagging Extension (MTE) instructions. 7432 7433(define_insn "irg" 7434 [(set (match_operand:DI 0 "register_operand" "=rk") 7435 (ior:DI 7436 (and:DI (match_operand:DI 1 "register_operand" "rk") 7437 (const_int -1080863910568919041)) ;; 0xf0ff... 7438 (ashift:DI (unspec:QI [(match_operand:DI 2 "register_operand" "r")] 7439 UNSPEC_GEN_TAG_RND) 7440 (const_int 56))))] 7441 "TARGET_MEMTAG" 7442 "irg\\t%0, %1, %2" 7443 [(set_attr "type" "memtag")] 7444) 7445 7446(define_insn "gmi" 7447 [(set (match_operand:DI 0 "register_operand" "=r") 7448 (ior:DI (ashift:DI 7449 (const_int 1) 7450 (and:QI (lshiftrt:DI 7451 (match_operand:DI 1 "register_operand" "rk") 7452 (const_int 56)) (const_int 15))) 7453 (match_operand:DI 2 "register_operand" "r")))] 7454 "TARGET_MEMTAG" 7455 "gmi\\t%0, %1, %2" 7456 [(set_attr "type" "memtag")] 7457) 7458 7459(define_insn "addg" 7460 [(set (match_operand:DI 0 "register_operand" "=rk") 7461 (ior:DI 7462 (and:DI (plus:DI (match_operand:DI 1 "register_operand" "rk") 7463 (match_operand:DI 2 "aarch64_granule16_uimm6" "i")) 7464 (const_int -1080863910568919041)) ;; 0xf0ff... 7465 (ashift:DI 7466 (unspec:QI 7467 [(and:QI (lshiftrt:DI (match_dup 1) (const_int 56)) (const_int 15)) 7468 (match_operand:QI 3 "aarch64_memtag_tag_offset" "i")] 7469 UNSPEC_GEN_TAG) 7470 (const_int 56))))] 7471 "TARGET_MEMTAG" 7472 "addg\\t%0, %1, #%2, #%3" 7473 [(set_attr "type" "memtag")] 7474) 7475 7476(define_insn "subp" 7477 [(set (match_operand:DI 0 "register_operand" "=r") 7478 (minus:DI 7479 (and:DI (match_operand:DI 1 "register_operand" "rk") 7480 (const_int 72057594037927935)) ;; 0x00ff... 7481 (and:DI (match_operand:DI 2 "register_operand" "rk") 7482 (const_int 72057594037927935))))] ;; 0x00ff... 7483 "TARGET_MEMTAG" 7484 "subp\\t%0, %1, %2" 7485 [(set_attr "type" "memtag")] 7486) 7487 7488;; LDG will use the 16-byte aligned value of the address. 7489(define_insn "ldg" 7490 [(set (match_operand:DI 0 "register_operand" "+r") 7491 (ior:DI 7492 (and:DI (match_dup 0) (const_int -1080863910568919041)) ;; 0xf0ff... 7493 (ashift:DI 7494 (mem:QI (unspec:DI 7495 [(and:DI (plus:DI (match_operand:DI 1 "register_operand" "rk") 7496 (match_operand:DI 2 "aarch64_granule16_simm9" "i")) 7497 (const_int -16))] UNSPEC_TAG_SPACE)) 7498 (const_int 56))))] 7499 "TARGET_MEMTAG" 7500 "ldg\\t%0, [%1, #%2]" 7501 [(set_attr "type" "memtag")] 7502) 7503 7504;; STG doesn't align the address but aborts with alignment fault 7505;; when the address is not 16-byte aligned. 7506(define_insn "stg" 7507 [(set (mem:QI (unspec:DI 7508 [(plus:DI (match_operand:DI 1 "register_operand" "rk") 7509 (match_operand:DI 2 "aarch64_granule16_simm9" "i"))] 7510 UNSPEC_TAG_SPACE)) 7511 (and:QI (lshiftrt:DI (match_operand:DI 0 "register_operand" "rk") 7512 (const_int 56)) (const_int 15)))] 7513 "TARGET_MEMTAG" 7514 "stg\\t%0, [%1, #%2]" 7515 [(set_attr "type" "memtag")] 7516) 7517 7518;; AdvSIMD Stuff 7519(include "aarch64-simd.md") 7520 7521;; Atomic Operations 7522(include "atomics.md") 7523 7524;; ldp/stp peephole patterns 7525(include "aarch64-ldpstp.md") 7526 7527;; SVE. 7528(include "aarch64-sve.md") 7529 7530;; SVE2. 7531(include "aarch64-sve2.md") 7532