1;; ARM NEON coprocessor Machine Description 2;; Copyright (C) 2006-2013 Free Software Foundation, Inc. 3;; Written by CodeSourcery. 4;; 5;; This file is part of GCC. 6;; 7;; GCC is free software; you can redistribute it and/or modify it 8;; under the terms of the GNU General Public License as published by 9;; the Free Software Foundation; either version 3, or (at your option) 10;; any later version. 11;; 12;; GCC is distributed in the hope that it will be useful, but 13;; WITHOUT ANY WARRANTY; without even the implied warranty of 14;; MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU 15;; General Public License for more details. 16;; 17;; You should have received a copy of the GNU General Public License 18;; along with GCC; see the file COPYING3. If not see 19;; <http://www.gnu.org/licenses/>. 20 21 22;; Attribute used to permit string comparisons against <VQH_mnem> in 23;; neon_type attribute definitions. 24(define_attr "vqh_mnem" "vadd,vmin,vmax" (const_string "vadd")) 25 26(define_insn "*neon_mov<mode>" 27 [(set (match_operand:VDX 0 "nonimmediate_operand" 28 "=w,Un,w, w, ?r,?w,?r,?r, ?Us") 29 (match_operand:VDX 1 "general_operand" 30 " w,w, Dn,Uni, w, r, r, Usi,r"))] 31 "TARGET_NEON 32 && (register_operand (operands[0], <MODE>mode) 33 || register_operand (operands[1], <MODE>mode))" 34{ 35 if (which_alternative == 2) 36 { 37 int width, is_valid; 38 static char templ[40]; 39 40 is_valid = neon_immediate_valid_for_move (operands[1], <MODE>mode, 41 &operands[1], &width); 42 43 gcc_assert (is_valid != 0); 44 45 if (width == 0) 46 return "vmov.f32\t%P0, %1 @ <mode>"; 47 else 48 sprintf (templ, "vmov.i%d\t%%P0, %%x1 @ <mode>", width); 49 50 return templ; 51 } 52 53 switch (which_alternative) 54 { 55 case 0: return "vmov\t%P0, %P1 @ <mode>"; 56 case 1: case 3: return output_move_neon (operands); 57 case 2: gcc_unreachable (); 58 case 4: return "vmov\t%Q0, %R0, %P1 @ <mode>"; 59 case 5: return "vmov\t%P0, %Q1, %R1 @ <mode>"; 60 default: return output_move_double (operands, true, NULL); 61 } 62} 63 [(set_attr "neon_type" "neon_int_1,*,neon_vmov,*,neon_mrrc,neon_mcr_2_mcrr,*,*,*") 64 (set_attr "type" "*,f_stored,*,f_loadd,*,*,alu_reg,load2,store2") 65 (set_attr "insn" "*,*,*,*,*,*,mov,*,*") 66 (set_attr "length" "4,4,4,4,4,4,8,8,8") 67 (set_attr "arm_pool_range" "*,*,*,1020,*,*,*,1020,*") 68 (set_attr "thumb2_pool_range" "*,*,*,1018,*,*,*,1018,*") 69 (set_attr "neg_pool_range" "*,*,*,1004,*,*,*,1004,*")]) 70 71(define_insn "*neon_mov<mode>" 72 [(set (match_operand:VQXMOV 0 "nonimmediate_operand" 73 "=w,Un,w, w, ?r,?w,?r,?r, ?Us") 74 (match_operand:VQXMOV 1 "general_operand" 75 " w,w, Dn,Uni, w, r, r, Usi, r"))] 76 "TARGET_NEON 77 && (register_operand (operands[0], <MODE>mode) 78 || register_operand (operands[1], <MODE>mode))" 79{ 80 if (which_alternative == 2) 81 { 82 int width, is_valid; 83 static char templ[40]; 84 85 is_valid = neon_immediate_valid_for_move (operands[1], <MODE>mode, 86 &operands[1], &width); 87 88 gcc_assert (is_valid != 0); 89 90 if (width == 0) 91 return "vmov.f32\t%q0, %1 @ <mode>"; 92 else 93 sprintf (templ, "vmov.i%d\t%%q0, %%1 @ <mode>", width); 94 95 return templ; 96 } 97 98 switch (which_alternative) 99 { 100 case 0: return "vmov\t%q0, %q1 @ <mode>"; 101 case 1: case 3: return output_move_neon (operands); 102 case 2: gcc_unreachable (); 103 case 4: return "vmov\t%Q0, %R0, %e1 @ <mode>\;vmov\t%J0, %K0, %f1"; 104 case 5: return "vmov\t%e0, %Q1, %R1 @ <mode>\;vmov\t%f0, %J1, %K1"; 105 default: return output_move_quad (operands); 106 } 107} 108 [(set_attr "neon_type" "neon_int_1,neon_stm_2,neon_vmov,neon_ldm_2,\ 109 neon_mrrc,neon_mcr_2_mcrr,*,*,*") 110 (set_attr "type" "*,*,*,*,*,*,alu_reg,load4,store4") 111 (set_attr "insn" "*,*,*,*,*,*,mov,*,*") 112 (set_attr "length" "4,8,4,8,8,8,16,8,16") 113 (set_attr "arm_pool_range" "*,*,*,1020,*,*,*,1020,*") 114 (set_attr "thumb2_pool_range" "*,*,*,1018,*,*,*,1018,*") 115 (set_attr "neg_pool_range" "*,*,*,996,*,*,*,996,*")]) 116 117(define_expand "movti" 118 [(set (match_operand:TI 0 "nonimmediate_operand" "") 119 (match_operand:TI 1 "general_operand" ""))] 120 "TARGET_NEON" 121{ 122 if (can_create_pseudo_p ()) 123 { 124 if (!REG_P (operands[0])) 125 operands[1] = force_reg (TImode, operands[1]); 126 } 127}) 128 129(define_expand "mov<mode>" 130 [(set (match_operand:VSTRUCT 0 "nonimmediate_operand" "") 131 (match_operand:VSTRUCT 1 "general_operand" ""))] 132 "TARGET_NEON" 133{ 134 if (can_create_pseudo_p ()) 135 { 136 if (!REG_P (operands[0])) 137 operands[1] = force_reg (<MODE>mode, operands[1]); 138 } 139}) 140 141(define_insn "*neon_mov<mode>" 142 [(set (match_operand:VSTRUCT 0 "nonimmediate_operand" "=w,Ut,w") 143 (match_operand:VSTRUCT 1 "general_operand" " w,w, Ut"))] 144 "TARGET_NEON 145 && (register_operand (operands[0], <MODE>mode) 146 || register_operand (operands[1], <MODE>mode))" 147{ 148 switch (which_alternative) 149 { 150 case 0: return "#"; 151 case 1: case 2: return output_move_neon (operands); 152 default: gcc_unreachable (); 153 } 154} 155 [(set_attr "neon_type" "neon_int_1,neon_stm_2,neon_ldm_2") 156 (set (attr "length") (symbol_ref "arm_attr_length_move_neon (insn)"))]) 157 158(define_split 159 [(set (match_operand:EI 0 "s_register_operand" "") 160 (match_operand:EI 1 "s_register_operand" ""))] 161 "TARGET_NEON && reload_completed" 162 [(set (match_dup 0) (match_dup 1)) 163 (set (match_dup 2) (match_dup 3))] 164{ 165 int rdest = REGNO (operands[0]); 166 int rsrc = REGNO (operands[1]); 167 rtx dest[2], src[2]; 168 169 dest[0] = gen_rtx_REG (TImode, rdest); 170 src[0] = gen_rtx_REG (TImode, rsrc); 171 dest[1] = gen_rtx_REG (DImode, rdest + 4); 172 src[1] = gen_rtx_REG (DImode, rsrc + 4); 173 174 neon_disambiguate_copy (operands, dest, src, 2); 175}) 176 177(define_split 178 [(set (match_operand:OI 0 "s_register_operand" "") 179 (match_operand:OI 1 "s_register_operand" ""))] 180 "TARGET_NEON && reload_completed" 181 [(set (match_dup 0) (match_dup 1)) 182 (set (match_dup 2) (match_dup 3))] 183{ 184 int rdest = REGNO (operands[0]); 185 int rsrc = REGNO (operands[1]); 186 rtx dest[2], src[2]; 187 188 dest[0] = gen_rtx_REG (TImode, rdest); 189 src[0] = gen_rtx_REG (TImode, rsrc); 190 dest[1] = gen_rtx_REG (TImode, rdest + 4); 191 src[1] = gen_rtx_REG (TImode, rsrc + 4); 192 193 neon_disambiguate_copy (operands, dest, src, 2); 194}) 195 196(define_split 197 [(set (match_operand:CI 0 "s_register_operand" "") 198 (match_operand:CI 1 "s_register_operand" ""))] 199 "TARGET_NEON && reload_completed" 200 [(set (match_dup 0) (match_dup 1)) 201 (set (match_dup 2) (match_dup 3)) 202 (set (match_dup 4) (match_dup 5))] 203{ 204 int rdest = REGNO (operands[0]); 205 int rsrc = REGNO (operands[1]); 206 rtx dest[3], src[3]; 207 208 dest[0] = gen_rtx_REG (TImode, rdest); 209 src[0] = gen_rtx_REG (TImode, rsrc); 210 dest[1] = gen_rtx_REG (TImode, rdest + 4); 211 src[1] = gen_rtx_REG (TImode, rsrc + 4); 212 dest[2] = gen_rtx_REG (TImode, rdest + 8); 213 src[2] = gen_rtx_REG (TImode, rsrc + 8); 214 215 neon_disambiguate_copy (operands, dest, src, 3); 216}) 217 218(define_split 219 [(set (match_operand:XI 0 "s_register_operand" "") 220 (match_operand:XI 1 "s_register_operand" ""))] 221 "TARGET_NEON && reload_completed" 222 [(set (match_dup 0) (match_dup 1)) 223 (set (match_dup 2) (match_dup 3)) 224 (set (match_dup 4) (match_dup 5)) 225 (set (match_dup 6) (match_dup 7))] 226{ 227 int rdest = REGNO (operands[0]); 228 int rsrc = REGNO (operands[1]); 229 rtx dest[4], src[4]; 230 231 dest[0] = gen_rtx_REG (TImode, rdest); 232 src[0] = gen_rtx_REG (TImode, rsrc); 233 dest[1] = gen_rtx_REG (TImode, rdest + 4); 234 src[1] = gen_rtx_REG (TImode, rsrc + 4); 235 dest[2] = gen_rtx_REG (TImode, rdest + 8); 236 src[2] = gen_rtx_REG (TImode, rsrc + 8); 237 dest[3] = gen_rtx_REG (TImode, rdest + 12); 238 src[3] = gen_rtx_REG (TImode, rsrc + 12); 239 240 neon_disambiguate_copy (operands, dest, src, 4); 241}) 242 243(define_expand "movmisalign<mode>" 244 [(set (match_operand:VDQX 0 "neon_struct_or_register_operand") 245 (unspec:VDQX [(match_operand:VDQX 1 "neon_struct_or_register_operand")] 246 UNSPEC_MISALIGNED_ACCESS))] 247 "TARGET_NEON && !BYTES_BIG_ENDIAN && unaligned_access" 248{ 249 /* This pattern is not permitted to fail during expansion: if both arguments 250 are non-registers (e.g. memory := constant, which can be created by the 251 auto-vectorizer), force operand 1 into a register. */ 252 if (!s_register_operand (operands[0], <MODE>mode) 253 && !s_register_operand (operands[1], <MODE>mode)) 254 operands[1] = force_reg (<MODE>mode, operands[1]); 255}) 256 257(define_insn "*movmisalign<mode>_neon_store" 258 [(set (match_operand:VDX 0 "neon_struct_operand" "=Um") 259 (unspec:VDX [(match_operand:VDX 1 "s_register_operand" " w")] 260 UNSPEC_MISALIGNED_ACCESS))] 261 "TARGET_NEON && !BYTES_BIG_ENDIAN && unaligned_access" 262 "vst1.<V_sz_elem>\t{%P1}, %A0" 263 [(set_attr "neon_type" "neon_vst1_1_2_regs_vst2_2_regs")]) 264 265(define_insn "*movmisalign<mode>_neon_load" 266 [(set (match_operand:VDX 0 "s_register_operand" "=w") 267 (unspec:VDX [(match_operand:VDX 1 "neon_struct_operand" " Um")] 268 UNSPEC_MISALIGNED_ACCESS))] 269 "TARGET_NEON && !BYTES_BIG_ENDIAN && unaligned_access" 270 "vld1.<V_sz_elem>\t{%P0}, %A1" 271 [(set_attr "neon_type" "neon_vld1_1_2_regs")]) 272 273(define_insn "*movmisalign<mode>_neon_store" 274 [(set (match_operand:VQX 0 "neon_struct_operand" "=Um") 275 (unspec:VQX [(match_operand:VQX 1 "s_register_operand" " w")] 276 UNSPEC_MISALIGNED_ACCESS))] 277 "TARGET_NEON && !BYTES_BIG_ENDIAN && unaligned_access" 278 "vst1.<V_sz_elem>\t{%q1}, %A0" 279 [(set_attr "neon_type" "neon_vst1_1_2_regs_vst2_2_regs")]) 280 281(define_insn "*movmisalign<mode>_neon_load" 282 [(set (match_operand:VQX 0 "s_register_operand" "=w") 283 (unspec:VQX [(match_operand:VQX 1 "neon_struct_operand" " Um")] 284 UNSPEC_MISALIGNED_ACCESS))] 285 "TARGET_NEON && !BYTES_BIG_ENDIAN && unaligned_access" 286 "vld1.<V_sz_elem>\t{%q0}, %A1" 287 [(set_attr "neon_type" "neon_vld1_1_2_regs")]) 288 289(define_insn "vec_set<mode>_internal" 290 [(set (match_operand:VD 0 "s_register_operand" "=w,w") 291 (vec_merge:VD 292 (vec_duplicate:VD 293 (match_operand:<V_elem> 1 "nonimmediate_operand" "Um,r")) 294 (match_operand:VD 3 "s_register_operand" "0,0") 295 (match_operand:SI 2 "immediate_operand" "i,i")))] 296 "TARGET_NEON" 297{ 298 int elt = ffs ((int) INTVAL (operands[2])) - 1; 299 if (BYTES_BIG_ENDIAN) 300 elt = GET_MODE_NUNITS (<MODE>mode) - 1 - elt; 301 operands[2] = GEN_INT (elt); 302 303 if (which_alternative == 0) 304 return "vld1.<V_sz_elem>\t{%P0[%c2]}, %A1"; 305 else 306 return "vmov.<V_sz_elem>\t%P0[%c2], %1"; 307} 308 [(set_attr "neon_type" "neon_vld1_vld2_lane,neon_mcr")]) 309 310(define_insn "vec_set<mode>_internal" 311 [(set (match_operand:VQ 0 "s_register_operand" "=w,w") 312 (vec_merge:VQ 313 (vec_duplicate:VQ 314 (match_operand:<V_elem> 1 "nonimmediate_operand" "Um,r")) 315 (match_operand:VQ 3 "s_register_operand" "0,0") 316 (match_operand:SI 2 "immediate_operand" "i,i")))] 317 "TARGET_NEON" 318{ 319 HOST_WIDE_INT elem = ffs ((int) INTVAL (operands[2])) - 1; 320 int half_elts = GET_MODE_NUNITS (<MODE>mode) / 2; 321 int elt = elem % half_elts; 322 int hi = (elem / half_elts) * 2; 323 int regno = REGNO (operands[0]); 324 325 if (BYTES_BIG_ENDIAN) 326 elt = half_elts - 1 - elt; 327 328 operands[0] = gen_rtx_REG (<V_HALF>mode, regno + hi); 329 operands[2] = GEN_INT (elt); 330 331 if (which_alternative == 0) 332 return "vld1.<V_sz_elem>\t{%P0[%c2]}, %A1"; 333 else 334 return "vmov.<V_sz_elem>\t%P0[%c2], %1"; 335} 336 [(set_attr "neon_type" "neon_vld1_vld2_lane,neon_mcr")] 337) 338 339(define_insn "vec_setv2di_internal" 340 [(set (match_operand:V2DI 0 "s_register_operand" "=w,w") 341 (vec_merge:V2DI 342 (vec_duplicate:V2DI 343 (match_operand:DI 1 "nonimmediate_operand" "Um,r")) 344 (match_operand:V2DI 3 "s_register_operand" "0,0") 345 (match_operand:SI 2 "immediate_operand" "i,i")))] 346 "TARGET_NEON" 347{ 348 HOST_WIDE_INT elem = ffs ((int) INTVAL (operands[2])) - 1; 349 int regno = REGNO (operands[0]) + 2 * elem; 350 351 operands[0] = gen_rtx_REG (DImode, regno); 352 353 if (which_alternative == 0) 354 return "vld1.64\t%P0, %A1"; 355 else 356 return "vmov\t%P0, %Q1, %R1"; 357} 358 [(set_attr "neon_type" "neon_vld1_1_2_regs,neon_mcr_2_mcrr")] 359) 360 361(define_expand "vec_set<mode>" 362 [(match_operand:VDQ 0 "s_register_operand" "") 363 (match_operand:<V_elem> 1 "s_register_operand" "") 364 (match_operand:SI 2 "immediate_operand" "")] 365 "TARGET_NEON" 366{ 367 HOST_WIDE_INT elem = (HOST_WIDE_INT) 1 << INTVAL (operands[2]); 368 emit_insn (gen_vec_set<mode>_internal (operands[0], operands[1], 369 GEN_INT (elem), operands[0])); 370 DONE; 371}) 372 373(define_insn "vec_extract<mode>" 374 [(set (match_operand:<V_elem> 0 "nonimmediate_operand" "=Um,r") 375 (vec_select:<V_elem> 376 (match_operand:VD 1 "s_register_operand" "w,w") 377 (parallel [(match_operand:SI 2 "immediate_operand" "i,i")])))] 378 "TARGET_NEON" 379{ 380 if (BYTES_BIG_ENDIAN) 381 { 382 int elt = INTVAL (operands[2]); 383 elt = GET_MODE_NUNITS (<MODE>mode) - 1 - elt; 384 operands[2] = GEN_INT (elt); 385 } 386 387 if (which_alternative == 0) 388 return "vst1.<V_sz_elem>\t{%P1[%c2]}, %A0"; 389 else 390 return "vmov.<V_uf_sclr>\t%0, %P1[%c2]"; 391} 392 [(set_attr "neon_type" "neon_vst1_vst2_lane,neon_bp_simple")] 393) 394 395(define_insn "vec_extract<mode>" 396 [(set (match_operand:<V_elem> 0 "nonimmediate_operand" "=Um,r") 397 (vec_select:<V_elem> 398 (match_operand:VQ 1 "s_register_operand" "w,w") 399 (parallel [(match_operand:SI 2 "immediate_operand" "i,i")])))] 400 "TARGET_NEON" 401{ 402 int half_elts = GET_MODE_NUNITS (<MODE>mode) / 2; 403 int elt = INTVAL (operands[2]) % half_elts; 404 int hi = (INTVAL (operands[2]) / half_elts) * 2; 405 int regno = REGNO (operands[1]); 406 407 if (BYTES_BIG_ENDIAN) 408 elt = half_elts - 1 - elt; 409 410 operands[1] = gen_rtx_REG (<V_HALF>mode, regno + hi); 411 operands[2] = GEN_INT (elt); 412 413 if (which_alternative == 0) 414 return "vst1.<V_sz_elem>\t{%P1[%c2]}, %A0"; 415 else 416 return "vmov.<V_uf_sclr>\t%0, %P1[%c2]"; 417} 418 [(set_attr "neon_type" "neon_vst1_vst2_lane,neon_bp_simple")] 419) 420 421(define_insn "vec_extractv2di" 422 [(set (match_operand:DI 0 "nonimmediate_operand" "=Um,r") 423 (vec_select:DI 424 (match_operand:V2DI 1 "s_register_operand" "w,w") 425 (parallel [(match_operand:SI 2 "immediate_operand" "i,i")])))] 426 "TARGET_NEON" 427{ 428 int regno = REGNO (operands[1]) + 2 * INTVAL (operands[2]); 429 430 operands[1] = gen_rtx_REG (DImode, regno); 431 432 if (which_alternative == 0) 433 return "vst1.64\t{%P1}, %A0 @ v2di"; 434 else 435 return "vmov\t%Q0, %R0, %P1 @ v2di"; 436} 437 [(set_attr "neon_type" "neon_vst1_vst2_lane,neon_int_1")] 438) 439 440(define_expand "vec_init<mode>" 441 [(match_operand:VDQ 0 "s_register_operand" "") 442 (match_operand 1 "" "")] 443 "TARGET_NEON" 444{ 445 neon_expand_vector_init (operands[0], operands[1]); 446 DONE; 447}) 448 449;; Doubleword and quadword arithmetic. 450 451;; NOTE: some other instructions also support 64-bit integer 452;; element size, which we could potentially use for "long long" operations. 453 454(define_insn "*add<mode>3_neon" 455 [(set (match_operand:VDQ 0 "s_register_operand" "=w") 456 (plus:VDQ (match_operand:VDQ 1 "s_register_operand" "w") 457 (match_operand:VDQ 2 "s_register_operand" "w")))] 458 "TARGET_NEON && (!<Is_float_mode> || flag_unsafe_math_optimizations)" 459 "vadd.<V_if_elem>\t%<V_reg>0, %<V_reg>1, %<V_reg>2" 460 [(set (attr "neon_type") 461 (if_then_else (match_test "<Is_float_mode>") 462 (if_then_else (match_test "<Is_d_reg>") 463 (const_string "neon_fp_vadd_ddd_vabs_dd") 464 (const_string "neon_fp_vadd_qqq_vabs_qq")) 465 (const_string "neon_int_1")))] 466) 467 468(define_insn "adddi3_neon" 469 [(set (match_operand:DI 0 "s_register_operand" "=w,?&r,?&r,?w,?&r,?&r,?&r") 470 (plus:DI (match_operand:DI 1 "s_register_operand" "%w,0,0,w,r,0,r") 471 (match_operand:DI 2 "arm_adddi_operand" "w,r,0,w,r,Dd,Dd"))) 472 (clobber (reg:CC CC_REGNUM))] 473 "TARGET_NEON" 474{ 475 switch (which_alternative) 476 { 477 case 0: /* fall through */ 478 case 3: return "vadd.i64\t%P0, %P1, %P2"; 479 case 1: return "#"; 480 case 2: return "#"; 481 case 4: return "#"; 482 case 5: return "#"; 483 case 6: return "#"; 484 default: gcc_unreachable (); 485 } 486} 487 [(set_attr "neon_type" "neon_int_1,*,*,neon_int_1,*,*,*") 488 (set_attr "conds" "*,clob,clob,*,clob,clob,clob") 489 (set_attr "length" "*,8,8,*,8,8,8") 490 (set_attr "arch" "nota8,*,*,onlya8,*,*,*")] 491) 492 493(define_insn "*sub<mode>3_neon" 494 [(set (match_operand:VDQ 0 "s_register_operand" "=w") 495 (minus:VDQ (match_operand:VDQ 1 "s_register_operand" "w") 496 (match_operand:VDQ 2 "s_register_operand" "w")))] 497 "TARGET_NEON && (!<Is_float_mode> || flag_unsafe_math_optimizations)" 498 "vsub.<V_if_elem>\t%<V_reg>0, %<V_reg>1, %<V_reg>2" 499 [(set (attr "neon_type") 500 (if_then_else (match_test "<Is_float_mode>") 501 (if_then_else (match_test "<Is_d_reg>") 502 (const_string "neon_fp_vadd_ddd_vabs_dd") 503 (const_string "neon_fp_vadd_qqq_vabs_qq")) 504 (const_string "neon_int_2")))] 505) 506 507(define_insn "subdi3_neon" 508 [(set (match_operand:DI 0 "s_register_operand" "=w,?&r,?&r,?&r,?w") 509 (minus:DI (match_operand:DI 1 "s_register_operand" "w,0,r,0,w") 510 (match_operand:DI 2 "s_register_operand" "w,r,0,0,w"))) 511 (clobber (reg:CC CC_REGNUM))] 512 "TARGET_NEON" 513{ 514 switch (which_alternative) 515 { 516 case 0: /* fall through */ 517 case 4: return "vsub.i64\t%P0, %P1, %P2"; 518 case 1: /* fall through */ 519 case 2: /* fall through */ 520 case 3: return "subs\\t%Q0, %Q1, %Q2\;sbc\\t%R0, %R1, %R2"; 521 default: gcc_unreachable (); 522 } 523} 524 [(set_attr "neon_type" "neon_int_2,*,*,*,neon_int_2") 525 (set_attr "conds" "*,clob,clob,clob,*") 526 (set_attr "length" "*,8,8,8,*") 527 (set_attr "arch" "nota8,*,*,*,onlya8")] 528) 529 530(define_insn "*mul<mode>3_neon" 531 [(set (match_operand:VDQ 0 "s_register_operand" "=w") 532 (mult:VDQ (match_operand:VDQ 1 "s_register_operand" "w") 533 (match_operand:VDQ 2 "s_register_operand" "w")))] 534 "TARGET_NEON && (!<Is_float_mode> || flag_unsafe_math_optimizations)" 535 "vmul.<V_if_elem>\t%<V_reg>0, %<V_reg>1, %<V_reg>2" 536 [(set (attr "neon_type") 537 (if_then_else (match_test "<Is_float_mode>") 538 (if_then_else (match_test "<Is_d_reg>") 539 (const_string "neon_fp_vadd_ddd_vabs_dd") 540 (const_string "neon_fp_vadd_qqq_vabs_qq")) 541 (if_then_else (match_test "<Is_d_reg>") 542 (if_then_else 543 (match_test "<Scalar_mul_8_16>") 544 (const_string "neon_mul_ddd_8_16_qdd_16_8_long_32_16_long") 545 (const_string "neon_mul_qqq_8_16_32_ddd_32")) 546 (if_then_else (match_test "<Scalar_mul_8_16>") 547 (const_string "neon_mul_qqq_8_16_32_ddd_32") 548 (const_string "neon_mul_qqq_8_16_32_ddd_32")))))] 549) 550 551(define_insn "mul<mode>3add<mode>_neon" 552 [(set (match_operand:VDQ 0 "s_register_operand" "=w") 553 (plus:VDQ (mult:VDQ (match_operand:VDQ 2 "s_register_operand" "w") 554 (match_operand:VDQ 3 "s_register_operand" "w")) 555 (match_operand:VDQ 1 "s_register_operand" "0")))] 556 "TARGET_NEON && (!<Is_float_mode> || flag_unsafe_math_optimizations)" 557 "vmla.<V_if_elem>\t%<V_reg>0, %<V_reg>2, %<V_reg>3" 558 [(set (attr "neon_type") 559 (if_then_else (match_test "<Is_float_mode>") 560 (if_then_else (match_test "<Is_d_reg>") 561 (const_string "neon_fp_vmla_ddd") 562 (const_string "neon_fp_vmla_qqq")) 563 (if_then_else (match_test "<Is_d_reg>") 564 (if_then_else 565 (match_test "<Scalar_mul_8_16>") 566 (const_string "neon_mla_ddd_8_16_qdd_16_8_long_32_16_long") 567 (const_string "neon_mla_ddd_32_qqd_16_ddd_32_scalar_qdd_64_32_long_scalar_qdd_64_32_long")) 568 (if_then_else (match_test "<Scalar_mul_8_16>") 569 (const_string "neon_mla_qqq_8_16") 570 (const_string "neon_mla_qqq_32_qqd_32_scalar")))))] 571) 572 573(define_insn "mul<mode>3neg<mode>add<mode>_neon" 574 [(set (match_operand:VDQ 0 "s_register_operand" "=w") 575 (minus:VDQ (match_operand:VDQ 1 "s_register_operand" "0") 576 (mult:VDQ (match_operand:VDQ 2 "s_register_operand" "w") 577 (match_operand:VDQ 3 "s_register_operand" "w"))))] 578 "TARGET_NEON && (!<Is_float_mode> || flag_unsafe_math_optimizations)" 579 "vmls.<V_if_elem>\t%<V_reg>0, %<V_reg>2, %<V_reg>3" 580 [(set (attr "neon_type") 581 (if_then_else (match_test "<Is_float_mode>") 582 (if_then_else (match_test "<Is_d_reg>") 583 (const_string "neon_fp_vmla_ddd") 584 (const_string "neon_fp_vmla_qqq")) 585 (if_then_else (match_test "<Is_d_reg>") 586 (if_then_else 587 (match_test "<Scalar_mul_8_16>") 588 (const_string "neon_mla_ddd_8_16_qdd_16_8_long_32_16_long") 589 (const_string "neon_mla_ddd_32_qqd_16_ddd_32_scalar_qdd_64_32_long_scalar_qdd_64_32_long")) 590 (if_then_else (match_test "<Scalar_mul_8_16>") 591 (const_string "neon_mla_qqq_8_16") 592 (const_string "neon_mla_qqq_32_qqd_32_scalar")))))] 593) 594 595;; Fused multiply-accumulate 596;; We define each insn twice here: 597;; 1: with flag_unsafe_math_optimizations for the widening multiply phase 598;; to be able to use when converting to FMA. 599;; 2: without flag_unsafe_math_optimizations for the intrinsics to use. 600(define_insn "fma<VCVTF:mode>4" 601 [(set (match_operand:VCVTF 0 "register_operand" "=w") 602 (fma:VCVTF (match_operand:VCVTF 1 "register_operand" "w") 603 (match_operand:VCVTF 2 "register_operand" "w") 604 (match_operand:VCVTF 3 "register_operand" "0")))] 605 "TARGET_NEON && TARGET_FMA && flag_unsafe_math_optimizations" 606 "vfma%?.<V_if_elem>\\t%<V_reg>0, %<V_reg>1, %<V_reg>2" 607 [(set (attr "neon_type") 608 (if_then_else (match_test "<Is_d_reg>") 609 (const_string "neon_fp_vmla_ddd") 610 (const_string "neon_fp_vmla_qqq")))] 611) 612 613(define_insn "fma<VCVTF:mode>4_intrinsic" 614 [(set (match_operand:VCVTF 0 "register_operand" "=w") 615 (fma:VCVTF (match_operand:VCVTF 1 "register_operand" "w") 616 (match_operand:VCVTF 2 "register_operand" "w") 617 (match_operand:VCVTF 3 "register_operand" "0")))] 618 "TARGET_NEON && TARGET_FMA" 619 "vfma%?.<V_if_elem>\\t%<V_reg>0, %<V_reg>1, %<V_reg>2" 620 [(set (attr "neon_type") 621 (if_then_else (match_test "<Is_d_reg>") 622 (const_string "neon_fp_vmla_ddd") 623 (const_string "neon_fp_vmla_qqq")))] 624) 625 626(define_insn "*fmsub<VCVTF:mode>4" 627 [(set (match_operand:VCVTF 0 "register_operand" "=w") 628 (fma:VCVTF (neg:VCVTF (match_operand:VCVTF 1 "register_operand" "w")) 629 (match_operand:VCVTF 2 "register_operand" "w") 630 (match_operand:VCVTF 3 "register_operand" "0")))] 631 "TARGET_NEON && TARGET_FMA && flag_unsafe_math_optimizations" 632 "vfms%?.<V_if_elem>\\t%<V_reg>0, %<V_reg>1, %<V_reg>2" 633 [(set (attr "neon_type") 634 (if_then_else (match_test "<Is_d_reg>") 635 (const_string "neon_fp_vmla_ddd") 636 (const_string "neon_fp_vmla_qqq")))] 637) 638 639(define_insn "fmsub<VCVTF:mode>4_intrinsic" 640 [(set (match_operand:VCVTF 0 "register_operand" "=w") 641 (fma:VCVTF (neg:VCVTF (match_operand:VCVTF 1 "register_operand" "w")) 642 (match_operand:VCVTF 2 "register_operand" "w") 643 (match_operand:VCVTF 3 "register_operand" "0")))] 644 "TARGET_NEON && TARGET_FMA" 645 "vfms%?.<V_if_elem>\\t%<V_reg>0, %<V_reg>1, %<V_reg>2" 646 [(set (attr "neon_type") 647 (if_then_else (match_test "<Is_d_reg>") 648 (const_string "neon_fp_vmla_ddd") 649 (const_string "neon_fp_vmla_qqq")))] 650) 651 652(define_insn "neon_vrint<NEON_VRINT:nvrint_variant><VCVTF:mode>" 653 [(set (match_operand:VCVTF 0 "s_register_operand" "=w") 654 (unspec:VCVTF [(match_operand:VCVTF 1 655 "s_register_operand" "w")] 656 NEON_VRINT))] 657 "TARGET_NEON && TARGET_FPU_ARMV8" 658 "vrint<nvrint_variant>%?.f32\\t%<V_reg>0, %<V_reg>1" 659 [(set (attr "neon_type") 660 (if_then_else (match_test "<Is_d_reg>") 661 (const_string "neon_fp_vadd_ddd_vabs_dd") 662 (const_string "neon_fp_vadd_qqq_vabs_qq")))] 663) 664 665(define_insn "ior<mode>3" 666 [(set (match_operand:VDQ 0 "s_register_operand" "=w,w") 667 (ior:VDQ (match_operand:VDQ 1 "s_register_operand" "w,0") 668 (match_operand:VDQ 2 "neon_logic_op2" "w,Dl")))] 669 "TARGET_NEON" 670{ 671 switch (which_alternative) 672 { 673 case 0: return "vorr\t%<V_reg>0, %<V_reg>1, %<V_reg>2"; 674 case 1: return neon_output_logic_immediate ("vorr", &operands[2], 675 <MODE>mode, 0, VALID_NEON_QREG_MODE (<MODE>mode)); 676 default: gcc_unreachable (); 677 } 678} 679 [(set_attr "neon_type" "neon_int_1")] 680) 681 682(define_insn "iordi3_neon" 683 [(set (match_operand:DI 0 "s_register_operand" "=w,w,?&r,?&r,?w,?w") 684 (ior:DI (match_operand:DI 1 "s_register_operand" "%w,0,0,r,w,0") 685 (match_operand:DI 2 "neon_logic_op2" "w,Dl,r,r,w,Dl")))] 686 "TARGET_NEON" 687{ 688 switch (which_alternative) 689 { 690 case 0: /* fall through */ 691 case 4: return "vorr\t%P0, %P1, %P2"; 692 case 1: /* fall through */ 693 case 5: return neon_output_logic_immediate ("vorr", &operands[2], 694 DImode, 0, VALID_NEON_QREG_MODE (DImode)); 695 case 2: return "#"; 696 case 3: return "#"; 697 default: gcc_unreachable (); 698 } 699} 700 [(set_attr "neon_type" "neon_int_1,neon_int_1,*,*,neon_int_1,neon_int_1") 701 (set_attr "length" "*,*,8,8,*,*") 702 (set_attr "arch" "nota8,nota8,*,*,onlya8,onlya8")] 703) 704 705;; The concrete forms of the Neon immediate-logic instructions are vbic and 706;; vorr. We support the pseudo-instruction vand instead, because that 707;; corresponds to the canonical form the middle-end expects to use for 708;; immediate bitwise-ANDs. 709 710(define_insn "and<mode>3" 711 [(set (match_operand:VDQ 0 "s_register_operand" "=w,w") 712 (and:VDQ (match_operand:VDQ 1 "s_register_operand" "w,0") 713 (match_operand:VDQ 2 "neon_inv_logic_op2" "w,DL")))] 714 "TARGET_NEON" 715{ 716 switch (which_alternative) 717 { 718 case 0: return "vand\t%<V_reg>0, %<V_reg>1, %<V_reg>2"; 719 case 1: return neon_output_logic_immediate ("vand", &operands[2], 720 <MODE>mode, 1, VALID_NEON_QREG_MODE (<MODE>mode)); 721 default: gcc_unreachable (); 722 } 723} 724 [(set_attr "neon_type" "neon_int_1")] 725) 726 727(define_insn "anddi3_neon" 728 [(set (match_operand:DI 0 "s_register_operand" "=w,w,?&r,?&r,?w,?w") 729 (and:DI (match_operand:DI 1 "s_register_operand" "%w,0,0,r,w,0") 730 (match_operand:DI 2 "neon_inv_logic_op2" "w,DL,r,r,w,DL")))] 731 "TARGET_NEON" 732{ 733 switch (which_alternative) 734 { 735 case 0: /* fall through */ 736 case 4: return "vand\t%P0, %P1, %P2"; 737 case 1: /* fall through */ 738 case 5: return neon_output_logic_immediate ("vand", &operands[2], 739 DImode, 1, VALID_NEON_QREG_MODE (DImode)); 740 case 2: return "#"; 741 case 3: return "#"; 742 default: gcc_unreachable (); 743 } 744} 745 [(set_attr "neon_type" "neon_int_1,neon_int_1,*,*,neon_int_1,neon_int_1") 746 (set_attr "length" "*,*,8,8,*,*") 747 (set_attr "arch" "nota8,nota8,*,*,onlya8,onlya8")] 748) 749 750(define_insn "orn<mode>3_neon" 751 [(set (match_operand:VDQ 0 "s_register_operand" "=w") 752 (ior:VDQ (not:VDQ (match_operand:VDQ 2 "s_register_operand" "w")) 753 (match_operand:VDQ 1 "s_register_operand" "w")))] 754 "TARGET_NEON" 755 "vorn\t%<V_reg>0, %<V_reg>1, %<V_reg>2" 756 [(set_attr "neon_type" "neon_int_1")] 757) 758 759;; TODO: investigate whether we should disable 760;; this and bicdi3_neon for the A8 in line with the other 761;; changes above. 762(define_insn_and_split "orndi3_neon" 763 [(set (match_operand:DI 0 "s_register_operand" "=w,?&r,?&r,?&r") 764 (ior:DI (not:DI (match_operand:DI 2 "s_register_operand" "w,0,0,r")) 765 (match_operand:DI 1 "s_register_operand" "w,r,r,0")))] 766 "TARGET_NEON" 767 "@ 768 vorn\t%P0, %P1, %P2 769 # 770 # 771 #" 772 "reload_completed && 773 (TARGET_NEON && !(IS_VFP_REGNUM (REGNO (operands[0]))))" 774 [(set (match_dup 0) (ior:SI (not:SI (match_dup 2)) (match_dup 1))) 775 (set (match_dup 3) (ior:SI (not:SI (match_dup 4)) (match_dup 5)))] 776 " 777 { 778 if (TARGET_THUMB2) 779 { 780 operands[3] = gen_highpart (SImode, operands[0]); 781 operands[0] = gen_lowpart (SImode, operands[0]); 782 operands[4] = gen_highpart (SImode, operands[2]); 783 operands[2] = gen_lowpart (SImode, operands[2]); 784 operands[5] = gen_highpart (SImode, operands[1]); 785 operands[1] = gen_lowpart (SImode, operands[1]); 786 } 787 else 788 { 789 emit_insn (gen_one_cmpldi2 (operands[0], operands[2])); 790 emit_insn (gen_iordi3 (operands[0], operands[1], operands[0])); 791 DONE; 792 } 793 }" 794 [(set_attr "neon_type" "neon_int_1,*,*,*") 795 (set_attr "length" "*,16,8,8") 796 (set_attr "arch" "any,a,t2,t2")] 797) 798 799(define_insn "bic<mode>3_neon" 800 [(set (match_operand:VDQ 0 "s_register_operand" "=w") 801 (and:VDQ (not:VDQ (match_operand:VDQ 2 "s_register_operand" "w")) 802 (match_operand:VDQ 1 "s_register_operand" "w")))] 803 "TARGET_NEON" 804 "vbic\t%<V_reg>0, %<V_reg>1, %<V_reg>2" 805 [(set_attr "neon_type" "neon_int_1")] 806) 807 808;; Compare to *anddi_notdi_di. 809(define_insn "bicdi3_neon" 810 [(set (match_operand:DI 0 "s_register_operand" "=w,?=&r,?&r") 811 (and:DI (not:DI (match_operand:DI 2 "s_register_operand" "w,r,0")) 812 (match_operand:DI 1 "s_register_operand" "w,0,r")))] 813 "TARGET_NEON" 814 "@ 815 vbic\t%P0, %P1, %P2 816 # 817 #" 818 [(set_attr "neon_type" "neon_int_1,*,*") 819 (set_attr "length" "*,8,8")] 820) 821 822(define_insn "xor<mode>3" 823 [(set (match_operand:VDQ 0 "s_register_operand" "=w") 824 (xor:VDQ (match_operand:VDQ 1 "s_register_operand" "w") 825 (match_operand:VDQ 2 "s_register_operand" "w")))] 826 "TARGET_NEON" 827 "veor\t%<V_reg>0, %<V_reg>1, %<V_reg>2" 828 [(set_attr "neon_type" "neon_int_1")] 829) 830 831(define_insn "xordi3_neon" 832 [(set (match_operand:DI 0 "s_register_operand" "=w,?&r,?&r,?w") 833 (xor:DI (match_operand:DI 1 "s_register_operand" "%w,0,r,w") 834 (match_operand:DI 2 "s_register_operand" "w,r,r,w")))] 835 "TARGET_NEON" 836 "@ 837 veor\t%P0, %P1, %P2 838 # 839 # 840 veor\t%P0, %P1, %P2" 841 [(set_attr "neon_type" "neon_int_1,*,*,neon_int_1") 842 (set_attr "length" "*,8,8,*") 843 (set_attr "arch" "nota8,*,*,onlya8")] 844) 845 846(define_insn "one_cmpl<mode>2" 847 [(set (match_operand:VDQ 0 "s_register_operand" "=w") 848 (not:VDQ (match_operand:VDQ 1 "s_register_operand" "w")))] 849 "TARGET_NEON" 850 "vmvn\t%<V_reg>0, %<V_reg>1" 851 [(set_attr "neon_type" "neon_int_1")] 852) 853 854(define_insn "abs<mode>2" 855 [(set (match_operand:VDQW 0 "s_register_operand" "=w") 856 (abs:VDQW (match_operand:VDQW 1 "s_register_operand" "w")))] 857 "TARGET_NEON" 858 "vabs.<V_s_elem>\t%<V_reg>0, %<V_reg>1" 859 [(set (attr "neon_type") 860 (if_then_else (match_test "<Is_float_mode>") 861 (if_then_else (match_test "<Is_d_reg>") 862 (const_string "neon_fp_vadd_ddd_vabs_dd") 863 (const_string "neon_fp_vadd_qqq_vabs_qq")) 864 (const_string "neon_int_3")))] 865) 866 867(define_insn "neg<mode>2" 868 [(set (match_operand:VDQW 0 "s_register_operand" "=w") 869 (neg:VDQW (match_operand:VDQW 1 "s_register_operand" "w")))] 870 "TARGET_NEON" 871 "vneg.<V_s_elem>\t%<V_reg>0, %<V_reg>1" 872 [(set (attr "neon_type") 873 (if_then_else (match_test "<Is_float_mode>") 874 (if_then_else (match_test "<Is_d_reg>") 875 (const_string "neon_fp_vadd_ddd_vabs_dd") 876 (const_string "neon_fp_vadd_qqq_vabs_qq")) 877 (const_string "neon_int_3")))] 878) 879 880(define_insn "negdi2_neon" 881 [(set (match_operand:DI 0 "s_register_operand" "=&w, w,r,&r") 882 (neg:DI (match_operand:DI 1 "s_register_operand" " w, w,0, r"))) 883 (clobber (match_scratch:DI 2 "= X,&w,X, X")) 884 (clobber (reg:CC CC_REGNUM))] 885 "TARGET_NEON" 886 "#" 887 [(set_attr "length" "8")] 888) 889 890; Split negdi2_neon for vfp registers 891(define_split 892 [(set (match_operand:DI 0 "s_register_operand" "") 893 (neg:DI (match_operand:DI 1 "s_register_operand" ""))) 894 (clobber (match_scratch:DI 2 "")) 895 (clobber (reg:CC CC_REGNUM))] 896 "TARGET_NEON && reload_completed && IS_VFP_REGNUM (REGNO (operands[0]))" 897 [(set (match_dup 2) (const_int 0)) 898 (parallel [(set (match_dup 0) (minus:DI (match_dup 2) (match_dup 1))) 899 (clobber (reg:CC CC_REGNUM))])] 900 { 901 if (!REG_P (operands[2])) 902 operands[2] = operands[0]; 903 } 904) 905 906; Split negdi2_neon for core registers 907(define_split 908 [(set (match_operand:DI 0 "s_register_operand" "") 909 (neg:DI (match_operand:DI 1 "s_register_operand" ""))) 910 (clobber (match_scratch:DI 2 "")) 911 (clobber (reg:CC CC_REGNUM))] 912 "TARGET_32BIT && reload_completed 913 && arm_general_register_operand (operands[0], DImode)" 914 [(parallel [(set (match_dup 0) (neg:DI (match_dup 1))) 915 (clobber (reg:CC CC_REGNUM))])] 916 "" 917) 918 919(define_insn "*umin<mode>3_neon" 920 [(set (match_operand:VDQIW 0 "s_register_operand" "=w") 921 (umin:VDQIW (match_operand:VDQIW 1 "s_register_operand" "w") 922 (match_operand:VDQIW 2 "s_register_operand" "w")))] 923 "TARGET_NEON" 924 "vmin.<V_u_elem>\t%<V_reg>0, %<V_reg>1, %<V_reg>2" 925 [(set_attr "neon_type" "neon_int_5")] 926) 927 928(define_insn "*umax<mode>3_neon" 929 [(set (match_operand:VDQIW 0 "s_register_operand" "=w") 930 (umax:VDQIW (match_operand:VDQIW 1 "s_register_operand" "w") 931 (match_operand:VDQIW 2 "s_register_operand" "w")))] 932 "TARGET_NEON" 933 "vmax.<V_u_elem>\t%<V_reg>0, %<V_reg>1, %<V_reg>2" 934 [(set_attr "neon_type" "neon_int_5")] 935) 936 937(define_insn "*smin<mode>3_neon" 938 [(set (match_operand:VDQW 0 "s_register_operand" "=w") 939 (smin:VDQW (match_operand:VDQW 1 "s_register_operand" "w") 940 (match_operand:VDQW 2 "s_register_operand" "w")))] 941 "TARGET_NEON" 942 "vmin.<V_s_elem>\t%<V_reg>0, %<V_reg>1, %<V_reg>2" 943 [(set (attr "neon_type") 944 (if_then_else (match_test "<Is_float_mode>") 945 (const_string "neon_fp_vadd_ddd_vabs_dd") 946 (const_string "neon_int_5")))] 947) 948 949(define_insn "*smax<mode>3_neon" 950 [(set (match_operand:VDQW 0 "s_register_operand" "=w") 951 (smax:VDQW (match_operand:VDQW 1 "s_register_operand" "w") 952 (match_operand:VDQW 2 "s_register_operand" "w")))] 953 "TARGET_NEON" 954 "vmax.<V_s_elem>\t%<V_reg>0, %<V_reg>1, %<V_reg>2" 955 [(set (attr "neon_type") 956 (if_then_else (match_test "<Is_float_mode>") 957 (const_string "neon_fp_vadd_ddd_vabs_dd") 958 (const_string "neon_int_5")))] 959) 960 961; TODO: V2DI shifts are current disabled because there are bugs in the 962; generic vectorizer code. It ends up creating a V2DI constructor with 963; SImode elements. 964 965(define_insn "vashl<mode>3" 966 [(set (match_operand:VDQIW 0 "s_register_operand" "=w,w") 967 (ashift:VDQIW (match_operand:VDQIW 1 "s_register_operand" "w,w") 968 (match_operand:VDQIW 2 "imm_lshift_or_reg_neon" "w,Dn")))] 969 "TARGET_NEON" 970 { 971 switch (which_alternative) 972 { 973 case 0: return "vshl.<V_s_elem>\t%<V_reg>0, %<V_reg>1, %<V_reg>2"; 974 case 1: return neon_output_shift_immediate ("vshl", 'i', &operands[2], 975 <MODE>mode, 976 VALID_NEON_QREG_MODE (<MODE>mode), 977 true); 978 default: gcc_unreachable (); 979 } 980 } 981 [(set (attr "neon_type") 982 (if_then_else (match_test "<Is_d_reg>") 983 (const_string "neon_vshl_ddd") 984 (const_string "neon_shift_3")))] 985) 986 987(define_insn "vashr<mode>3_imm" 988 [(set (match_operand:VDQIW 0 "s_register_operand" "=w") 989 (ashiftrt:VDQIW (match_operand:VDQIW 1 "s_register_operand" "w") 990 (match_operand:VDQIW 2 "imm_for_neon_rshift_operand" "Dn")))] 991 "TARGET_NEON" 992 { 993 return neon_output_shift_immediate ("vshr", 's', &operands[2], 994 <MODE>mode, VALID_NEON_QREG_MODE (<MODE>mode), 995 false); 996 } 997 [(set (attr "neon_type") 998 (if_then_else (match_test "<Is_d_reg>") 999 (const_string "neon_vshl_ddd") 1000 (const_string "neon_shift_3")))] 1001) 1002 1003(define_insn "vlshr<mode>3_imm" 1004 [(set (match_operand:VDQIW 0 "s_register_operand" "=w") 1005 (lshiftrt:VDQIW (match_operand:VDQIW 1 "s_register_operand" "w") 1006 (match_operand:VDQIW 2 "imm_for_neon_rshift_operand" "Dn")))] 1007 "TARGET_NEON" 1008 { 1009 return neon_output_shift_immediate ("vshr", 'u', &operands[2], 1010 <MODE>mode, VALID_NEON_QREG_MODE (<MODE>mode), 1011 false); 1012 } 1013 [(set (attr "neon_type") 1014 (if_then_else (match_test "<Is_d_reg>") 1015 (const_string "neon_vshl_ddd") 1016 (const_string "neon_shift_3")))] 1017) 1018 1019; Used for implementing logical shift-right, which is a left-shift by a negative 1020; amount, with signed operands. This is essentially the same as ashl<mode>3 1021; above, but using an unspec in case GCC tries anything tricky with negative 1022; shift amounts. 1023 1024(define_insn "ashl<mode>3_signed" 1025 [(set (match_operand:VDQI 0 "s_register_operand" "=w") 1026 (unspec:VDQI [(match_operand:VDQI 1 "s_register_operand" "w") 1027 (match_operand:VDQI 2 "s_register_operand" "w")] 1028 UNSPEC_ASHIFT_SIGNED))] 1029 "TARGET_NEON" 1030 "vshl.<V_s_elem>\t%<V_reg>0, %<V_reg>1, %<V_reg>2" 1031 [(set (attr "neon_type") 1032 (if_then_else (match_test "<Is_d_reg>") 1033 (const_string "neon_vshl_ddd") 1034 (const_string "neon_shift_3")))] 1035) 1036 1037; Used for implementing logical shift-right, which is a left-shift by a negative 1038; amount, with unsigned operands. 1039 1040(define_insn "ashl<mode>3_unsigned" 1041 [(set (match_operand:VDQI 0 "s_register_operand" "=w") 1042 (unspec:VDQI [(match_operand:VDQI 1 "s_register_operand" "w") 1043 (match_operand:VDQI 2 "s_register_operand" "w")] 1044 UNSPEC_ASHIFT_UNSIGNED))] 1045 "TARGET_NEON" 1046 "vshl.<V_u_elem>\t%<V_reg>0, %<V_reg>1, %<V_reg>2" 1047 [(set (attr "neon_type") 1048 (if_then_else (match_test "<Is_d_reg>") 1049 (const_string "neon_vshl_ddd") 1050 (const_string "neon_shift_3")))] 1051) 1052 1053(define_expand "vashr<mode>3" 1054 [(set (match_operand:VDQIW 0 "s_register_operand" "") 1055 (ashiftrt:VDQIW (match_operand:VDQIW 1 "s_register_operand" "") 1056 (match_operand:VDQIW 2 "imm_rshift_or_reg_neon" "")))] 1057 "TARGET_NEON" 1058{ 1059 if (s_register_operand (operands[2], <MODE>mode)) 1060 { 1061 rtx neg = gen_reg_rtx (<MODE>mode); 1062 emit_insn (gen_neg<mode>2 (neg, operands[2])); 1063 emit_insn (gen_ashl<mode>3_signed (operands[0], operands[1], neg)); 1064 } 1065 else 1066 emit_insn (gen_vashr<mode>3_imm (operands[0], operands[1], operands[2])); 1067 DONE; 1068}) 1069 1070(define_expand "vlshr<mode>3" 1071 [(set (match_operand:VDQIW 0 "s_register_operand" "") 1072 (lshiftrt:VDQIW (match_operand:VDQIW 1 "s_register_operand" "") 1073 (match_operand:VDQIW 2 "imm_rshift_or_reg_neon" "")))] 1074 "TARGET_NEON" 1075{ 1076 if (s_register_operand (operands[2], <MODE>mode)) 1077 { 1078 rtx neg = gen_reg_rtx (<MODE>mode); 1079 emit_insn (gen_neg<mode>2 (neg, operands[2])); 1080 emit_insn (gen_ashl<mode>3_unsigned (operands[0], operands[1], neg)); 1081 } 1082 else 1083 emit_insn (gen_vlshr<mode>3_imm (operands[0], operands[1], operands[2])); 1084 DONE; 1085}) 1086 1087;; 64-bit shifts 1088 1089;; This pattern loads a 32-bit shift count into a 64-bit NEON register, 1090;; leaving the upper half uninitalized. This is OK since the shift 1091;; instruction only looks at the low 8 bits anyway. To avoid confusing 1092;; data flow analysis however, we pretend the full register is set 1093;; using an unspec. 1094(define_insn "neon_load_count" 1095 [(set (match_operand:DI 0 "s_register_operand" "=w,w") 1096 (unspec:DI [(match_operand:SI 1 "nonimmediate_operand" "Um,r")] 1097 UNSPEC_LOAD_COUNT))] 1098 "TARGET_NEON" 1099 "@ 1100 vld1.32\t{%P0[0]}, %A1 1101 vmov.32\t%P0[0], %1" 1102 [(set_attr "neon_type" "neon_vld1_vld2_lane,neon_mcr")] 1103) 1104 1105(define_insn "ashldi3_neon_noclobber" 1106 [(set (match_operand:DI 0 "s_register_operand" "=w,w") 1107 (ashift:DI (match_operand:DI 1 "s_register_operand" " w,w") 1108 (match_operand:DI 2 "reg_or_int_operand" " i,w")))] 1109 "TARGET_NEON && reload_completed 1110 && (!CONST_INT_P (operands[2]) 1111 || (INTVAL (operands[2]) >= 0 && INTVAL (operands[2]) < 64))" 1112 "@ 1113 vshl.u64\t%P0, %P1, %2 1114 vshl.u64\t%P0, %P1, %P2" 1115 [(set_attr "neon_type" "neon_vshl_ddd,neon_vshl_ddd")] 1116) 1117 1118(define_insn_and_split "ashldi3_neon" 1119 [(set (match_operand:DI 0 "s_register_operand" "= w, w,?&r,?r, ?w,w") 1120 (ashift:DI (match_operand:DI 1 "s_register_operand" " 0w, w, 0r, r, 0w,w") 1121 (match_operand:SI 2 "general_operand" "rUm, i, r, i,rUm,i"))) 1122 (clobber (match_scratch:SI 3 "= X, X,?&r, X, X,X")) 1123 (clobber (match_scratch:SI 4 "= X, X,?&r, X, X,X")) 1124 (clobber (match_scratch:DI 5 "=&w, X, X, X, &w,X")) 1125 (clobber (reg:CC_C CC_REGNUM))] 1126 "TARGET_NEON" 1127 "#" 1128 "TARGET_NEON && reload_completed" 1129 [(const_int 0)] 1130 " 1131 { 1132 if (IS_VFP_REGNUM (REGNO (operands[0]))) 1133 { 1134 if (CONST_INT_P (operands[2])) 1135 { 1136 if (INTVAL (operands[2]) < 1) 1137 { 1138 emit_insn (gen_movdi (operands[0], operands[1])); 1139 DONE; 1140 } 1141 else if (INTVAL (operands[2]) > 63) 1142 operands[2] = gen_rtx_CONST_INT (VOIDmode, 63); 1143 } 1144 else 1145 { 1146 emit_insn (gen_neon_load_count (operands[5], operands[2])); 1147 operands[2] = operands[5]; 1148 } 1149 1150 /* Ditch the unnecessary clobbers. */ 1151 emit_insn (gen_ashldi3_neon_noclobber (operands[0], operands[1], 1152 operands[2])); 1153 } 1154 else 1155 { 1156 if (CONST_INT_P (operands[2]) && INTVAL (operands[2]) == 1) 1157 /* This clobbers CC. */ 1158 emit_insn (gen_arm_ashldi3_1bit (operands[0], operands[1])); 1159 else 1160 arm_emit_coreregs_64bit_shift (ASHIFT, operands[0], operands[1], 1161 operands[2], operands[3], operands[4]); 1162 } 1163 DONE; 1164 }" 1165 [(set_attr "arch" "nota8,nota8,*,*,onlya8,onlya8") 1166 (set_attr "opt" "*,*,speed,speed,*,*")] 1167) 1168 1169; The shift amount needs to be negated for right-shifts 1170(define_insn "signed_shift_di3_neon" 1171 [(set (match_operand:DI 0 "s_register_operand" "=w") 1172 (unspec:DI [(match_operand:DI 1 "s_register_operand" " w") 1173 (match_operand:DI 2 "s_register_operand" " w")] 1174 UNSPEC_ASHIFT_SIGNED))] 1175 "TARGET_NEON && reload_completed" 1176 "vshl.s64\t%P0, %P1, %P2" 1177 [(set_attr "neon_type" "neon_vshl_ddd")] 1178) 1179 1180; The shift amount needs to be negated for right-shifts 1181(define_insn "unsigned_shift_di3_neon" 1182 [(set (match_operand:DI 0 "s_register_operand" "=w") 1183 (unspec:DI [(match_operand:DI 1 "s_register_operand" " w") 1184 (match_operand:DI 2 "s_register_operand" " w")] 1185 UNSPEC_ASHIFT_UNSIGNED))] 1186 "TARGET_NEON && reload_completed" 1187 "vshl.u64\t%P0, %P1, %P2" 1188 [(set_attr "neon_type" "neon_vshl_ddd")] 1189) 1190 1191(define_insn "ashrdi3_neon_imm_noclobber" 1192 [(set (match_operand:DI 0 "s_register_operand" "=w") 1193 (ashiftrt:DI (match_operand:DI 1 "s_register_operand" " w") 1194 (match_operand:DI 2 "const_int_operand" " i")))] 1195 "TARGET_NEON && reload_completed 1196 && INTVAL (operands[2]) > 0 && INTVAL (operands[2]) <= 64" 1197 "vshr.s64\t%P0, %P1, %2" 1198 [(set_attr "neon_type" "neon_vshl_ddd")] 1199) 1200 1201(define_insn "lshrdi3_neon_imm_noclobber" 1202 [(set (match_operand:DI 0 "s_register_operand" "=w") 1203 (lshiftrt:DI (match_operand:DI 1 "s_register_operand" " w") 1204 (match_operand:DI 2 "const_int_operand" " i")))] 1205 "TARGET_NEON && reload_completed 1206 && INTVAL (operands[2]) > 0 && INTVAL (operands[2]) <= 64" 1207 "vshr.u64\t%P0, %P1, %2" 1208 [(set_attr "neon_type" "neon_vshl_ddd")] 1209) 1210 1211;; ashrdi3_neon 1212;; lshrdi3_neon 1213(define_insn_and_split "<shift>di3_neon" 1214 [(set (match_operand:DI 0 "s_register_operand" "= w, w,?&r,?r,?w,?w") 1215 (rshifts:DI (match_operand:DI 1 "s_register_operand" " 0w, w, 0r, r,0w, w") 1216 (match_operand:SI 2 "reg_or_int_operand" " r, i, r, i, r, i"))) 1217 (clobber (match_scratch:SI 3 "=2r, X, &r, X,2r, X")) 1218 (clobber (match_scratch:SI 4 "= X, X, &r, X, X, X")) 1219 (clobber (match_scratch:DI 5 "=&w, X, X, X,&w, X")) 1220 (clobber (reg:CC CC_REGNUM))] 1221 "TARGET_NEON" 1222 "#" 1223 "TARGET_NEON && reload_completed" 1224 [(const_int 0)] 1225 " 1226 { 1227 if (IS_VFP_REGNUM (REGNO (operands[0]))) 1228 { 1229 if (CONST_INT_P (operands[2])) 1230 { 1231 if (INTVAL (operands[2]) < 1) 1232 { 1233 emit_insn (gen_movdi (operands[0], operands[1])); 1234 DONE; 1235 } 1236 else if (INTVAL (operands[2]) > 64) 1237 operands[2] = gen_rtx_CONST_INT (VOIDmode, 64); 1238 1239 /* Ditch the unnecessary clobbers. */ 1240 emit_insn (gen_<shift>di3_neon_imm_noclobber (operands[0], 1241 operands[1], 1242 operands[2])); 1243 } 1244 else 1245 { 1246 /* We must use a negative left-shift. */ 1247 emit_insn (gen_negsi2 (operands[3], operands[2])); 1248 emit_insn (gen_neon_load_count (operands[5], operands[3])); 1249 emit_insn (gen_<shifttype>_shift_di3_neon (operands[0], operands[1], 1250 operands[5])); 1251 } 1252 } 1253 else 1254 { 1255 if (CONST_INT_P (operands[2]) && INTVAL (operands[2]) == 1) 1256 /* This clobbers CC. */ 1257 emit_insn (gen_arm_<shift>di3_1bit (operands[0], operands[1])); 1258 else 1259 /* This clobbers CC (ASHIFTRT by register only). */ 1260 arm_emit_coreregs_64bit_shift (<CODE>, operands[0], operands[1], 1261 operands[2], operands[3], operands[4]); 1262 } 1263 1264 DONE; 1265 }" 1266 [(set_attr "arch" "nota8,nota8,*,*,onlya8,onlya8") 1267 (set_attr "opt" "*,*,speed,speed,*,*")] 1268) 1269 1270;; Widening operations 1271 1272(define_insn "widen_ssum<mode>3" 1273 [(set (match_operand:<V_widen> 0 "s_register_operand" "=w") 1274 (plus:<V_widen> (sign_extend:<V_widen> 1275 (match_operand:VW 1 "s_register_operand" "%w")) 1276 (match_operand:<V_widen> 2 "s_register_operand" "w")))] 1277 "TARGET_NEON" 1278 "vaddw.<V_s_elem>\t%q0, %q2, %P1" 1279 [(set_attr "neon_type" "neon_int_3")] 1280) 1281 1282(define_insn "widen_usum<mode>3" 1283 [(set (match_operand:<V_widen> 0 "s_register_operand" "=w") 1284 (plus:<V_widen> (zero_extend:<V_widen> 1285 (match_operand:VW 1 "s_register_operand" "%w")) 1286 (match_operand:<V_widen> 2 "s_register_operand" "w")))] 1287 "TARGET_NEON" 1288 "vaddw.<V_u_elem>\t%q0, %q2, %P1" 1289 [(set_attr "neon_type" "neon_int_3")] 1290) 1291 1292;; VEXT can be used to synthesize coarse whole-vector shifts with 8-bit 1293;; shift-count granularity. That's good enough for the middle-end's current 1294;; needs. 1295 1296;; Note that it's not safe to perform such an operation in big-endian mode, 1297;; due to element-ordering issues. 1298 1299(define_expand "vec_shr_<mode>" 1300 [(match_operand:VDQ 0 "s_register_operand" "") 1301 (match_operand:VDQ 1 "s_register_operand" "") 1302 (match_operand:SI 2 "const_multiple_of_8_operand" "")] 1303 "TARGET_NEON && !BYTES_BIG_ENDIAN" 1304{ 1305 rtx zero_reg; 1306 HOST_WIDE_INT num_bits = INTVAL (operands[2]); 1307 const int width = GET_MODE_BITSIZE (<MODE>mode); 1308 const enum machine_mode bvecmode = (width == 128) ? V16QImode : V8QImode; 1309 rtx (*gen_ext) (rtx, rtx, rtx, rtx) = 1310 (width == 128) ? gen_neon_vextv16qi : gen_neon_vextv8qi; 1311 1312 if (num_bits == width) 1313 { 1314 emit_move_insn (operands[0], operands[1]); 1315 DONE; 1316 } 1317 1318 zero_reg = force_reg (bvecmode, CONST0_RTX (bvecmode)); 1319 operands[0] = gen_lowpart (bvecmode, operands[0]); 1320 operands[1] = gen_lowpart (bvecmode, operands[1]); 1321 1322 emit_insn (gen_ext (operands[0], operands[1], zero_reg, 1323 GEN_INT (num_bits / BITS_PER_UNIT))); 1324 DONE; 1325}) 1326 1327(define_expand "vec_shl_<mode>" 1328 [(match_operand:VDQ 0 "s_register_operand" "") 1329 (match_operand:VDQ 1 "s_register_operand" "") 1330 (match_operand:SI 2 "const_multiple_of_8_operand" "")] 1331 "TARGET_NEON && !BYTES_BIG_ENDIAN" 1332{ 1333 rtx zero_reg; 1334 HOST_WIDE_INT num_bits = INTVAL (operands[2]); 1335 const int width = GET_MODE_BITSIZE (<MODE>mode); 1336 const enum machine_mode bvecmode = (width == 128) ? V16QImode : V8QImode; 1337 rtx (*gen_ext) (rtx, rtx, rtx, rtx) = 1338 (width == 128) ? gen_neon_vextv16qi : gen_neon_vextv8qi; 1339 1340 if (num_bits == 0) 1341 { 1342 emit_move_insn (operands[0], CONST0_RTX (<MODE>mode)); 1343 DONE; 1344 } 1345 1346 num_bits = width - num_bits; 1347 1348 zero_reg = force_reg (bvecmode, CONST0_RTX (bvecmode)); 1349 operands[0] = gen_lowpart (bvecmode, operands[0]); 1350 operands[1] = gen_lowpart (bvecmode, operands[1]); 1351 1352 emit_insn (gen_ext (operands[0], zero_reg, operands[1], 1353 GEN_INT (num_bits / BITS_PER_UNIT))); 1354 DONE; 1355}) 1356 1357;; Helpers for quad-word reduction operations 1358 1359; Add (or smin, smax...) the low N/2 elements of the N-element vector 1360; operand[1] to the high N/2 elements of same. Put the result in operand[0], an 1361; N/2-element vector. 1362 1363(define_insn "quad_halves_<code>v4si" 1364 [(set (match_operand:V2SI 0 "s_register_operand" "=w") 1365 (vqh_ops:V2SI 1366 (vec_select:V2SI (match_operand:V4SI 1 "s_register_operand" "w") 1367 (parallel [(const_int 0) (const_int 1)])) 1368 (vec_select:V2SI (match_dup 1) 1369 (parallel [(const_int 2) (const_int 3)]))))] 1370 "TARGET_NEON" 1371 "<VQH_mnem>.<VQH_sign>32\t%P0, %e1, %f1" 1372 [(set_attr "vqh_mnem" "<VQH_mnem>") 1373 (set (attr "neon_type") 1374 (if_then_else (eq_attr "vqh_mnem" "vadd") 1375 (const_string "neon_int_1") (const_string "neon_int_5")))] 1376) 1377 1378(define_insn "quad_halves_<code>v4sf" 1379 [(set (match_operand:V2SF 0 "s_register_operand" "=w") 1380 (vqhs_ops:V2SF 1381 (vec_select:V2SF (match_operand:V4SF 1 "s_register_operand" "w") 1382 (parallel [(const_int 0) (const_int 1)])) 1383 (vec_select:V2SF (match_dup 1) 1384 (parallel [(const_int 2) (const_int 3)]))))] 1385 "TARGET_NEON && flag_unsafe_math_optimizations" 1386 "<VQH_mnem>.f32\t%P0, %e1, %f1" 1387 [(set_attr "vqh_mnem" "<VQH_mnem>") 1388 (set (attr "neon_type") 1389 (if_then_else (eq_attr "vqh_mnem" "vadd") 1390 (const_string "neon_int_1") (const_string "neon_int_5")))] 1391) 1392 1393(define_insn "quad_halves_<code>v8hi" 1394 [(set (match_operand:V4HI 0 "s_register_operand" "+w") 1395 (vqh_ops:V4HI 1396 (vec_select:V4HI (match_operand:V8HI 1 "s_register_operand" "w") 1397 (parallel [(const_int 0) (const_int 1) 1398 (const_int 2) (const_int 3)])) 1399 (vec_select:V4HI (match_dup 1) 1400 (parallel [(const_int 4) (const_int 5) 1401 (const_int 6) (const_int 7)]))))] 1402 "TARGET_NEON" 1403 "<VQH_mnem>.<VQH_sign>16\t%P0, %e1, %f1" 1404 [(set_attr "vqh_mnem" "<VQH_mnem>") 1405 (set (attr "neon_type") 1406 (if_then_else (eq_attr "vqh_mnem" "vadd") 1407 (const_string "neon_int_1") (const_string "neon_int_5")))] 1408) 1409 1410(define_insn "quad_halves_<code>v16qi" 1411 [(set (match_operand:V8QI 0 "s_register_operand" "+w") 1412 (vqh_ops:V8QI 1413 (vec_select:V8QI (match_operand:V16QI 1 "s_register_operand" "w") 1414 (parallel [(const_int 0) (const_int 1) 1415 (const_int 2) (const_int 3) 1416 (const_int 4) (const_int 5) 1417 (const_int 6) (const_int 7)])) 1418 (vec_select:V8QI (match_dup 1) 1419 (parallel [(const_int 8) (const_int 9) 1420 (const_int 10) (const_int 11) 1421 (const_int 12) (const_int 13) 1422 (const_int 14) (const_int 15)]))))] 1423 "TARGET_NEON" 1424 "<VQH_mnem>.<VQH_sign>8\t%P0, %e1, %f1" 1425 [(set_attr "vqh_mnem" "<VQH_mnem>") 1426 (set (attr "neon_type") 1427 (if_then_else (eq_attr "vqh_mnem" "vadd") 1428 (const_string "neon_int_1") (const_string "neon_int_5")))] 1429) 1430 1431(define_expand "move_hi_quad_<mode>" 1432 [(match_operand:ANY128 0 "s_register_operand" "") 1433 (match_operand:<V_HALF> 1 "s_register_operand" "")] 1434 "TARGET_NEON" 1435{ 1436 emit_move_insn (simplify_gen_subreg (<V_HALF>mode, operands[0], <MODE>mode, 1437 GET_MODE_SIZE (<V_HALF>mode)), 1438 operands[1]); 1439 DONE; 1440}) 1441 1442(define_expand "move_lo_quad_<mode>" 1443 [(match_operand:ANY128 0 "s_register_operand" "") 1444 (match_operand:<V_HALF> 1 "s_register_operand" "")] 1445 "TARGET_NEON" 1446{ 1447 emit_move_insn (simplify_gen_subreg (<V_HALF>mode, operands[0], 1448 <MODE>mode, 0), 1449 operands[1]); 1450 DONE; 1451}) 1452 1453;; Reduction operations 1454 1455(define_expand "reduc_splus_<mode>" 1456 [(match_operand:VD 0 "s_register_operand" "") 1457 (match_operand:VD 1 "s_register_operand" "")] 1458 "TARGET_NEON && (!<Is_float_mode> || flag_unsafe_math_optimizations)" 1459{ 1460 neon_pairwise_reduce (operands[0], operands[1], <MODE>mode, 1461 &gen_neon_vpadd_internal<mode>); 1462 DONE; 1463}) 1464 1465(define_expand "reduc_splus_<mode>" 1466 [(match_operand:VQ 0 "s_register_operand" "") 1467 (match_operand:VQ 1 "s_register_operand" "")] 1468 "TARGET_NEON && (!<Is_float_mode> || flag_unsafe_math_optimizations) 1469 && !BYTES_BIG_ENDIAN" 1470{ 1471 rtx step1 = gen_reg_rtx (<V_HALF>mode); 1472 rtx res_d = gen_reg_rtx (<V_HALF>mode); 1473 1474 emit_insn (gen_quad_halves_plus<mode> (step1, operands[1])); 1475 emit_insn (gen_reduc_splus_<V_half> (res_d, step1)); 1476 emit_insn (gen_move_lo_quad_<mode> (operands[0], res_d)); 1477 1478 DONE; 1479}) 1480 1481(define_insn "reduc_splus_v2di" 1482 [(set (match_operand:V2DI 0 "s_register_operand" "=w") 1483 (unspec:V2DI [(match_operand:V2DI 1 "s_register_operand" "w")] 1484 UNSPEC_VPADD))] 1485 "TARGET_NEON && !BYTES_BIG_ENDIAN" 1486 "vadd.i64\t%e0, %e1, %f1" 1487 [(set_attr "neon_type" "neon_int_1")] 1488) 1489 1490;; NEON does not distinguish between signed and unsigned addition except on 1491;; widening operations. 1492(define_expand "reduc_uplus_<mode>" 1493 [(match_operand:VDQI 0 "s_register_operand" "") 1494 (match_operand:VDQI 1 "s_register_operand" "")] 1495 "TARGET_NEON && (<Is_d_reg> || !BYTES_BIG_ENDIAN)" 1496{ 1497 emit_insn (gen_reduc_splus_<mode> (operands[0], operands[1])); 1498 DONE; 1499}) 1500 1501(define_expand "reduc_smin_<mode>" 1502 [(match_operand:VD 0 "s_register_operand" "") 1503 (match_operand:VD 1 "s_register_operand" "")] 1504 "TARGET_NEON && (!<Is_float_mode> || flag_unsafe_math_optimizations)" 1505{ 1506 neon_pairwise_reduce (operands[0], operands[1], <MODE>mode, 1507 &gen_neon_vpsmin<mode>); 1508 DONE; 1509}) 1510 1511(define_expand "reduc_smin_<mode>" 1512 [(match_operand:VQ 0 "s_register_operand" "") 1513 (match_operand:VQ 1 "s_register_operand" "")] 1514 "TARGET_NEON && (!<Is_float_mode> || flag_unsafe_math_optimizations) 1515 && !BYTES_BIG_ENDIAN" 1516{ 1517 rtx step1 = gen_reg_rtx (<V_HALF>mode); 1518 rtx res_d = gen_reg_rtx (<V_HALF>mode); 1519 1520 emit_insn (gen_quad_halves_smin<mode> (step1, operands[1])); 1521 emit_insn (gen_reduc_smin_<V_half> (res_d, step1)); 1522 emit_insn (gen_move_lo_quad_<mode> (operands[0], res_d)); 1523 1524 DONE; 1525}) 1526 1527(define_expand "reduc_smax_<mode>" 1528 [(match_operand:VD 0 "s_register_operand" "") 1529 (match_operand:VD 1 "s_register_operand" "")] 1530 "TARGET_NEON && (!<Is_float_mode> || flag_unsafe_math_optimizations)" 1531{ 1532 neon_pairwise_reduce (operands[0], operands[1], <MODE>mode, 1533 &gen_neon_vpsmax<mode>); 1534 DONE; 1535}) 1536 1537(define_expand "reduc_smax_<mode>" 1538 [(match_operand:VQ 0 "s_register_operand" "") 1539 (match_operand:VQ 1 "s_register_operand" "")] 1540 "TARGET_NEON && (!<Is_float_mode> || flag_unsafe_math_optimizations) 1541 && !BYTES_BIG_ENDIAN" 1542{ 1543 rtx step1 = gen_reg_rtx (<V_HALF>mode); 1544 rtx res_d = gen_reg_rtx (<V_HALF>mode); 1545 1546 emit_insn (gen_quad_halves_smax<mode> (step1, operands[1])); 1547 emit_insn (gen_reduc_smax_<V_half> (res_d, step1)); 1548 emit_insn (gen_move_lo_quad_<mode> (operands[0], res_d)); 1549 1550 DONE; 1551}) 1552 1553(define_expand "reduc_umin_<mode>" 1554 [(match_operand:VDI 0 "s_register_operand" "") 1555 (match_operand:VDI 1 "s_register_operand" "")] 1556 "TARGET_NEON" 1557{ 1558 neon_pairwise_reduce (operands[0], operands[1], <MODE>mode, 1559 &gen_neon_vpumin<mode>); 1560 DONE; 1561}) 1562 1563(define_expand "reduc_umin_<mode>" 1564 [(match_operand:VQI 0 "s_register_operand" "") 1565 (match_operand:VQI 1 "s_register_operand" "")] 1566 "TARGET_NEON && !BYTES_BIG_ENDIAN" 1567{ 1568 rtx step1 = gen_reg_rtx (<V_HALF>mode); 1569 rtx res_d = gen_reg_rtx (<V_HALF>mode); 1570 1571 emit_insn (gen_quad_halves_umin<mode> (step1, operands[1])); 1572 emit_insn (gen_reduc_umin_<V_half> (res_d, step1)); 1573 emit_insn (gen_move_lo_quad_<mode> (operands[0], res_d)); 1574 1575 DONE; 1576}) 1577 1578(define_expand "reduc_umax_<mode>" 1579 [(match_operand:VDI 0 "s_register_operand" "") 1580 (match_operand:VDI 1 "s_register_operand" "")] 1581 "TARGET_NEON" 1582{ 1583 neon_pairwise_reduce (operands[0], operands[1], <MODE>mode, 1584 &gen_neon_vpumax<mode>); 1585 DONE; 1586}) 1587 1588(define_expand "reduc_umax_<mode>" 1589 [(match_operand:VQI 0 "s_register_operand" "") 1590 (match_operand:VQI 1 "s_register_operand" "")] 1591 "TARGET_NEON && !BYTES_BIG_ENDIAN" 1592{ 1593 rtx step1 = gen_reg_rtx (<V_HALF>mode); 1594 rtx res_d = gen_reg_rtx (<V_HALF>mode); 1595 1596 emit_insn (gen_quad_halves_umax<mode> (step1, operands[1])); 1597 emit_insn (gen_reduc_umax_<V_half> (res_d, step1)); 1598 emit_insn (gen_move_lo_quad_<mode> (operands[0], res_d)); 1599 1600 DONE; 1601}) 1602 1603(define_insn "neon_vpadd_internal<mode>" 1604 [(set (match_operand:VD 0 "s_register_operand" "=w") 1605 (unspec:VD [(match_operand:VD 1 "s_register_operand" "w") 1606 (match_operand:VD 2 "s_register_operand" "w")] 1607 UNSPEC_VPADD))] 1608 "TARGET_NEON" 1609 "vpadd.<V_if_elem>\t%P0, %P1, %P2" 1610 ;; Assume this schedules like vadd. 1611 [(set (attr "neon_type") 1612 (if_then_else (match_test "<Is_float_mode>") 1613 (if_then_else (match_test "<Is_d_reg>") 1614 (const_string "neon_fp_vadd_ddd_vabs_dd") 1615 (const_string "neon_fp_vadd_qqq_vabs_qq")) 1616 (const_string "neon_int_1")))] 1617) 1618 1619(define_insn "neon_vpsmin<mode>" 1620 [(set (match_operand:VD 0 "s_register_operand" "=w") 1621 (unspec:VD [(match_operand:VD 1 "s_register_operand" "w") 1622 (match_operand:VD 2 "s_register_operand" "w")] 1623 UNSPEC_VPSMIN))] 1624 "TARGET_NEON" 1625 "vpmin.<V_s_elem>\t%P0, %P1, %P2" 1626 ;; Assume this schedules like vmin. 1627 [(set (attr "neon_type") 1628 (if_then_else (match_test "<Is_float_mode>") 1629 (const_string "neon_fp_vadd_ddd_vabs_dd") 1630 (const_string "neon_int_5")))] 1631) 1632 1633(define_insn "neon_vpsmax<mode>" 1634 [(set (match_operand:VD 0 "s_register_operand" "=w") 1635 (unspec:VD [(match_operand:VD 1 "s_register_operand" "w") 1636 (match_operand:VD 2 "s_register_operand" "w")] 1637 UNSPEC_VPSMAX))] 1638 "TARGET_NEON" 1639 "vpmax.<V_s_elem>\t%P0, %P1, %P2" 1640 ;; Assume this schedules like vmax. 1641 [(set (attr "neon_type") 1642 (if_then_else (match_test "<Is_float_mode>") 1643 (const_string "neon_fp_vadd_ddd_vabs_dd") 1644 (const_string "neon_int_5")))] 1645) 1646 1647(define_insn "neon_vpumin<mode>" 1648 [(set (match_operand:VDI 0 "s_register_operand" "=w") 1649 (unspec:VDI [(match_operand:VDI 1 "s_register_operand" "w") 1650 (match_operand:VDI 2 "s_register_operand" "w")] 1651 UNSPEC_VPUMIN))] 1652 "TARGET_NEON" 1653 "vpmin.<V_u_elem>\t%P0, %P1, %P2" 1654 ;; Assume this schedules like umin. 1655 [(set_attr "neon_type" "neon_int_5")] 1656) 1657 1658(define_insn "neon_vpumax<mode>" 1659 [(set (match_operand:VDI 0 "s_register_operand" "=w") 1660 (unspec:VDI [(match_operand:VDI 1 "s_register_operand" "w") 1661 (match_operand:VDI 2 "s_register_operand" "w")] 1662 UNSPEC_VPUMAX))] 1663 "TARGET_NEON" 1664 "vpmax.<V_u_elem>\t%P0, %P1, %P2" 1665 ;; Assume this schedules like umax. 1666 [(set_attr "neon_type" "neon_int_5")] 1667) 1668 1669;; Saturating arithmetic 1670 1671; NOTE: Neon supports many more saturating variants of instructions than the 1672; following, but these are all GCC currently understands. 1673; FIXME: Actually, GCC doesn't know how to create saturating add/sub by itself 1674; yet either, although these patterns may be used by intrinsics when they're 1675; added. 1676 1677(define_insn "*ss_add<mode>_neon" 1678 [(set (match_operand:VD 0 "s_register_operand" "=w") 1679 (ss_plus:VD (match_operand:VD 1 "s_register_operand" "w") 1680 (match_operand:VD 2 "s_register_operand" "w")))] 1681 "TARGET_NEON" 1682 "vqadd.<V_s_elem>\t%P0, %P1, %P2" 1683 [(set_attr "neon_type" "neon_int_4")] 1684) 1685 1686(define_insn "*us_add<mode>_neon" 1687 [(set (match_operand:VD 0 "s_register_operand" "=w") 1688 (us_plus:VD (match_operand:VD 1 "s_register_operand" "w") 1689 (match_operand:VD 2 "s_register_operand" "w")))] 1690 "TARGET_NEON" 1691 "vqadd.<V_u_elem>\t%P0, %P1, %P2" 1692 [(set_attr "neon_type" "neon_int_4")] 1693) 1694 1695(define_insn "*ss_sub<mode>_neon" 1696 [(set (match_operand:VD 0 "s_register_operand" "=w") 1697 (ss_minus:VD (match_operand:VD 1 "s_register_operand" "w") 1698 (match_operand:VD 2 "s_register_operand" "w")))] 1699 "TARGET_NEON" 1700 "vqsub.<V_s_elem>\t%P0, %P1, %P2" 1701 [(set_attr "neon_type" "neon_int_5")] 1702) 1703 1704(define_insn "*us_sub<mode>_neon" 1705 [(set (match_operand:VD 0 "s_register_operand" "=w") 1706 (us_minus:VD (match_operand:VD 1 "s_register_operand" "w") 1707 (match_operand:VD 2 "s_register_operand" "w")))] 1708 "TARGET_NEON" 1709 "vqsub.<V_u_elem>\t%P0, %P1, %P2" 1710 [(set_attr "neon_type" "neon_int_5")] 1711) 1712 1713;; Conditional instructions. These are comparisons with conditional moves for 1714;; vectors. They perform the assignment: 1715;; 1716;; Vop0 = (Vop4 <op3> Vop5) ? Vop1 : Vop2; 1717;; 1718;; where op3 is <, <=, ==, !=, >= or >. Operations are performed 1719;; element-wise. 1720 1721(define_expand "vcond<mode><mode>" 1722 [(set (match_operand:VDQW 0 "s_register_operand" "") 1723 (if_then_else:VDQW 1724 (match_operator 3 "comparison_operator" 1725 [(match_operand:VDQW 4 "s_register_operand" "") 1726 (match_operand:VDQW 5 "nonmemory_operand" "")]) 1727 (match_operand:VDQW 1 "s_register_operand" "") 1728 (match_operand:VDQW 2 "s_register_operand" "")))] 1729 "TARGET_NEON && (!<Is_float_mode> || flag_unsafe_math_optimizations)" 1730{ 1731 HOST_WIDE_INT magic_word = (<MODE>mode == V2SFmode || <MODE>mode == V4SFmode) 1732 ? 3 : 1; 1733 rtx magic_rtx = GEN_INT (magic_word); 1734 int inverse = 0; 1735 int use_zero_form = 0; 1736 int swap_bsl_operands = 0; 1737 rtx mask = gen_reg_rtx (<V_cmp_result>mode); 1738 rtx tmp = gen_reg_rtx (<V_cmp_result>mode); 1739 1740 rtx (*base_comparison) (rtx, rtx, rtx, rtx); 1741 rtx (*complimentary_comparison) (rtx, rtx, rtx, rtx); 1742 1743 switch (GET_CODE (operands[3])) 1744 { 1745 case GE: 1746 case GT: 1747 case LE: 1748 case LT: 1749 case EQ: 1750 if (operands[5] == CONST0_RTX (<MODE>mode)) 1751 { 1752 use_zero_form = 1; 1753 break; 1754 } 1755 /* Fall through. */ 1756 default: 1757 if (!REG_P (operands[5])) 1758 operands[5] = force_reg (<MODE>mode, operands[5]); 1759 } 1760 1761 switch (GET_CODE (operands[3])) 1762 { 1763 case LT: 1764 case UNLT: 1765 inverse = 1; 1766 /* Fall through. */ 1767 case GE: 1768 case UNGE: 1769 case ORDERED: 1770 case UNORDERED: 1771 base_comparison = gen_neon_vcge<mode>; 1772 complimentary_comparison = gen_neon_vcgt<mode>; 1773 break; 1774 case LE: 1775 case UNLE: 1776 inverse = 1; 1777 /* Fall through. */ 1778 case GT: 1779 case UNGT: 1780 base_comparison = gen_neon_vcgt<mode>; 1781 complimentary_comparison = gen_neon_vcge<mode>; 1782 break; 1783 case EQ: 1784 case NE: 1785 case UNEQ: 1786 base_comparison = gen_neon_vceq<mode>; 1787 complimentary_comparison = gen_neon_vceq<mode>; 1788 break; 1789 default: 1790 gcc_unreachable (); 1791 } 1792 1793 switch (GET_CODE (operands[3])) 1794 { 1795 case LT: 1796 case LE: 1797 case GT: 1798 case GE: 1799 case EQ: 1800 /* The easy case. Here we emit one of vcge, vcgt or vceq. 1801 As a LT b <=> b GE a && a LE b <=> b GT a. Our transformations are: 1802 a GE b -> a GE b 1803 a GT b -> a GT b 1804 a LE b -> b GE a 1805 a LT b -> b GT a 1806 a EQ b -> a EQ b 1807 Note that there also exist direct comparison against 0 forms, 1808 so catch those as a special case. */ 1809 if (use_zero_form) 1810 { 1811 inverse = 0; 1812 switch (GET_CODE (operands[3])) 1813 { 1814 case LT: 1815 base_comparison = gen_neon_vclt<mode>; 1816 break; 1817 case LE: 1818 base_comparison = gen_neon_vcle<mode>; 1819 break; 1820 default: 1821 /* Do nothing, other zero form cases already have the correct 1822 base_comparison. */ 1823 break; 1824 } 1825 } 1826 1827 if (!inverse) 1828 emit_insn (base_comparison (mask, operands[4], operands[5], magic_rtx)); 1829 else 1830 emit_insn (complimentary_comparison (mask, operands[5], operands[4], magic_rtx)); 1831 break; 1832 case UNLT: 1833 case UNLE: 1834 case UNGT: 1835 case UNGE: 1836 case NE: 1837 /* Vector compare returns false for lanes which are unordered, so if we use 1838 the inverse of the comparison we actually want to emit, then 1839 swap the operands to BSL, we will end up with the correct result. 1840 Note that a NE NaN and NaN NE b are true for all a, b. 1841 1842 Our transformations are: 1843 a GE b -> !(b GT a) 1844 a GT b -> !(b GE a) 1845 a LE b -> !(a GT b) 1846 a LT b -> !(a GE b) 1847 a NE b -> !(a EQ b) */ 1848 1849 if (inverse) 1850 emit_insn (base_comparison (mask, operands[4], operands[5], magic_rtx)); 1851 else 1852 emit_insn (complimentary_comparison (mask, operands[5], operands[4], magic_rtx)); 1853 1854 swap_bsl_operands = 1; 1855 break; 1856 case UNEQ: 1857 /* We check (a > b || b > a). combining these comparisons give us 1858 true iff !(a != b && a ORDERED b), swapping the operands to BSL 1859 will then give us (a == b || a UNORDERED b) as intended. */ 1860 1861 emit_insn (gen_neon_vcgt<mode> (mask, operands[4], operands[5], magic_rtx)); 1862 emit_insn (gen_neon_vcgt<mode> (tmp, operands[5], operands[4], magic_rtx)); 1863 emit_insn (gen_ior<v_cmp_result>3 (mask, mask, tmp)); 1864 swap_bsl_operands = 1; 1865 break; 1866 case UNORDERED: 1867 /* Operands are ORDERED iff (a > b || b >= a). 1868 Swapping the operands to BSL will give the UNORDERED case. */ 1869 swap_bsl_operands = 1; 1870 /* Fall through. */ 1871 case ORDERED: 1872 emit_insn (gen_neon_vcgt<mode> (tmp, operands[4], operands[5], magic_rtx)); 1873 emit_insn (gen_neon_vcge<mode> (mask, operands[5], operands[4], magic_rtx)); 1874 emit_insn (gen_ior<v_cmp_result>3 (mask, mask, tmp)); 1875 break; 1876 default: 1877 gcc_unreachable (); 1878 } 1879 1880 if (swap_bsl_operands) 1881 emit_insn (gen_neon_vbsl<mode> (operands[0], mask, operands[2], 1882 operands[1])); 1883 else 1884 emit_insn (gen_neon_vbsl<mode> (operands[0], mask, operands[1], 1885 operands[2])); 1886 DONE; 1887}) 1888 1889(define_expand "vcondu<mode><mode>" 1890 [(set (match_operand:VDQIW 0 "s_register_operand" "") 1891 (if_then_else:VDQIW 1892 (match_operator 3 "arm_comparison_operator" 1893 [(match_operand:VDQIW 4 "s_register_operand" "") 1894 (match_operand:VDQIW 5 "s_register_operand" "")]) 1895 (match_operand:VDQIW 1 "s_register_operand" "") 1896 (match_operand:VDQIW 2 "s_register_operand" "")))] 1897 "TARGET_NEON" 1898{ 1899 rtx mask; 1900 int inverse = 0, immediate_zero = 0; 1901 1902 mask = gen_reg_rtx (<V_cmp_result>mode); 1903 1904 if (operands[5] == CONST0_RTX (<MODE>mode)) 1905 immediate_zero = 1; 1906 else if (!REG_P (operands[5])) 1907 operands[5] = force_reg (<MODE>mode, operands[5]); 1908 1909 switch (GET_CODE (operands[3])) 1910 { 1911 case GEU: 1912 emit_insn (gen_neon_vcge<mode> (mask, operands[4], operands[5], 1913 const0_rtx)); 1914 break; 1915 1916 case GTU: 1917 emit_insn (gen_neon_vcgt<mode> (mask, operands[4], operands[5], 1918 const0_rtx)); 1919 break; 1920 1921 case EQ: 1922 emit_insn (gen_neon_vceq<mode> (mask, operands[4], operands[5], 1923 const0_rtx)); 1924 break; 1925 1926 case LEU: 1927 if (immediate_zero) 1928 emit_insn (gen_neon_vcle<mode> (mask, operands[4], operands[5], 1929 const0_rtx)); 1930 else 1931 emit_insn (gen_neon_vcge<mode> (mask, operands[5], operands[4], 1932 const0_rtx)); 1933 break; 1934 1935 case LTU: 1936 if (immediate_zero) 1937 emit_insn (gen_neon_vclt<mode> (mask, operands[4], operands[5], 1938 const0_rtx)); 1939 else 1940 emit_insn (gen_neon_vcgt<mode> (mask, operands[5], operands[4], 1941 const0_rtx)); 1942 break; 1943 1944 case NE: 1945 emit_insn (gen_neon_vceq<mode> (mask, operands[4], operands[5], 1946 const0_rtx)); 1947 inverse = 1; 1948 break; 1949 1950 default: 1951 gcc_unreachable (); 1952 } 1953 1954 if (inverse) 1955 emit_insn (gen_neon_vbsl<mode> (operands[0], mask, operands[2], 1956 operands[1])); 1957 else 1958 emit_insn (gen_neon_vbsl<mode> (operands[0], mask, operands[1], 1959 operands[2])); 1960 1961 DONE; 1962}) 1963 1964;; Patterns for builtins. 1965 1966; good for plain vadd, vaddq. 1967 1968(define_expand "neon_vadd<mode>" 1969 [(match_operand:VDQX 0 "s_register_operand" "=w") 1970 (match_operand:VDQX 1 "s_register_operand" "w") 1971 (match_operand:VDQX 2 "s_register_operand" "w") 1972 (match_operand:SI 3 "immediate_operand" "i")] 1973 "TARGET_NEON" 1974{ 1975 if (!<Is_float_mode> || flag_unsafe_math_optimizations) 1976 emit_insn (gen_add<mode>3 (operands[0], operands[1], operands[2])); 1977 else 1978 emit_insn (gen_neon_vadd<mode>_unspec (operands[0], operands[1], 1979 operands[2])); 1980 DONE; 1981}) 1982 1983; Note that NEON operations don't support the full IEEE 754 standard: in 1984; particular, denormal values are flushed to zero. This means that GCC cannot 1985; use those instructions for autovectorization, etc. unless 1986; -funsafe-math-optimizations is in effect (in which case flush-to-zero 1987; behaviour is permissible). Intrinsic operations (provided by the arm_neon.h 1988; header) must work in either case: if -funsafe-math-optimizations is given, 1989; intrinsics expand to "canonical" RTL where possible, otherwise intrinsics 1990; expand to unspecs (which may potentially limit the extent to which they might 1991; be optimized by generic code). 1992 1993; Used for intrinsics when flag_unsafe_math_optimizations is false. 1994 1995(define_insn "neon_vadd<mode>_unspec" 1996 [(set (match_operand:VDQX 0 "s_register_operand" "=w") 1997 (unspec:VDQX [(match_operand:VDQX 1 "s_register_operand" "w") 1998 (match_operand:VDQX 2 "s_register_operand" "w")] 1999 UNSPEC_VADD))] 2000 "TARGET_NEON" 2001 "vadd.<V_if_elem>\t%<V_reg>0, %<V_reg>1, %<V_reg>2" 2002 [(set (attr "neon_type") 2003 (if_then_else (match_test "<Is_float_mode>") 2004 (if_then_else (match_test "<Is_d_reg>") 2005 (const_string "neon_fp_vadd_ddd_vabs_dd") 2006 (const_string "neon_fp_vadd_qqq_vabs_qq")) 2007 (const_string "neon_int_1")))] 2008) 2009 2010; operand 3 represents in bits: 2011; bit 0: signed (vs unsigned). 2012; bit 1: rounding (vs none). 2013 2014(define_insn "neon_vaddl<mode>" 2015 [(set (match_operand:<V_widen> 0 "s_register_operand" "=w") 2016 (unspec:<V_widen> [(match_operand:VDI 1 "s_register_operand" "w") 2017 (match_operand:VDI 2 "s_register_operand" "w") 2018 (match_operand:SI 3 "immediate_operand" "i")] 2019 UNSPEC_VADDL))] 2020 "TARGET_NEON" 2021 "vaddl.%T3%#<V_sz_elem>\t%q0, %P1, %P2" 2022 [(set_attr "neon_type" "neon_int_3")] 2023) 2024 2025(define_insn "neon_vaddw<mode>" 2026 [(set (match_operand:<V_widen> 0 "s_register_operand" "=w") 2027 (unspec:<V_widen> [(match_operand:<V_widen> 1 "s_register_operand" "w") 2028 (match_operand:VDI 2 "s_register_operand" "w") 2029 (match_operand:SI 3 "immediate_operand" "i")] 2030 UNSPEC_VADDW))] 2031 "TARGET_NEON" 2032 "vaddw.%T3%#<V_sz_elem>\t%q0, %q1, %P2" 2033 [(set_attr "neon_type" "neon_int_2")] 2034) 2035 2036; vhadd and vrhadd. 2037 2038(define_insn "neon_vhadd<mode>" 2039 [(set (match_operand:VDQIW 0 "s_register_operand" "=w") 2040 (unspec:VDQIW [(match_operand:VDQIW 1 "s_register_operand" "w") 2041 (match_operand:VDQIW 2 "s_register_operand" "w") 2042 (match_operand:SI 3 "immediate_operand" "i")] 2043 UNSPEC_VHADD))] 2044 "TARGET_NEON" 2045 "v%O3hadd.%T3%#<V_sz_elem>\t%<V_reg>0, %<V_reg>1, %<V_reg>2" 2046 [(set_attr "neon_type" "neon_int_4")] 2047) 2048 2049(define_insn "neon_vqadd<mode>" 2050 [(set (match_operand:VDQIX 0 "s_register_operand" "=w") 2051 (unspec:VDQIX [(match_operand:VDQIX 1 "s_register_operand" "w") 2052 (match_operand:VDQIX 2 "s_register_operand" "w") 2053 (match_operand:SI 3 "immediate_operand" "i")] 2054 UNSPEC_VQADD))] 2055 "TARGET_NEON" 2056 "vqadd.%T3%#<V_sz_elem>\t%<V_reg>0, %<V_reg>1, %<V_reg>2" 2057 [(set_attr "neon_type" "neon_int_4")] 2058) 2059 2060(define_insn "neon_vaddhn<mode>" 2061 [(set (match_operand:<V_narrow> 0 "s_register_operand" "=w") 2062 (unspec:<V_narrow> [(match_operand:VN 1 "s_register_operand" "w") 2063 (match_operand:VN 2 "s_register_operand" "w") 2064 (match_operand:SI 3 "immediate_operand" "i")] 2065 UNSPEC_VADDHN))] 2066 "TARGET_NEON" 2067 "v%O3addhn.<V_if_elem>\t%P0, %q1, %q2" 2068 [(set_attr "neon_type" "neon_int_4")] 2069) 2070 2071;; We cannot replace this unspec with mul<mode>3 because of the odd 2072;; polynomial multiplication case that can specified by operand 3. 2073(define_insn "neon_vmul<mode>" 2074 [(set (match_operand:VDQW 0 "s_register_operand" "=w") 2075 (unspec:VDQW [(match_operand:VDQW 1 "s_register_operand" "w") 2076 (match_operand:VDQW 2 "s_register_operand" "w") 2077 (match_operand:SI 3 "immediate_operand" "i")] 2078 UNSPEC_VMUL))] 2079 "TARGET_NEON" 2080 "vmul.%F3%#<V_sz_elem>\t%<V_reg>0, %<V_reg>1, %<V_reg>2" 2081 [(set (attr "neon_type") 2082 (if_then_else (match_test "<Is_float_mode>") 2083 (if_then_else (match_test "<Is_d_reg>") 2084 (const_string "neon_fp_vadd_ddd_vabs_dd") 2085 (const_string "neon_fp_vadd_qqq_vabs_qq")) 2086 (if_then_else (match_test "<Is_d_reg>") 2087 (if_then_else 2088 (match_test "<Scalar_mul_8_16>") 2089 (const_string "neon_mul_ddd_8_16_qdd_16_8_long_32_16_long") 2090 (const_string "neon_mul_qqq_8_16_32_ddd_32")) 2091 (if_then_else (match_test "<Scalar_mul_8_16>") 2092 (const_string "neon_mul_qqq_8_16_32_ddd_32") 2093 (const_string "neon_mul_qqq_8_16_32_ddd_32")))))] 2094) 2095 2096(define_expand "neon_vmla<mode>" 2097 [(match_operand:VDQW 0 "s_register_operand" "=w") 2098 (match_operand:VDQW 1 "s_register_operand" "0") 2099 (match_operand:VDQW 2 "s_register_operand" "w") 2100 (match_operand:VDQW 3 "s_register_operand" "w") 2101 (match_operand:SI 4 "immediate_operand" "i")] 2102 "TARGET_NEON" 2103{ 2104 if (!<Is_float_mode> || flag_unsafe_math_optimizations) 2105 emit_insn (gen_mul<mode>3add<mode>_neon (operands[0], operands[1], 2106 operands[2], operands[3])); 2107 else 2108 emit_insn (gen_neon_vmla<mode>_unspec (operands[0], operands[1], 2109 operands[2], operands[3])); 2110 DONE; 2111}) 2112 2113(define_expand "neon_vfma<VCVTF:mode>" 2114 [(match_operand:VCVTF 0 "s_register_operand") 2115 (match_operand:VCVTF 1 "s_register_operand") 2116 (match_operand:VCVTF 2 "s_register_operand") 2117 (match_operand:VCVTF 3 "s_register_operand") 2118 (match_operand:SI 4 "immediate_operand")] 2119 "TARGET_NEON && TARGET_FMA" 2120{ 2121 emit_insn (gen_fma<mode>4_intrinsic (operands[0], operands[2], operands[3], 2122 operands[1])); 2123 DONE; 2124}) 2125 2126(define_expand "neon_vfms<VCVTF:mode>" 2127 [(match_operand:VCVTF 0 "s_register_operand") 2128 (match_operand:VCVTF 1 "s_register_operand") 2129 (match_operand:VCVTF 2 "s_register_operand") 2130 (match_operand:VCVTF 3 "s_register_operand") 2131 (match_operand:SI 4 "immediate_operand")] 2132 "TARGET_NEON && TARGET_FMA" 2133{ 2134 emit_insn (gen_fmsub<mode>4_intrinsic (operands[0], operands[2], operands[3], 2135 operands[1])); 2136 DONE; 2137}) 2138 2139; Used for intrinsics when flag_unsafe_math_optimizations is false. 2140 2141(define_insn "neon_vmla<mode>_unspec" 2142 [(set (match_operand:VDQ 0 "s_register_operand" "=w") 2143 (unspec:VDQ [(match_operand:VDQ 1 "s_register_operand" "0") 2144 (match_operand:VDQ 2 "s_register_operand" "w") 2145 (match_operand:VDQ 3 "s_register_operand" "w")] 2146 UNSPEC_VMLA))] 2147 "TARGET_NEON" 2148 "vmla.<V_if_elem>\t%<V_reg>0, %<V_reg>2, %<V_reg>3" 2149 [(set (attr "neon_type") 2150 (if_then_else (match_test "<Is_float_mode>") 2151 (if_then_else (match_test "<Is_d_reg>") 2152 (const_string "neon_fp_vmla_ddd") 2153 (const_string "neon_fp_vmla_qqq")) 2154 (if_then_else (match_test "<Is_d_reg>") 2155 (if_then_else 2156 (match_test "<Scalar_mul_8_16>") 2157 (const_string "neon_mla_ddd_8_16_qdd_16_8_long_32_16_long") 2158 (const_string "neon_mla_ddd_32_qqd_16_ddd_32_scalar_qdd_64_32_long_scalar_qdd_64_32_long")) 2159 (if_then_else (match_test "<Scalar_mul_8_16>") 2160 (const_string "neon_mla_qqq_8_16") 2161 (const_string "neon_mla_qqq_32_qqd_32_scalar")))))] 2162) 2163 2164(define_insn "neon_vmlal<mode>" 2165 [(set (match_operand:<V_widen> 0 "s_register_operand" "=w") 2166 (unspec:<V_widen> [(match_operand:<V_widen> 1 "s_register_operand" "0") 2167 (match_operand:VW 2 "s_register_operand" "w") 2168 (match_operand:VW 3 "s_register_operand" "w") 2169 (match_operand:SI 4 "immediate_operand" "i")] 2170 UNSPEC_VMLAL))] 2171 "TARGET_NEON" 2172 "vmlal.%T4%#<V_sz_elem>\t%q0, %P2, %P3" 2173 [(set (attr "neon_type") 2174 (if_then_else (match_test "<Scalar_mul_8_16>") 2175 (const_string "neon_mla_ddd_8_16_qdd_16_8_long_32_16_long") 2176 (const_string "neon_mla_ddd_32_qqd_16_ddd_32_scalar_qdd_64_32_long_scalar_qdd_64_32_long")))] 2177) 2178 2179(define_expand "neon_vmls<mode>" 2180 [(match_operand:VDQW 0 "s_register_operand" "=w") 2181 (match_operand:VDQW 1 "s_register_operand" "0") 2182 (match_operand:VDQW 2 "s_register_operand" "w") 2183 (match_operand:VDQW 3 "s_register_operand" "w") 2184 (match_operand:SI 4 "immediate_operand" "i")] 2185 "TARGET_NEON" 2186{ 2187 if (!<Is_float_mode> || flag_unsafe_math_optimizations) 2188 emit_insn (gen_mul<mode>3neg<mode>add<mode>_neon (operands[0], 2189 operands[1], operands[2], operands[3])); 2190 else 2191 emit_insn (gen_neon_vmls<mode>_unspec (operands[0], operands[1], 2192 operands[2], operands[3])); 2193 DONE; 2194}) 2195 2196; Used for intrinsics when flag_unsafe_math_optimizations is false. 2197 2198(define_insn "neon_vmls<mode>_unspec" 2199 [(set (match_operand:VDQ 0 "s_register_operand" "=w") 2200 (unspec:VDQ [(match_operand:VDQ 1 "s_register_operand" "0") 2201 (match_operand:VDQ 2 "s_register_operand" "w") 2202 (match_operand:VDQ 3 "s_register_operand" "w")] 2203 UNSPEC_VMLS))] 2204 "TARGET_NEON" 2205 "vmls.<V_if_elem>\t%<V_reg>0, %<V_reg>2, %<V_reg>3" 2206 [(set (attr "neon_type") 2207 (if_then_else (match_test "<Is_float_mode>") 2208 (if_then_else (match_test "<Is_d_reg>") 2209 (const_string "neon_fp_vmla_ddd") 2210 (const_string "neon_fp_vmla_qqq")) 2211 (if_then_else (match_test "<Is_d_reg>") 2212 (if_then_else 2213 (match_test "<Scalar_mul_8_16>") 2214 (const_string "neon_mla_ddd_8_16_qdd_16_8_long_32_16_long") 2215 (const_string "neon_mla_ddd_32_qqd_16_ddd_32_scalar_qdd_64_32_long_scalar_qdd_64_32_long")) 2216 (if_then_else 2217 (match_test "<Scalar_mul_8_16>") 2218 (const_string "neon_mla_qqq_8_16") 2219 (const_string "neon_mla_qqq_32_qqd_32_scalar")))))] 2220) 2221 2222(define_insn "neon_vmlsl<mode>" 2223 [(set (match_operand:<V_widen> 0 "s_register_operand" "=w") 2224 (unspec:<V_widen> [(match_operand:<V_widen> 1 "s_register_operand" "0") 2225 (match_operand:VW 2 "s_register_operand" "w") 2226 (match_operand:VW 3 "s_register_operand" "w") 2227 (match_operand:SI 4 "immediate_operand" "i")] 2228 UNSPEC_VMLSL))] 2229 "TARGET_NEON" 2230 "vmlsl.%T4%#<V_sz_elem>\t%q0, %P2, %P3" 2231 [(set (attr "neon_type") 2232 (if_then_else (match_test "<Scalar_mul_8_16>") 2233 (const_string "neon_mla_ddd_8_16_qdd_16_8_long_32_16_long") 2234 (const_string "neon_mla_ddd_32_qqd_16_ddd_32_scalar_qdd_64_32_long_scalar_qdd_64_32_long")))] 2235) 2236 2237(define_insn "neon_vqdmulh<mode>" 2238 [(set (match_operand:VMDQI 0 "s_register_operand" "=w") 2239 (unspec:VMDQI [(match_operand:VMDQI 1 "s_register_operand" "w") 2240 (match_operand:VMDQI 2 "s_register_operand" "w") 2241 (match_operand:SI 3 "immediate_operand" "i")] 2242 UNSPEC_VQDMULH))] 2243 "TARGET_NEON" 2244 "vq%O3dmulh.<V_s_elem>\t%<V_reg>0, %<V_reg>1, %<V_reg>2" 2245 [(set (attr "neon_type") 2246 (if_then_else (match_test "<Is_d_reg>") 2247 (if_then_else (match_test "<Scalar_mul_8_16>") 2248 (const_string "neon_mul_ddd_8_16_qdd_16_8_long_32_16_long") 2249 (const_string "neon_mul_qqq_8_16_32_ddd_32")) 2250 (if_then_else (match_test "<Scalar_mul_8_16>") 2251 (const_string "neon_mul_qqq_8_16_32_ddd_32") 2252 (const_string "neon_mul_qqq_8_16_32_ddd_32"))))] 2253) 2254 2255(define_insn "neon_vqdmlal<mode>" 2256 [(set (match_operand:<V_widen> 0 "s_register_operand" "=w") 2257 (unspec:<V_widen> [(match_operand:<V_widen> 1 "s_register_operand" "0") 2258 (match_operand:VMDI 2 "s_register_operand" "w") 2259 (match_operand:VMDI 3 "s_register_operand" "w") 2260 (match_operand:SI 4 "immediate_operand" "i")] 2261 UNSPEC_VQDMLAL))] 2262 "TARGET_NEON" 2263 "vqdmlal.<V_s_elem>\t%q0, %P2, %P3" 2264 [(set (attr "neon_type") 2265 (if_then_else (match_test "<Scalar_mul_8_16>") 2266 (const_string "neon_mla_ddd_8_16_qdd_16_8_long_32_16_long") 2267 (const_string "neon_mla_ddd_32_qqd_16_ddd_32_scalar_qdd_64_32_long_scalar_qdd_64_32_long")))] 2268) 2269 2270(define_insn "neon_vqdmlsl<mode>" 2271 [(set (match_operand:<V_widen> 0 "s_register_operand" "=w") 2272 (unspec:<V_widen> [(match_operand:<V_widen> 1 "s_register_operand" "0") 2273 (match_operand:VMDI 2 "s_register_operand" "w") 2274 (match_operand:VMDI 3 "s_register_operand" "w") 2275 (match_operand:SI 4 "immediate_operand" "i")] 2276 UNSPEC_VQDMLSL))] 2277 "TARGET_NEON" 2278 "vqdmlsl.<V_s_elem>\t%q0, %P2, %P3" 2279 [(set (attr "neon_type") 2280 (if_then_else (match_test "<Scalar_mul_8_16>") 2281 (const_string "neon_mla_ddd_8_16_qdd_16_8_long_32_16_long") 2282 (const_string "neon_mla_ddd_32_qqd_16_ddd_32_scalar_qdd_64_32_long_scalar_qdd_64_32_long")))] 2283) 2284 2285(define_insn "neon_vmull<mode>" 2286 [(set (match_operand:<V_widen> 0 "s_register_operand" "=w") 2287 (unspec:<V_widen> [(match_operand:VW 1 "s_register_operand" "w") 2288 (match_operand:VW 2 "s_register_operand" "w") 2289 (match_operand:SI 3 "immediate_operand" "i")] 2290 UNSPEC_VMULL))] 2291 "TARGET_NEON" 2292 "vmull.%T3%#<V_sz_elem>\t%q0, %P1, %P2" 2293 [(set (attr "neon_type") 2294 (if_then_else (match_test "<Scalar_mul_8_16>") 2295 (const_string "neon_mul_ddd_8_16_qdd_16_8_long_32_16_long") 2296 (const_string "neon_mul_qdd_64_32_long_qqd_16_ddd_32_scalar_64_32_long_scalar")))] 2297) 2298 2299(define_insn "neon_vqdmull<mode>" 2300 [(set (match_operand:<V_widen> 0 "s_register_operand" "=w") 2301 (unspec:<V_widen> [(match_operand:VMDI 1 "s_register_operand" "w") 2302 (match_operand:VMDI 2 "s_register_operand" "w") 2303 (match_operand:SI 3 "immediate_operand" "i")] 2304 UNSPEC_VQDMULL))] 2305 "TARGET_NEON" 2306 "vqdmull.<V_s_elem>\t%q0, %P1, %P2" 2307 [(set (attr "neon_type") 2308 (if_then_else (match_test "<Scalar_mul_8_16>") 2309 (const_string "neon_mul_ddd_8_16_qdd_16_8_long_32_16_long") 2310 (const_string "neon_mul_qdd_64_32_long_qqd_16_ddd_32_scalar_64_32_long_scalar")))] 2311) 2312 2313(define_expand "neon_vsub<mode>" 2314 [(match_operand:VDQX 0 "s_register_operand" "=w") 2315 (match_operand:VDQX 1 "s_register_operand" "w") 2316 (match_operand:VDQX 2 "s_register_operand" "w") 2317 (match_operand:SI 3 "immediate_operand" "i")] 2318 "TARGET_NEON" 2319{ 2320 if (!<Is_float_mode> || flag_unsafe_math_optimizations) 2321 emit_insn (gen_sub<mode>3 (operands[0], operands[1], operands[2])); 2322 else 2323 emit_insn (gen_neon_vsub<mode>_unspec (operands[0], operands[1], 2324 operands[2])); 2325 DONE; 2326}) 2327 2328; Used for intrinsics when flag_unsafe_math_optimizations is false. 2329 2330(define_insn "neon_vsub<mode>_unspec" 2331 [(set (match_operand:VDQX 0 "s_register_operand" "=w") 2332 (unspec:VDQX [(match_operand:VDQX 1 "s_register_operand" "w") 2333 (match_operand:VDQX 2 "s_register_operand" "w")] 2334 UNSPEC_VSUB))] 2335 "TARGET_NEON" 2336 "vsub.<V_if_elem>\t%<V_reg>0, %<V_reg>1, %<V_reg>2" 2337 [(set (attr "neon_type") 2338 (if_then_else (match_test "<Is_float_mode>") 2339 (if_then_else (match_test "<Is_d_reg>") 2340 (const_string "neon_fp_vadd_ddd_vabs_dd") 2341 (const_string "neon_fp_vadd_qqq_vabs_qq")) 2342 (const_string "neon_int_2")))] 2343) 2344 2345(define_insn "neon_vsubl<mode>" 2346 [(set (match_operand:<V_widen> 0 "s_register_operand" "=w") 2347 (unspec:<V_widen> [(match_operand:VDI 1 "s_register_operand" "w") 2348 (match_operand:VDI 2 "s_register_operand" "w") 2349 (match_operand:SI 3 "immediate_operand" "i")] 2350 UNSPEC_VSUBL))] 2351 "TARGET_NEON" 2352 "vsubl.%T3%#<V_sz_elem>\t%q0, %P1, %P2" 2353 [(set_attr "neon_type" "neon_int_2")] 2354) 2355 2356(define_insn "neon_vsubw<mode>" 2357 [(set (match_operand:<V_widen> 0 "s_register_operand" "=w") 2358 (unspec:<V_widen> [(match_operand:<V_widen> 1 "s_register_operand" "w") 2359 (match_operand:VDI 2 "s_register_operand" "w") 2360 (match_operand:SI 3 "immediate_operand" "i")] 2361 UNSPEC_VSUBW))] 2362 "TARGET_NEON" 2363 "vsubw.%T3%#<V_sz_elem>\t%q0, %q1, %P2" 2364 [(set_attr "neon_type" "neon_int_2")] 2365) 2366 2367(define_insn "neon_vqsub<mode>" 2368 [(set (match_operand:VDQIX 0 "s_register_operand" "=w") 2369 (unspec:VDQIX [(match_operand:VDQIX 1 "s_register_operand" "w") 2370 (match_operand:VDQIX 2 "s_register_operand" "w") 2371 (match_operand:SI 3 "immediate_operand" "i")] 2372 UNSPEC_VQSUB))] 2373 "TARGET_NEON" 2374 "vqsub.%T3%#<V_sz_elem>\t%<V_reg>0, %<V_reg>1, %<V_reg>2" 2375 [(set_attr "neon_type" "neon_int_5")] 2376) 2377 2378(define_insn "neon_vhsub<mode>" 2379 [(set (match_operand:VDQIW 0 "s_register_operand" "=w") 2380 (unspec:VDQIW [(match_operand:VDQIW 1 "s_register_operand" "w") 2381 (match_operand:VDQIW 2 "s_register_operand" "w") 2382 (match_operand:SI 3 "immediate_operand" "i")] 2383 UNSPEC_VHSUB))] 2384 "TARGET_NEON" 2385 "vhsub.%T3%#<V_sz_elem>\t%<V_reg>0, %<V_reg>1, %<V_reg>2" 2386 [(set_attr "neon_type" "neon_int_5")] 2387) 2388 2389(define_insn "neon_vsubhn<mode>" 2390 [(set (match_operand:<V_narrow> 0 "s_register_operand" "=w") 2391 (unspec:<V_narrow> [(match_operand:VN 1 "s_register_operand" "w") 2392 (match_operand:VN 2 "s_register_operand" "w") 2393 (match_operand:SI 3 "immediate_operand" "i")] 2394 UNSPEC_VSUBHN))] 2395 "TARGET_NEON" 2396 "v%O3subhn.<V_if_elem>\t%P0, %q1, %q2" 2397 [(set_attr "neon_type" "neon_int_4")] 2398) 2399 2400(define_insn "neon_vceq<mode>" 2401 [(set (match_operand:<V_cmp_result> 0 "s_register_operand" "=w,w") 2402 (unspec:<V_cmp_result> 2403 [(match_operand:VDQW 1 "s_register_operand" "w,w") 2404 (match_operand:VDQW 2 "reg_or_zero_operand" "w,Dz") 2405 (match_operand:SI 3 "immediate_operand" "i,i")] 2406 UNSPEC_VCEQ))] 2407 "TARGET_NEON" 2408 "@ 2409 vceq.<V_if_elem>\t%<V_reg>0, %<V_reg>1, %<V_reg>2 2410 vceq.<V_if_elem>\t%<V_reg>0, %<V_reg>1, #0" 2411 [(set (attr "neon_type") 2412 (if_then_else (match_test "<Is_float_mode>") 2413 (if_then_else (match_test "<Is_d_reg>") 2414 (const_string "neon_fp_vadd_ddd_vabs_dd") 2415 (const_string "neon_fp_vadd_qqq_vabs_qq")) 2416 (const_string "neon_int_5")))] 2417) 2418 2419(define_insn "neon_vcge<mode>" 2420 [(set (match_operand:<V_cmp_result> 0 "s_register_operand" "=w,w") 2421 (unspec:<V_cmp_result> 2422 [(match_operand:VDQW 1 "s_register_operand" "w,w") 2423 (match_operand:VDQW 2 "reg_or_zero_operand" "w,Dz") 2424 (match_operand:SI 3 "immediate_operand" "i,i")] 2425 UNSPEC_VCGE))] 2426 "TARGET_NEON" 2427 "@ 2428 vcge.%T3%#<V_sz_elem>\t%<V_reg>0, %<V_reg>1, %<V_reg>2 2429 vcge.%T3%#<V_sz_elem>\t%<V_reg>0, %<V_reg>1, #0" 2430 [(set (attr "neon_type") 2431 (if_then_else (match_test "<Is_float_mode>") 2432 (if_then_else (match_test "<Is_d_reg>") 2433 (const_string "neon_fp_vadd_ddd_vabs_dd") 2434 (const_string "neon_fp_vadd_qqq_vabs_qq")) 2435 (const_string "neon_int_5")))] 2436) 2437 2438(define_insn "neon_vcgeu<mode>" 2439 [(set (match_operand:<V_cmp_result> 0 "s_register_operand" "=w") 2440 (unspec:<V_cmp_result> 2441 [(match_operand:VDQIW 1 "s_register_operand" "w") 2442 (match_operand:VDQIW 2 "s_register_operand" "w") 2443 (match_operand:SI 3 "immediate_operand" "i")] 2444 UNSPEC_VCGEU))] 2445 "TARGET_NEON" 2446 "vcge.%T3%#<V_sz_elem>\t%<V_reg>0, %<V_reg>1, %<V_reg>2" 2447 [(set_attr "neon_type" "neon_int_5")] 2448) 2449 2450(define_insn "neon_vcgt<mode>" 2451 [(set (match_operand:<V_cmp_result> 0 "s_register_operand" "=w,w") 2452 (unspec:<V_cmp_result> 2453 [(match_operand:VDQW 1 "s_register_operand" "w,w") 2454 (match_operand:VDQW 2 "reg_or_zero_operand" "w,Dz") 2455 (match_operand:SI 3 "immediate_operand" "i,i")] 2456 UNSPEC_VCGT))] 2457 "TARGET_NEON" 2458 "@ 2459 vcgt.%T3%#<V_sz_elem>\t%<V_reg>0, %<V_reg>1, %<V_reg>2 2460 vcgt.%T3%#<V_sz_elem>\t%<V_reg>0, %<V_reg>1, #0" 2461 [(set (attr "neon_type") 2462 (if_then_else (match_test "<Is_float_mode>") 2463 (if_then_else (match_test "<Is_d_reg>") 2464 (const_string "neon_fp_vadd_ddd_vabs_dd") 2465 (const_string "neon_fp_vadd_qqq_vabs_qq")) 2466 (const_string "neon_int_5")))] 2467) 2468 2469(define_insn "neon_vcgtu<mode>" 2470 [(set (match_operand:<V_cmp_result> 0 "s_register_operand" "=w") 2471 (unspec:<V_cmp_result> 2472 [(match_operand:VDQIW 1 "s_register_operand" "w") 2473 (match_operand:VDQIW 2 "s_register_operand" "w") 2474 (match_operand:SI 3 "immediate_operand" "i")] 2475 UNSPEC_VCGTU))] 2476 "TARGET_NEON" 2477 "vcgt.%T3%#<V_sz_elem>\t%<V_reg>0, %<V_reg>1, %<V_reg>2" 2478 [(set_attr "neon_type" "neon_int_5")] 2479) 2480 2481;; VCLE and VCLT only support comparisons with immediate zero (register 2482;; variants are VCGE and VCGT with operands reversed). 2483 2484(define_insn "neon_vcle<mode>" 2485 [(set (match_operand:<V_cmp_result> 0 "s_register_operand" "=w") 2486 (unspec:<V_cmp_result> 2487 [(match_operand:VDQW 1 "s_register_operand" "w") 2488 (match_operand:VDQW 2 "zero_operand" "Dz") 2489 (match_operand:SI 3 "immediate_operand" "i")] 2490 UNSPEC_VCLE))] 2491 "TARGET_NEON" 2492 "vcle.%T3%#<V_sz_elem>\t%<V_reg>0, %<V_reg>1, #0" 2493 [(set (attr "neon_type") 2494 (if_then_else (match_test "<Is_float_mode>") 2495 (if_then_else (match_test "<Is_d_reg>") 2496 (const_string "neon_fp_vadd_ddd_vabs_dd") 2497 (const_string "neon_fp_vadd_qqq_vabs_qq")) 2498 (const_string "neon_int_5")))] 2499) 2500 2501(define_insn "neon_vclt<mode>" 2502 [(set (match_operand:<V_cmp_result> 0 "s_register_operand" "=w") 2503 (unspec:<V_cmp_result> 2504 [(match_operand:VDQW 1 "s_register_operand" "w") 2505 (match_operand:VDQW 2 "zero_operand" "Dz") 2506 (match_operand:SI 3 "immediate_operand" "i")] 2507 UNSPEC_VCLT))] 2508 "TARGET_NEON" 2509 "vclt.%T3%#<V_sz_elem>\t%<V_reg>0, %<V_reg>1, #0" 2510 [(set (attr "neon_type") 2511 (if_then_else (match_test "<Is_float_mode>") 2512 (if_then_else (match_test "<Is_d_reg>") 2513 (const_string "neon_fp_vadd_ddd_vabs_dd") 2514 (const_string "neon_fp_vadd_qqq_vabs_qq")) 2515 (const_string "neon_int_5")))] 2516) 2517 2518(define_insn "neon_vcage<mode>" 2519 [(set (match_operand:<V_cmp_result> 0 "s_register_operand" "=w") 2520 (unspec:<V_cmp_result> [(match_operand:VCVTF 1 "s_register_operand" "w") 2521 (match_operand:VCVTF 2 "s_register_operand" "w") 2522 (match_operand:SI 3 "immediate_operand" "i")] 2523 UNSPEC_VCAGE))] 2524 "TARGET_NEON" 2525 "vacge.<V_if_elem>\t%<V_reg>0, %<V_reg>1, %<V_reg>2" 2526 [(set (attr "neon_type") 2527 (if_then_else (match_test "<Is_d_reg>") 2528 (const_string "neon_fp_vadd_ddd_vabs_dd") 2529 (const_string "neon_fp_vadd_qqq_vabs_qq")))] 2530) 2531 2532(define_insn "neon_vcagt<mode>" 2533 [(set (match_operand:<V_cmp_result> 0 "s_register_operand" "=w") 2534 (unspec:<V_cmp_result> [(match_operand:VCVTF 1 "s_register_operand" "w") 2535 (match_operand:VCVTF 2 "s_register_operand" "w") 2536 (match_operand:SI 3 "immediate_operand" "i")] 2537 UNSPEC_VCAGT))] 2538 "TARGET_NEON" 2539 "vacgt.<V_if_elem>\t%<V_reg>0, %<V_reg>1, %<V_reg>2" 2540 [(set (attr "neon_type") 2541 (if_then_else (match_test "<Is_d_reg>") 2542 (const_string "neon_fp_vadd_ddd_vabs_dd") 2543 (const_string "neon_fp_vadd_qqq_vabs_qq")))] 2544) 2545 2546(define_insn "neon_vtst<mode>" 2547 [(set (match_operand:VDQIW 0 "s_register_operand" "=w") 2548 (unspec:VDQIW [(match_operand:VDQIW 1 "s_register_operand" "w") 2549 (match_operand:VDQIW 2 "s_register_operand" "w") 2550 (match_operand:SI 3 "immediate_operand" "i")] 2551 UNSPEC_VTST))] 2552 "TARGET_NEON" 2553 "vtst.<V_sz_elem>\t%<V_reg>0, %<V_reg>1, %<V_reg>2" 2554 [(set_attr "neon_type" "neon_int_4")] 2555) 2556 2557(define_insn "neon_vabd<mode>" 2558 [(set (match_operand:VDQW 0 "s_register_operand" "=w") 2559 (unspec:VDQW [(match_operand:VDQW 1 "s_register_operand" "w") 2560 (match_operand:VDQW 2 "s_register_operand" "w") 2561 (match_operand:SI 3 "immediate_operand" "i")] 2562 UNSPEC_VABD))] 2563 "TARGET_NEON" 2564 "vabd.%T3%#<V_sz_elem>\t%<V_reg>0, %<V_reg>1, %<V_reg>2" 2565 [(set (attr "neon_type") 2566 (if_then_else (match_test "<Is_float_mode>") 2567 (if_then_else (match_test "<Is_d_reg>") 2568 (const_string "neon_fp_vadd_ddd_vabs_dd") 2569 (const_string "neon_fp_vadd_qqq_vabs_qq")) 2570 (const_string "neon_int_5")))] 2571) 2572 2573(define_insn "neon_vabdl<mode>" 2574 [(set (match_operand:<V_widen> 0 "s_register_operand" "=w") 2575 (unspec:<V_widen> [(match_operand:VW 1 "s_register_operand" "w") 2576 (match_operand:VW 2 "s_register_operand" "w") 2577 (match_operand:SI 3 "immediate_operand" "i")] 2578 UNSPEC_VABDL))] 2579 "TARGET_NEON" 2580 "vabdl.%T3%#<V_sz_elem>\t%q0, %P1, %P2" 2581 [(set_attr "neon_type" "neon_int_5")] 2582) 2583 2584(define_insn "neon_vaba<mode>" 2585 [(set (match_operand:VDQIW 0 "s_register_operand" "=w") 2586 (plus:VDQIW (unspec:VDQIW [(match_operand:VDQIW 2 "s_register_operand" "w") 2587 (match_operand:VDQIW 3 "s_register_operand" "w") 2588 (match_operand:SI 4 "immediate_operand" "i")] 2589 UNSPEC_VABD) 2590 (match_operand:VDQIW 1 "s_register_operand" "0")))] 2591 "TARGET_NEON" 2592 "vaba.%T4%#<V_sz_elem>\t%<V_reg>0, %<V_reg>2, %<V_reg>3" 2593 [(set (attr "neon_type") 2594 (if_then_else (match_test "<Is_d_reg>") 2595 (const_string "neon_vaba") (const_string "neon_vaba_qqq")))] 2596) 2597 2598(define_insn "neon_vabal<mode>" 2599 [(set (match_operand:<V_widen> 0 "s_register_operand" "=w") 2600 (plus:<V_widen> (unspec:<V_widen> [(match_operand:VW 2 "s_register_operand" "w") 2601 (match_operand:VW 3 "s_register_operand" "w") 2602 (match_operand:SI 4 "immediate_operand" "i")] 2603 UNSPEC_VABDL) 2604 (match_operand:<V_widen> 1 "s_register_operand" "0")))] 2605 "TARGET_NEON" 2606 "vabal.%T4%#<V_sz_elem>\t%q0, %P2, %P3" 2607 [(set_attr "neon_type" "neon_vaba")] 2608) 2609 2610(define_insn "neon_vmax<mode>" 2611 [(set (match_operand:VDQW 0 "s_register_operand" "=w") 2612 (unspec:VDQW [(match_operand:VDQW 1 "s_register_operand" "w") 2613 (match_operand:VDQW 2 "s_register_operand" "w") 2614 (match_operand:SI 3 "immediate_operand" "i")] 2615 UNSPEC_VMAX))] 2616 "TARGET_NEON" 2617 "vmax.%T3%#<V_sz_elem>\t%<V_reg>0, %<V_reg>1, %<V_reg>2" 2618 [(set (attr "neon_type") 2619 (if_then_else (match_test "<Is_float_mode>") 2620 (if_then_else (match_test "<Is_d_reg>") 2621 (const_string "neon_fp_vadd_ddd_vabs_dd") 2622 (const_string "neon_fp_vadd_qqq_vabs_qq")) 2623 (const_string "neon_int_5")))] 2624) 2625 2626(define_insn "neon_vmin<mode>" 2627 [(set (match_operand:VDQW 0 "s_register_operand" "=w") 2628 (unspec:VDQW [(match_operand:VDQW 1 "s_register_operand" "w") 2629 (match_operand:VDQW 2 "s_register_operand" "w") 2630 (match_operand:SI 3 "immediate_operand" "i")] 2631 UNSPEC_VMIN))] 2632 "TARGET_NEON" 2633 "vmin.%T3%#<V_sz_elem>\t%<V_reg>0, %<V_reg>1, %<V_reg>2" 2634 [(set (attr "neon_type") 2635 (if_then_else (match_test "<Is_float_mode>") 2636 (if_then_else (match_test "<Is_d_reg>") 2637 (const_string "neon_fp_vadd_ddd_vabs_dd") 2638 (const_string "neon_fp_vadd_qqq_vabs_qq")) 2639 (const_string "neon_int_5")))] 2640) 2641 2642(define_expand "neon_vpadd<mode>" 2643 [(match_operand:VD 0 "s_register_operand" "=w") 2644 (match_operand:VD 1 "s_register_operand" "w") 2645 (match_operand:VD 2 "s_register_operand" "w") 2646 (match_operand:SI 3 "immediate_operand" "i")] 2647 "TARGET_NEON" 2648{ 2649 emit_insn (gen_neon_vpadd_internal<mode> (operands[0], operands[1], 2650 operands[2])); 2651 DONE; 2652}) 2653 2654(define_insn "neon_vpaddl<mode>" 2655 [(set (match_operand:<V_double_width> 0 "s_register_operand" "=w") 2656 (unspec:<V_double_width> [(match_operand:VDQIW 1 "s_register_operand" "w") 2657 (match_operand:SI 2 "immediate_operand" "i")] 2658 UNSPEC_VPADDL))] 2659 "TARGET_NEON" 2660 "vpaddl.%T2%#<V_sz_elem>\t%<V_reg>0, %<V_reg>1" 2661 ;; Assume this schedules like vaddl. 2662 [(set_attr "neon_type" "neon_int_3")] 2663) 2664 2665(define_insn "neon_vpadal<mode>" 2666 [(set (match_operand:<V_double_width> 0 "s_register_operand" "=w") 2667 (unspec:<V_double_width> [(match_operand:<V_double_width> 1 "s_register_operand" "0") 2668 (match_operand:VDQIW 2 "s_register_operand" "w") 2669 (match_operand:SI 3 "immediate_operand" "i")] 2670 UNSPEC_VPADAL))] 2671 "TARGET_NEON" 2672 "vpadal.%T3%#<V_sz_elem>\t%<V_reg>0, %<V_reg>2" 2673 ;; Assume this schedules like vpadd. 2674 [(set_attr "neon_type" "neon_int_1")] 2675) 2676 2677(define_insn "neon_vpmax<mode>" 2678 [(set (match_operand:VD 0 "s_register_operand" "=w") 2679 (unspec:VD [(match_operand:VD 1 "s_register_operand" "w") 2680 (match_operand:VD 2 "s_register_operand" "w") 2681 (match_operand:SI 3 "immediate_operand" "i")] 2682 UNSPEC_VPMAX))] 2683 "TARGET_NEON" 2684 "vpmax.%T3%#<V_sz_elem>\t%<V_reg>0, %<V_reg>1, %<V_reg>2" 2685 ;; Assume this schedules like vmax. 2686 [(set (attr "neon_type") 2687 (if_then_else (match_test "<Is_float_mode>") 2688 (const_string "neon_fp_vadd_ddd_vabs_dd") 2689 (const_string "neon_int_5")))] 2690) 2691 2692(define_insn "neon_vpmin<mode>" 2693 [(set (match_operand:VD 0 "s_register_operand" "=w") 2694 (unspec:VD [(match_operand:VD 1 "s_register_operand" "w") 2695 (match_operand:VD 2 "s_register_operand" "w") 2696 (match_operand:SI 3 "immediate_operand" "i")] 2697 UNSPEC_VPMIN))] 2698 "TARGET_NEON" 2699 "vpmin.%T3%#<V_sz_elem>\t%<V_reg>0, %<V_reg>1, %<V_reg>2" 2700 ;; Assume this schedules like vmin. 2701 [(set (attr "neon_type") 2702 (if_then_else (match_test "<Is_float_mode>") 2703 (const_string "neon_fp_vadd_ddd_vabs_dd") 2704 (const_string "neon_int_5")))] 2705) 2706 2707(define_insn "neon_vrecps<mode>" 2708 [(set (match_operand:VCVTF 0 "s_register_operand" "=w") 2709 (unspec:VCVTF [(match_operand:VCVTF 1 "s_register_operand" "w") 2710 (match_operand:VCVTF 2 "s_register_operand" "w") 2711 (match_operand:SI 3 "immediate_operand" "i")] 2712 UNSPEC_VRECPS))] 2713 "TARGET_NEON" 2714 "vrecps.<V_if_elem>\t%<V_reg>0, %<V_reg>1, %<V_reg>2" 2715 [(set (attr "neon_type") 2716 (if_then_else (match_test "<Is_d_reg>") 2717 (const_string "neon_fp_vrecps_vrsqrts_ddd") 2718 (const_string "neon_fp_vrecps_vrsqrts_qqq")))] 2719) 2720 2721(define_insn "neon_vrsqrts<mode>" 2722 [(set (match_operand:VCVTF 0 "s_register_operand" "=w") 2723 (unspec:VCVTF [(match_operand:VCVTF 1 "s_register_operand" "w") 2724 (match_operand:VCVTF 2 "s_register_operand" "w") 2725 (match_operand:SI 3 "immediate_operand" "i")] 2726 UNSPEC_VRSQRTS))] 2727 "TARGET_NEON" 2728 "vrsqrts.<V_if_elem>\t%<V_reg>0, %<V_reg>1, %<V_reg>2" 2729 [(set (attr "neon_type") 2730 (if_then_else (match_test "<Is_d_reg>") 2731 (const_string "neon_fp_vrecps_vrsqrts_ddd") 2732 (const_string "neon_fp_vrecps_vrsqrts_qqq")))] 2733) 2734 2735(define_expand "neon_vabs<mode>" 2736 [(match_operand:VDQW 0 "s_register_operand" "") 2737 (match_operand:VDQW 1 "s_register_operand" "") 2738 (match_operand:SI 2 "immediate_operand" "")] 2739 "TARGET_NEON" 2740{ 2741 emit_insn (gen_abs<mode>2 (operands[0], operands[1])); 2742 DONE; 2743}) 2744 2745(define_insn "neon_vqabs<mode>" 2746 [(set (match_operand:VDQIW 0 "s_register_operand" "=w") 2747 (unspec:VDQIW [(match_operand:VDQIW 1 "s_register_operand" "w") 2748 (match_operand:SI 2 "immediate_operand" "i")] 2749 UNSPEC_VQABS))] 2750 "TARGET_NEON" 2751 "vqabs.<V_s_elem>\t%<V_reg>0, %<V_reg>1" 2752 [(set_attr "neon_type" "neon_vqneg_vqabs")] 2753) 2754 2755(define_expand "neon_vneg<mode>" 2756 [(match_operand:VDQW 0 "s_register_operand" "") 2757 (match_operand:VDQW 1 "s_register_operand" "") 2758 (match_operand:SI 2 "immediate_operand" "")] 2759 "TARGET_NEON" 2760{ 2761 emit_insn (gen_neg<mode>2 (operands[0], operands[1])); 2762 DONE; 2763}) 2764 2765(define_insn "neon_vqneg<mode>" 2766 [(set (match_operand:VDQIW 0 "s_register_operand" "=w") 2767 (unspec:VDQIW [(match_operand:VDQIW 1 "s_register_operand" "w") 2768 (match_operand:SI 2 "immediate_operand" "i")] 2769 UNSPEC_VQNEG))] 2770 "TARGET_NEON" 2771 "vqneg.<V_s_elem>\t%<V_reg>0, %<V_reg>1" 2772 [(set_attr "neon_type" "neon_vqneg_vqabs")] 2773) 2774 2775(define_insn "neon_vcls<mode>" 2776 [(set (match_operand:VDQIW 0 "s_register_operand" "=w") 2777 (unspec:VDQIW [(match_operand:VDQIW 1 "s_register_operand" "w") 2778 (match_operand:SI 2 "immediate_operand" "i")] 2779 UNSPEC_VCLS))] 2780 "TARGET_NEON" 2781 "vcls.<V_s_elem>\t%<V_reg>0, %<V_reg>1" 2782 [(set_attr "neon_type" "neon_int_1")] 2783) 2784 2785(define_insn "clz<mode>2" 2786 [(set (match_operand:VDQIW 0 "s_register_operand" "=w") 2787 (clz:VDQIW (match_operand:VDQIW 1 "s_register_operand" "w")))] 2788 "TARGET_NEON" 2789 "vclz.<V_if_elem>\t%<V_reg>0, %<V_reg>1" 2790 [(set_attr "neon_type" "neon_int_1")] 2791) 2792 2793(define_expand "neon_vclz<mode>" 2794 [(match_operand:VDQIW 0 "s_register_operand" "") 2795 (match_operand:VDQIW 1 "s_register_operand" "") 2796 (match_operand:SI 2 "immediate_operand" "")] 2797 "TARGET_NEON" 2798{ 2799 emit_insn (gen_clz<mode>2 (operands[0], operands[1])); 2800 DONE; 2801}) 2802 2803(define_insn "popcount<mode>2" 2804 [(set (match_operand:VE 0 "s_register_operand" "=w") 2805 (popcount:VE (match_operand:VE 1 "s_register_operand" "w")))] 2806 "TARGET_NEON" 2807 "vcnt.<V_sz_elem>\t%<V_reg>0, %<V_reg>1" 2808 [(set_attr "neon_type" "neon_int_1")] 2809) 2810 2811(define_expand "neon_vcnt<mode>" 2812 [(match_operand:VE 0 "s_register_operand" "=w") 2813 (match_operand:VE 1 "s_register_operand" "w") 2814 (match_operand:SI 2 "immediate_operand" "i")] 2815 "TARGET_NEON" 2816{ 2817 emit_insn (gen_popcount<mode>2 (operands[0], operands[1])); 2818 DONE; 2819}) 2820 2821(define_insn "neon_vrecpe<mode>" 2822 [(set (match_operand:V32 0 "s_register_operand" "=w") 2823 (unspec:V32 [(match_operand:V32 1 "s_register_operand" "w") 2824 (match_operand:SI 2 "immediate_operand" "i")] 2825 UNSPEC_VRECPE))] 2826 "TARGET_NEON" 2827 "vrecpe.<V_u_elem>\t%<V_reg>0, %<V_reg>1" 2828 [(set (attr "neon_type") 2829 (if_then_else (match_test "<Is_d_reg>") 2830 (const_string "neon_fp_vadd_ddd_vabs_dd") 2831 (const_string "neon_fp_vadd_qqq_vabs_qq")))] 2832) 2833 2834(define_insn "neon_vrsqrte<mode>" 2835 [(set (match_operand:V32 0 "s_register_operand" "=w") 2836 (unspec:V32 [(match_operand:V32 1 "s_register_operand" "w") 2837 (match_operand:SI 2 "immediate_operand" "i")] 2838 UNSPEC_VRSQRTE))] 2839 "TARGET_NEON" 2840 "vrsqrte.<V_u_elem>\t%<V_reg>0, %<V_reg>1" 2841 [(set (attr "neon_type") 2842 (if_then_else (match_test "<Is_d_reg>") 2843 (const_string "neon_fp_vadd_ddd_vabs_dd") 2844 (const_string "neon_fp_vadd_qqq_vabs_qq")))] 2845) 2846 2847(define_expand "neon_vmvn<mode>" 2848 [(match_operand:VDQIW 0 "s_register_operand" "") 2849 (match_operand:VDQIW 1 "s_register_operand" "") 2850 (match_operand:SI 2 "immediate_operand" "")] 2851 "TARGET_NEON" 2852{ 2853 emit_insn (gen_one_cmpl<mode>2 (operands[0], operands[1])); 2854 DONE; 2855}) 2856 2857(define_insn "neon_vget_lane<mode>_sext_internal" 2858 [(set (match_operand:SI 0 "s_register_operand" "=r") 2859 (sign_extend:SI 2860 (vec_select:<V_elem> 2861 (match_operand:VD 1 "s_register_operand" "w") 2862 (parallel [(match_operand:SI 2 "immediate_operand" "i")]))))] 2863 "TARGET_NEON" 2864{ 2865 if (BYTES_BIG_ENDIAN) 2866 { 2867 int elt = INTVAL (operands[2]); 2868 elt = GET_MODE_NUNITS (<MODE>mode) - 1 - elt; 2869 operands[2] = GEN_INT (elt); 2870 } 2871 return "vmov.s<V_sz_elem>\t%0, %P1[%c2]"; 2872} 2873 [(set_attr "neon_type" "neon_bp_simple")] 2874) 2875 2876(define_insn "neon_vget_lane<mode>_zext_internal" 2877 [(set (match_operand:SI 0 "s_register_operand" "=r") 2878 (zero_extend:SI 2879 (vec_select:<V_elem> 2880 (match_operand:VD 1 "s_register_operand" "w") 2881 (parallel [(match_operand:SI 2 "immediate_operand" "i")]))))] 2882 "TARGET_NEON" 2883{ 2884 if (BYTES_BIG_ENDIAN) 2885 { 2886 int elt = INTVAL (operands[2]); 2887 elt = GET_MODE_NUNITS (<MODE>mode) - 1 - elt; 2888 operands[2] = GEN_INT (elt); 2889 } 2890 return "vmov.u<V_sz_elem>\t%0, %P1[%c2]"; 2891} 2892 [(set_attr "neon_type" "neon_bp_simple")] 2893) 2894 2895(define_insn "neon_vget_lane<mode>_sext_internal" 2896 [(set (match_operand:SI 0 "s_register_operand" "=r") 2897 (sign_extend:SI 2898 (vec_select:<V_elem> 2899 (match_operand:VQ 1 "s_register_operand" "w") 2900 (parallel [(match_operand:SI 2 "immediate_operand" "i")]))))] 2901 "TARGET_NEON" 2902{ 2903 rtx ops[3]; 2904 int regno = REGNO (operands[1]); 2905 unsigned int halfelts = GET_MODE_NUNITS (<MODE>mode) / 2; 2906 unsigned int elt = INTVAL (operands[2]); 2907 unsigned int elt_adj = elt % halfelts; 2908 2909 if (BYTES_BIG_ENDIAN) 2910 elt_adj = halfelts - 1 - elt_adj; 2911 2912 ops[0] = operands[0]; 2913 ops[1] = gen_rtx_REG (<V_HALF>mode, regno + 2 * (elt / halfelts)); 2914 ops[2] = GEN_INT (elt_adj); 2915 output_asm_insn ("vmov.s<V_sz_elem>\t%0, %P1[%c2]", ops); 2916 2917 return ""; 2918} 2919 [(set_attr "neon_type" "neon_bp_simple")] 2920) 2921 2922(define_insn "neon_vget_lane<mode>_zext_internal" 2923 [(set (match_operand:SI 0 "s_register_operand" "=r") 2924 (zero_extend:SI 2925 (vec_select:<V_elem> 2926 (match_operand:VQ 1 "s_register_operand" "w") 2927 (parallel [(match_operand:SI 2 "immediate_operand" "i")]))))] 2928 "TARGET_NEON" 2929{ 2930 rtx ops[3]; 2931 int regno = REGNO (operands[1]); 2932 unsigned int halfelts = GET_MODE_NUNITS (<MODE>mode) / 2; 2933 unsigned int elt = INTVAL (operands[2]); 2934 unsigned int elt_adj = elt % halfelts; 2935 2936 if (BYTES_BIG_ENDIAN) 2937 elt_adj = halfelts - 1 - elt_adj; 2938 2939 ops[0] = operands[0]; 2940 ops[1] = gen_rtx_REG (<V_HALF>mode, regno + 2 * (elt / halfelts)); 2941 ops[2] = GEN_INT (elt_adj); 2942 output_asm_insn ("vmov.u<V_sz_elem>\t%0, %P1[%c2]", ops); 2943 2944 return ""; 2945} 2946 [(set_attr "neon_type" "neon_bp_simple")] 2947) 2948 2949(define_expand "neon_vget_lane<mode>" 2950 [(match_operand:<V_ext> 0 "s_register_operand" "") 2951 (match_operand:VDQW 1 "s_register_operand" "") 2952 (match_operand:SI 2 "immediate_operand" "") 2953 (match_operand:SI 3 "immediate_operand" "")] 2954 "TARGET_NEON" 2955{ 2956 HOST_WIDE_INT magic = INTVAL (operands[3]); 2957 rtx insn; 2958 2959 neon_lane_bounds (operands[2], 0, GET_MODE_NUNITS (<MODE>mode)); 2960 2961 if (BYTES_BIG_ENDIAN) 2962 { 2963 /* The intrinsics are defined in terms of a model where the 2964 element ordering in memory is vldm order, whereas the generic 2965 RTL is defined in terms of a model where the element ordering 2966 in memory is array order. Convert the lane number to conform 2967 to this model. */ 2968 unsigned int elt = INTVAL (operands[2]); 2969 unsigned int reg_nelts 2970 = 64 / GET_MODE_BITSIZE (GET_MODE_INNER (<MODE>mode)); 2971 elt ^= reg_nelts - 1; 2972 operands[2] = GEN_INT (elt); 2973 } 2974 2975 if ((magic & 3) == 3 || GET_MODE_BITSIZE (GET_MODE_INNER (<MODE>mode)) == 32) 2976 insn = gen_vec_extract<mode> (operands[0], operands[1], operands[2]); 2977 else 2978 { 2979 if ((magic & 1) != 0) 2980 insn = gen_neon_vget_lane<mode>_sext_internal (operands[0], operands[1], 2981 operands[2]); 2982 else 2983 insn = gen_neon_vget_lane<mode>_zext_internal (operands[0], operands[1], 2984 operands[2]); 2985 } 2986 emit_insn (insn); 2987 DONE; 2988}) 2989 2990; Operand 3 (info word) is ignored because it does nothing useful with 64-bit 2991; elements. 2992 2993(define_expand "neon_vget_lanedi" 2994 [(match_operand:DI 0 "s_register_operand" "=r") 2995 (match_operand:DI 1 "s_register_operand" "w") 2996 (match_operand:SI 2 "immediate_operand" "i") 2997 (match_operand:SI 3 "immediate_operand" "i")] 2998 "TARGET_NEON" 2999{ 3000 neon_lane_bounds (operands[2], 0, 1); 3001 emit_move_insn (operands[0], operands[1]); 3002 DONE; 3003}) 3004 3005(define_expand "neon_vget_lanev2di" 3006 [(match_operand:DI 0 "s_register_operand" "") 3007 (match_operand:V2DI 1 "s_register_operand" "") 3008 (match_operand:SI 2 "immediate_operand" "") 3009 (match_operand:SI 3 "immediate_operand" "")] 3010 "TARGET_NEON" 3011{ 3012 switch (INTVAL (operands[2])) 3013 { 3014 case 0: 3015 emit_move_insn (operands[0], gen_lowpart (DImode, operands[1])); 3016 break; 3017 case 1: 3018 emit_move_insn (operands[0], gen_highpart (DImode, operands[1])); 3019 break; 3020 default: 3021 neon_lane_bounds (operands[2], 0, 1); 3022 FAIL; 3023 } 3024 DONE; 3025}) 3026 3027(define_expand "neon_vset_lane<mode>" 3028 [(match_operand:VDQ 0 "s_register_operand" "=w") 3029 (match_operand:<V_elem> 1 "s_register_operand" "r") 3030 (match_operand:VDQ 2 "s_register_operand" "0") 3031 (match_operand:SI 3 "immediate_operand" "i")] 3032 "TARGET_NEON" 3033{ 3034 unsigned int elt = INTVAL (operands[3]); 3035 neon_lane_bounds (operands[3], 0, GET_MODE_NUNITS (<MODE>mode)); 3036 3037 if (BYTES_BIG_ENDIAN) 3038 { 3039 unsigned int reg_nelts 3040 = 64 / GET_MODE_BITSIZE (GET_MODE_INNER (<MODE>mode)); 3041 elt ^= reg_nelts - 1; 3042 } 3043 3044 emit_insn (gen_vec_set<mode>_internal (operands[0], operands[1], 3045 GEN_INT (1 << elt), operands[2])); 3046 DONE; 3047}) 3048 3049; See neon_vget_lanedi comment for reasons operands 2 & 3 are ignored. 3050 3051(define_expand "neon_vset_lanedi" 3052 [(match_operand:DI 0 "s_register_operand" "=w") 3053 (match_operand:DI 1 "s_register_operand" "r") 3054 (match_operand:DI 2 "s_register_operand" "0") 3055 (match_operand:SI 3 "immediate_operand" "i")] 3056 "TARGET_NEON" 3057{ 3058 neon_lane_bounds (operands[3], 0, 1); 3059 emit_move_insn (operands[0], operands[1]); 3060 DONE; 3061}) 3062 3063(define_expand "neon_vcreate<mode>" 3064 [(match_operand:VDX 0 "s_register_operand" "") 3065 (match_operand:DI 1 "general_operand" "")] 3066 "TARGET_NEON" 3067{ 3068 rtx src = gen_lowpart (<MODE>mode, operands[1]); 3069 emit_move_insn (operands[0], src); 3070 DONE; 3071}) 3072 3073(define_insn "neon_vdup_n<mode>" 3074 [(set (match_operand:VX 0 "s_register_operand" "=w") 3075 (vec_duplicate:VX (match_operand:<V_elem> 1 "s_register_operand" "r")))] 3076 "TARGET_NEON" 3077 "vdup.<V_sz_elem>\t%<V_reg>0, %1" 3078 ;; Assume this schedules like vmov. 3079 [(set_attr "neon_type" "neon_bp_simple")] 3080) 3081 3082(define_insn "neon_vdup_n<mode>" 3083 [(set (match_operand:V32 0 "s_register_operand" "=w,w") 3084 (vec_duplicate:V32 (match_operand:<V_elem> 1 "s_register_operand" "r,t")))] 3085 "TARGET_NEON" 3086 "@ 3087 vdup.<V_sz_elem>\t%<V_reg>0, %1 3088 vdup.<V_sz_elem>\t%<V_reg>0, %y1" 3089 ;; Assume this schedules like vmov. 3090 [(set_attr "neon_type" "neon_bp_simple")] 3091) 3092 3093(define_expand "neon_vdup_ndi" 3094 [(match_operand:DI 0 "s_register_operand" "=w") 3095 (match_operand:DI 1 "s_register_operand" "r")] 3096 "TARGET_NEON" 3097{ 3098 emit_move_insn (operands[0], operands[1]); 3099 DONE; 3100} 3101) 3102 3103(define_insn "neon_vdup_nv2di" 3104 [(set (match_operand:V2DI 0 "s_register_operand" "=w,w") 3105 (vec_duplicate:V2DI (match_operand:DI 1 "s_register_operand" "r,w")))] 3106 "TARGET_NEON" 3107 "@ 3108 vmov\t%e0, %Q1, %R1\;vmov\t%f0, %Q1, %R1 3109 vmov\t%e0, %P1\;vmov\t%f0, %P1" 3110 [(set_attr "length" "8") 3111 (set_attr "neon_type" "neon_bp_simple")] 3112) 3113 3114(define_insn "neon_vdup_lane<mode>_internal" 3115 [(set (match_operand:VDQW 0 "s_register_operand" "=w") 3116 (vec_duplicate:VDQW 3117 (vec_select:<V_elem> 3118 (match_operand:<V_double_vector_mode> 1 "s_register_operand" "w") 3119 (parallel [(match_operand:SI 2 "immediate_operand" "i")]))))] 3120 "TARGET_NEON" 3121{ 3122 if (BYTES_BIG_ENDIAN) 3123 { 3124 int elt = INTVAL (operands[2]); 3125 elt = GET_MODE_NUNITS (<V_double_vector_mode>mode) - 1 - elt; 3126 operands[2] = GEN_INT (elt); 3127 } 3128 if (<Is_d_reg>) 3129 return "vdup.<V_sz_elem>\t%P0, %P1[%c2]"; 3130 else 3131 return "vdup.<V_sz_elem>\t%q0, %P1[%c2]"; 3132} 3133 ;; Assume this schedules like vmov. 3134 [(set_attr "neon_type" "neon_bp_simple")] 3135) 3136 3137(define_expand "neon_vdup_lane<mode>" 3138 [(match_operand:VDQW 0 "s_register_operand" "=w") 3139 (match_operand:<V_double_vector_mode> 1 "s_register_operand" "w") 3140 (match_operand:SI 2 "immediate_operand" "i")] 3141 "TARGET_NEON" 3142{ 3143 neon_lane_bounds (operands[2], 0, GET_MODE_NUNITS (<V_double_vector_mode>mode)); 3144 if (BYTES_BIG_ENDIAN) 3145 { 3146 unsigned int elt = INTVAL (operands[2]); 3147 unsigned int reg_nelts 3148 = 64 / GET_MODE_BITSIZE (GET_MODE_INNER (<V_double_vector_mode>mode)); 3149 elt ^= reg_nelts - 1; 3150 operands[2] = GEN_INT (elt); 3151 } 3152 emit_insn (gen_neon_vdup_lane<mode>_internal (operands[0], operands[1], 3153 operands[2])); 3154 DONE; 3155}) 3156 3157; Scalar index is ignored, since only zero is valid here. 3158(define_expand "neon_vdup_lanedi" 3159 [(match_operand:DI 0 "s_register_operand" "=w") 3160 (match_operand:DI 1 "s_register_operand" "w") 3161 (match_operand:SI 2 "immediate_operand" "i")] 3162 "TARGET_NEON" 3163{ 3164 neon_lane_bounds (operands[2], 0, 1); 3165 emit_move_insn (operands[0], operands[1]); 3166 DONE; 3167}) 3168 3169; Likewise for v2di, as the DImode second operand has only a single element. 3170(define_expand "neon_vdup_lanev2di" 3171 [(match_operand:V2DI 0 "s_register_operand" "=w") 3172 (match_operand:DI 1 "s_register_operand" "w") 3173 (match_operand:SI 2 "immediate_operand" "i")] 3174 "TARGET_NEON" 3175{ 3176 neon_lane_bounds (operands[2], 0, 1); 3177 emit_insn (gen_neon_vdup_nv2di (operands[0], operands[1])); 3178 DONE; 3179}) 3180 3181; Disabled before reload because we don't want combine doing something silly, 3182; but used by the post-reload expansion of neon_vcombine. 3183(define_insn "*neon_vswp<mode>" 3184 [(set (match_operand:VDQX 0 "s_register_operand" "+w") 3185 (match_operand:VDQX 1 "s_register_operand" "+w")) 3186 (set (match_dup 1) (match_dup 0))] 3187 "TARGET_NEON && reload_completed" 3188 "vswp\t%<V_reg>0, %<V_reg>1" 3189 [(set (attr "neon_type") 3190 (if_then_else (match_test "<Is_d_reg>") 3191 (const_string "neon_bp_simple") 3192 (const_string "neon_bp_2cycle")))] 3193) 3194 3195;; In this insn, operand 1 should be low, and operand 2 the high part of the 3196;; dest vector. 3197;; FIXME: A different implementation of this builtin could make it much 3198;; more likely that we wouldn't actually need to output anything (we could make 3199;; it so that the reg allocator puts things in the right places magically 3200;; instead). Lack of subregs for vectors makes that tricky though, I think. 3201 3202(define_insn_and_split "neon_vcombine<mode>" 3203 [(set (match_operand:<V_DOUBLE> 0 "s_register_operand" "=w") 3204 (vec_concat:<V_DOUBLE> 3205 (match_operand:VDX 1 "s_register_operand" "w") 3206 (match_operand:VDX 2 "s_register_operand" "w")))] 3207 "TARGET_NEON" 3208 "#" 3209 "&& reload_completed" 3210 [(const_int 0)] 3211{ 3212 neon_split_vcombine (operands); 3213 DONE; 3214}) 3215 3216(define_expand "neon_vget_high<mode>" 3217 [(match_operand:<V_HALF> 0 "s_register_operand") 3218 (match_operand:VQX 1 "s_register_operand")] 3219 "TARGET_NEON" 3220{ 3221 emit_move_insn (operands[0], 3222 simplify_gen_subreg (<V_HALF>mode, operands[1], <MODE>mode, 3223 GET_MODE_SIZE (<V_HALF>mode))); 3224 DONE; 3225}) 3226 3227(define_expand "neon_vget_low<mode>" 3228 [(match_operand:<V_HALF> 0 "s_register_operand") 3229 (match_operand:VQX 1 "s_register_operand")] 3230 "TARGET_NEON" 3231{ 3232 emit_move_insn (operands[0], 3233 simplify_gen_subreg (<V_HALF>mode, operands[1], 3234 <MODE>mode, 0)); 3235 DONE; 3236}) 3237 3238(define_insn "float<mode><V_cvtto>2" 3239 [(set (match_operand:<V_CVTTO> 0 "s_register_operand" "=w") 3240 (float:<V_CVTTO> (match_operand:VCVTI 1 "s_register_operand" "w")))] 3241 "TARGET_NEON && !flag_rounding_math" 3242 "vcvt.f32.s32\t%<V_reg>0, %<V_reg>1" 3243 [(set (attr "neon_type") 3244 (if_then_else (match_test "<Is_d_reg>") 3245 (const_string "neon_fp_vadd_ddd_vabs_dd") 3246 (const_string "neon_fp_vadd_qqq_vabs_qq")))] 3247) 3248 3249(define_insn "floatuns<mode><V_cvtto>2" 3250 [(set (match_operand:<V_CVTTO> 0 "s_register_operand" "=w") 3251 (unsigned_float:<V_CVTTO> (match_operand:VCVTI 1 "s_register_operand" "w")))] 3252 "TARGET_NEON && !flag_rounding_math" 3253 "vcvt.f32.u32\t%<V_reg>0, %<V_reg>1" 3254 [(set (attr "neon_type") 3255 (if_then_else (match_test "<Is_d_reg>") 3256 (const_string "neon_fp_vadd_ddd_vabs_dd") 3257 (const_string "neon_fp_vadd_qqq_vabs_qq")))] 3258) 3259 3260(define_insn "fix_trunc<mode><V_cvtto>2" 3261 [(set (match_operand:<V_CVTTO> 0 "s_register_operand" "=w") 3262 (fix:<V_CVTTO> (match_operand:VCVTF 1 "s_register_operand" "w")))] 3263 "TARGET_NEON" 3264 "vcvt.s32.f32\t%<V_reg>0, %<V_reg>1" 3265 [(set (attr "neon_type") 3266 (if_then_else (match_test "<Is_d_reg>") 3267 (const_string "neon_fp_vadd_ddd_vabs_dd") 3268 (const_string "neon_fp_vadd_qqq_vabs_qq")))] 3269) 3270 3271(define_insn "fixuns_trunc<mode><V_cvtto>2" 3272 [(set (match_operand:<V_CVTTO> 0 "s_register_operand" "=w") 3273 (unsigned_fix:<V_CVTTO> (match_operand:VCVTF 1 "s_register_operand" "w")))] 3274 "TARGET_NEON" 3275 "vcvt.u32.f32\t%<V_reg>0, %<V_reg>1" 3276 [(set (attr "neon_type") 3277 (if_then_else (match_test "<Is_d_reg>") 3278 (const_string "neon_fp_vadd_ddd_vabs_dd") 3279 (const_string "neon_fp_vadd_qqq_vabs_qq")))] 3280) 3281 3282(define_insn "neon_vcvt<mode>" 3283 [(set (match_operand:<V_CVTTO> 0 "s_register_operand" "=w") 3284 (unspec:<V_CVTTO> [(match_operand:VCVTF 1 "s_register_operand" "w") 3285 (match_operand:SI 2 "immediate_operand" "i")] 3286 UNSPEC_VCVT))] 3287 "TARGET_NEON" 3288 "vcvt.%T2%#32.f32\t%<V_reg>0, %<V_reg>1" 3289 [(set (attr "neon_type") 3290 (if_then_else (match_test "<Is_d_reg>") 3291 (const_string "neon_fp_vadd_ddd_vabs_dd") 3292 (const_string "neon_fp_vadd_qqq_vabs_qq")))] 3293) 3294 3295(define_insn "neon_vcvt<mode>" 3296 [(set (match_operand:<V_CVTTO> 0 "s_register_operand" "=w") 3297 (unspec:<V_CVTTO> [(match_operand:VCVTI 1 "s_register_operand" "w") 3298 (match_operand:SI 2 "immediate_operand" "i")] 3299 UNSPEC_VCVT))] 3300 "TARGET_NEON" 3301 "vcvt.f32.%T2%#32\t%<V_reg>0, %<V_reg>1" 3302 [(set (attr "neon_type") 3303 (if_then_else (match_test "<Is_d_reg>") 3304 (const_string "neon_fp_vadd_ddd_vabs_dd") 3305 (const_string "neon_fp_vadd_qqq_vabs_qq")))] 3306) 3307 3308(define_insn "neon_vcvt_n<mode>" 3309 [(set (match_operand:<V_CVTTO> 0 "s_register_operand" "=w") 3310 (unspec:<V_CVTTO> [(match_operand:VCVTF 1 "s_register_operand" "w") 3311 (match_operand:SI 2 "immediate_operand" "i") 3312 (match_operand:SI 3 "immediate_operand" "i")] 3313 UNSPEC_VCVT_N))] 3314 "TARGET_NEON" 3315{ 3316 neon_const_bounds (operands[2], 1, 33); 3317 return "vcvt.%T3%#32.f32\t%<V_reg>0, %<V_reg>1, %2"; 3318} 3319 [(set (attr "neon_type") 3320 (if_then_else (match_test "<Is_d_reg>") 3321 (const_string "neon_fp_vadd_ddd_vabs_dd") 3322 (const_string "neon_fp_vadd_qqq_vabs_qq")))] 3323) 3324 3325(define_insn "neon_vcvt_n<mode>" 3326 [(set (match_operand:<V_CVTTO> 0 "s_register_operand" "=w") 3327 (unspec:<V_CVTTO> [(match_operand:VCVTI 1 "s_register_operand" "w") 3328 (match_operand:SI 2 "immediate_operand" "i") 3329 (match_operand:SI 3 "immediate_operand" "i")] 3330 UNSPEC_VCVT_N))] 3331 "TARGET_NEON" 3332{ 3333 neon_const_bounds (operands[2], 1, 33); 3334 return "vcvt.f32.%T3%#32\t%<V_reg>0, %<V_reg>1, %2"; 3335} 3336 [(set (attr "neon_type") 3337 (if_then_else (match_test "<Is_d_reg>") 3338 (const_string "neon_fp_vadd_ddd_vabs_dd") 3339 (const_string "neon_fp_vadd_qqq_vabs_qq")))] 3340) 3341 3342(define_insn "neon_vmovn<mode>" 3343 [(set (match_operand:<V_narrow> 0 "s_register_operand" "=w") 3344 (unspec:<V_narrow> [(match_operand:VN 1 "s_register_operand" "w") 3345 (match_operand:SI 2 "immediate_operand" "i")] 3346 UNSPEC_VMOVN))] 3347 "TARGET_NEON" 3348 "vmovn.<V_if_elem>\t%P0, %q1" 3349 [(set_attr "neon_type" "neon_bp_simple")] 3350) 3351 3352(define_insn "neon_vqmovn<mode>" 3353 [(set (match_operand:<V_narrow> 0 "s_register_operand" "=w") 3354 (unspec:<V_narrow> [(match_operand:VN 1 "s_register_operand" "w") 3355 (match_operand:SI 2 "immediate_operand" "i")] 3356 UNSPEC_VQMOVN))] 3357 "TARGET_NEON" 3358 "vqmovn.%T2%#<V_sz_elem>\t%P0, %q1" 3359 [(set_attr "neon_type" "neon_shift_2")] 3360) 3361 3362(define_insn "neon_vqmovun<mode>" 3363 [(set (match_operand:<V_narrow> 0 "s_register_operand" "=w") 3364 (unspec:<V_narrow> [(match_operand:VN 1 "s_register_operand" "w") 3365 (match_operand:SI 2 "immediate_operand" "i")] 3366 UNSPEC_VQMOVUN))] 3367 "TARGET_NEON" 3368 "vqmovun.<V_s_elem>\t%P0, %q1" 3369 [(set_attr "neon_type" "neon_shift_2")] 3370) 3371 3372(define_insn "neon_vmovl<mode>" 3373 [(set (match_operand:<V_widen> 0 "s_register_operand" "=w") 3374 (unspec:<V_widen> [(match_operand:VW 1 "s_register_operand" "w") 3375 (match_operand:SI 2 "immediate_operand" "i")] 3376 UNSPEC_VMOVL))] 3377 "TARGET_NEON" 3378 "vmovl.%T2%#<V_sz_elem>\t%q0, %P1" 3379 [(set_attr "neon_type" "neon_shift_1")] 3380) 3381 3382(define_insn "neon_vmul_lane<mode>" 3383 [(set (match_operand:VMD 0 "s_register_operand" "=w") 3384 (unspec:VMD [(match_operand:VMD 1 "s_register_operand" "w") 3385 (match_operand:VMD 2 "s_register_operand" 3386 "<scalar_mul_constraint>") 3387 (match_operand:SI 3 "immediate_operand" "i") 3388 (match_operand:SI 4 "immediate_operand" "i")] 3389 UNSPEC_VMUL_LANE))] 3390 "TARGET_NEON" 3391{ 3392 neon_lane_bounds (operands[3], 0, GET_MODE_NUNITS (<MODE>mode)); 3393 return "vmul.<V_if_elem>\t%P0, %P1, %P2[%c3]"; 3394} 3395 [(set (attr "neon_type") 3396 (if_then_else (match_test "<Is_float_mode>") 3397 (const_string "neon_fp_vmul_ddd") 3398 (if_then_else (match_test "<Scalar_mul_8_16>") 3399 (const_string "neon_mul_ddd_16_scalar_32_16_long_scalar") 3400 (const_string "neon_mul_qdd_64_32_long_qqd_16_ddd_32_scalar_64_32_long_scalar"))))] 3401) 3402 3403(define_insn "neon_vmul_lane<mode>" 3404 [(set (match_operand:VMQ 0 "s_register_operand" "=w") 3405 (unspec:VMQ [(match_operand:VMQ 1 "s_register_operand" "w") 3406 (match_operand:<V_HALF> 2 "s_register_operand" 3407 "<scalar_mul_constraint>") 3408 (match_operand:SI 3 "immediate_operand" "i") 3409 (match_operand:SI 4 "immediate_operand" "i")] 3410 UNSPEC_VMUL_LANE))] 3411 "TARGET_NEON" 3412{ 3413 neon_lane_bounds (operands[3], 0, GET_MODE_NUNITS (<V_HALF>mode)); 3414 return "vmul.<V_if_elem>\t%q0, %q1, %P2[%c3]"; 3415} 3416 [(set (attr "neon_type") 3417 (if_then_else (match_test "<Is_float_mode>") 3418 (const_string "neon_fp_vmul_qqd") 3419 (if_then_else (match_test "<Scalar_mul_8_16>") 3420 (const_string "neon_mul_qdd_64_32_long_qqd_16_ddd_32_scalar_64_32_long_scalar") 3421 (const_string "neon_mul_qqd_32_scalar"))))] 3422) 3423 3424(define_insn "neon_vmull_lane<mode>" 3425 [(set (match_operand:<V_widen> 0 "s_register_operand" "=w") 3426 (unspec:<V_widen> [(match_operand:VMDI 1 "s_register_operand" "w") 3427 (match_operand:VMDI 2 "s_register_operand" 3428 "<scalar_mul_constraint>") 3429 (match_operand:SI 3 "immediate_operand" "i") 3430 (match_operand:SI 4 "immediate_operand" "i")] 3431 UNSPEC_VMULL_LANE))] 3432 "TARGET_NEON" 3433{ 3434 neon_lane_bounds (operands[3], 0, GET_MODE_NUNITS (<MODE>mode)); 3435 return "vmull.%T4%#<V_sz_elem>\t%q0, %P1, %P2[%c3]"; 3436} 3437 [(set (attr "neon_type") 3438 (if_then_else (match_test "<Scalar_mul_8_16>") 3439 (const_string "neon_mul_ddd_16_scalar_32_16_long_scalar") 3440 (const_string "neon_mul_qdd_64_32_long_qqd_16_ddd_32_scalar_64_32_long_scalar")))] 3441) 3442 3443(define_insn "neon_vqdmull_lane<mode>" 3444 [(set (match_operand:<V_widen> 0 "s_register_operand" "=w") 3445 (unspec:<V_widen> [(match_operand:VMDI 1 "s_register_operand" "w") 3446 (match_operand:VMDI 2 "s_register_operand" 3447 "<scalar_mul_constraint>") 3448 (match_operand:SI 3 "immediate_operand" "i") 3449 (match_operand:SI 4 "immediate_operand" "i")] 3450 UNSPEC_VQDMULL_LANE))] 3451 "TARGET_NEON" 3452{ 3453 neon_lane_bounds (operands[3], 0, GET_MODE_NUNITS (<MODE>mode)); 3454 return "vqdmull.<V_s_elem>\t%q0, %P1, %P2[%c3]"; 3455} 3456 [(set (attr "neon_type") 3457 (if_then_else (match_test "<Scalar_mul_8_16>") 3458 (const_string "neon_mul_ddd_16_scalar_32_16_long_scalar") 3459 (const_string "neon_mul_qdd_64_32_long_qqd_16_ddd_32_scalar_64_32_long_scalar")))] 3460) 3461 3462(define_insn "neon_vqdmulh_lane<mode>" 3463 [(set (match_operand:VMQI 0 "s_register_operand" "=w") 3464 (unspec:VMQI [(match_operand:VMQI 1 "s_register_operand" "w") 3465 (match_operand:<V_HALF> 2 "s_register_operand" 3466 "<scalar_mul_constraint>") 3467 (match_operand:SI 3 "immediate_operand" "i") 3468 (match_operand:SI 4 "immediate_operand" "i")] 3469 UNSPEC_VQDMULH_LANE))] 3470 "TARGET_NEON" 3471{ 3472 neon_lane_bounds (operands[3], 0, GET_MODE_NUNITS (<MODE>mode)); 3473 return "vq%O4dmulh.%T4%#<V_sz_elem>\t%q0, %q1, %P2[%c3]"; 3474} 3475 [(set (attr "neon_type") 3476 (if_then_else (match_test "<Scalar_mul_8_16>") 3477 (const_string "neon_mul_qdd_64_32_long_qqd_16_ddd_32_scalar_64_32_long_scalar") 3478 (const_string "neon_mul_qqd_32_scalar")))] 3479) 3480 3481(define_insn "neon_vqdmulh_lane<mode>" 3482 [(set (match_operand:VMDI 0 "s_register_operand" "=w") 3483 (unspec:VMDI [(match_operand:VMDI 1 "s_register_operand" "w") 3484 (match_operand:VMDI 2 "s_register_operand" 3485 "<scalar_mul_constraint>") 3486 (match_operand:SI 3 "immediate_operand" "i") 3487 (match_operand:SI 4 "immediate_operand" "i")] 3488 UNSPEC_VQDMULH_LANE))] 3489 "TARGET_NEON" 3490{ 3491 neon_lane_bounds (operands[3], 0, GET_MODE_NUNITS (<MODE>mode)); 3492 return "vq%O4dmulh.%T4%#<V_sz_elem>\t%P0, %P1, %P2[%c3]"; 3493} 3494 [(set (attr "neon_type") 3495 (if_then_else (match_test "<Scalar_mul_8_16>") 3496 (const_string "neon_mul_ddd_16_scalar_32_16_long_scalar") 3497 (const_string "neon_mul_qdd_64_32_long_qqd_16_ddd_32_scalar_64_32_long_scalar")))] 3498) 3499 3500(define_insn "neon_vmla_lane<mode>" 3501 [(set (match_operand:VMD 0 "s_register_operand" "=w") 3502 (unspec:VMD [(match_operand:VMD 1 "s_register_operand" "0") 3503 (match_operand:VMD 2 "s_register_operand" "w") 3504 (match_operand:VMD 3 "s_register_operand" 3505 "<scalar_mul_constraint>") 3506 (match_operand:SI 4 "immediate_operand" "i") 3507 (match_operand:SI 5 "immediate_operand" "i")] 3508 UNSPEC_VMLA_LANE))] 3509 "TARGET_NEON" 3510{ 3511 neon_lane_bounds (operands[4], 0, GET_MODE_NUNITS (<MODE>mode)); 3512 return "vmla.<V_if_elem>\t%P0, %P2, %P3[%c4]"; 3513} 3514 [(set (attr "neon_type") 3515 (if_then_else (match_test "<Is_float_mode>") 3516 (const_string "neon_fp_vmla_ddd_scalar") 3517 (if_then_else (match_test "<Scalar_mul_8_16>") 3518 (const_string "neon_mla_ddd_16_scalar_qdd_32_16_long_scalar") 3519 (const_string "neon_mla_ddd_32_qqd_16_ddd_32_scalar_qdd_64_32_long_scalar_qdd_64_32_long"))))] 3520) 3521 3522(define_insn "neon_vmla_lane<mode>" 3523 [(set (match_operand:VMQ 0 "s_register_operand" "=w") 3524 (unspec:VMQ [(match_operand:VMQ 1 "s_register_operand" "0") 3525 (match_operand:VMQ 2 "s_register_operand" "w") 3526 (match_operand:<V_HALF> 3 "s_register_operand" 3527 "<scalar_mul_constraint>") 3528 (match_operand:SI 4 "immediate_operand" "i") 3529 (match_operand:SI 5 "immediate_operand" "i")] 3530 UNSPEC_VMLA_LANE))] 3531 "TARGET_NEON" 3532{ 3533 neon_lane_bounds (operands[4], 0, GET_MODE_NUNITS (<MODE>mode)); 3534 return "vmla.<V_if_elem>\t%q0, %q2, %P3[%c4]"; 3535} 3536 [(set (attr "neon_type") 3537 (if_then_else (match_test "<Is_float_mode>") 3538 (const_string "neon_fp_vmla_qqq_scalar") 3539 (if_then_else (match_test "<Scalar_mul_8_16>") 3540 (const_string "neon_mla_ddd_32_qqd_16_ddd_32_scalar_qdd_64_32_long_scalar_qdd_64_32_long") 3541 (const_string "neon_mla_qqq_32_qqd_32_scalar"))))] 3542) 3543 3544(define_insn "neon_vmlal_lane<mode>" 3545 [(set (match_operand:<V_widen> 0 "s_register_operand" "=w") 3546 (unspec:<V_widen> [(match_operand:<V_widen> 1 "s_register_operand" "0") 3547 (match_operand:VMDI 2 "s_register_operand" "w") 3548 (match_operand:VMDI 3 "s_register_operand" 3549 "<scalar_mul_constraint>") 3550 (match_operand:SI 4 "immediate_operand" "i") 3551 (match_operand:SI 5 "immediate_operand" "i")] 3552 UNSPEC_VMLAL_LANE))] 3553 "TARGET_NEON" 3554{ 3555 neon_lane_bounds (operands[4], 0, GET_MODE_NUNITS (<MODE>mode)); 3556 return "vmlal.%T5%#<V_sz_elem>\t%q0, %P2, %P3[%c4]"; 3557} 3558 [(set (attr "neon_type") 3559 (if_then_else (match_test "<Scalar_mul_8_16>") 3560 (const_string "neon_mla_ddd_16_scalar_qdd_32_16_long_scalar") 3561 (const_string "neon_mla_ddd_32_qqd_16_ddd_32_scalar_qdd_64_32_long_scalar_qdd_64_32_long")))] 3562) 3563 3564(define_insn "neon_vqdmlal_lane<mode>" 3565 [(set (match_operand:<V_widen> 0 "s_register_operand" "=w") 3566 (unspec:<V_widen> [(match_operand:<V_widen> 1 "s_register_operand" "0") 3567 (match_operand:VMDI 2 "s_register_operand" "w") 3568 (match_operand:VMDI 3 "s_register_operand" 3569 "<scalar_mul_constraint>") 3570 (match_operand:SI 4 "immediate_operand" "i") 3571 (match_operand:SI 5 "immediate_operand" "i")] 3572 UNSPEC_VQDMLAL_LANE))] 3573 "TARGET_NEON" 3574{ 3575 neon_lane_bounds (operands[4], 0, GET_MODE_NUNITS (<MODE>mode)); 3576 return "vqdmlal.<V_s_elem>\t%q0, %P2, %P3[%c4]"; 3577} 3578 [(set (attr "neon_type") 3579 (if_then_else (match_test "<Scalar_mul_8_16>") 3580 (const_string "neon_mla_ddd_16_scalar_qdd_32_16_long_scalar") 3581 (const_string "neon_mla_ddd_32_qqd_16_ddd_32_scalar_qdd_64_32_long_scalar_qdd_64_32_long")))] 3582) 3583 3584(define_insn "neon_vmls_lane<mode>" 3585 [(set (match_operand:VMD 0 "s_register_operand" "=w") 3586 (unspec:VMD [(match_operand:VMD 1 "s_register_operand" "0") 3587 (match_operand:VMD 2 "s_register_operand" "w") 3588 (match_operand:VMD 3 "s_register_operand" 3589 "<scalar_mul_constraint>") 3590 (match_operand:SI 4 "immediate_operand" "i") 3591 (match_operand:SI 5 "immediate_operand" "i")] 3592 UNSPEC_VMLS_LANE))] 3593 "TARGET_NEON" 3594{ 3595 neon_lane_bounds (operands[4], 0, GET_MODE_NUNITS (<MODE>mode)); 3596 return "vmls.<V_if_elem>\t%P0, %P2, %P3[%c4]"; 3597} 3598 [(set (attr "neon_type") 3599 (if_then_else (match_test "<Is_float_mode>") 3600 (const_string "neon_fp_vmla_ddd_scalar") 3601 (if_then_else (match_test "<Scalar_mul_8_16>") 3602 (const_string "neon_mla_ddd_16_scalar_qdd_32_16_long_scalar") 3603 (const_string "neon_mla_ddd_32_qqd_16_ddd_32_scalar_qdd_64_32_long_scalar_qdd_64_32_long"))))] 3604) 3605 3606(define_insn "neon_vmls_lane<mode>" 3607 [(set (match_operand:VMQ 0 "s_register_operand" "=w") 3608 (unspec:VMQ [(match_operand:VMQ 1 "s_register_operand" "0") 3609 (match_operand:VMQ 2 "s_register_operand" "w") 3610 (match_operand:<V_HALF> 3 "s_register_operand" 3611 "<scalar_mul_constraint>") 3612 (match_operand:SI 4 "immediate_operand" "i") 3613 (match_operand:SI 5 "immediate_operand" "i")] 3614 UNSPEC_VMLS_LANE))] 3615 "TARGET_NEON" 3616{ 3617 neon_lane_bounds (operands[4], 0, GET_MODE_NUNITS (<MODE>mode)); 3618 return "vmls.<V_if_elem>\t%q0, %q2, %P3[%c4]"; 3619} 3620 [(set (attr "neon_type") 3621 (if_then_else (match_test "<Is_float_mode>") 3622 (const_string "neon_fp_vmla_qqq_scalar") 3623 (if_then_else (match_test "<Scalar_mul_8_16>") 3624 (const_string "neon_mla_ddd_32_qqd_16_ddd_32_scalar_qdd_64_32_long_scalar_qdd_64_32_long") 3625 (const_string "neon_mla_qqq_32_qqd_32_scalar"))))] 3626) 3627 3628(define_insn "neon_vmlsl_lane<mode>" 3629 [(set (match_operand:<V_widen> 0 "s_register_operand" "=w") 3630 (unspec:<V_widen> [(match_operand:<V_widen> 1 "s_register_operand" "0") 3631 (match_operand:VMDI 2 "s_register_operand" "w") 3632 (match_operand:VMDI 3 "s_register_operand" 3633 "<scalar_mul_constraint>") 3634 (match_operand:SI 4 "immediate_operand" "i") 3635 (match_operand:SI 5 "immediate_operand" "i")] 3636 UNSPEC_VMLSL_LANE))] 3637 "TARGET_NEON" 3638{ 3639 neon_lane_bounds (operands[4], 0, GET_MODE_NUNITS (<MODE>mode)); 3640 return "vmlsl.%T5%#<V_sz_elem>\t%q0, %P2, %P3[%c4]"; 3641} 3642 [(set (attr "neon_type") 3643 (if_then_else (match_test "<Scalar_mul_8_16>") 3644 (const_string "neon_mla_ddd_16_scalar_qdd_32_16_long_scalar") 3645 (const_string "neon_mla_ddd_32_qqd_16_ddd_32_scalar_qdd_64_32_long_scalar_qdd_64_32_long")))] 3646) 3647 3648(define_insn "neon_vqdmlsl_lane<mode>" 3649 [(set (match_operand:<V_widen> 0 "s_register_operand" "=w") 3650 (unspec:<V_widen> [(match_operand:<V_widen> 1 "s_register_operand" "0") 3651 (match_operand:VMDI 2 "s_register_operand" "w") 3652 (match_operand:VMDI 3 "s_register_operand" 3653 "<scalar_mul_constraint>") 3654 (match_operand:SI 4 "immediate_operand" "i") 3655 (match_operand:SI 5 "immediate_operand" "i")] 3656 UNSPEC_VQDMLSL_LANE))] 3657 "TARGET_NEON" 3658{ 3659 neon_lane_bounds (operands[4], 0, GET_MODE_NUNITS (<MODE>mode)); 3660 return "vqdmlsl.<V_s_elem>\t%q0, %P2, %P3[%c4]"; 3661} 3662 [(set (attr "neon_type") 3663 (if_then_else (match_test "<Scalar_mul_8_16>") 3664 (const_string "neon_mla_ddd_16_scalar_qdd_32_16_long_scalar") 3665 (const_string "neon_mla_ddd_32_qqd_16_ddd_32_scalar_qdd_64_32_long_scalar_qdd_64_32_long")))] 3666) 3667 3668; FIXME: For the "_n" multiply/multiply-accumulate insns, we copy a value in a 3669; core register into a temp register, then use a scalar taken from that. This 3670; isn't an optimal solution if e.g. the scalar has just been read from memory 3671; or extracted from another vector. The latter case it's currently better to 3672; use the "_lane" variant, and the former case can probably be implemented 3673; using vld1_lane, but that hasn't been done yet. 3674 3675(define_expand "neon_vmul_n<mode>" 3676 [(match_operand:VMD 0 "s_register_operand" "") 3677 (match_operand:VMD 1 "s_register_operand" "") 3678 (match_operand:<V_elem> 2 "s_register_operand" "") 3679 (match_operand:SI 3 "immediate_operand" "")] 3680 "TARGET_NEON" 3681{ 3682 rtx tmp = gen_reg_rtx (<MODE>mode); 3683 emit_insn (gen_neon_vset_lane<mode> (tmp, operands[2], tmp, const0_rtx)); 3684 emit_insn (gen_neon_vmul_lane<mode> (operands[0], operands[1], tmp, 3685 const0_rtx, const0_rtx)); 3686 DONE; 3687}) 3688 3689(define_expand "neon_vmul_n<mode>" 3690 [(match_operand:VMQ 0 "s_register_operand" "") 3691 (match_operand:VMQ 1 "s_register_operand" "") 3692 (match_operand:<V_elem> 2 "s_register_operand" "") 3693 (match_operand:SI 3 "immediate_operand" "")] 3694 "TARGET_NEON" 3695{ 3696 rtx tmp = gen_reg_rtx (<V_HALF>mode); 3697 emit_insn (gen_neon_vset_lane<V_half> (tmp, operands[2], tmp, const0_rtx)); 3698 emit_insn (gen_neon_vmul_lane<mode> (operands[0], operands[1], tmp, 3699 const0_rtx, const0_rtx)); 3700 DONE; 3701}) 3702 3703(define_expand "neon_vmull_n<mode>" 3704 [(match_operand:<V_widen> 0 "s_register_operand" "") 3705 (match_operand:VMDI 1 "s_register_operand" "") 3706 (match_operand:<V_elem> 2 "s_register_operand" "") 3707 (match_operand:SI 3 "immediate_operand" "")] 3708 "TARGET_NEON" 3709{ 3710 rtx tmp = gen_reg_rtx (<MODE>mode); 3711 emit_insn (gen_neon_vset_lane<mode> (tmp, operands[2], tmp, const0_rtx)); 3712 emit_insn (gen_neon_vmull_lane<mode> (operands[0], operands[1], tmp, 3713 const0_rtx, operands[3])); 3714 DONE; 3715}) 3716 3717(define_expand "neon_vqdmull_n<mode>" 3718 [(match_operand:<V_widen> 0 "s_register_operand" "") 3719 (match_operand:VMDI 1 "s_register_operand" "") 3720 (match_operand:<V_elem> 2 "s_register_operand" "") 3721 (match_operand:SI 3 "immediate_operand" "")] 3722 "TARGET_NEON" 3723{ 3724 rtx tmp = gen_reg_rtx (<MODE>mode); 3725 emit_insn (gen_neon_vset_lane<mode> (tmp, operands[2], tmp, const0_rtx)); 3726 emit_insn (gen_neon_vqdmull_lane<mode> (operands[0], operands[1], tmp, 3727 const0_rtx, const0_rtx)); 3728 DONE; 3729}) 3730 3731(define_expand "neon_vqdmulh_n<mode>" 3732 [(match_operand:VMDI 0 "s_register_operand" "") 3733 (match_operand:VMDI 1 "s_register_operand" "") 3734 (match_operand:<V_elem> 2 "s_register_operand" "") 3735 (match_operand:SI 3 "immediate_operand" "")] 3736 "TARGET_NEON" 3737{ 3738 rtx tmp = gen_reg_rtx (<MODE>mode); 3739 emit_insn (gen_neon_vset_lane<mode> (tmp, operands[2], tmp, const0_rtx)); 3740 emit_insn (gen_neon_vqdmulh_lane<mode> (operands[0], operands[1], tmp, 3741 const0_rtx, operands[3])); 3742 DONE; 3743}) 3744 3745(define_expand "neon_vqdmulh_n<mode>" 3746 [(match_operand:VMQI 0 "s_register_operand" "") 3747 (match_operand:VMQI 1 "s_register_operand" "") 3748 (match_operand:<V_elem> 2 "s_register_operand" "") 3749 (match_operand:SI 3 "immediate_operand" "")] 3750 "TARGET_NEON" 3751{ 3752 rtx tmp = gen_reg_rtx (<V_HALF>mode); 3753 emit_insn (gen_neon_vset_lane<V_half> (tmp, operands[2], tmp, const0_rtx)); 3754 emit_insn (gen_neon_vqdmulh_lane<mode> (operands[0], operands[1], tmp, 3755 const0_rtx, operands[3])); 3756 DONE; 3757}) 3758 3759(define_expand "neon_vmla_n<mode>" 3760 [(match_operand:VMD 0 "s_register_operand" "") 3761 (match_operand:VMD 1 "s_register_operand" "") 3762 (match_operand:VMD 2 "s_register_operand" "") 3763 (match_operand:<V_elem> 3 "s_register_operand" "") 3764 (match_operand:SI 4 "immediate_operand" "")] 3765 "TARGET_NEON" 3766{ 3767 rtx tmp = gen_reg_rtx (<MODE>mode); 3768 emit_insn (gen_neon_vset_lane<mode> (tmp, operands[3], tmp, const0_rtx)); 3769 emit_insn (gen_neon_vmla_lane<mode> (operands[0], operands[1], operands[2], 3770 tmp, const0_rtx, operands[4])); 3771 DONE; 3772}) 3773 3774(define_expand "neon_vmla_n<mode>" 3775 [(match_operand:VMQ 0 "s_register_operand" "") 3776 (match_operand:VMQ 1 "s_register_operand" "") 3777 (match_operand:VMQ 2 "s_register_operand" "") 3778 (match_operand:<V_elem> 3 "s_register_operand" "") 3779 (match_operand:SI 4 "immediate_operand" "")] 3780 "TARGET_NEON" 3781{ 3782 rtx tmp = gen_reg_rtx (<V_HALF>mode); 3783 emit_insn (gen_neon_vset_lane<V_half> (tmp, operands[3], tmp, const0_rtx)); 3784 emit_insn (gen_neon_vmla_lane<mode> (operands[0], operands[1], operands[2], 3785 tmp, const0_rtx, operands[4])); 3786 DONE; 3787}) 3788 3789(define_expand "neon_vmlal_n<mode>" 3790 [(match_operand:<V_widen> 0 "s_register_operand" "") 3791 (match_operand:<V_widen> 1 "s_register_operand" "") 3792 (match_operand:VMDI 2 "s_register_operand" "") 3793 (match_operand:<V_elem> 3 "s_register_operand" "") 3794 (match_operand:SI 4 "immediate_operand" "")] 3795 "TARGET_NEON" 3796{ 3797 rtx tmp = gen_reg_rtx (<MODE>mode); 3798 emit_insn (gen_neon_vset_lane<mode> (tmp, operands[3], tmp, const0_rtx)); 3799 emit_insn (gen_neon_vmlal_lane<mode> (operands[0], operands[1], operands[2], 3800 tmp, const0_rtx, operands[4])); 3801 DONE; 3802}) 3803 3804(define_expand "neon_vqdmlal_n<mode>" 3805 [(match_operand:<V_widen> 0 "s_register_operand" "") 3806 (match_operand:<V_widen> 1 "s_register_operand" "") 3807 (match_operand:VMDI 2 "s_register_operand" "") 3808 (match_operand:<V_elem> 3 "s_register_operand" "") 3809 (match_operand:SI 4 "immediate_operand" "")] 3810 "TARGET_NEON" 3811{ 3812 rtx tmp = gen_reg_rtx (<MODE>mode); 3813 emit_insn (gen_neon_vset_lane<mode> (tmp, operands[3], tmp, const0_rtx)); 3814 emit_insn (gen_neon_vqdmlal_lane<mode> (operands[0], operands[1], operands[2], 3815 tmp, const0_rtx, operands[4])); 3816 DONE; 3817}) 3818 3819(define_expand "neon_vmls_n<mode>" 3820 [(match_operand:VMD 0 "s_register_operand" "") 3821 (match_operand:VMD 1 "s_register_operand" "") 3822 (match_operand:VMD 2 "s_register_operand" "") 3823 (match_operand:<V_elem> 3 "s_register_operand" "") 3824 (match_operand:SI 4 "immediate_operand" "")] 3825 "TARGET_NEON" 3826{ 3827 rtx tmp = gen_reg_rtx (<MODE>mode); 3828 emit_insn (gen_neon_vset_lane<mode> (tmp, operands[3], tmp, const0_rtx)); 3829 emit_insn (gen_neon_vmls_lane<mode> (operands[0], operands[1], operands[2], 3830 tmp, const0_rtx, operands[4])); 3831 DONE; 3832}) 3833 3834(define_expand "neon_vmls_n<mode>" 3835 [(match_operand:VMQ 0 "s_register_operand" "") 3836 (match_operand:VMQ 1 "s_register_operand" "") 3837 (match_operand:VMQ 2 "s_register_operand" "") 3838 (match_operand:<V_elem> 3 "s_register_operand" "") 3839 (match_operand:SI 4 "immediate_operand" "")] 3840 "TARGET_NEON" 3841{ 3842 rtx tmp = gen_reg_rtx (<V_HALF>mode); 3843 emit_insn (gen_neon_vset_lane<V_half> (tmp, operands[3], tmp, const0_rtx)); 3844 emit_insn (gen_neon_vmls_lane<mode> (operands[0], operands[1], operands[2], 3845 tmp, const0_rtx, operands[4])); 3846 DONE; 3847}) 3848 3849(define_expand "neon_vmlsl_n<mode>" 3850 [(match_operand:<V_widen> 0 "s_register_operand" "") 3851 (match_operand:<V_widen> 1 "s_register_operand" "") 3852 (match_operand:VMDI 2 "s_register_operand" "") 3853 (match_operand:<V_elem> 3 "s_register_operand" "") 3854 (match_operand:SI 4 "immediate_operand" "")] 3855 "TARGET_NEON" 3856{ 3857 rtx tmp = gen_reg_rtx (<MODE>mode); 3858 emit_insn (gen_neon_vset_lane<mode> (tmp, operands[3], tmp, const0_rtx)); 3859 emit_insn (gen_neon_vmlsl_lane<mode> (operands[0], operands[1], operands[2], 3860 tmp, const0_rtx, operands[4])); 3861 DONE; 3862}) 3863 3864(define_expand "neon_vqdmlsl_n<mode>" 3865 [(match_operand:<V_widen> 0 "s_register_operand" "") 3866 (match_operand:<V_widen> 1 "s_register_operand" "") 3867 (match_operand:VMDI 2 "s_register_operand" "") 3868 (match_operand:<V_elem> 3 "s_register_operand" "") 3869 (match_operand:SI 4 "immediate_operand" "")] 3870 "TARGET_NEON" 3871{ 3872 rtx tmp = gen_reg_rtx (<MODE>mode); 3873 emit_insn (gen_neon_vset_lane<mode> (tmp, operands[3], tmp, const0_rtx)); 3874 emit_insn (gen_neon_vqdmlsl_lane<mode> (operands[0], operands[1], operands[2], 3875 tmp, const0_rtx, operands[4])); 3876 DONE; 3877}) 3878 3879(define_insn "neon_vext<mode>" 3880 [(set (match_operand:VDQX 0 "s_register_operand" "=w") 3881 (unspec:VDQX [(match_operand:VDQX 1 "s_register_operand" "w") 3882 (match_operand:VDQX 2 "s_register_operand" "w") 3883 (match_operand:SI 3 "immediate_operand" "i")] 3884 UNSPEC_VEXT))] 3885 "TARGET_NEON" 3886{ 3887 neon_const_bounds (operands[3], 0, GET_MODE_NUNITS (<MODE>mode)); 3888 return "vext.<V_sz_elem>\t%<V_reg>0, %<V_reg>1, %<V_reg>2, %3"; 3889} 3890 [(set (attr "neon_type") 3891 (if_then_else (match_test "<Is_d_reg>") 3892 (const_string "neon_bp_simple") 3893 (const_string "neon_bp_2cycle")))] 3894) 3895 3896(define_insn "neon_vrev64<mode>" 3897 [(set (match_operand:VDQ 0 "s_register_operand" "=w") 3898 (unspec:VDQ [(match_operand:VDQ 1 "s_register_operand" "w") 3899 (match_operand:SI 2 "immediate_operand" "i")] 3900 UNSPEC_VREV64))] 3901 "TARGET_NEON" 3902 "vrev64.<V_sz_elem>\t%<V_reg>0, %<V_reg>1" 3903 [(set_attr "neon_type" "neon_bp_simple")] 3904) 3905 3906(define_insn "neon_vrev32<mode>" 3907 [(set (match_operand:VX 0 "s_register_operand" "=w") 3908 (unspec:VX [(match_operand:VX 1 "s_register_operand" "w") 3909 (match_operand:SI 2 "immediate_operand" "i")] 3910 UNSPEC_VREV32))] 3911 "TARGET_NEON" 3912 "vrev32.<V_sz_elem>\t%<V_reg>0, %<V_reg>1" 3913 [(set_attr "neon_type" "neon_bp_simple")] 3914) 3915 3916(define_insn "neon_vrev16<mode>" 3917 [(set (match_operand:VE 0 "s_register_operand" "=w") 3918 (unspec:VE [(match_operand:VE 1 "s_register_operand" "w") 3919 (match_operand:SI 2 "immediate_operand" "i")] 3920 UNSPEC_VREV16))] 3921 "TARGET_NEON" 3922 "vrev16.<V_sz_elem>\t%<V_reg>0, %<V_reg>1" 3923 [(set_attr "neon_type" "neon_bp_simple")] 3924) 3925 3926; vbsl_* intrinsics may compile to any of vbsl/vbif/vbit depending on register 3927; allocation. For an intrinsic of form: 3928; rD = vbsl_* (rS, rN, rM) 3929; We can use any of: 3930; vbsl rS, rN, rM (if D = S) 3931; vbit rD, rN, rS (if D = M, so 1-bits in rS choose bits from rN, else rM) 3932; vbif rD, rM, rS (if D = N, so 0-bits in rS choose bits from rM, else rN) 3933 3934(define_insn "neon_vbsl<mode>_internal" 3935 [(set (match_operand:VDQX 0 "s_register_operand" "=w,w,w") 3936 (unspec:VDQX [(match_operand:VDQX 1 "s_register_operand" " 0,w,w") 3937 (match_operand:VDQX 2 "s_register_operand" " w,w,0") 3938 (match_operand:VDQX 3 "s_register_operand" " w,0,w")] 3939 UNSPEC_VBSL))] 3940 "TARGET_NEON" 3941 "@ 3942 vbsl\t%<V_reg>0, %<V_reg>2, %<V_reg>3 3943 vbit\t%<V_reg>0, %<V_reg>2, %<V_reg>1 3944 vbif\t%<V_reg>0, %<V_reg>3, %<V_reg>1" 3945 [(set_attr "neon_type" "neon_int_1")] 3946) 3947 3948(define_expand "neon_vbsl<mode>" 3949 [(set (match_operand:VDQX 0 "s_register_operand" "") 3950 (unspec:VDQX [(match_operand:<V_cmp_result> 1 "s_register_operand" "") 3951 (match_operand:VDQX 2 "s_register_operand" "") 3952 (match_operand:VDQX 3 "s_register_operand" "")] 3953 UNSPEC_VBSL))] 3954 "TARGET_NEON" 3955{ 3956 /* We can't alias operands together if they have different modes. */ 3957 operands[1] = gen_lowpart (<MODE>mode, operands[1]); 3958}) 3959 3960(define_insn "neon_vshl<mode>" 3961 [(set (match_operand:VDQIX 0 "s_register_operand" "=w") 3962 (unspec:VDQIX [(match_operand:VDQIX 1 "s_register_operand" "w") 3963 (match_operand:VDQIX 2 "s_register_operand" "w") 3964 (match_operand:SI 3 "immediate_operand" "i")] 3965 UNSPEC_VSHL))] 3966 "TARGET_NEON" 3967 "v%O3shl.%T3%#<V_sz_elem>\t%<V_reg>0, %<V_reg>1, %<V_reg>2" 3968 [(set (attr "neon_type") 3969 (if_then_else (match_test "<Is_d_reg>") 3970 (const_string "neon_vshl_ddd") 3971 (const_string "neon_shift_3")))] 3972) 3973 3974(define_insn "neon_vqshl<mode>" 3975 [(set (match_operand:VDQIX 0 "s_register_operand" "=w") 3976 (unspec:VDQIX [(match_operand:VDQIX 1 "s_register_operand" "w") 3977 (match_operand:VDQIX 2 "s_register_operand" "w") 3978 (match_operand:SI 3 "immediate_operand" "i")] 3979 UNSPEC_VQSHL))] 3980 "TARGET_NEON" 3981 "vq%O3shl.%T3%#<V_sz_elem>\t%<V_reg>0, %<V_reg>1, %<V_reg>2" 3982 [(set (attr "neon_type") 3983 (if_then_else (match_test "<Is_d_reg>") 3984 (const_string "neon_shift_2") 3985 (const_string "neon_vqshl_vrshl_vqrshl_qqq")))] 3986) 3987 3988(define_insn "neon_vshr_n<mode>" 3989 [(set (match_operand:VDQIX 0 "s_register_operand" "=w") 3990 (unspec:VDQIX [(match_operand:VDQIX 1 "s_register_operand" "w") 3991 (match_operand:SI 2 "immediate_operand" "i") 3992 (match_operand:SI 3 "immediate_operand" "i")] 3993 UNSPEC_VSHR_N))] 3994 "TARGET_NEON" 3995{ 3996 neon_const_bounds (operands[2], 1, neon_element_bits (<MODE>mode) + 1); 3997 return "v%O3shr.%T3%#<V_sz_elem>\t%<V_reg>0, %<V_reg>1, %2"; 3998} 3999 [(set_attr "neon_type" "neon_shift_1")] 4000) 4001 4002(define_insn "neon_vshrn_n<mode>" 4003 [(set (match_operand:<V_narrow> 0 "s_register_operand" "=w") 4004 (unspec:<V_narrow> [(match_operand:VN 1 "s_register_operand" "w") 4005 (match_operand:SI 2 "immediate_operand" "i") 4006 (match_operand:SI 3 "immediate_operand" "i")] 4007 UNSPEC_VSHRN_N))] 4008 "TARGET_NEON" 4009{ 4010 neon_const_bounds (operands[2], 1, neon_element_bits (<MODE>mode) / 2 + 1); 4011 return "v%O3shrn.<V_if_elem>\t%P0, %q1, %2"; 4012} 4013 [(set_attr "neon_type" "neon_shift_1")] 4014) 4015 4016(define_insn "neon_vqshrn_n<mode>" 4017 [(set (match_operand:<V_narrow> 0 "s_register_operand" "=w") 4018 (unspec:<V_narrow> [(match_operand:VN 1 "s_register_operand" "w") 4019 (match_operand:SI 2 "immediate_operand" "i") 4020 (match_operand:SI 3 "immediate_operand" "i")] 4021 UNSPEC_VQSHRN_N))] 4022 "TARGET_NEON" 4023{ 4024 neon_const_bounds (operands[2], 1, neon_element_bits (<MODE>mode) / 2 + 1); 4025 return "vq%O3shrn.%T3%#<V_sz_elem>\t%P0, %q1, %2"; 4026} 4027 [(set_attr "neon_type" "neon_shift_2")] 4028) 4029 4030(define_insn "neon_vqshrun_n<mode>" 4031 [(set (match_operand:<V_narrow> 0 "s_register_operand" "=w") 4032 (unspec:<V_narrow> [(match_operand:VN 1 "s_register_operand" "w") 4033 (match_operand:SI 2 "immediate_operand" "i") 4034 (match_operand:SI 3 "immediate_operand" "i")] 4035 UNSPEC_VQSHRUN_N))] 4036 "TARGET_NEON" 4037{ 4038 neon_const_bounds (operands[2], 1, neon_element_bits (<MODE>mode) / 2 + 1); 4039 return "vq%O3shrun.%T3%#<V_sz_elem>\t%P0, %q1, %2"; 4040} 4041 [(set_attr "neon_type" "neon_shift_2")] 4042) 4043 4044(define_insn "neon_vshl_n<mode>" 4045 [(set (match_operand:VDQIX 0 "s_register_operand" "=w") 4046 (unspec:VDQIX [(match_operand:VDQIX 1 "s_register_operand" "w") 4047 (match_operand:SI 2 "immediate_operand" "i") 4048 (match_operand:SI 3 "immediate_operand" "i")] 4049 UNSPEC_VSHL_N))] 4050 "TARGET_NEON" 4051{ 4052 neon_const_bounds (operands[2], 0, neon_element_bits (<MODE>mode)); 4053 return "vshl.<V_if_elem>\t%<V_reg>0, %<V_reg>1, %2"; 4054} 4055 [(set_attr "neon_type" "neon_shift_1")] 4056) 4057 4058(define_insn "neon_vqshl_n<mode>" 4059 [(set (match_operand:VDQIX 0 "s_register_operand" "=w") 4060 (unspec:VDQIX [(match_operand:VDQIX 1 "s_register_operand" "w") 4061 (match_operand:SI 2 "immediate_operand" "i") 4062 (match_operand:SI 3 "immediate_operand" "i")] 4063 UNSPEC_VQSHL_N))] 4064 "TARGET_NEON" 4065{ 4066 neon_const_bounds (operands[2], 0, neon_element_bits (<MODE>mode)); 4067 return "vqshl.%T3%#<V_sz_elem>\t%<V_reg>0, %<V_reg>1, %2"; 4068} 4069 [(set_attr "neon_type" "neon_shift_2")] 4070) 4071 4072(define_insn "neon_vqshlu_n<mode>" 4073 [(set (match_operand:VDQIX 0 "s_register_operand" "=w") 4074 (unspec:VDQIX [(match_operand:VDQIX 1 "s_register_operand" "w") 4075 (match_operand:SI 2 "immediate_operand" "i") 4076 (match_operand:SI 3 "immediate_operand" "i")] 4077 UNSPEC_VQSHLU_N))] 4078 "TARGET_NEON" 4079{ 4080 neon_const_bounds (operands[2], 0, neon_element_bits (<MODE>mode)); 4081 return "vqshlu.%T3%#<V_sz_elem>\t%<V_reg>0, %<V_reg>1, %2"; 4082} 4083 [(set_attr "neon_type" "neon_shift_2")] 4084) 4085 4086(define_insn "neon_vshll_n<mode>" 4087 [(set (match_operand:<V_widen> 0 "s_register_operand" "=w") 4088 (unspec:<V_widen> [(match_operand:VW 1 "s_register_operand" "w") 4089 (match_operand:SI 2 "immediate_operand" "i") 4090 (match_operand:SI 3 "immediate_operand" "i")] 4091 UNSPEC_VSHLL_N))] 4092 "TARGET_NEON" 4093{ 4094 /* The boundaries are: 0 < imm <= size. */ 4095 neon_const_bounds (operands[2], 0, neon_element_bits (<MODE>mode) + 1); 4096 return "vshll.%T3%#<V_sz_elem>\t%q0, %P1, %2"; 4097} 4098 [(set_attr "neon_type" "neon_shift_1")] 4099) 4100 4101(define_insn "neon_vsra_n<mode>" 4102 [(set (match_operand:VDQIX 0 "s_register_operand" "=w") 4103 (unspec:VDQIX [(match_operand:VDQIX 1 "s_register_operand" "0") 4104 (match_operand:VDQIX 2 "s_register_operand" "w") 4105 (match_operand:SI 3 "immediate_operand" "i") 4106 (match_operand:SI 4 "immediate_operand" "i")] 4107 UNSPEC_VSRA_N))] 4108 "TARGET_NEON" 4109{ 4110 neon_const_bounds (operands[3], 1, neon_element_bits (<MODE>mode) + 1); 4111 return "v%O4sra.%T4%#<V_sz_elem>\t%<V_reg>0, %<V_reg>2, %3"; 4112} 4113 [(set_attr "neon_type" "neon_vsra_vrsra")] 4114) 4115 4116(define_insn "neon_vsri_n<mode>" 4117 [(set (match_operand:VDQIX 0 "s_register_operand" "=w") 4118 (unspec:VDQIX [(match_operand:VDQIX 1 "s_register_operand" "0") 4119 (match_operand:VDQIX 2 "s_register_operand" "w") 4120 (match_operand:SI 3 "immediate_operand" "i")] 4121 UNSPEC_VSRI))] 4122 "TARGET_NEON" 4123{ 4124 neon_const_bounds (operands[3], 1, neon_element_bits (<MODE>mode) + 1); 4125 return "vsri.<V_sz_elem>\t%<V_reg>0, %<V_reg>2, %3"; 4126} 4127 [(set (attr "neon_type") 4128 (if_then_else (match_test "<Is_d_reg>") 4129 (const_string "neon_shift_1") 4130 (const_string "neon_shift_3")))] 4131) 4132 4133(define_insn "neon_vsli_n<mode>" 4134 [(set (match_operand:VDQIX 0 "s_register_operand" "=w") 4135 (unspec:VDQIX [(match_operand:VDQIX 1 "s_register_operand" "0") 4136 (match_operand:VDQIX 2 "s_register_operand" "w") 4137 (match_operand:SI 3 "immediate_operand" "i")] 4138 UNSPEC_VSLI))] 4139 "TARGET_NEON" 4140{ 4141 neon_const_bounds (operands[3], 0, neon_element_bits (<MODE>mode)); 4142 return "vsli.<V_sz_elem>\t%<V_reg>0, %<V_reg>2, %3"; 4143} 4144 [(set (attr "neon_type") 4145 (if_then_else (match_test "<Is_d_reg>") 4146 (const_string "neon_shift_1") 4147 (const_string "neon_shift_3")))] 4148) 4149 4150(define_insn "neon_vtbl1v8qi" 4151 [(set (match_operand:V8QI 0 "s_register_operand" "=w") 4152 (unspec:V8QI [(match_operand:V8QI 1 "s_register_operand" "w") 4153 (match_operand:V8QI 2 "s_register_operand" "w")] 4154 UNSPEC_VTBL))] 4155 "TARGET_NEON" 4156 "vtbl.8\t%P0, {%P1}, %P2" 4157 [(set_attr "neon_type" "neon_bp_2cycle")] 4158) 4159 4160(define_insn "neon_vtbl2v8qi" 4161 [(set (match_operand:V8QI 0 "s_register_operand" "=w") 4162 (unspec:V8QI [(match_operand:TI 1 "s_register_operand" "w") 4163 (match_operand:V8QI 2 "s_register_operand" "w")] 4164 UNSPEC_VTBL))] 4165 "TARGET_NEON" 4166{ 4167 rtx ops[4]; 4168 int tabbase = REGNO (operands[1]); 4169 4170 ops[0] = operands[0]; 4171 ops[1] = gen_rtx_REG (V8QImode, tabbase); 4172 ops[2] = gen_rtx_REG (V8QImode, tabbase + 2); 4173 ops[3] = operands[2]; 4174 output_asm_insn ("vtbl.8\t%P0, {%P1, %P2}, %P3", ops); 4175 4176 return ""; 4177} 4178 [(set_attr "neon_type" "neon_bp_2cycle")] 4179) 4180 4181(define_insn "neon_vtbl3v8qi" 4182 [(set (match_operand:V8QI 0 "s_register_operand" "=w") 4183 (unspec:V8QI [(match_operand:EI 1 "s_register_operand" "w") 4184 (match_operand:V8QI 2 "s_register_operand" "w")] 4185 UNSPEC_VTBL))] 4186 "TARGET_NEON" 4187{ 4188 rtx ops[5]; 4189 int tabbase = REGNO (operands[1]); 4190 4191 ops[0] = operands[0]; 4192 ops[1] = gen_rtx_REG (V8QImode, tabbase); 4193 ops[2] = gen_rtx_REG (V8QImode, tabbase + 2); 4194 ops[3] = gen_rtx_REG (V8QImode, tabbase + 4); 4195 ops[4] = operands[2]; 4196 output_asm_insn ("vtbl.8\t%P0, {%P1, %P2, %P3}, %P4", ops); 4197 4198 return ""; 4199} 4200 [(set_attr "neon_type" "neon_bp_3cycle")] 4201) 4202 4203(define_insn "neon_vtbl4v8qi" 4204 [(set (match_operand:V8QI 0 "s_register_operand" "=w") 4205 (unspec:V8QI [(match_operand:OI 1 "s_register_operand" "w") 4206 (match_operand:V8QI 2 "s_register_operand" "w")] 4207 UNSPEC_VTBL))] 4208 "TARGET_NEON" 4209{ 4210 rtx ops[6]; 4211 int tabbase = REGNO (operands[1]); 4212 4213 ops[0] = operands[0]; 4214 ops[1] = gen_rtx_REG (V8QImode, tabbase); 4215 ops[2] = gen_rtx_REG (V8QImode, tabbase + 2); 4216 ops[3] = gen_rtx_REG (V8QImode, tabbase + 4); 4217 ops[4] = gen_rtx_REG (V8QImode, tabbase + 6); 4218 ops[5] = operands[2]; 4219 output_asm_insn ("vtbl.8\t%P0, {%P1, %P2, %P3, %P4}, %P5", ops); 4220 4221 return ""; 4222} 4223 [(set_attr "neon_type" "neon_bp_3cycle")] 4224) 4225 4226;; These three are used by the vec_perm infrastructure for V16QImode. 4227(define_insn_and_split "neon_vtbl1v16qi" 4228 [(set (match_operand:V16QI 0 "s_register_operand" "=&w") 4229 (unspec:V16QI [(match_operand:V16QI 1 "s_register_operand" "w") 4230 (match_operand:V16QI 2 "s_register_operand" "w")] 4231 UNSPEC_VTBL))] 4232 "TARGET_NEON" 4233 "#" 4234 "&& reload_completed" 4235 [(const_int 0)] 4236{ 4237 rtx op0, op1, op2, part0, part2; 4238 unsigned ofs; 4239 4240 op0 = operands[0]; 4241 op1 = gen_lowpart (TImode, operands[1]); 4242 op2 = operands[2]; 4243 4244 ofs = subreg_lowpart_offset (V8QImode, V16QImode); 4245 part0 = simplify_subreg (V8QImode, op0, V16QImode, ofs); 4246 part2 = simplify_subreg (V8QImode, op2, V16QImode, ofs); 4247 emit_insn (gen_neon_vtbl2v8qi (part0, op1, part2)); 4248 4249 ofs = subreg_highpart_offset (V8QImode, V16QImode); 4250 part0 = simplify_subreg (V8QImode, op0, V16QImode, ofs); 4251 part2 = simplify_subreg (V8QImode, op2, V16QImode, ofs); 4252 emit_insn (gen_neon_vtbl2v8qi (part0, op1, part2)); 4253 DONE; 4254}) 4255 4256(define_insn_and_split "neon_vtbl2v16qi" 4257 [(set (match_operand:V16QI 0 "s_register_operand" "=&w") 4258 (unspec:V16QI [(match_operand:OI 1 "s_register_operand" "w") 4259 (match_operand:V16QI 2 "s_register_operand" "w")] 4260 UNSPEC_VTBL))] 4261 "TARGET_NEON" 4262 "#" 4263 "&& reload_completed" 4264 [(const_int 0)] 4265{ 4266 rtx op0, op1, op2, part0, part2; 4267 unsigned ofs; 4268 4269 op0 = operands[0]; 4270 op1 = operands[1]; 4271 op2 = operands[2]; 4272 4273 ofs = subreg_lowpart_offset (V8QImode, V16QImode); 4274 part0 = simplify_subreg (V8QImode, op0, V16QImode, ofs); 4275 part2 = simplify_subreg (V8QImode, op2, V16QImode, ofs); 4276 emit_insn (gen_neon_vtbl2v8qi (part0, op1, part2)); 4277 4278 ofs = subreg_highpart_offset (V8QImode, V16QImode); 4279 part0 = simplify_subreg (V8QImode, op0, V16QImode, ofs); 4280 part2 = simplify_subreg (V8QImode, op2, V16QImode, ofs); 4281 emit_insn (gen_neon_vtbl2v8qi (part0, op1, part2)); 4282 DONE; 4283}) 4284 4285;; ??? Logically we should extend the regular neon_vcombine pattern to 4286;; handle quad-word input modes, producing octa-word output modes. But 4287;; that requires us to add support for octa-word vector modes in moves. 4288;; That seems overkill for this one use in vec_perm. 4289(define_insn_and_split "neon_vcombinev16qi" 4290 [(set (match_operand:OI 0 "s_register_operand" "=w") 4291 (unspec:OI [(match_operand:V16QI 1 "s_register_operand" "w") 4292 (match_operand:V16QI 2 "s_register_operand" "w")] 4293 UNSPEC_VCONCAT))] 4294 "TARGET_NEON" 4295 "#" 4296 "&& reload_completed" 4297 [(const_int 0)] 4298{ 4299 neon_split_vcombine (operands); 4300 DONE; 4301}) 4302 4303(define_insn "neon_vtbx1v8qi" 4304 [(set (match_operand:V8QI 0 "s_register_operand" "=w") 4305 (unspec:V8QI [(match_operand:V8QI 1 "s_register_operand" "0") 4306 (match_operand:V8QI 2 "s_register_operand" "w") 4307 (match_operand:V8QI 3 "s_register_operand" "w")] 4308 UNSPEC_VTBX))] 4309 "TARGET_NEON" 4310 "vtbx.8\t%P0, {%P2}, %P3" 4311 [(set_attr "neon_type" "neon_bp_2cycle")] 4312) 4313 4314(define_insn "neon_vtbx2v8qi" 4315 [(set (match_operand:V8QI 0 "s_register_operand" "=w") 4316 (unspec:V8QI [(match_operand:V8QI 1 "s_register_operand" "0") 4317 (match_operand:TI 2 "s_register_operand" "w") 4318 (match_operand:V8QI 3 "s_register_operand" "w")] 4319 UNSPEC_VTBX))] 4320 "TARGET_NEON" 4321{ 4322 rtx ops[4]; 4323 int tabbase = REGNO (operands[2]); 4324 4325 ops[0] = operands[0]; 4326 ops[1] = gen_rtx_REG (V8QImode, tabbase); 4327 ops[2] = gen_rtx_REG (V8QImode, tabbase + 2); 4328 ops[3] = operands[3]; 4329 output_asm_insn ("vtbx.8\t%P0, {%P1, %P2}, %P3", ops); 4330 4331 return ""; 4332} 4333 [(set_attr "neon_type" "neon_bp_2cycle")] 4334) 4335 4336(define_insn "neon_vtbx3v8qi" 4337 [(set (match_operand:V8QI 0 "s_register_operand" "=w") 4338 (unspec:V8QI [(match_operand:V8QI 1 "s_register_operand" "0") 4339 (match_operand:EI 2 "s_register_operand" "w") 4340 (match_operand:V8QI 3 "s_register_operand" "w")] 4341 UNSPEC_VTBX))] 4342 "TARGET_NEON" 4343{ 4344 rtx ops[5]; 4345 int tabbase = REGNO (operands[2]); 4346 4347 ops[0] = operands[0]; 4348 ops[1] = gen_rtx_REG (V8QImode, tabbase); 4349 ops[2] = gen_rtx_REG (V8QImode, tabbase + 2); 4350 ops[3] = gen_rtx_REG (V8QImode, tabbase + 4); 4351 ops[4] = operands[3]; 4352 output_asm_insn ("vtbx.8\t%P0, {%P1, %P2, %P3}, %P4", ops); 4353 4354 return ""; 4355} 4356 [(set_attr "neon_type" "neon_bp_3cycle")] 4357) 4358 4359(define_insn "neon_vtbx4v8qi" 4360 [(set (match_operand:V8QI 0 "s_register_operand" "=w") 4361 (unspec:V8QI [(match_operand:V8QI 1 "s_register_operand" "0") 4362 (match_operand:OI 2 "s_register_operand" "w") 4363 (match_operand:V8QI 3 "s_register_operand" "w")] 4364 UNSPEC_VTBX))] 4365 "TARGET_NEON" 4366{ 4367 rtx ops[6]; 4368 int tabbase = REGNO (operands[2]); 4369 4370 ops[0] = operands[0]; 4371 ops[1] = gen_rtx_REG (V8QImode, tabbase); 4372 ops[2] = gen_rtx_REG (V8QImode, tabbase + 2); 4373 ops[3] = gen_rtx_REG (V8QImode, tabbase + 4); 4374 ops[4] = gen_rtx_REG (V8QImode, tabbase + 6); 4375 ops[5] = operands[3]; 4376 output_asm_insn ("vtbx.8\t%P0, {%P1, %P2, %P3, %P4}, %P5", ops); 4377 4378 return ""; 4379} 4380 [(set_attr "neon_type" "neon_bp_3cycle")] 4381) 4382 4383(define_expand "neon_vtrn<mode>_internal" 4384 [(parallel 4385 [(set (match_operand:VDQW 0 "s_register_operand" "") 4386 (unspec:VDQW [(match_operand:VDQW 1 "s_register_operand" "") 4387 (match_operand:VDQW 2 "s_register_operand" "")] 4388 UNSPEC_VTRN1)) 4389 (set (match_operand:VDQW 3 "s_register_operand" "") 4390 (unspec:VDQW [(match_dup 1) (match_dup 2)] UNSPEC_VTRN2))])] 4391 "TARGET_NEON" 4392 "" 4393) 4394 4395;; Note: Different operand numbering to handle tied registers correctly. 4396(define_insn "*neon_vtrn<mode>_insn" 4397 [(set (match_operand:VDQW 0 "s_register_operand" "=w") 4398 (unspec:VDQW [(match_operand:VDQW 1 "s_register_operand" "0") 4399 (match_operand:VDQW 3 "s_register_operand" "2")] 4400 UNSPEC_VTRN1)) 4401 (set (match_operand:VDQW 2 "s_register_operand" "=w") 4402 (unspec:VDQW [(match_dup 1) (match_dup 3)] 4403 UNSPEC_VTRN2))] 4404 "TARGET_NEON" 4405 "vtrn.<V_sz_elem>\t%<V_reg>0, %<V_reg>2" 4406 [(set (attr "neon_type") 4407 (if_then_else (match_test "<Is_d_reg>") 4408 (const_string "neon_bp_simple") 4409 (const_string "neon_bp_3cycle")))] 4410) 4411 4412(define_expand "neon_vtrn<mode>" 4413 [(match_operand:SI 0 "s_register_operand" "r") 4414 (match_operand:VDQW 1 "s_register_operand" "w") 4415 (match_operand:VDQW 2 "s_register_operand" "w")] 4416 "TARGET_NEON" 4417{ 4418 neon_emit_pair_result_insn (<MODE>mode, gen_neon_vtrn<mode>_internal, 4419 operands[0], operands[1], operands[2]); 4420 DONE; 4421}) 4422 4423(define_expand "neon_vzip<mode>_internal" 4424 [(parallel 4425 [(set (match_operand:VDQW 0 "s_register_operand" "") 4426 (unspec:VDQW [(match_operand:VDQW 1 "s_register_operand" "") 4427 (match_operand:VDQW 2 "s_register_operand" "")] 4428 UNSPEC_VZIP1)) 4429 (set (match_operand:VDQW 3 "s_register_operand" "") 4430 (unspec:VDQW [(match_dup 1) (match_dup 2)] UNSPEC_VZIP2))])] 4431 "TARGET_NEON" 4432 "" 4433) 4434 4435;; Note: Different operand numbering to handle tied registers correctly. 4436(define_insn "*neon_vzip<mode>_insn" 4437 [(set (match_operand:VDQW 0 "s_register_operand" "=w") 4438 (unspec:VDQW [(match_operand:VDQW 1 "s_register_operand" "0") 4439 (match_operand:VDQW 3 "s_register_operand" "2")] 4440 UNSPEC_VZIP1)) 4441 (set (match_operand:VDQW 2 "s_register_operand" "=w") 4442 (unspec:VDQW [(match_dup 1) (match_dup 3)] 4443 UNSPEC_VZIP2))] 4444 "TARGET_NEON" 4445 "vzip.<V_sz_elem>\t%<V_reg>0, %<V_reg>2" 4446 [(set (attr "neon_type") 4447 (if_then_else (match_test "<Is_d_reg>") 4448 (const_string "neon_bp_simple") 4449 (const_string "neon_bp_3cycle")))] 4450) 4451 4452(define_expand "neon_vzip<mode>" 4453 [(match_operand:SI 0 "s_register_operand" "r") 4454 (match_operand:VDQW 1 "s_register_operand" "w") 4455 (match_operand:VDQW 2 "s_register_operand" "w")] 4456 "TARGET_NEON" 4457{ 4458 neon_emit_pair_result_insn (<MODE>mode, gen_neon_vzip<mode>_internal, 4459 operands[0], operands[1], operands[2]); 4460 DONE; 4461}) 4462 4463(define_expand "neon_vuzp<mode>_internal" 4464 [(parallel 4465 [(set (match_operand:VDQW 0 "s_register_operand" "") 4466 (unspec:VDQW [(match_operand:VDQW 1 "s_register_operand" "") 4467 (match_operand:VDQW 2 "s_register_operand" "")] 4468 UNSPEC_VUZP1)) 4469 (set (match_operand:VDQW 3 "s_register_operand" "") 4470 (unspec:VDQW [(match_dup 1) (match_dup 2)] UNSPEC_VUZP2))])] 4471 "TARGET_NEON" 4472 "" 4473) 4474 4475;; Note: Different operand numbering to handle tied registers correctly. 4476(define_insn "*neon_vuzp<mode>_insn" 4477 [(set (match_operand:VDQW 0 "s_register_operand" "=w") 4478 (unspec:VDQW [(match_operand:VDQW 1 "s_register_operand" "0") 4479 (match_operand:VDQW 3 "s_register_operand" "2")] 4480 UNSPEC_VUZP1)) 4481 (set (match_operand:VDQW 2 "s_register_operand" "=w") 4482 (unspec:VDQW [(match_dup 1) (match_dup 3)] 4483 UNSPEC_VUZP2))] 4484 "TARGET_NEON" 4485 "vuzp.<V_sz_elem>\t%<V_reg>0, %<V_reg>2" 4486 [(set (attr "neon_type") 4487 (if_then_else (match_test "<Is_d_reg>") 4488 (const_string "neon_bp_simple") 4489 (const_string "neon_bp_3cycle")))] 4490) 4491 4492(define_expand "neon_vuzp<mode>" 4493 [(match_operand:SI 0 "s_register_operand" "r") 4494 (match_operand:VDQW 1 "s_register_operand" "w") 4495 (match_operand:VDQW 2 "s_register_operand" "w")] 4496 "TARGET_NEON" 4497{ 4498 neon_emit_pair_result_insn (<MODE>mode, gen_neon_vuzp<mode>_internal, 4499 operands[0], operands[1], operands[2]); 4500 DONE; 4501}) 4502 4503(define_expand "neon_vreinterpretv8qi<mode>" 4504 [(match_operand:V8QI 0 "s_register_operand" "") 4505 (match_operand:VDX 1 "s_register_operand" "")] 4506 "TARGET_NEON" 4507{ 4508 neon_reinterpret (operands[0], operands[1]); 4509 DONE; 4510}) 4511 4512(define_expand "neon_vreinterpretv4hi<mode>" 4513 [(match_operand:V4HI 0 "s_register_operand" "") 4514 (match_operand:VDX 1 "s_register_operand" "")] 4515 "TARGET_NEON" 4516{ 4517 neon_reinterpret (operands[0], operands[1]); 4518 DONE; 4519}) 4520 4521(define_expand "neon_vreinterpretv2si<mode>" 4522 [(match_operand:V2SI 0 "s_register_operand" "") 4523 (match_operand:VDX 1 "s_register_operand" "")] 4524 "TARGET_NEON" 4525{ 4526 neon_reinterpret (operands[0], operands[1]); 4527 DONE; 4528}) 4529 4530(define_expand "neon_vreinterpretv2sf<mode>" 4531 [(match_operand:V2SF 0 "s_register_operand" "") 4532 (match_operand:VDX 1 "s_register_operand" "")] 4533 "TARGET_NEON" 4534{ 4535 neon_reinterpret (operands[0], operands[1]); 4536 DONE; 4537}) 4538 4539(define_expand "neon_vreinterpretdi<mode>" 4540 [(match_operand:DI 0 "s_register_operand" "") 4541 (match_operand:VDX 1 "s_register_operand" "")] 4542 "TARGET_NEON" 4543{ 4544 neon_reinterpret (operands[0], operands[1]); 4545 DONE; 4546}) 4547 4548(define_expand "neon_vreinterpretv16qi<mode>" 4549 [(match_operand:V16QI 0 "s_register_operand" "") 4550 (match_operand:VQX 1 "s_register_operand" "")] 4551 "TARGET_NEON" 4552{ 4553 neon_reinterpret (operands[0], operands[1]); 4554 DONE; 4555}) 4556 4557(define_expand "neon_vreinterpretv8hi<mode>" 4558 [(match_operand:V8HI 0 "s_register_operand" "") 4559 (match_operand:VQX 1 "s_register_operand" "")] 4560 "TARGET_NEON" 4561{ 4562 neon_reinterpret (operands[0], operands[1]); 4563 DONE; 4564}) 4565 4566(define_expand "neon_vreinterpretv4si<mode>" 4567 [(match_operand:V4SI 0 "s_register_operand" "") 4568 (match_operand:VQX 1 "s_register_operand" "")] 4569 "TARGET_NEON" 4570{ 4571 neon_reinterpret (operands[0], operands[1]); 4572 DONE; 4573}) 4574 4575(define_expand "neon_vreinterpretv4sf<mode>" 4576 [(match_operand:V4SF 0 "s_register_operand" "") 4577 (match_operand:VQX 1 "s_register_operand" "")] 4578 "TARGET_NEON" 4579{ 4580 neon_reinterpret (operands[0], operands[1]); 4581 DONE; 4582}) 4583 4584(define_expand "neon_vreinterpretv2di<mode>" 4585 [(match_operand:V2DI 0 "s_register_operand" "") 4586 (match_operand:VQX 1 "s_register_operand" "")] 4587 "TARGET_NEON" 4588{ 4589 neon_reinterpret (operands[0], operands[1]); 4590 DONE; 4591}) 4592 4593(define_expand "vec_load_lanes<mode><mode>" 4594 [(set (match_operand:VDQX 0 "s_register_operand") 4595 (unspec:VDQX [(match_operand:VDQX 1 "neon_struct_operand")] 4596 UNSPEC_VLD1))] 4597 "TARGET_NEON") 4598 4599(define_insn "neon_vld1<mode>" 4600 [(set (match_operand:VDQX 0 "s_register_operand" "=w") 4601 (unspec:VDQX [(match_operand:VDQX 1 "neon_struct_operand" "Um")] 4602 UNSPEC_VLD1))] 4603 "TARGET_NEON" 4604 "vld1.<V_sz_elem>\t%h0, %A1" 4605 [(set_attr "neon_type" "neon_vld1_1_2_regs")] 4606) 4607 4608(define_insn "neon_vld1_lane<mode>" 4609 [(set (match_operand:VDX 0 "s_register_operand" "=w") 4610 (unspec:VDX [(match_operand:<V_elem> 1 "neon_struct_operand" "Um") 4611 (match_operand:VDX 2 "s_register_operand" "0") 4612 (match_operand:SI 3 "immediate_operand" "i")] 4613 UNSPEC_VLD1_LANE))] 4614 "TARGET_NEON" 4615{ 4616 HOST_WIDE_INT lane = INTVAL (operands[3]); 4617 HOST_WIDE_INT max = GET_MODE_NUNITS (<MODE>mode); 4618 if (lane < 0 || lane >= max) 4619 error ("lane out of range"); 4620 if (max == 1) 4621 return "vld1.<V_sz_elem>\t%P0, %A1"; 4622 else 4623 return "vld1.<V_sz_elem>\t{%P0[%c3]}, %A1"; 4624} 4625 [(set (attr "neon_type") 4626 (if_then_else (eq (const_string "<V_mode_nunits>") (const_int 2)) 4627 (const_string "neon_vld1_1_2_regs") 4628 (const_string "neon_vld1_vld2_lane")))] 4629) 4630 4631(define_insn "neon_vld1_lane<mode>" 4632 [(set (match_operand:VQX 0 "s_register_operand" "=w") 4633 (unspec:VQX [(match_operand:<V_elem> 1 "neon_struct_operand" "Um") 4634 (match_operand:VQX 2 "s_register_operand" "0") 4635 (match_operand:SI 3 "immediate_operand" "i")] 4636 UNSPEC_VLD1_LANE))] 4637 "TARGET_NEON" 4638{ 4639 HOST_WIDE_INT lane = INTVAL (operands[3]); 4640 HOST_WIDE_INT max = GET_MODE_NUNITS (<MODE>mode); 4641 int regno = REGNO (operands[0]); 4642 if (lane < 0 || lane >= max) 4643 error ("lane out of range"); 4644 else if (lane >= max / 2) 4645 { 4646 lane -= max / 2; 4647 regno += 2; 4648 operands[3] = GEN_INT (lane); 4649 } 4650 operands[0] = gen_rtx_REG (<V_HALF>mode, regno); 4651 if (max == 2) 4652 return "vld1.<V_sz_elem>\t%P0, %A1"; 4653 else 4654 return "vld1.<V_sz_elem>\t{%P0[%c3]}, %A1"; 4655} 4656 [(set (attr "neon_type") 4657 (if_then_else (eq (const_string "<V_mode_nunits>") (const_int 2)) 4658 (const_string "neon_vld1_1_2_regs") 4659 (const_string "neon_vld1_vld2_lane")))] 4660) 4661 4662(define_insn "neon_vld1_dup<mode>" 4663 [(set (match_operand:VDX 0 "s_register_operand" "=w") 4664 (vec_duplicate:VDX (match_operand:<V_elem> 1 "neon_struct_operand" "Um")))] 4665 "TARGET_NEON" 4666{ 4667 if (GET_MODE_NUNITS (<MODE>mode) > 1) 4668 return "vld1.<V_sz_elem>\t{%P0[]}, %A1"; 4669 else 4670 return "vld1.<V_sz_elem>\t%h0, %A1"; 4671} 4672 [(set (attr "neon_type") 4673 (if_then_else (gt (const_string "<V_mode_nunits>") (const_string "1")) 4674 (const_string "neon_vld2_2_regs_vld1_vld2_all_lanes") 4675 (const_string "neon_vld1_1_2_regs")))] 4676) 4677 4678(define_insn "neon_vld1_dup<mode>" 4679 [(set (match_operand:VQ 0 "s_register_operand" "=w") 4680 (vec_duplicate:VQ (match_operand:<V_elem> 1 "neon_struct_operand" "Um")))] 4681 "TARGET_NEON" 4682{ 4683 return "vld1.<V_sz_elem>\t{%e0[], %f0[]}, %A1"; 4684} 4685 [(set_attr "neon_type" "neon_vld2_2_regs_vld1_vld2_all_lanes")] 4686) 4687 4688(define_insn_and_split "neon_vld1_dupv2di" 4689 [(set (match_operand:V2DI 0 "s_register_operand" "=w") 4690 (vec_duplicate:V2DI (match_operand:DI 1 "neon_struct_operand" "Um")))] 4691 "TARGET_NEON" 4692 "#" 4693 "&& reload_completed" 4694 [(const_int 0)] 4695 { 4696 rtx tmprtx = gen_lowpart (DImode, operands[0]); 4697 emit_insn (gen_neon_vld1_dupdi (tmprtx, operands[1])); 4698 emit_move_insn (gen_highpart (DImode, operands[0]), tmprtx ); 4699 DONE; 4700 } 4701 [(set_attr "length" "8") 4702 (set_attr "neon_type" "neon_vld2_2_regs_vld1_vld2_all_lanes")] 4703) 4704 4705(define_expand "vec_store_lanes<mode><mode>" 4706 [(set (match_operand:VDQX 0 "neon_struct_operand") 4707 (unspec:VDQX [(match_operand:VDQX 1 "s_register_operand")] 4708 UNSPEC_VST1))] 4709 "TARGET_NEON") 4710 4711(define_insn "neon_vst1<mode>" 4712 [(set (match_operand:VDQX 0 "neon_struct_operand" "=Um") 4713 (unspec:VDQX [(match_operand:VDQX 1 "s_register_operand" "w")] 4714 UNSPEC_VST1))] 4715 "TARGET_NEON" 4716 "vst1.<V_sz_elem>\t%h1, %A0" 4717 [(set_attr "neon_type" "neon_vst1_1_2_regs_vst2_2_regs")]) 4718 4719(define_insn "neon_vst1_lane<mode>" 4720 [(set (match_operand:<V_elem> 0 "neon_struct_operand" "=Um") 4721 (unspec:<V_elem> 4722 [(match_operand:VDX 1 "s_register_operand" "w") 4723 (match_operand:SI 2 "immediate_operand" "i")] 4724 UNSPEC_VST1_LANE))] 4725 "TARGET_NEON" 4726{ 4727 HOST_WIDE_INT lane = INTVAL (operands[2]); 4728 HOST_WIDE_INT max = GET_MODE_NUNITS (<MODE>mode); 4729 if (lane < 0 || lane >= max) 4730 error ("lane out of range"); 4731 if (max == 1) 4732 return "vst1.<V_sz_elem>\t{%P1}, %A0"; 4733 else 4734 return "vst1.<V_sz_elem>\t{%P1[%c2]}, %A0"; 4735} 4736 [(set (attr "neon_type") 4737 (if_then_else (eq (const_string "<V_mode_nunits>") (const_int 1)) 4738 (const_string "neon_vst1_1_2_regs_vst2_2_regs") 4739 (const_string "neon_vst1_vst2_lane")))]) 4740 4741(define_insn "neon_vst1_lane<mode>" 4742 [(set (match_operand:<V_elem> 0 "neon_struct_operand" "=Um") 4743 (unspec:<V_elem> 4744 [(match_operand:VQX 1 "s_register_operand" "w") 4745 (match_operand:SI 2 "immediate_operand" "i")] 4746 UNSPEC_VST1_LANE))] 4747 "TARGET_NEON" 4748{ 4749 HOST_WIDE_INT lane = INTVAL (operands[2]); 4750 HOST_WIDE_INT max = GET_MODE_NUNITS (<MODE>mode); 4751 int regno = REGNO (operands[1]); 4752 if (lane < 0 || lane >= max) 4753 error ("lane out of range"); 4754 else if (lane >= max / 2) 4755 { 4756 lane -= max / 2; 4757 regno += 2; 4758 operands[2] = GEN_INT (lane); 4759 } 4760 operands[1] = gen_rtx_REG (<V_HALF>mode, regno); 4761 if (max == 2) 4762 return "vst1.<V_sz_elem>\t{%P1}, %A0"; 4763 else 4764 return "vst1.<V_sz_elem>\t{%P1[%c2]}, %A0"; 4765} 4766 [(set_attr "neon_type" "neon_vst1_vst2_lane")] 4767) 4768 4769(define_expand "vec_load_lanesti<mode>" 4770 [(set (match_operand:TI 0 "s_register_operand") 4771 (unspec:TI [(match_operand:TI 1 "neon_struct_operand") 4772 (unspec:VDX [(const_int 0)] UNSPEC_VSTRUCTDUMMY)] 4773 UNSPEC_VLD2))] 4774 "TARGET_NEON") 4775 4776(define_insn "neon_vld2<mode>" 4777 [(set (match_operand:TI 0 "s_register_operand" "=w") 4778 (unspec:TI [(match_operand:TI 1 "neon_struct_operand" "Um") 4779 (unspec:VDX [(const_int 0)] UNSPEC_VSTRUCTDUMMY)] 4780 UNSPEC_VLD2))] 4781 "TARGET_NEON" 4782{ 4783 if (<V_sz_elem> == 64) 4784 return "vld1.64\t%h0, %A1"; 4785 else 4786 return "vld2.<V_sz_elem>\t%h0, %A1"; 4787} 4788 [(set (attr "neon_type") 4789 (if_then_else (eq (const_string "<V_sz_elem>") (const_string "64")) 4790 (const_string "neon_vld1_1_2_regs") 4791 (const_string "neon_vld2_2_regs_vld1_vld2_all_lanes")))] 4792) 4793 4794(define_expand "vec_load_lanesoi<mode>" 4795 [(set (match_operand:OI 0 "s_register_operand") 4796 (unspec:OI [(match_operand:OI 1 "neon_struct_operand") 4797 (unspec:VQ [(const_int 0)] UNSPEC_VSTRUCTDUMMY)] 4798 UNSPEC_VLD2))] 4799 "TARGET_NEON") 4800 4801(define_insn "neon_vld2<mode>" 4802 [(set (match_operand:OI 0 "s_register_operand" "=w") 4803 (unspec:OI [(match_operand:OI 1 "neon_struct_operand" "Um") 4804 (unspec:VQ [(const_int 0)] UNSPEC_VSTRUCTDUMMY)] 4805 UNSPEC_VLD2))] 4806 "TARGET_NEON" 4807 "vld2.<V_sz_elem>\t%h0, %A1" 4808 [(set_attr "neon_type" "neon_vld2_2_regs_vld1_vld2_all_lanes")]) 4809 4810(define_insn "neon_vld2_lane<mode>" 4811 [(set (match_operand:TI 0 "s_register_operand" "=w") 4812 (unspec:TI [(match_operand:<V_two_elem> 1 "neon_struct_operand" "Um") 4813 (match_operand:TI 2 "s_register_operand" "0") 4814 (match_operand:SI 3 "immediate_operand" "i") 4815 (unspec:VD [(const_int 0)] UNSPEC_VSTRUCTDUMMY)] 4816 UNSPEC_VLD2_LANE))] 4817 "TARGET_NEON" 4818{ 4819 HOST_WIDE_INT lane = INTVAL (operands[3]); 4820 HOST_WIDE_INT max = GET_MODE_NUNITS (<MODE>mode); 4821 int regno = REGNO (operands[0]); 4822 rtx ops[4]; 4823 if (lane < 0 || lane >= max) 4824 error ("lane out of range"); 4825 ops[0] = gen_rtx_REG (DImode, regno); 4826 ops[1] = gen_rtx_REG (DImode, regno + 2); 4827 ops[2] = operands[1]; 4828 ops[3] = operands[3]; 4829 output_asm_insn ("vld2.<V_sz_elem>\t{%P0[%c3], %P1[%c3]}, %A2", ops); 4830 return ""; 4831} 4832 [(set_attr "neon_type" "neon_vld1_vld2_lane")] 4833) 4834 4835(define_insn "neon_vld2_lane<mode>" 4836 [(set (match_operand:OI 0 "s_register_operand" "=w") 4837 (unspec:OI [(match_operand:<V_two_elem> 1 "neon_struct_operand" "Um") 4838 (match_operand:OI 2 "s_register_operand" "0") 4839 (match_operand:SI 3 "immediate_operand" "i") 4840 (unspec:VMQ [(const_int 0)] UNSPEC_VSTRUCTDUMMY)] 4841 UNSPEC_VLD2_LANE))] 4842 "TARGET_NEON" 4843{ 4844 HOST_WIDE_INT lane = INTVAL (operands[3]); 4845 HOST_WIDE_INT max = GET_MODE_NUNITS (<MODE>mode); 4846 int regno = REGNO (operands[0]); 4847 rtx ops[4]; 4848 if (lane < 0 || lane >= max) 4849 error ("lane out of range"); 4850 else if (lane >= max / 2) 4851 { 4852 lane -= max / 2; 4853 regno += 2; 4854 } 4855 ops[0] = gen_rtx_REG (DImode, regno); 4856 ops[1] = gen_rtx_REG (DImode, regno + 4); 4857 ops[2] = operands[1]; 4858 ops[3] = GEN_INT (lane); 4859 output_asm_insn ("vld2.<V_sz_elem>\t{%P0[%c3], %P1[%c3]}, %A2", ops); 4860 return ""; 4861} 4862 [(set_attr "neon_type" "neon_vld1_vld2_lane")] 4863) 4864 4865(define_insn "neon_vld2_dup<mode>" 4866 [(set (match_operand:TI 0 "s_register_operand" "=w") 4867 (unspec:TI [(match_operand:<V_two_elem> 1 "neon_struct_operand" "Um") 4868 (unspec:VDX [(const_int 0)] UNSPEC_VSTRUCTDUMMY)] 4869 UNSPEC_VLD2_DUP))] 4870 "TARGET_NEON" 4871{ 4872 if (GET_MODE_NUNITS (<MODE>mode) > 1) 4873 return "vld2.<V_sz_elem>\t{%e0[], %f0[]}, %A1"; 4874 else 4875 return "vld1.<V_sz_elem>\t%h0, %A1"; 4876} 4877 [(set (attr "neon_type") 4878 (if_then_else (gt (const_string "<V_mode_nunits>") (const_string "1")) 4879 (const_string "neon_vld2_2_regs_vld1_vld2_all_lanes") 4880 (const_string "neon_vld1_1_2_regs")))] 4881) 4882 4883(define_expand "vec_store_lanesti<mode>" 4884 [(set (match_operand:TI 0 "neon_struct_operand") 4885 (unspec:TI [(match_operand:TI 1 "s_register_operand") 4886 (unspec:VDX [(const_int 0)] UNSPEC_VSTRUCTDUMMY)] 4887 UNSPEC_VST2))] 4888 "TARGET_NEON") 4889 4890(define_insn "neon_vst2<mode>" 4891 [(set (match_operand:TI 0 "neon_struct_operand" "=Um") 4892 (unspec:TI [(match_operand:TI 1 "s_register_operand" "w") 4893 (unspec:VDX [(const_int 0)] UNSPEC_VSTRUCTDUMMY)] 4894 UNSPEC_VST2))] 4895 "TARGET_NEON" 4896{ 4897 if (<V_sz_elem> == 64) 4898 return "vst1.64\t%h1, %A0"; 4899 else 4900 return "vst2.<V_sz_elem>\t%h1, %A0"; 4901} 4902 [(set (attr "neon_type") 4903 (if_then_else (eq (const_string "<V_sz_elem>") (const_string "64")) 4904 (const_string "neon_vst1_1_2_regs_vst2_2_regs") 4905 (const_string "neon_vst1_1_2_regs_vst2_2_regs")))] 4906) 4907 4908(define_expand "vec_store_lanesoi<mode>" 4909 [(set (match_operand:OI 0 "neon_struct_operand") 4910 (unspec:OI [(match_operand:OI 1 "s_register_operand") 4911 (unspec:VQ [(const_int 0)] UNSPEC_VSTRUCTDUMMY)] 4912 UNSPEC_VST2))] 4913 "TARGET_NEON") 4914 4915(define_insn "neon_vst2<mode>" 4916 [(set (match_operand:OI 0 "neon_struct_operand" "=Um") 4917 (unspec:OI [(match_operand:OI 1 "s_register_operand" "w") 4918 (unspec:VQ [(const_int 0)] UNSPEC_VSTRUCTDUMMY)] 4919 UNSPEC_VST2))] 4920 "TARGET_NEON" 4921 "vst2.<V_sz_elem>\t%h1, %A0" 4922 [(set_attr "neon_type" "neon_vst1_1_2_regs_vst2_2_regs")] 4923) 4924 4925(define_insn "neon_vst2_lane<mode>" 4926 [(set (match_operand:<V_two_elem> 0 "neon_struct_operand" "=Um") 4927 (unspec:<V_two_elem> 4928 [(match_operand:TI 1 "s_register_operand" "w") 4929 (match_operand:SI 2 "immediate_operand" "i") 4930 (unspec:VD [(const_int 0)] UNSPEC_VSTRUCTDUMMY)] 4931 UNSPEC_VST2_LANE))] 4932 "TARGET_NEON" 4933{ 4934 HOST_WIDE_INT lane = INTVAL (operands[2]); 4935 HOST_WIDE_INT max = GET_MODE_NUNITS (<MODE>mode); 4936 int regno = REGNO (operands[1]); 4937 rtx ops[4]; 4938 if (lane < 0 || lane >= max) 4939 error ("lane out of range"); 4940 ops[0] = operands[0]; 4941 ops[1] = gen_rtx_REG (DImode, regno); 4942 ops[2] = gen_rtx_REG (DImode, regno + 2); 4943 ops[3] = operands[2]; 4944 output_asm_insn ("vst2.<V_sz_elem>\t{%P1[%c3], %P2[%c3]}, %A0", ops); 4945 return ""; 4946} 4947 [(set_attr "neon_type" "neon_vst1_vst2_lane")] 4948) 4949 4950(define_insn "neon_vst2_lane<mode>" 4951 [(set (match_operand:<V_two_elem> 0 "neon_struct_operand" "=Um") 4952 (unspec:<V_two_elem> 4953 [(match_operand:OI 1 "s_register_operand" "w") 4954 (match_operand:SI 2 "immediate_operand" "i") 4955 (unspec:VMQ [(const_int 0)] UNSPEC_VSTRUCTDUMMY)] 4956 UNSPEC_VST2_LANE))] 4957 "TARGET_NEON" 4958{ 4959 HOST_WIDE_INT lane = INTVAL (operands[2]); 4960 HOST_WIDE_INT max = GET_MODE_NUNITS (<MODE>mode); 4961 int regno = REGNO (operands[1]); 4962 rtx ops[4]; 4963 if (lane < 0 || lane >= max) 4964 error ("lane out of range"); 4965 else if (lane >= max / 2) 4966 { 4967 lane -= max / 2; 4968 regno += 2; 4969 } 4970 ops[0] = operands[0]; 4971 ops[1] = gen_rtx_REG (DImode, regno); 4972 ops[2] = gen_rtx_REG (DImode, regno + 4); 4973 ops[3] = GEN_INT (lane); 4974 output_asm_insn ("vst2.<V_sz_elem>\t{%P1[%c3], %P2[%c3]}, %A0", ops); 4975 return ""; 4976} 4977 [(set_attr "neon_type" "neon_vst1_vst2_lane")] 4978) 4979 4980(define_expand "vec_load_lanesei<mode>" 4981 [(set (match_operand:EI 0 "s_register_operand") 4982 (unspec:EI [(match_operand:EI 1 "neon_struct_operand") 4983 (unspec:VDX [(const_int 0)] UNSPEC_VSTRUCTDUMMY)] 4984 UNSPEC_VLD3))] 4985 "TARGET_NEON") 4986 4987(define_insn "neon_vld3<mode>" 4988 [(set (match_operand:EI 0 "s_register_operand" "=w") 4989 (unspec:EI [(match_operand:EI 1 "neon_struct_operand" "Um") 4990 (unspec:VDX [(const_int 0)] UNSPEC_VSTRUCTDUMMY)] 4991 UNSPEC_VLD3))] 4992 "TARGET_NEON" 4993{ 4994 if (<V_sz_elem> == 64) 4995 return "vld1.64\t%h0, %A1"; 4996 else 4997 return "vld3.<V_sz_elem>\t%h0, %A1"; 4998} 4999 [(set (attr "neon_type") 5000 (if_then_else (eq (const_string "<V_sz_elem>") (const_string "64")) 5001 (const_string "neon_vld1_1_2_regs") 5002 (const_string "neon_vld3_vld4")))] 5003) 5004 5005(define_expand "vec_load_lanesci<mode>" 5006 [(match_operand:CI 0 "s_register_operand") 5007 (match_operand:CI 1 "neon_struct_operand") 5008 (unspec:VQ [(const_int 0)] UNSPEC_VSTRUCTDUMMY)] 5009 "TARGET_NEON" 5010{ 5011 emit_insn (gen_neon_vld3<mode> (operands[0], operands[1])); 5012 DONE; 5013}) 5014 5015(define_expand "neon_vld3<mode>" 5016 [(match_operand:CI 0 "s_register_operand") 5017 (match_operand:CI 1 "neon_struct_operand") 5018 (unspec:VQ [(const_int 0)] UNSPEC_VSTRUCTDUMMY)] 5019 "TARGET_NEON" 5020{ 5021 rtx mem; 5022 5023 mem = adjust_address (operands[1], EImode, 0); 5024 emit_insn (gen_neon_vld3qa<mode> (operands[0], mem)); 5025 mem = adjust_address (mem, EImode, GET_MODE_SIZE (EImode)); 5026 emit_insn (gen_neon_vld3qb<mode> (operands[0], mem, operands[0])); 5027 DONE; 5028}) 5029 5030(define_insn "neon_vld3qa<mode>" 5031 [(set (match_operand:CI 0 "s_register_operand" "=w") 5032 (unspec:CI [(match_operand:EI 1 "neon_struct_operand" "Um") 5033 (unspec:VQ [(const_int 0)] UNSPEC_VSTRUCTDUMMY)] 5034 UNSPEC_VLD3A))] 5035 "TARGET_NEON" 5036{ 5037 int regno = REGNO (operands[0]); 5038 rtx ops[4]; 5039 ops[0] = gen_rtx_REG (DImode, regno); 5040 ops[1] = gen_rtx_REG (DImode, regno + 4); 5041 ops[2] = gen_rtx_REG (DImode, regno + 8); 5042 ops[3] = operands[1]; 5043 output_asm_insn ("vld3.<V_sz_elem>\t{%P0, %P1, %P2}, %A3", ops); 5044 return ""; 5045} 5046 [(set_attr "neon_type" "neon_vld3_vld4")] 5047) 5048 5049(define_insn "neon_vld3qb<mode>" 5050 [(set (match_operand:CI 0 "s_register_operand" "=w") 5051 (unspec:CI [(match_operand:EI 1 "neon_struct_operand" "Um") 5052 (match_operand:CI 2 "s_register_operand" "0") 5053 (unspec:VQ [(const_int 0)] UNSPEC_VSTRUCTDUMMY)] 5054 UNSPEC_VLD3B))] 5055 "TARGET_NEON" 5056{ 5057 int regno = REGNO (operands[0]); 5058 rtx ops[4]; 5059 ops[0] = gen_rtx_REG (DImode, regno + 2); 5060 ops[1] = gen_rtx_REG (DImode, regno + 6); 5061 ops[2] = gen_rtx_REG (DImode, regno + 10); 5062 ops[3] = operands[1]; 5063 output_asm_insn ("vld3.<V_sz_elem>\t{%P0, %P1, %P2}, %A3", ops); 5064 return ""; 5065} 5066 [(set_attr "neon_type" "neon_vld3_vld4")] 5067) 5068 5069(define_insn "neon_vld3_lane<mode>" 5070 [(set (match_operand:EI 0 "s_register_operand" "=w") 5071 (unspec:EI [(match_operand:<V_three_elem> 1 "neon_struct_operand" "Um") 5072 (match_operand:EI 2 "s_register_operand" "0") 5073 (match_operand:SI 3 "immediate_operand" "i") 5074 (unspec:VD [(const_int 0)] UNSPEC_VSTRUCTDUMMY)] 5075 UNSPEC_VLD3_LANE))] 5076 "TARGET_NEON" 5077{ 5078 HOST_WIDE_INT lane = INTVAL (operands[3]); 5079 HOST_WIDE_INT max = GET_MODE_NUNITS (<MODE>mode); 5080 int regno = REGNO (operands[0]); 5081 rtx ops[5]; 5082 if (lane < 0 || lane >= max) 5083 error ("lane out of range"); 5084 ops[0] = gen_rtx_REG (DImode, regno); 5085 ops[1] = gen_rtx_REG (DImode, regno + 2); 5086 ops[2] = gen_rtx_REG (DImode, regno + 4); 5087 ops[3] = operands[1]; 5088 ops[4] = operands[3]; 5089 output_asm_insn ("vld3.<V_sz_elem>\t{%P0[%c4], %P1[%c4], %P2[%c4]}, %3", 5090 ops); 5091 return ""; 5092} 5093 [(set_attr "neon_type" "neon_vld3_vld4_lane")] 5094) 5095 5096(define_insn "neon_vld3_lane<mode>" 5097 [(set (match_operand:CI 0 "s_register_operand" "=w") 5098 (unspec:CI [(match_operand:<V_three_elem> 1 "neon_struct_operand" "Um") 5099 (match_operand:CI 2 "s_register_operand" "0") 5100 (match_operand:SI 3 "immediate_operand" "i") 5101 (unspec:VMQ [(const_int 0)] UNSPEC_VSTRUCTDUMMY)] 5102 UNSPEC_VLD3_LANE))] 5103 "TARGET_NEON" 5104{ 5105 HOST_WIDE_INT lane = INTVAL (operands[3]); 5106 HOST_WIDE_INT max = GET_MODE_NUNITS (<MODE>mode); 5107 int regno = REGNO (operands[0]); 5108 rtx ops[5]; 5109 if (lane < 0 || lane >= max) 5110 error ("lane out of range"); 5111 else if (lane >= max / 2) 5112 { 5113 lane -= max / 2; 5114 regno += 2; 5115 } 5116 ops[0] = gen_rtx_REG (DImode, regno); 5117 ops[1] = gen_rtx_REG (DImode, regno + 4); 5118 ops[2] = gen_rtx_REG (DImode, regno + 8); 5119 ops[3] = operands[1]; 5120 ops[4] = GEN_INT (lane); 5121 output_asm_insn ("vld3.<V_sz_elem>\t{%P0[%c4], %P1[%c4], %P2[%c4]}, %3", 5122 ops); 5123 return ""; 5124} 5125 [(set_attr "neon_type" "neon_vld3_vld4_lane")] 5126) 5127 5128(define_insn "neon_vld3_dup<mode>" 5129 [(set (match_operand:EI 0 "s_register_operand" "=w") 5130 (unspec:EI [(match_operand:<V_three_elem> 1 "neon_struct_operand" "Um") 5131 (unspec:VDX [(const_int 0)] UNSPEC_VSTRUCTDUMMY)] 5132 UNSPEC_VLD3_DUP))] 5133 "TARGET_NEON" 5134{ 5135 if (GET_MODE_NUNITS (<MODE>mode) > 1) 5136 { 5137 int regno = REGNO (operands[0]); 5138 rtx ops[4]; 5139 ops[0] = gen_rtx_REG (DImode, regno); 5140 ops[1] = gen_rtx_REG (DImode, regno + 2); 5141 ops[2] = gen_rtx_REG (DImode, regno + 4); 5142 ops[3] = operands[1]; 5143 output_asm_insn ("vld3.<V_sz_elem>\t{%P0[], %P1[], %P2[]}, %3", ops); 5144 return ""; 5145 } 5146 else 5147 return "vld1.<V_sz_elem>\t%h0, %A1"; 5148} 5149 [(set (attr "neon_type") 5150 (if_then_else (gt (const_string "<V_mode_nunits>") (const_string "1")) 5151 (const_string "neon_vld3_vld4_all_lanes") 5152 (const_string "neon_vld1_1_2_regs")))]) 5153 5154(define_expand "vec_store_lanesei<mode>" 5155 [(set (match_operand:EI 0 "neon_struct_operand") 5156 (unspec:EI [(match_operand:EI 1 "s_register_operand") 5157 (unspec:VDX [(const_int 0)] UNSPEC_VSTRUCTDUMMY)] 5158 UNSPEC_VST3))] 5159 "TARGET_NEON") 5160 5161(define_insn "neon_vst3<mode>" 5162 [(set (match_operand:EI 0 "neon_struct_operand" "=Um") 5163 (unspec:EI [(match_operand:EI 1 "s_register_operand" "w") 5164 (unspec:VDX [(const_int 0)] UNSPEC_VSTRUCTDUMMY)] 5165 UNSPEC_VST3))] 5166 "TARGET_NEON" 5167{ 5168 if (<V_sz_elem> == 64) 5169 return "vst1.64\t%h1, %A0"; 5170 else 5171 return "vst3.<V_sz_elem>\t%h1, %A0"; 5172} 5173 [(set (attr "neon_type") 5174 (if_then_else (eq (const_string "<V_sz_elem>") (const_string "64")) 5175 (const_string "neon_vst1_1_2_regs_vst2_2_regs") 5176 (const_string "neon_vst2_4_regs_vst3_vst4")))]) 5177 5178(define_expand "vec_store_lanesci<mode>" 5179 [(match_operand:CI 0 "neon_struct_operand") 5180 (match_operand:CI 1 "s_register_operand") 5181 (unspec:VQ [(const_int 0)] UNSPEC_VSTRUCTDUMMY)] 5182 "TARGET_NEON" 5183{ 5184 emit_insn (gen_neon_vst3<mode> (operands[0], operands[1])); 5185 DONE; 5186}) 5187 5188(define_expand "neon_vst3<mode>" 5189 [(match_operand:CI 0 "neon_struct_operand") 5190 (match_operand:CI 1 "s_register_operand") 5191 (unspec:VQ [(const_int 0)] UNSPEC_VSTRUCTDUMMY)] 5192 "TARGET_NEON" 5193{ 5194 rtx mem; 5195 5196 mem = adjust_address (operands[0], EImode, 0); 5197 emit_insn (gen_neon_vst3qa<mode> (mem, operands[1])); 5198 mem = adjust_address (mem, EImode, GET_MODE_SIZE (EImode)); 5199 emit_insn (gen_neon_vst3qb<mode> (mem, operands[1])); 5200 DONE; 5201}) 5202 5203(define_insn "neon_vst3qa<mode>" 5204 [(set (match_operand:EI 0 "neon_struct_operand" "=Um") 5205 (unspec:EI [(match_operand:CI 1 "s_register_operand" "w") 5206 (unspec:VQ [(const_int 0)] UNSPEC_VSTRUCTDUMMY)] 5207 UNSPEC_VST3A))] 5208 "TARGET_NEON" 5209{ 5210 int regno = REGNO (operands[1]); 5211 rtx ops[4]; 5212 ops[0] = operands[0]; 5213 ops[1] = gen_rtx_REG (DImode, regno); 5214 ops[2] = gen_rtx_REG (DImode, regno + 4); 5215 ops[3] = gen_rtx_REG (DImode, regno + 8); 5216 output_asm_insn ("vst3.<V_sz_elem>\t{%P1, %P2, %P3}, %A0", ops); 5217 return ""; 5218} 5219 [(set_attr "neon_type" "neon_vst2_4_regs_vst3_vst4")] 5220) 5221 5222(define_insn "neon_vst3qb<mode>" 5223 [(set (match_operand:EI 0 "neon_struct_operand" "=Um") 5224 (unspec:EI [(match_operand:CI 1 "s_register_operand" "w") 5225 (unspec:VQ [(const_int 0)] UNSPEC_VSTRUCTDUMMY)] 5226 UNSPEC_VST3B))] 5227 "TARGET_NEON" 5228{ 5229 int regno = REGNO (operands[1]); 5230 rtx ops[4]; 5231 ops[0] = operands[0]; 5232 ops[1] = gen_rtx_REG (DImode, regno + 2); 5233 ops[2] = gen_rtx_REG (DImode, regno + 6); 5234 ops[3] = gen_rtx_REG (DImode, regno + 10); 5235 output_asm_insn ("vst3.<V_sz_elem>\t{%P1, %P2, %P3}, %A0", ops); 5236 return ""; 5237} 5238 [(set_attr "neon_type" "neon_vst2_4_regs_vst3_vst4")] 5239) 5240 5241(define_insn "neon_vst3_lane<mode>" 5242 [(set (match_operand:<V_three_elem> 0 "neon_struct_operand" "=Um") 5243 (unspec:<V_three_elem> 5244 [(match_operand:EI 1 "s_register_operand" "w") 5245 (match_operand:SI 2 "immediate_operand" "i") 5246 (unspec:VD [(const_int 0)] UNSPEC_VSTRUCTDUMMY)] 5247 UNSPEC_VST3_LANE))] 5248 "TARGET_NEON" 5249{ 5250 HOST_WIDE_INT lane = INTVAL (operands[2]); 5251 HOST_WIDE_INT max = GET_MODE_NUNITS (<MODE>mode); 5252 int regno = REGNO (operands[1]); 5253 rtx ops[5]; 5254 if (lane < 0 || lane >= max) 5255 error ("lane out of range"); 5256 ops[0] = operands[0]; 5257 ops[1] = gen_rtx_REG (DImode, regno); 5258 ops[2] = gen_rtx_REG (DImode, regno + 2); 5259 ops[3] = gen_rtx_REG (DImode, regno + 4); 5260 ops[4] = operands[2]; 5261 output_asm_insn ("vst3.<V_sz_elem>\t{%P1[%c4], %P2[%c4], %P3[%c4]}, %0", 5262 ops); 5263 return ""; 5264} 5265 [(set_attr "neon_type" "neon_vst3_vst4_lane")] 5266) 5267 5268(define_insn "neon_vst3_lane<mode>" 5269 [(set (match_operand:<V_three_elem> 0 "neon_struct_operand" "=Um") 5270 (unspec:<V_three_elem> 5271 [(match_operand:CI 1 "s_register_operand" "w") 5272 (match_operand:SI 2 "immediate_operand" "i") 5273 (unspec:VMQ [(const_int 0)] UNSPEC_VSTRUCTDUMMY)] 5274 UNSPEC_VST3_LANE))] 5275 "TARGET_NEON" 5276{ 5277 HOST_WIDE_INT lane = INTVAL (operands[2]); 5278 HOST_WIDE_INT max = GET_MODE_NUNITS (<MODE>mode); 5279 int regno = REGNO (operands[1]); 5280 rtx ops[5]; 5281 if (lane < 0 || lane >= max) 5282 error ("lane out of range"); 5283 else if (lane >= max / 2) 5284 { 5285 lane -= max / 2; 5286 regno += 2; 5287 } 5288 ops[0] = operands[0]; 5289 ops[1] = gen_rtx_REG (DImode, regno); 5290 ops[2] = gen_rtx_REG (DImode, regno + 4); 5291 ops[3] = gen_rtx_REG (DImode, regno + 8); 5292 ops[4] = GEN_INT (lane); 5293 output_asm_insn ("vst3.<V_sz_elem>\t{%P1[%c4], %P2[%c4], %P3[%c4]}, %0", 5294 ops); 5295 return ""; 5296} 5297[(set_attr "neon_type" "neon_vst3_vst4_lane")]) 5298 5299(define_expand "vec_load_lanesoi<mode>" 5300 [(set (match_operand:OI 0 "s_register_operand") 5301 (unspec:OI [(match_operand:OI 1 "neon_struct_operand") 5302 (unspec:VDX [(const_int 0)] UNSPEC_VSTRUCTDUMMY)] 5303 UNSPEC_VLD4))] 5304 "TARGET_NEON") 5305 5306(define_insn "neon_vld4<mode>" 5307 [(set (match_operand:OI 0 "s_register_operand" "=w") 5308 (unspec:OI [(match_operand:OI 1 "neon_struct_operand" "Um") 5309 (unspec:VDX [(const_int 0)] UNSPEC_VSTRUCTDUMMY)] 5310 UNSPEC_VLD4))] 5311 "TARGET_NEON" 5312{ 5313 if (<V_sz_elem> == 64) 5314 return "vld1.64\t%h0, %A1"; 5315 else 5316 return "vld4.<V_sz_elem>\t%h0, %A1"; 5317} 5318 [(set (attr "neon_type") 5319 (if_then_else (eq (const_string "<V_sz_elem>") (const_string "64")) 5320 (const_string "neon_vld1_1_2_regs") 5321 (const_string "neon_vld3_vld4")))] 5322) 5323 5324(define_expand "vec_load_lanesxi<mode>" 5325 [(match_operand:XI 0 "s_register_operand") 5326 (match_operand:XI 1 "neon_struct_operand") 5327 (unspec:VQ [(const_int 0)] UNSPEC_VSTRUCTDUMMY)] 5328 "TARGET_NEON" 5329{ 5330 emit_insn (gen_neon_vld4<mode> (operands[0], operands[1])); 5331 DONE; 5332}) 5333 5334(define_expand "neon_vld4<mode>" 5335 [(match_operand:XI 0 "s_register_operand") 5336 (match_operand:XI 1 "neon_struct_operand") 5337 (unspec:VQ [(const_int 0)] UNSPEC_VSTRUCTDUMMY)] 5338 "TARGET_NEON" 5339{ 5340 rtx mem; 5341 5342 mem = adjust_address (operands[1], OImode, 0); 5343 emit_insn (gen_neon_vld4qa<mode> (operands[0], mem)); 5344 mem = adjust_address (mem, OImode, GET_MODE_SIZE (OImode)); 5345 emit_insn (gen_neon_vld4qb<mode> (operands[0], mem, operands[0])); 5346 DONE; 5347}) 5348 5349(define_insn "neon_vld4qa<mode>" 5350 [(set (match_operand:XI 0 "s_register_operand" "=w") 5351 (unspec:XI [(match_operand:OI 1 "neon_struct_operand" "Um") 5352 (unspec:VQ [(const_int 0)] UNSPEC_VSTRUCTDUMMY)] 5353 UNSPEC_VLD4A))] 5354 "TARGET_NEON" 5355{ 5356 int regno = REGNO (operands[0]); 5357 rtx ops[5]; 5358 ops[0] = gen_rtx_REG (DImode, regno); 5359 ops[1] = gen_rtx_REG (DImode, regno + 4); 5360 ops[2] = gen_rtx_REG (DImode, regno + 8); 5361 ops[3] = gen_rtx_REG (DImode, regno + 12); 5362 ops[4] = operands[1]; 5363 output_asm_insn ("vld4.<V_sz_elem>\t{%P0, %P1, %P2, %P3}, %A4", ops); 5364 return ""; 5365} 5366 [(set_attr "neon_type" "neon_vld3_vld4")] 5367) 5368 5369(define_insn "neon_vld4qb<mode>" 5370 [(set (match_operand:XI 0 "s_register_operand" "=w") 5371 (unspec:XI [(match_operand:OI 1 "neon_struct_operand" "Um") 5372 (match_operand:XI 2 "s_register_operand" "0") 5373 (unspec:VQ [(const_int 0)] UNSPEC_VSTRUCTDUMMY)] 5374 UNSPEC_VLD4B))] 5375 "TARGET_NEON" 5376{ 5377 int regno = REGNO (operands[0]); 5378 rtx ops[5]; 5379 ops[0] = gen_rtx_REG (DImode, regno + 2); 5380 ops[1] = gen_rtx_REG (DImode, regno + 6); 5381 ops[2] = gen_rtx_REG (DImode, regno + 10); 5382 ops[3] = gen_rtx_REG (DImode, regno + 14); 5383 ops[4] = operands[1]; 5384 output_asm_insn ("vld4.<V_sz_elem>\t{%P0, %P1, %P2, %P3}, %A4", ops); 5385 return ""; 5386} 5387 [(set_attr "neon_type" "neon_vld3_vld4")] 5388) 5389 5390(define_insn "neon_vld4_lane<mode>" 5391 [(set (match_operand:OI 0 "s_register_operand" "=w") 5392 (unspec:OI [(match_operand:<V_four_elem> 1 "neon_struct_operand" "Um") 5393 (match_operand:OI 2 "s_register_operand" "0") 5394 (match_operand:SI 3 "immediate_operand" "i") 5395 (unspec:VD [(const_int 0)] UNSPEC_VSTRUCTDUMMY)] 5396 UNSPEC_VLD4_LANE))] 5397 "TARGET_NEON" 5398{ 5399 HOST_WIDE_INT lane = INTVAL (operands[3]); 5400 HOST_WIDE_INT max = GET_MODE_NUNITS (<MODE>mode); 5401 int regno = REGNO (operands[0]); 5402 rtx ops[6]; 5403 if (lane < 0 || lane >= max) 5404 error ("lane out of range"); 5405 ops[0] = gen_rtx_REG (DImode, regno); 5406 ops[1] = gen_rtx_REG (DImode, regno + 2); 5407 ops[2] = gen_rtx_REG (DImode, regno + 4); 5408 ops[3] = gen_rtx_REG (DImode, regno + 6); 5409 ops[4] = operands[1]; 5410 ops[5] = operands[3]; 5411 output_asm_insn ("vld4.<V_sz_elem>\t{%P0[%c5], %P1[%c5], %P2[%c5], %P3[%c5]}, %A4", 5412 ops); 5413 return ""; 5414} 5415 [(set_attr "neon_type" "neon_vld3_vld4_lane")] 5416) 5417 5418(define_insn "neon_vld4_lane<mode>" 5419 [(set (match_operand:XI 0 "s_register_operand" "=w") 5420 (unspec:XI [(match_operand:<V_four_elem> 1 "neon_struct_operand" "Um") 5421 (match_operand:XI 2 "s_register_operand" "0") 5422 (match_operand:SI 3 "immediate_operand" "i") 5423 (unspec:VMQ [(const_int 0)] UNSPEC_VSTRUCTDUMMY)] 5424 UNSPEC_VLD4_LANE))] 5425 "TARGET_NEON" 5426{ 5427 HOST_WIDE_INT lane = INTVAL (operands[3]); 5428 HOST_WIDE_INT max = GET_MODE_NUNITS (<MODE>mode); 5429 int regno = REGNO (operands[0]); 5430 rtx ops[6]; 5431 if (lane < 0 || lane >= max) 5432 error ("lane out of range"); 5433 else if (lane >= max / 2) 5434 { 5435 lane -= max / 2; 5436 regno += 2; 5437 } 5438 ops[0] = gen_rtx_REG (DImode, regno); 5439 ops[1] = gen_rtx_REG (DImode, regno + 4); 5440 ops[2] = gen_rtx_REG (DImode, regno + 8); 5441 ops[3] = gen_rtx_REG (DImode, regno + 12); 5442 ops[4] = operands[1]; 5443 ops[5] = GEN_INT (lane); 5444 output_asm_insn ("vld4.<V_sz_elem>\t{%P0[%c5], %P1[%c5], %P2[%c5], %P3[%c5]}, %A4", 5445 ops); 5446 return ""; 5447} 5448 [(set_attr "neon_type" "neon_vld3_vld4_lane")] 5449) 5450 5451(define_insn "neon_vld4_dup<mode>" 5452 [(set (match_operand:OI 0 "s_register_operand" "=w") 5453 (unspec:OI [(match_operand:<V_four_elem> 1 "neon_struct_operand" "Um") 5454 (unspec:VDX [(const_int 0)] UNSPEC_VSTRUCTDUMMY)] 5455 UNSPEC_VLD4_DUP))] 5456 "TARGET_NEON" 5457{ 5458 if (GET_MODE_NUNITS (<MODE>mode) > 1) 5459 { 5460 int regno = REGNO (operands[0]); 5461 rtx ops[5]; 5462 ops[0] = gen_rtx_REG (DImode, regno); 5463 ops[1] = gen_rtx_REG (DImode, regno + 2); 5464 ops[2] = gen_rtx_REG (DImode, regno + 4); 5465 ops[3] = gen_rtx_REG (DImode, regno + 6); 5466 ops[4] = operands[1]; 5467 output_asm_insn ("vld4.<V_sz_elem>\t{%P0[], %P1[], %P2[], %P3[]}, %A4", 5468 ops); 5469 return ""; 5470 } 5471 else 5472 return "vld1.<V_sz_elem>\t%h0, %A1"; 5473} 5474 [(set (attr "neon_type") 5475 (if_then_else (gt (const_string "<V_mode_nunits>") (const_string "1")) 5476 (const_string "neon_vld3_vld4_all_lanes") 5477 (const_string "neon_vld1_1_2_regs")))] 5478) 5479 5480(define_expand "vec_store_lanesoi<mode>" 5481 [(set (match_operand:OI 0 "neon_struct_operand") 5482 (unspec:OI [(match_operand:OI 1 "s_register_operand") 5483 (unspec:VDX [(const_int 0)] UNSPEC_VSTRUCTDUMMY)] 5484 UNSPEC_VST4))] 5485 "TARGET_NEON") 5486 5487(define_insn "neon_vst4<mode>" 5488 [(set (match_operand:OI 0 "neon_struct_operand" "=Um") 5489 (unspec:OI [(match_operand:OI 1 "s_register_operand" "w") 5490 (unspec:VDX [(const_int 0)] UNSPEC_VSTRUCTDUMMY)] 5491 UNSPEC_VST4))] 5492 "TARGET_NEON" 5493{ 5494 if (<V_sz_elem> == 64) 5495 return "vst1.64\t%h1, %A0"; 5496 else 5497 return "vst4.<V_sz_elem>\t%h1, %A0"; 5498} 5499 [(set (attr "neon_type") 5500 (if_then_else (eq (const_string "<V_sz_elem>") (const_string "64")) 5501 (const_string "neon_vst1_1_2_regs_vst2_2_regs") 5502 (const_string "neon_vst2_4_regs_vst3_vst4")))] 5503) 5504 5505(define_expand "vec_store_lanesxi<mode>" 5506 [(match_operand:XI 0 "neon_struct_operand") 5507 (match_operand:XI 1 "s_register_operand") 5508 (unspec:VQ [(const_int 0)] UNSPEC_VSTRUCTDUMMY)] 5509 "TARGET_NEON" 5510{ 5511 emit_insn (gen_neon_vst4<mode> (operands[0], operands[1])); 5512 DONE; 5513}) 5514 5515(define_expand "neon_vst4<mode>" 5516 [(match_operand:XI 0 "neon_struct_operand") 5517 (match_operand:XI 1 "s_register_operand") 5518 (unspec:VQ [(const_int 0)] UNSPEC_VSTRUCTDUMMY)] 5519 "TARGET_NEON" 5520{ 5521 rtx mem; 5522 5523 mem = adjust_address (operands[0], OImode, 0); 5524 emit_insn (gen_neon_vst4qa<mode> (mem, operands[1])); 5525 mem = adjust_address (mem, OImode, GET_MODE_SIZE (OImode)); 5526 emit_insn (gen_neon_vst4qb<mode> (mem, operands[1])); 5527 DONE; 5528}) 5529 5530(define_insn "neon_vst4qa<mode>" 5531 [(set (match_operand:OI 0 "neon_struct_operand" "=Um") 5532 (unspec:OI [(match_operand:XI 1 "s_register_operand" "w") 5533 (unspec:VQ [(const_int 0)] UNSPEC_VSTRUCTDUMMY)] 5534 UNSPEC_VST4A))] 5535 "TARGET_NEON" 5536{ 5537 int regno = REGNO (operands[1]); 5538 rtx ops[5]; 5539 ops[0] = operands[0]; 5540 ops[1] = gen_rtx_REG (DImode, regno); 5541 ops[2] = gen_rtx_REG (DImode, regno + 4); 5542 ops[3] = gen_rtx_REG (DImode, regno + 8); 5543 ops[4] = gen_rtx_REG (DImode, regno + 12); 5544 output_asm_insn ("vst4.<V_sz_elem>\t{%P1, %P2, %P3, %P4}, %A0", ops); 5545 return ""; 5546} 5547 [(set_attr "neon_type" "neon_vst2_4_regs_vst3_vst4")] 5548) 5549 5550(define_insn "neon_vst4qb<mode>" 5551 [(set (match_operand:OI 0 "neon_struct_operand" "=Um") 5552 (unspec:OI [(match_operand:XI 1 "s_register_operand" "w") 5553 (unspec:VQ [(const_int 0)] UNSPEC_VSTRUCTDUMMY)] 5554 UNSPEC_VST4B))] 5555 "TARGET_NEON" 5556{ 5557 int regno = REGNO (operands[1]); 5558 rtx ops[5]; 5559 ops[0] = operands[0]; 5560 ops[1] = gen_rtx_REG (DImode, regno + 2); 5561 ops[2] = gen_rtx_REG (DImode, regno + 6); 5562 ops[3] = gen_rtx_REG (DImode, regno + 10); 5563 ops[4] = gen_rtx_REG (DImode, regno + 14); 5564 output_asm_insn ("vst4.<V_sz_elem>\t{%P1, %P2, %P3, %P4}, %A0", ops); 5565 return ""; 5566} 5567 [(set_attr "neon_type" "neon_vst2_4_regs_vst3_vst4")] 5568) 5569 5570(define_insn "neon_vst4_lane<mode>" 5571 [(set (match_operand:<V_four_elem> 0 "neon_struct_operand" "=Um") 5572 (unspec:<V_four_elem> 5573 [(match_operand:OI 1 "s_register_operand" "w") 5574 (match_operand:SI 2 "immediate_operand" "i") 5575 (unspec:VD [(const_int 0)] UNSPEC_VSTRUCTDUMMY)] 5576 UNSPEC_VST4_LANE))] 5577 "TARGET_NEON" 5578{ 5579 HOST_WIDE_INT lane = INTVAL (operands[2]); 5580 HOST_WIDE_INT max = GET_MODE_NUNITS (<MODE>mode); 5581 int regno = REGNO (operands[1]); 5582 rtx ops[6]; 5583 if (lane < 0 || lane >= max) 5584 error ("lane out of range"); 5585 ops[0] = operands[0]; 5586 ops[1] = gen_rtx_REG (DImode, regno); 5587 ops[2] = gen_rtx_REG (DImode, regno + 2); 5588 ops[3] = gen_rtx_REG (DImode, regno + 4); 5589 ops[4] = gen_rtx_REG (DImode, regno + 6); 5590 ops[5] = operands[2]; 5591 output_asm_insn ("vst4.<V_sz_elem>\t{%P1[%c5], %P2[%c5], %P3[%c5], %P4[%c5]}, %A0", 5592 ops); 5593 return ""; 5594} 5595 [(set_attr "neon_type" "neon_vst3_vst4_lane")] 5596) 5597 5598(define_insn "neon_vst4_lane<mode>" 5599 [(set (match_operand:<V_four_elem> 0 "neon_struct_operand" "=Um") 5600 (unspec:<V_four_elem> 5601 [(match_operand:XI 1 "s_register_operand" "w") 5602 (match_operand:SI 2 "immediate_operand" "i") 5603 (unspec:VMQ [(const_int 0)] UNSPEC_VSTRUCTDUMMY)] 5604 UNSPEC_VST4_LANE))] 5605 "TARGET_NEON" 5606{ 5607 HOST_WIDE_INT lane = INTVAL (operands[2]); 5608 HOST_WIDE_INT max = GET_MODE_NUNITS (<MODE>mode); 5609 int regno = REGNO (operands[1]); 5610 rtx ops[6]; 5611 if (lane < 0 || lane >= max) 5612 error ("lane out of range"); 5613 else if (lane >= max / 2) 5614 { 5615 lane -= max / 2; 5616 regno += 2; 5617 } 5618 ops[0] = operands[0]; 5619 ops[1] = gen_rtx_REG (DImode, regno); 5620 ops[2] = gen_rtx_REG (DImode, regno + 4); 5621 ops[3] = gen_rtx_REG (DImode, regno + 8); 5622 ops[4] = gen_rtx_REG (DImode, regno + 12); 5623 ops[5] = GEN_INT (lane); 5624 output_asm_insn ("vst4.<V_sz_elem>\t{%P1[%c5], %P2[%c5], %P3[%c5], %P4[%c5]}, %A0", 5625 ops); 5626 return ""; 5627} 5628 [(set_attr "neon_type" "neon_vst3_vst4_lane")] 5629) 5630 5631(define_expand "neon_vand<mode>" 5632 [(match_operand:VDQX 0 "s_register_operand" "") 5633 (match_operand:VDQX 1 "s_register_operand" "") 5634 (match_operand:VDQX 2 "neon_inv_logic_op2" "") 5635 (match_operand:SI 3 "immediate_operand" "")] 5636 "TARGET_NEON" 5637{ 5638 emit_insn (gen_and<mode>3<V_suf64> (operands[0], operands[1], operands[2])); 5639 DONE; 5640}) 5641 5642(define_expand "neon_vorr<mode>" 5643 [(match_operand:VDQX 0 "s_register_operand" "") 5644 (match_operand:VDQX 1 "s_register_operand" "") 5645 (match_operand:VDQX 2 "neon_logic_op2" "") 5646 (match_operand:SI 3 "immediate_operand" "")] 5647 "TARGET_NEON" 5648{ 5649 emit_insn (gen_ior<mode>3<V_suf64> (operands[0], operands[1], operands[2])); 5650 DONE; 5651}) 5652 5653(define_expand "neon_veor<mode>" 5654 [(match_operand:VDQX 0 "s_register_operand" "") 5655 (match_operand:VDQX 1 "s_register_operand" "") 5656 (match_operand:VDQX 2 "s_register_operand" "") 5657 (match_operand:SI 3 "immediate_operand" "")] 5658 "TARGET_NEON" 5659{ 5660 emit_insn (gen_xor<mode>3<V_suf64> (operands[0], operands[1], operands[2])); 5661 DONE; 5662}) 5663 5664(define_expand "neon_vbic<mode>" 5665 [(match_operand:VDQX 0 "s_register_operand" "") 5666 (match_operand:VDQX 1 "s_register_operand" "") 5667 (match_operand:VDQX 2 "neon_logic_op2" "") 5668 (match_operand:SI 3 "immediate_operand" "")] 5669 "TARGET_NEON" 5670{ 5671 emit_insn (gen_bic<mode>3_neon (operands[0], operands[1], operands[2])); 5672 DONE; 5673}) 5674 5675(define_expand "neon_vorn<mode>" 5676 [(match_operand:VDQX 0 "s_register_operand" "") 5677 (match_operand:VDQX 1 "s_register_operand" "") 5678 (match_operand:VDQX 2 "neon_inv_logic_op2" "") 5679 (match_operand:SI 3 "immediate_operand" "")] 5680 "TARGET_NEON" 5681{ 5682 emit_insn (gen_orn<mode>3_neon (operands[0], operands[1], operands[2])); 5683 DONE; 5684}) 5685 5686(define_insn "neon_vec_unpack<US>_lo_<mode>" 5687 [(set (match_operand:<V_unpack> 0 "register_operand" "=w") 5688 (SE:<V_unpack> (vec_select:<V_HALF> 5689 (match_operand:VU 1 "register_operand" "w") 5690 (match_operand:VU 2 "vect_par_constant_low" ""))))] 5691 "TARGET_NEON && !BYTES_BIG_ENDIAN" 5692 "vmovl.<US><V_sz_elem> %q0, %e1" 5693 [(set_attr "neon_type" "neon_shift_1")] 5694) 5695 5696(define_insn "neon_vec_unpack<US>_hi_<mode>" 5697 [(set (match_operand:<V_unpack> 0 "register_operand" "=w") 5698 (SE:<V_unpack> (vec_select:<V_HALF> 5699 (match_operand:VU 1 "register_operand" "w") 5700 (match_operand:VU 2 "vect_par_constant_high" ""))))] 5701 "TARGET_NEON && !BYTES_BIG_ENDIAN" 5702 "vmovl.<US><V_sz_elem> %q0, %f1" 5703 [(set_attr "neon_type" "neon_shift_1")] 5704) 5705 5706(define_expand "vec_unpack<US>_hi_<mode>" 5707 [(match_operand:<V_unpack> 0 "register_operand" "") 5708 (SE:<V_unpack> (match_operand:VU 1 "register_operand"))] 5709 "TARGET_NEON && !BYTES_BIG_ENDIAN" 5710 { 5711 rtvec v = rtvec_alloc (<V_mode_nunits>/2) ; 5712 rtx t1; 5713 int i; 5714 for (i = 0; i < (<V_mode_nunits>/2); i++) 5715 RTVEC_ELT (v, i) = GEN_INT ((<V_mode_nunits>/2) + i); 5716 5717 t1 = gen_rtx_PARALLEL (<MODE>mode, v); 5718 emit_insn (gen_neon_vec_unpack<US>_hi_<mode> (operands[0], 5719 operands[1], 5720 t1)); 5721 DONE; 5722 } 5723) 5724 5725(define_expand "vec_unpack<US>_lo_<mode>" 5726 [(match_operand:<V_unpack> 0 "register_operand" "") 5727 (SE:<V_unpack> (match_operand:VU 1 "register_operand" ""))] 5728 "TARGET_NEON && !BYTES_BIG_ENDIAN" 5729 { 5730 rtvec v = rtvec_alloc (<V_mode_nunits>/2) ; 5731 rtx t1; 5732 int i; 5733 for (i = 0; i < (<V_mode_nunits>/2) ; i++) 5734 RTVEC_ELT (v, i) = GEN_INT (i); 5735 t1 = gen_rtx_PARALLEL (<MODE>mode, v); 5736 emit_insn (gen_neon_vec_unpack<US>_lo_<mode> (operands[0], 5737 operands[1], 5738 t1)); 5739 DONE; 5740 } 5741) 5742 5743(define_insn "neon_vec_<US>mult_lo_<mode>" 5744 [(set (match_operand:<V_unpack> 0 "register_operand" "=w") 5745 (mult:<V_unpack> (SE:<V_unpack> (vec_select:<V_HALF> 5746 (match_operand:VU 1 "register_operand" "w") 5747 (match_operand:VU 2 "vect_par_constant_low" ""))) 5748 (SE:<V_unpack> (vec_select:<V_HALF> 5749 (match_operand:VU 3 "register_operand" "w") 5750 (match_dup 2)))))] 5751 "TARGET_NEON && !BYTES_BIG_ENDIAN" 5752 "vmull.<US><V_sz_elem> %q0, %e1, %e3" 5753 [(set_attr "neon_type" "neon_shift_1")] 5754) 5755 5756(define_expand "vec_widen_<US>mult_lo_<mode>" 5757 [(match_operand:<V_unpack> 0 "register_operand" "") 5758 (SE:<V_unpack> (match_operand:VU 1 "register_operand" "")) 5759 (SE:<V_unpack> (match_operand:VU 2 "register_operand" ""))] 5760 "TARGET_NEON && !BYTES_BIG_ENDIAN" 5761 { 5762 rtvec v = rtvec_alloc (<V_mode_nunits>/2) ; 5763 rtx t1; 5764 int i; 5765 for (i = 0; i < (<V_mode_nunits>/2) ; i++) 5766 RTVEC_ELT (v, i) = GEN_INT (i); 5767 t1 = gen_rtx_PARALLEL (<MODE>mode, v); 5768 5769 emit_insn (gen_neon_vec_<US>mult_lo_<mode> (operands[0], 5770 operands[1], 5771 t1, 5772 operands[2])); 5773 DONE; 5774 } 5775) 5776 5777(define_insn "neon_vec_<US>mult_hi_<mode>" 5778 [(set (match_operand:<V_unpack> 0 "register_operand" "=w") 5779 (mult:<V_unpack> (SE:<V_unpack> (vec_select:<V_HALF> 5780 (match_operand:VU 1 "register_operand" "w") 5781 (match_operand:VU 2 "vect_par_constant_high" ""))) 5782 (SE:<V_unpack> (vec_select:<V_HALF> 5783 (match_operand:VU 3 "register_operand" "w") 5784 (match_dup 2)))))] 5785 "TARGET_NEON && !BYTES_BIG_ENDIAN" 5786 "vmull.<US><V_sz_elem> %q0, %f1, %f3" 5787 [(set_attr "neon_type" "neon_shift_1")] 5788) 5789 5790(define_expand "vec_widen_<US>mult_hi_<mode>" 5791 [(match_operand:<V_unpack> 0 "register_operand" "") 5792 (SE:<V_unpack> (match_operand:VU 1 "register_operand" "")) 5793 (SE:<V_unpack> (match_operand:VU 2 "register_operand" ""))] 5794 "TARGET_NEON && !BYTES_BIG_ENDIAN" 5795 { 5796 rtvec v = rtvec_alloc (<V_mode_nunits>/2) ; 5797 rtx t1; 5798 int i; 5799 for (i = 0; i < (<V_mode_nunits>/2) ; i++) 5800 RTVEC_ELT (v, i) = GEN_INT (<V_mode_nunits>/2 + i); 5801 t1 = gen_rtx_PARALLEL (<MODE>mode, v); 5802 5803 emit_insn (gen_neon_vec_<US>mult_hi_<mode> (operands[0], 5804 operands[1], 5805 t1, 5806 operands[2])); 5807 DONE; 5808 5809 } 5810) 5811 5812(define_insn "neon_vec_<US>shiftl_<mode>" 5813 [(set (match_operand:<V_widen> 0 "register_operand" "=w") 5814 (SE:<V_widen> (ashift:VW (match_operand:VW 1 "register_operand" "w") 5815 (match_operand:<V_innermode> 2 "const_neon_scalar_shift_amount_operand" ""))))] 5816 "TARGET_NEON" 5817{ 5818 return "vshll.<US><V_sz_elem> %q0, %P1, %2"; 5819} 5820 [(set_attr "neon_type" "neon_shift_1")] 5821) 5822 5823(define_expand "vec_widen_<US>shiftl_lo_<mode>" 5824 [(match_operand:<V_unpack> 0 "register_operand" "") 5825 (SE:<V_unpack> (match_operand:VU 1 "register_operand" "")) 5826 (match_operand:SI 2 "immediate_operand" "i")] 5827 "TARGET_NEON && !BYTES_BIG_ENDIAN" 5828 { 5829 emit_insn (gen_neon_vec_<US>shiftl_<V_half> (operands[0], 5830 simplify_gen_subreg (<V_HALF>mode, operands[1], <MODE>mode, 0), 5831 operands[2])); 5832 DONE; 5833 } 5834) 5835 5836(define_expand "vec_widen_<US>shiftl_hi_<mode>" 5837 [(match_operand:<V_unpack> 0 "register_operand" "") 5838 (SE:<V_unpack> (match_operand:VU 1 "register_operand" "")) 5839 (match_operand:SI 2 "immediate_operand" "i")] 5840 "TARGET_NEON && !BYTES_BIG_ENDIAN" 5841 { 5842 emit_insn (gen_neon_vec_<US>shiftl_<V_half> (operands[0], 5843 simplify_gen_subreg (<V_HALF>mode, operands[1], <MODE>mode, 5844 GET_MODE_SIZE (<V_HALF>mode)), 5845 operands[2])); 5846 DONE; 5847 } 5848) 5849 5850;; Vectorize for non-neon-quad case 5851(define_insn "neon_unpack<US>_<mode>" 5852 [(set (match_operand:<V_widen> 0 "register_operand" "=w") 5853 (SE:<V_widen> (match_operand:VDI 1 "register_operand" "w")))] 5854 "TARGET_NEON" 5855 "vmovl.<US><V_sz_elem> %q0, %P1" 5856 [(set_attr "neon_type" "neon_shift_1")] 5857) 5858 5859(define_expand "vec_unpack<US>_lo_<mode>" 5860 [(match_operand:<V_double_width> 0 "register_operand" "") 5861 (SE:<V_double_width>(match_operand:VDI 1 "register_operand"))] 5862 "TARGET_NEON" 5863{ 5864 rtx tmpreg = gen_reg_rtx (<V_widen>mode); 5865 emit_insn (gen_neon_unpack<US>_<mode> (tmpreg, operands[1])); 5866 emit_insn (gen_neon_vget_low<V_widen_l> (operands[0], tmpreg)); 5867 5868 DONE; 5869} 5870) 5871 5872(define_expand "vec_unpack<US>_hi_<mode>" 5873 [(match_operand:<V_double_width> 0 "register_operand" "") 5874 (SE:<V_double_width>(match_operand:VDI 1 "register_operand"))] 5875 "TARGET_NEON" 5876{ 5877 rtx tmpreg = gen_reg_rtx (<V_widen>mode); 5878 emit_insn (gen_neon_unpack<US>_<mode> (tmpreg, operands[1])); 5879 emit_insn (gen_neon_vget_high<V_widen_l> (operands[0], tmpreg)); 5880 5881 DONE; 5882} 5883) 5884 5885(define_insn "neon_vec_<US>mult_<mode>" 5886 [(set (match_operand:<V_widen> 0 "register_operand" "=w") 5887 (mult:<V_widen> (SE:<V_widen> 5888 (match_operand:VDI 1 "register_operand" "w")) 5889 (SE:<V_widen> 5890 (match_operand:VDI 2 "register_operand" "w"))))] 5891 "TARGET_NEON" 5892 "vmull.<US><V_sz_elem> %q0, %P1, %P2" 5893 [(set_attr "neon_type" "neon_shift_1")] 5894) 5895 5896(define_expand "vec_widen_<US>mult_hi_<mode>" 5897 [(match_operand:<V_double_width> 0 "register_operand" "") 5898 (SE:<V_double_width> (match_operand:VDI 1 "register_operand" "")) 5899 (SE:<V_double_width> (match_operand:VDI 2 "register_operand" ""))] 5900 "TARGET_NEON" 5901 { 5902 rtx tmpreg = gen_reg_rtx (<V_widen>mode); 5903 emit_insn (gen_neon_vec_<US>mult_<mode> (tmpreg, operands[1], operands[2])); 5904 emit_insn (gen_neon_vget_high<V_widen_l> (operands[0], tmpreg)); 5905 5906 DONE; 5907 5908 } 5909) 5910 5911(define_expand "vec_widen_<US>mult_lo_<mode>" 5912 [(match_operand:<V_double_width> 0 "register_operand" "") 5913 (SE:<V_double_width> (match_operand:VDI 1 "register_operand" "")) 5914 (SE:<V_double_width> (match_operand:VDI 2 "register_operand" ""))] 5915 "TARGET_NEON" 5916 { 5917 rtx tmpreg = gen_reg_rtx (<V_widen>mode); 5918 emit_insn (gen_neon_vec_<US>mult_<mode> (tmpreg, operands[1], operands[2])); 5919 emit_insn (gen_neon_vget_low<V_widen_l> (operands[0], tmpreg)); 5920 5921 DONE; 5922 5923 } 5924) 5925 5926(define_expand "vec_widen_<US>shiftl_hi_<mode>" 5927 [(match_operand:<V_double_width> 0 "register_operand" "") 5928 (SE:<V_double_width> (match_operand:VDI 1 "register_operand" "")) 5929 (match_operand:SI 2 "immediate_operand" "i")] 5930 "TARGET_NEON" 5931 { 5932 rtx tmpreg = gen_reg_rtx (<V_widen>mode); 5933 emit_insn (gen_neon_vec_<US>shiftl_<mode> (tmpreg, operands[1], operands[2])); 5934 emit_insn (gen_neon_vget_high<V_widen_l> (operands[0], tmpreg)); 5935 5936 DONE; 5937 } 5938) 5939 5940(define_expand "vec_widen_<US>shiftl_lo_<mode>" 5941 [(match_operand:<V_double_width> 0 "register_operand" "") 5942 (SE:<V_double_width> (match_operand:VDI 1 "register_operand" "")) 5943 (match_operand:SI 2 "immediate_operand" "i")] 5944 "TARGET_NEON" 5945 { 5946 rtx tmpreg = gen_reg_rtx (<V_widen>mode); 5947 emit_insn (gen_neon_vec_<US>shiftl_<mode> (tmpreg, operands[1], operands[2])); 5948 emit_insn (gen_neon_vget_low<V_widen_l> (operands[0], tmpreg)); 5949 5950 DONE; 5951 } 5952) 5953 5954; FIXME: These instruction patterns can't be used safely in big-endian mode 5955; because the ordering of vector elements in Q registers is different from what 5956; the semantics of the instructions require. 5957 5958(define_insn "vec_pack_trunc_<mode>" 5959 [(set (match_operand:<V_narrow_pack> 0 "register_operand" "=&w") 5960 (vec_concat:<V_narrow_pack> 5961 (truncate:<V_narrow> 5962 (match_operand:VN 1 "register_operand" "w")) 5963 (truncate:<V_narrow> 5964 (match_operand:VN 2 "register_operand" "w"))))] 5965 "TARGET_NEON && !BYTES_BIG_ENDIAN" 5966 "vmovn.i<V_sz_elem>\t%e0, %q1\;vmovn.i<V_sz_elem>\t%f0, %q2" 5967 [(set_attr "neon_type" "neon_shift_1") 5968 (set_attr "length" "8")] 5969) 5970 5971;; For the non-quad case. 5972(define_insn "neon_vec_pack_trunc_<mode>" 5973 [(set (match_operand:<V_narrow> 0 "register_operand" "=w") 5974 (truncate:<V_narrow> (match_operand:VN 1 "register_operand" "w")))] 5975 "TARGET_NEON && !BYTES_BIG_ENDIAN" 5976 "vmovn.i<V_sz_elem>\t%P0, %q1" 5977 [(set_attr "neon_type" "neon_shift_1")] 5978) 5979 5980(define_expand "vec_pack_trunc_<mode>" 5981 [(match_operand:<V_narrow_pack> 0 "register_operand" "") 5982 (match_operand:VSHFT 1 "register_operand" "") 5983 (match_operand:VSHFT 2 "register_operand")] 5984 "TARGET_NEON && !BYTES_BIG_ENDIAN" 5985{ 5986 rtx tempreg = gen_reg_rtx (<V_DOUBLE>mode); 5987 5988 emit_insn (gen_move_lo_quad_<V_double> (tempreg, operands[1])); 5989 emit_insn (gen_move_hi_quad_<V_double> (tempreg, operands[2])); 5990 emit_insn (gen_neon_vec_pack_trunc_<V_double> (operands[0], tempreg)); 5991 DONE; 5992}) 5993 5994(define_insn "neon_vabd<mode>_2" 5995 [(set (match_operand:VDQ 0 "s_register_operand" "=w") 5996 (abs:VDQ (minus:VDQ (match_operand:VDQ 1 "s_register_operand" "w") 5997 (match_operand:VDQ 2 "s_register_operand" "w"))))] 5998 "TARGET_NEON && (!<Is_float_mode> || flag_unsafe_math_optimizations)" 5999 "vabd.<V_s_elem> %<V_reg>0, %<V_reg>1, %<V_reg>2" 6000 [(set (attr "neon_type") 6001 (if_then_else (ne (symbol_ref "<Is_float_mode>") (const_int 0)) 6002 (if_then_else (ne (symbol_ref "<Is_d_reg>") (const_int 0)) 6003 (const_string "neon_fp_vadd_ddd_vabs_dd") 6004 (const_string "neon_fp_vadd_qqq_vabs_qq")) 6005 (const_string "neon_int_5")))] 6006) 6007 6008(define_insn "neon_vabd<mode>_3" 6009 [(set (match_operand:VDQ 0 "s_register_operand" "=w") 6010 (abs:VDQ (unspec:VDQ [(match_operand:VDQ 1 "s_register_operand" "w") 6011 (match_operand:VDQ 2 "s_register_operand" "w")] 6012 UNSPEC_VSUB)))] 6013 "TARGET_NEON && (!<Is_float_mode> || flag_unsafe_math_optimizations)" 6014 "vabd.<V_if_elem> %<V_reg>0, %<V_reg>1, %<V_reg>2" 6015 [(set (attr "neon_type") 6016 (if_then_else (ne (symbol_ref "<Is_float_mode>") (const_int 0)) 6017 (if_then_else (ne (symbol_ref "<Is_d_reg>") (const_int 0)) 6018 (const_string "neon_fp_vadd_ddd_vabs_dd") 6019 (const_string "neon_fp_vadd_qqq_vabs_qq")) 6020 (const_string "neon_int_5")))] 6021) 6022 6023;; Copy from core-to-neon regs, then extend, not vice-versa 6024 6025(define_split 6026 [(set (match_operand:DI 0 "s_register_operand" "") 6027 (sign_extend:DI (match_operand:SI 1 "s_register_operand" "")))] 6028 "TARGET_NEON && reload_completed && IS_VFP_REGNUM (REGNO (operands[0]))" 6029 [(set (match_dup 2) (vec_duplicate:V2SI (match_dup 1))) 6030 (set (match_dup 0) (ashiftrt:DI (match_dup 0) (const_int 32)))] 6031 { 6032 operands[2] = gen_rtx_REG (V2SImode, REGNO (operands[0])); 6033 }) 6034 6035(define_split 6036 [(set (match_operand:DI 0 "s_register_operand" "") 6037 (sign_extend:DI (match_operand:HI 1 "s_register_operand" "")))] 6038 "TARGET_NEON && reload_completed && IS_VFP_REGNUM (REGNO (operands[0]))" 6039 [(set (match_dup 2) (vec_duplicate:V4HI (match_dup 1))) 6040 (set (match_dup 0) (ashiftrt:DI (match_dup 0) (const_int 48)))] 6041 { 6042 operands[2] = gen_rtx_REG (V4HImode, REGNO (operands[0])); 6043 }) 6044 6045(define_split 6046 [(set (match_operand:DI 0 "s_register_operand" "") 6047 (sign_extend:DI (match_operand:QI 1 "s_register_operand" "")))] 6048 "TARGET_NEON && reload_completed && IS_VFP_REGNUM (REGNO (operands[0]))" 6049 [(set (match_dup 2) (vec_duplicate:V8QI (match_dup 1))) 6050 (set (match_dup 0) (ashiftrt:DI (match_dup 0) (const_int 56)))] 6051 { 6052 operands[2] = gen_rtx_REG (V8QImode, REGNO (operands[0])); 6053 }) 6054 6055(define_split 6056 [(set (match_operand:DI 0 "s_register_operand" "") 6057 (zero_extend:DI (match_operand:SI 1 "s_register_operand" "")))] 6058 "TARGET_NEON && reload_completed && IS_VFP_REGNUM (REGNO (operands[0]))" 6059 [(set (match_dup 2) (vec_duplicate:V2SI (match_dup 1))) 6060 (set (match_dup 0) (lshiftrt:DI (match_dup 0) (const_int 32)))] 6061 { 6062 operands[2] = gen_rtx_REG (V2SImode, REGNO (operands[0])); 6063 }) 6064 6065(define_split 6066 [(set (match_operand:DI 0 "s_register_operand" "") 6067 (zero_extend:DI (match_operand:HI 1 "s_register_operand" "")))] 6068 "TARGET_NEON && reload_completed && IS_VFP_REGNUM (REGNO (operands[0]))" 6069 [(set (match_dup 2) (vec_duplicate:V4HI (match_dup 1))) 6070 (set (match_dup 0) (lshiftrt:DI (match_dup 0) (const_int 48)))] 6071 { 6072 operands[2] = gen_rtx_REG (V4HImode, REGNO (operands[0])); 6073 }) 6074 6075(define_split 6076 [(set (match_operand:DI 0 "s_register_operand" "") 6077 (zero_extend:DI (match_operand:QI 1 "s_register_operand" "")))] 6078 "TARGET_NEON && reload_completed && IS_VFP_REGNUM (REGNO (operands[0]))" 6079 [(set (match_dup 2) (vec_duplicate:V8QI (match_dup 1))) 6080 (set (match_dup 0) (lshiftrt:DI (match_dup 0) (const_int 56)))] 6081 { 6082 operands[2] = gen_rtx_REG (V8QImode, REGNO (operands[0])); 6083 }) 6084