1 /* Expand the basic unary and binary arithmetic operations, for GNU compiler. 2 Copyright (C) 1987-2018 Free Software Foundation, Inc. 3 4 This file is part of GCC. 5 6 GCC is free software; you can redistribute it and/or modify it under 7 the terms of the GNU General Public License as published by the Free 8 Software Foundation; either version 3, or (at your option) any later 9 version. 10 11 GCC is distributed in the hope that it will be useful, but WITHOUT ANY 12 WARRANTY; without even the implied warranty of MERCHANTABILITY or 13 FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License 14 for more details. 15 16 You should have received a copy of the GNU General Public License 17 along with GCC; see the file COPYING3. If not see 18 <http://www.gnu.org/licenses/>. */ 19 20 21 #include "config.h" 22 #include "system.h" 23 #include "coretypes.h" 24 #include "backend.h" 25 #include "target.h" 26 #include "rtl.h" 27 #include "tree.h" 28 #include "memmodel.h" 29 #include "predict.h" 30 #include "tm_p.h" 31 #include "expmed.h" 32 #include "optabs.h" 33 #include "emit-rtl.h" 34 #include "recog.h" 35 #include "diagnostic-core.h" 36 #include "rtx-vector-builder.h" 37 38 /* Include insn-config.h before expr.h so that HAVE_conditional_move 39 is properly defined. */ 40 #include "stor-layout.h" 41 #include "except.h" 42 #include "dojump.h" 43 #include "explow.h" 44 #include "expr.h" 45 #include "optabs-tree.h" 46 #include "libfuncs.h" 47 48 static void prepare_float_lib_cmp (rtx, rtx, enum rtx_code, rtx *, 49 machine_mode *); 50 static rtx expand_unop_direct (machine_mode, optab, rtx, rtx, int); 51 static void emit_libcall_block_1 (rtx_insn *, rtx, rtx, rtx, bool); 52 53 /* Debug facility for use in GDB. */ 54 void debug_optab_libfuncs (void); 55 56 /* Add a REG_EQUAL note to the last insn in INSNS. TARGET is being set to 57 the result of operation CODE applied to OP0 (and OP1 if it is a binary 58 operation). OP0_MODE is OP0's mode. 59 60 If the last insn does not set TARGET, don't do anything, but return 1. 61 62 If the last insn or a previous insn sets TARGET and TARGET is one of OP0 63 or OP1, don't add the REG_EQUAL note but return 0. Our caller can then 64 try again, ensuring that TARGET is not one of the operands. */ 65 66 static int 67 add_equal_note (rtx_insn *insns, rtx target, enum rtx_code code, rtx op0, 68 rtx op1, machine_mode op0_mode) 69 { 70 rtx_insn *last_insn; 71 rtx set; 72 rtx note; 73 74 gcc_assert (insns && INSN_P (insns) && NEXT_INSN (insns)); 75 76 if (GET_RTX_CLASS (code) != RTX_COMM_ARITH 77 && GET_RTX_CLASS (code) != RTX_BIN_ARITH 78 && GET_RTX_CLASS (code) != RTX_COMM_COMPARE 79 && GET_RTX_CLASS (code) != RTX_COMPARE 80 && GET_RTX_CLASS (code) != RTX_UNARY) 81 return 1; 82 83 if (GET_CODE (target) == ZERO_EXTRACT) 84 return 1; 85 86 for (last_insn = insns; 87 NEXT_INSN (last_insn) != NULL_RTX; 88 last_insn = NEXT_INSN (last_insn)) 89 ; 90 91 /* If TARGET is in OP0 or OP1, punt. We'd end up with a note referencing 92 a value changing in the insn, so the note would be invalid for CSE. */ 93 if (reg_overlap_mentioned_p (target, op0) 94 || (op1 && reg_overlap_mentioned_p (target, op1))) 95 { 96 if (MEM_P (target) 97 && (rtx_equal_p (target, op0) 98 || (op1 && rtx_equal_p (target, op1)))) 99 { 100 /* For MEM target, with MEM = MEM op X, prefer no REG_EQUAL note 101 over expanding it as temp = MEM op X, MEM = temp. If the target 102 supports MEM = MEM op X instructions, it is sometimes too hard 103 to reconstruct that form later, especially if X is also a memory, 104 and due to multiple occurrences of addresses the address might 105 be forced into register unnecessarily. 106 Note that not emitting the REG_EQUIV note might inhibit 107 CSE in some cases. */ 108 set = single_set (last_insn); 109 if (set 110 && GET_CODE (SET_SRC (set)) == code 111 && MEM_P (SET_DEST (set)) 112 && (rtx_equal_p (SET_DEST (set), XEXP (SET_SRC (set), 0)) 113 || (op1 && rtx_equal_p (SET_DEST (set), 114 XEXP (SET_SRC (set), 1))))) 115 return 1; 116 } 117 return 0; 118 } 119 120 set = set_for_reg_notes (last_insn); 121 if (set == NULL_RTX) 122 return 1; 123 124 if (! rtx_equal_p (SET_DEST (set), target) 125 /* For a STRICT_LOW_PART, the REG_NOTE applies to what is inside it. */ 126 && (GET_CODE (SET_DEST (set)) != STRICT_LOW_PART 127 || ! rtx_equal_p (XEXP (SET_DEST (set), 0), target))) 128 return 1; 129 130 if (GET_RTX_CLASS (code) == RTX_UNARY) 131 switch (code) 132 { 133 case FFS: 134 case CLZ: 135 case CTZ: 136 case CLRSB: 137 case POPCOUNT: 138 case PARITY: 139 case BSWAP: 140 if (op0_mode != VOIDmode && GET_MODE (target) != op0_mode) 141 { 142 note = gen_rtx_fmt_e (code, op0_mode, copy_rtx (op0)); 143 if (GET_MODE_UNIT_SIZE (op0_mode) 144 > GET_MODE_UNIT_SIZE (GET_MODE (target))) 145 note = simplify_gen_unary (TRUNCATE, GET_MODE (target), 146 note, op0_mode); 147 else 148 note = simplify_gen_unary (ZERO_EXTEND, GET_MODE (target), 149 note, op0_mode); 150 break; 151 } 152 /* FALLTHRU */ 153 default: 154 note = gen_rtx_fmt_e (code, GET_MODE (target), copy_rtx (op0)); 155 break; 156 } 157 else 158 note = gen_rtx_fmt_ee (code, GET_MODE (target), copy_rtx (op0), copy_rtx (op1)); 159 160 set_unique_reg_note (last_insn, REG_EQUAL, note); 161 162 return 1; 163 } 164 165 /* Given two input operands, OP0 and OP1, determine what the correct from_mode 166 for a widening operation would be. In most cases this would be OP0, but if 167 that's a constant it'll be VOIDmode, which isn't useful. */ 168 169 static machine_mode 170 widened_mode (machine_mode to_mode, rtx op0, rtx op1) 171 { 172 machine_mode m0 = GET_MODE (op0); 173 machine_mode m1 = GET_MODE (op1); 174 machine_mode result; 175 176 if (m0 == VOIDmode && m1 == VOIDmode) 177 return to_mode; 178 else if (m0 == VOIDmode || GET_MODE_UNIT_SIZE (m0) < GET_MODE_UNIT_SIZE (m1)) 179 result = m1; 180 else 181 result = m0; 182 183 if (GET_MODE_UNIT_SIZE (result) > GET_MODE_UNIT_SIZE (to_mode)) 184 return to_mode; 185 186 return result; 187 } 188 189 /* Widen OP to MODE and return the rtx for the widened operand. UNSIGNEDP 190 says whether OP is signed or unsigned. NO_EXTEND is nonzero if we need 191 not actually do a sign-extend or zero-extend, but can leave the 192 higher-order bits of the result rtx undefined, for example, in the case 193 of logical operations, but not right shifts. */ 194 195 static rtx 196 widen_operand (rtx op, machine_mode mode, machine_mode oldmode, 197 int unsignedp, int no_extend) 198 { 199 rtx result; 200 scalar_int_mode int_mode; 201 202 /* If we don't have to extend and this is a constant, return it. */ 203 if (no_extend && GET_MODE (op) == VOIDmode) 204 return op; 205 206 /* If we must extend do so. If OP is a SUBREG for a promoted object, also 207 extend since it will be more efficient to do so unless the signedness of 208 a promoted object differs from our extension. */ 209 if (! no_extend 210 || !is_a <scalar_int_mode> (mode, &int_mode) 211 || (GET_CODE (op) == SUBREG && SUBREG_PROMOTED_VAR_P (op) 212 && SUBREG_CHECK_PROMOTED_SIGN (op, unsignedp))) 213 return convert_modes (mode, oldmode, op, unsignedp); 214 215 /* If MODE is no wider than a single word, we return a lowpart or paradoxical 216 SUBREG. */ 217 if (GET_MODE_SIZE (int_mode) <= UNITS_PER_WORD) 218 return gen_lowpart (int_mode, force_reg (GET_MODE (op), op)); 219 220 /* Otherwise, get an object of MODE, clobber it, and set the low-order 221 part to OP. */ 222 223 result = gen_reg_rtx (int_mode); 224 emit_clobber (result); 225 emit_move_insn (gen_lowpart (GET_MODE (op), result), op); 226 return result; 227 } 228 229 /* Expand vector widening operations. 230 231 There are two different classes of operations handled here: 232 1) Operations whose result is wider than all the arguments to the operation. 233 Examples: VEC_UNPACK_HI/LO_EXPR, VEC_WIDEN_MULT_HI/LO_EXPR 234 In this case OP0 and optionally OP1 would be initialized, 235 but WIDE_OP wouldn't (not relevant for this case). 236 2) Operations whose result is of the same size as the last argument to the 237 operation, but wider than all the other arguments to the operation. 238 Examples: WIDEN_SUM_EXPR, VEC_DOT_PROD_EXPR. 239 In the case WIDE_OP, OP0 and optionally OP1 would be initialized. 240 241 E.g, when called to expand the following operations, this is how 242 the arguments will be initialized: 243 nops OP0 OP1 WIDE_OP 244 widening-sum 2 oprnd0 - oprnd1 245 widening-dot-product 3 oprnd0 oprnd1 oprnd2 246 widening-mult 2 oprnd0 oprnd1 - 247 type-promotion (vec-unpack) 1 oprnd0 - - */ 248 249 rtx 250 expand_widen_pattern_expr (sepops ops, rtx op0, rtx op1, rtx wide_op, 251 rtx target, int unsignedp) 252 { 253 struct expand_operand eops[4]; 254 tree oprnd0, oprnd1, oprnd2; 255 machine_mode wmode = VOIDmode, tmode0, tmode1 = VOIDmode; 256 optab widen_pattern_optab; 257 enum insn_code icode; 258 int nops = TREE_CODE_LENGTH (ops->code); 259 int op; 260 261 oprnd0 = ops->op0; 262 tmode0 = TYPE_MODE (TREE_TYPE (oprnd0)); 263 widen_pattern_optab = 264 optab_for_tree_code (ops->code, TREE_TYPE (oprnd0), optab_default); 265 if (ops->code == WIDEN_MULT_PLUS_EXPR 266 || ops->code == WIDEN_MULT_MINUS_EXPR) 267 icode = find_widening_optab_handler (widen_pattern_optab, 268 TYPE_MODE (TREE_TYPE (ops->op2)), 269 tmode0); 270 else 271 icode = optab_handler (widen_pattern_optab, tmode0); 272 gcc_assert (icode != CODE_FOR_nothing); 273 274 if (nops >= 2) 275 { 276 oprnd1 = ops->op1; 277 tmode1 = TYPE_MODE (TREE_TYPE (oprnd1)); 278 } 279 280 /* The last operand is of a wider mode than the rest of the operands. */ 281 if (nops == 2) 282 wmode = tmode1; 283 else if (nops == 3) 284 { 285 gcc_assert (tmode1 == tmode0); 286 gcc_assert (op1); 287 oprnd2 = ops->op2; 288 wmode = TYPE_MODE (TREE_TYPE (oprnd2)); 289 } 290 291 op = 0; 292 create_output_operand (&eops[op++], target, TYPE_MODE (ops->type)); 293 create_convert_operand_from (&eops[op++], op0, tmode0, unsignedp); 294 if (op1) 295 create_convert_operand_from (&eops[op++], op1, tmode1, unsignedp); 296 if (wide_op) 297 create_convert_operand_from (&eops[op++], wide_op, wmode, unsignedp); 298 expand_insn (icode, op, eops); 299 return eops[0].value; 300 } 301 302 /* Generate code to perform an operation specified by TERNARY_OPTAB 303 on operands OP0, OP1 and OP2, with result having machine-mode MODE. 304 305 UNSIGNEDP is for the case where we have to widen the operands 306 to perform the operation. It says to use zero-extension. 307 308 If TARGET is nonzero, the value 309 is generated there, if it is convenient to do so. 310 In all cases an rtx is returned for the locus of the value; 311 this may or may not be TARGET. */ 312 313 rtx 314 expand_ternary_op (machine_mode mode, optab ternary_optab, rtx op0, 315 rtx op1, rtx op2, rtx target, int unsignedp) 316 { 317 struct expand_operand ops[4]; 318 enum insn_code icode = optab_handler (ternary_optab, mode); 319 320 gcc_assert (optab_handler (ternary_optab, mode) != CODE_FOR_nothing); 321 322 create_output_operand (&ops[0], target, mode); 323 create_convert_operand_from (&ops[1], op0, mode, unsignedp); 324 create_convert_operand_from (&ops[2], op1, mode, unsignedp); 325 create_convert_operand_from (&ops[3], op2, mode, unsignedp); 326 expand_insn (icode, 4, ops); 327 return ops[0].value; 328 } 329 330 331 /* Like expand_binop, but return a constant rtx if the result can be 332 calculated at compile time. The arguments and return value are 333 otherwise the same as for expand_binop. */ 334 335 rtx 336 simplify_expand_binop (machine_mode mode, optab binoptab, 337 rtx op0, rtx op1, rtx target, int unsignedp, 338 enum optab_methods methods) 339 { 340 if (CONSTANT_P (op0) && CONSTANT_P (op1)) 341 { 342 rtx x = simplify_binary_operation (optab_to_code (binoptab), 343 mode, op0, op1); 344 if (x) 345 return x; 346 } 347 348 return expand_binop (mode, binoptab, op0, op1, target, unsignedp, methods); 349 } 350 351 /* Like simplify_expand_binop, but always put the result in TARGET. 352 Return true if the expansion succeeded. */ 353 354 bool 355 force_expand_binop (machine_mode mode, optab binoptab, 356 rtx op0, rtx op1, rtx target, int unsignedp, 357 enum optab_methods methods) 358 { 359 rtx x = simplify_expand_binop (mode, binoptab, op0, op1, 360 target, unsignedp, methods); 361 if (x == 0) 362 return false; 363 if (x != target) 364 emit_move_insn (target, x); 365 return true; 366 } 367 368 /* Create a new vector value in VMODE with all elements set to OP. The 369 mode of OP must be the element mode of VMODE. If OP is a constant, 370 then the return value will be a constant. */ 371 372 rtx 373 expand_vector_broadcast (machine_mode vmode, rtx op) 374 { 375 int n; 376 rtvec vec; 377 378 gcc_checking_assert (VECTOR_MODE_P (vmode)); 379 380 if (valid_for_const_vector_p (vmode, op)) 381 return gen_const_vec_duplicate (vmode, op); 382 383 insn_code icode = optab_handler (vec_duplicate_optab, vmode); 384 if (icode != CODE_FOR_nothing) 385 { 386 struct expand_operand ops[2]; 387 create_output_operand (&ops[0], NULL_RTX, vmode); 388 create_input_operand (&ops[1], op, GET_MODE (op)); 389 expand_insn (icode, 2, ops); 390 return ops[0].value; 391 } 392 393 if (!GET_MODE_NUNITS (vmode).is_constant (&n)) 394 return NULL; 395 396 /* ??? If the target doesn't have a vec_init, then we have no easy way 397 of performing this operation. Most of this sort of generic support 398 is hidden away in the vector lowering support in gimple. */ 399 icode = convert_optab_handler (vec_init_optab, vmode, 400 GET_MODE_INNER (vmode)); 401 if (icode == CODE_FOR_nothing) 402 return NULL; 403 404 vec = rtvec_alloc (n); 405 for (int i = 0; i < n; ++i) 406 RTVEC_ELT (vec, i) = op; 407 rtx ret = gen_reg_rtx (vmode); 408 emit_insn (GEN_FCN (icode) (ret, gen_rtx_PARALLEL (vmode, vec))); 409 410 return ret; 411 } 412 413 /* This subroutine of expand_doubleword_shift handles the cases in which 414 the effective shift value is >= BITS_PER_WORD. The arguments and return 415 value are the same as for the parent routine, except that SUPERWORD_OP1 416 is the shift count to use when shifting OUTOF_INPUT into INTO_TARGET. 417 INTO_TARGET may be null if the caller has decided to calculate it. */ 418 419 static bool 420 expand_superword_shift (optab binoptab, rtx outof_input, rtx superword_op1, 421 rtx outof_target, rtx into_target, 422 int unsignedp, enum optab_methods methods) 423 { 424 if (into_target != 0) 425 if (!force_expand_binop (word_mode, binoptab, outof_input, superword_op1, 426 into_target, unsignedp, methods)) 427 return false; 428 429 if (outof_target != 0) 430 { 431 /* For a signed right shift, we must fill OUTOF_TARGET with copies 432 of the sign bit, otherwise we must fill it with zeros. */ 433 if (binoptab != ashr_optab) 434 emit_move_insn (outof_target, CONST0_RTX (word_mode)); 435 else 436 if (!force_expand_binop (word_mode, binoptab, outof_input, 437 gen_int_shift_amount (word_mode, 438 BITS_PER_WORD - 1), 439 outof_target, unsignedp, methods)) 440 return false; 441 } 442 return true; 443 } 444 445 /* This subroutine of expand_doubleword_shift handles the cases in which 446 the effective shift value is < BITS_PER_WORD. The arguments and return 447 value are the same as for the parent routine. */ 448 449 static bool 450 expand_subword_shift (scalar_int_mode op1_mode, optab binoptab, 451 rtx outof_input, rtx into_input, rtx op1, 452 rtx outof_target, rtx into_target, 453 int unsignedp, enum optab_methods methods, 454 unsigned HOST_WIDE_INT shift_mask) 455 { 456 optab reverse_unsigned_shift, unsigned_shift; 457 rtx tmp, carries; 458 459 reverse_unsigned_shift = (binoptab == ashl_optab ? lshr_optab : ashl_optab); 460 unsigned_shift = (binoptab == ashl_optab ? ashl_optab : lshr_optab); 461 462 /* The low OP1 bits of INTO_TARGET come from the high bits of OUTOF_INPUT. 463 We therefore need to shift OUTOF_INPUT by (BITS_PER_WORD - OP1) bits in 464 the opposite direction to BINOPTAB. */ 465 if (CONSTANT_P (op1) || shift_mask >= BITS_PER_WORD) 466 { 467 carries = outof_input; 468 tmp = immed_wide_int_const (wi::shwi (BITS_PER_WORD, 469 op1_mode), op1_mode); 470 tmp = simplify_expand_binop (op1_mode, sub_optab, tmp, op1, 471 0, true, methods); 472 } 473 else 474 { 475 /* We must avoid shifting by BITS_PER_WORD bits since that is either 476 the same as a zero shift (if shift_mask == BITS_PER_WORD - 1) or 477 has unknown behavior. Do a single shift first, then shift by the 478 remainder. It's OK to use ~OP1 as the remainder if shift counts 479 are truncated to the mode size. */ 480 carries = expand_binop (word_mode, reverse_unsigned_shift, 481 outof_input, const1_rtx, 0, unsignedp, methods); 482 if (shift_mask == BITS_PER_WORD - 1) 483 { 484 tmp = immed_wide_int_const 485 (wi::minus_one (GET_MODE_PRECISION (op1_mode)), op1_mode); 486 tmp = simplify_expand_binop (op1_mode, xor_optab, op1, tmp, 487 0, true, methods); 488 } 489 else 490 { 491 tmp = immed_wide_int_const (wi::shwi (BITS_PER_WORD - 1, 492 op1_mode), op1_mode); 493 tmp = simplify_expand_binop (op1_mode, sub_optab, tmp, op1, 494 0, true, methods); 495 } 496 } 497 if (tmp == 0 || carries == 0) 498 return false; 499 carries = expand_binop (word_mode, reverse_unsigned_shift, 500 carries, tmp, 0, unsignedp, methods); 501 if (carries == 0) 502 return false; 503 504 /* Shift INTO_INPUT logically by OP1. This is the last use of INTO_INPUT 505 so the result can go directly into INTO_TARGET if convenient. */ 506 tmp = expand_binop (word_mode, unsigned_shift, into_input, op1, 507 into_target, unsignedp, methods); 508 if (tmp == 0) 509 return false; 510 511 /* Now OR in the bits carried over from OUTOF_INPUT. */ 512 if (!force_expand_binop (word_mode, ior_optab, tmp, carries, 513 into_target, unsignedp, methods)) 514 return false; 515 516 /* Use a standard word_mode shift for the out-of half. */ 517 if (outof_target != 0) 518 if (!force_expand_binop (word_mode, binoptab, outof_input, op1, 519 outof_target, unsignedp, methods)) 520 return false; 521 522 return true; 523 } 524 525 526 /* Try implementing expand_doubleword_shift using conditional moves. 527 The shift is by < BITS_PER_WORD if (CMP_CODE CMP1 CMP2) is true, 528 otherwise it is by >= BITS_PER_WORD. SUBWORD_OP1 and SUPERWORD_OP1 529 are the shift counts to use in the former and latter case. All other 530 arguments are the same as the parent routine. */ 531 532 static bool 533 expand_doubleword_shift_condmove (scalar_int_mode op1_mode, optab binoptab, 534 enum rtx_code cmp_code, rtx cmp1, rtx cmp2, 535 rtx outof_input, rtx into_input, 536 rtx subword_op1, rtx superword_op1, 537 rtx outof_target, rtx into_target, 538 int unsignedp, enum optab_methods methods, 539 unsigned HOST_WIDE_INT shift_mask) 540 { 541 rtx outof_superword, into_superword; 542 543 /* Put the superword version of the output into OUTOF_SUPERWORD and 544 INTO_SUPERWORD. */ 545 outof_superword = outof_target != 0 ? gen_reg_rtx (word_mode) : 0; 546 if (outof_target != 0 && subword_op1 == superword_op1) 547 { 548 /* The value INTO_TARGET >> SUBWORD_OP1, which we later store in 549 OUTOF_TARGET, is the same as the value of INTO_SUPERWORD. */ 550 into_superword = outof_target; 551 if (!expand_superword_shift (binoptab, outof_input, superword_op1, 552 outof_superword, 0, unsignedp, methods)) 553 return false; 554 } 555 else 556 { 557 into_superword = gen_reg_rtx (word_mode); 558 if (!expand_superword_shift (binoptab, outof_input, superword_op1, 559 outof_superword, into_superword, 560 unsignedp, methods)) 561 return false; 562 } 563 564 /* Put the subword version directly in OUTOF_TARGET and INTO_TARGET. */ 565 if (!expand_subword_shift (op1_mode, binoptab, 566 outof_input, into_input, subword_op1, 567 outof_target, into_target, 568 unsignedp, methods, shift_mask)) 569 return false; 570 571 /* Select between them. Do the INTO half first because INTO_SUPERWORD 572 might be the current value of OUTOF_TARGET. */ 573 if (!emit_conditional_move (into_target, cmp_code, cmp1, cmp2, op1_mode, 574 into_target, into_superword, word_mode, false)) 575 return false; 576 577 if (outof_target != 0) 578 if (!emit_conditional_move (outof_target, cmp_code, cmp1, cmp2, op1_mode, 579 outof_target, outof_superword, 580 word_mode, false)) 581 return false; 582 583 return true; 584 } 585 586 /* Expand a doubleword shift (ashl, ashr or lshr) using word-mode shifts. 587 OUTOF_INPUT and INTO_INPUT are the two word-sized halves of the first 588 input operand; the shift moves bits in the direction OUTOF_INPUT-> 589 INTO_TARGET. OUTOF_TARGET and INTO_TARGET are the equivalent words 590 of the target. OP1 is the shift count and OP1_MODE is its mode. 591 If OP1 is constant, it will have been truncated as appropriate 592 and is known to be nonzero. 593 594 If SHIFT_MASK is zero, the result of word shifts is undefined when the 595 shift count is outside the range [0, BITS_PER_WORD). This routine must 596 avoid generating such shifts for OP1s in the range [0, BITS_PER_WORD * 2). 597 598 If SHIFT_MASK is nonzero, all word-mode shift counts are effectively 599 masked by it and shifts in the range [BITS_PER_WORD, SHIFT_MASK) will 600 fill with zeros or sign bits as appropriate. 601 602 If SHIFT_MASK is BITS_PER_WORD - 1, this routine will synthesize 603 a doubleword shift whose equivalent mask is BITS_PER_WORD * 2 - 1. 604 Doing this preserves semantics required by SHIFT_COUNT_TRUNCATED. 605 In all other cases, shifts by values outside [0, BITS_PER_UNIT * 2) 606 are undefined. 607 608 BINOPTAB, UNSIGNEDP and METHODS are as for expand_binop. This function 609 may not use INTO_INPUT after modifying INTO_TARGET, and similarly for 610 OUTOF_INPUT and OUTOF_TARGET. OUTOF_TARGET can be null if the parent 611 function wants to calculate it itself. 612 613 Return true if the shift could be successfully synthesized. */ 614 615 static bool 616 expand_doubleword_shift (scalar_int_mode op1_mode, optab binoptab, 617 rtx outof_input, rtx into_input, rtx op1, 618 rtx outof_target, rtx into_target, 619 int unsignedp, enum optab_methods methods, 620 unsigned HOST_WIDE_INT shift_mask) 621 { 622 rtx superword_op1, tmp, cmp1, cmp2; 623 enum rtx_code cmp_code; 624 625 /* See if word-mode shifts by BITS_PER_WORD...BITS_PER_WORD * 2 - 1 will 626 fill the result with sign or zero bits as appropriate. If so, the value 627 of OUTOF_TARGET will always be (SHIFT OUTOF_INPUT OP1). Recursively call 628 this routine to calculate INTO_TARGET (which depends on both OUTOF_INPUT 629 and INTO_INPUT), then emit code to set up OUTOF_TARGET. 630 631 This isn't worthwhile for constant shifts since the optimizers will 632 cope better with in-range shift counts. */ 633 if (shift_mask >= BITS_PER_WORD 634 && outof_target != 0 635 && !CONSTANT_P (op1)) 636 { 637 if (!expand_doubleword_shift (op1_mode, binoptab, 638 outof_input, into_input, op1, 639 0, into_target, 640 unsignedp, methods, shift_mask)) 641 return false; 642 if (!force_expand_binop (word_mode, binoptab, outof_input, op1, 643 outof_target, unsignedp, methods)) 644 return false; 645 return true; 646 } 647 648 /* Set CMP_CODE, CMP1 and CMP2 so that the rtx (CMP_CODE CMP1 CMP2) 649 is true when the effective shift value is less than BITS_PER_WORD. 650 Set SUPERWORD_OP1 to the shift count that should be used to shift 651 OUTOF_INPUT into INTO_TARGET when the condition is false. */ 652 tmp = immed_wide_int_const (wi::shwi (BITS_PER_WORD, op1_mode), op1_mode); 653 if (!CONSTANT_P (op1) && shift_mask == BITS_PER_WORD - 1) 654 { 655 /* Set CMP1 to OP1 & BITS_PER_WORD. The result is zero iff OP1 656 is a subword shift count. */ 657 cmp1 = simplify_expand_binop (op1_mode, and_optab, op1, tmp, 658 0, true, methods); 659 cmp2 = CONST0_RTX (op1_mode); 660 cmp_code = EQ; 661 superword_op1 = op1; 662 } 663 else 664 { 665 /* Set CMP1 to OP1 - BITS_PER_WORD. */ 666 cmp1 = simplify_expand_binop (op1_mode, sub_optab, op1, tmp, 667 0, true, methods); 668 cmp2 = CONST0_RTX (op1_mode); 669 cmp_code = LT; 670 superword_op1 = cmp1; 671 } 672 if (cmp1 == 0) 673 return false; 674 675 /* If we can compute the condition at compile time, pick the 676 appropriate subroutine. */ 677 tmp = simplify_relational_operation (cmp_code, SImode, op1_mode, cmp1, cmp2); 678 if (tmp != 0 && CONST_INT_P (tmp)) 679 { 680 if (tmp == const0_rtx) 681 return expand_superword_shift (binoptab, outof_input, superword_op1, 682 outof_target, into_target, 683 unsignedp, methods); 684 else 685 return expand_subword_shift (op1_mode, binoptab, 686 outof_input, into_input, op1, 687 outof_target, into_target, 688 unsignedp, methods, shift_mask); 689 } 690 691 /* Try using conditional moves to generate straight-line code. */ 692 if (HAVE_conditional_move) 693 { 694 rtx_insn *start = get_last_insn (); 695 if (expand_doubleword_shift_condmove (op1_mode, binoptab, 696 cmp_code, cmp1, cmp2, 697 outof_input, into_input, 698 op1, superword_op1, 699 outof_target, into_target, 700 unsignedp, methods, shift_mask)) 701 return true; 702 delete_insns_since (start); 703 } 704 705 /* As a last resort, use branches to select the correct alternative. */ 706 rtx_code_label *subword_label = gen_label_rtx (); 707 rtx_code_label *done_label = gen_label_rtx (); 708 709 NO_DEFER_POP; 710 do_compare_rtx_and_jump (cmp1, cmp2, cmp_code, false, op1_mode, 711 0, 0, subword_label, 712 profile_probability::uninitialized ()); 713 OK_DEFER_POP; 714 715 if (!expand_superword_shift (binoptab, outof_input, superword_op1, 716 outof_target, into_target, 717 unsignedp, methods)) 718 return false; 719 720 emit_jump_insn (targetm.gen_jump (done_label)); 721 emit_barrier (); 722 emit_label (subword_label); 723 724 if (!expand_subword_shift (op1_mode, binoptab, 725 outof_input, into_input, op1, 726 outof_target, into_target, 727 unsignedp, methods, shift_mask)) 728 return false; 729 730 emit_label (done_label); 731 return true; 732 } 733 734 /* Subroutine of expand_binop. Perform a double word multiplication of 735 operands OP0 and OP1 both of mode MODE, which is exactly twice as wide 736 as the target's word_mode. This function return NULL_RTX if anything 737 goes wrong, in which case it may have already emitted instructions 738 which need to be deleted. 739 740 If we want to multiply two two-word values and have normal and widening 741 multiplies of single-word values, we can do this with three smaller 742 multiplications. 743 744 The multiplication proceeds as follows: 745 _______________________ 746 [__op0_high_|__op0_low__] 747 _______________________ 748 * [__op1_high_|__op1_low__] 749 _______________________________________________ 750 _______________________ 751 (1) [__op0_low__*__op1_low__] 752 _______________________ 753 (2a) [__op0_low__*__op1_high_] 754 _______________________ 755 (2b) [__op0_high_*__op1_low__] 756 _______________________ 757 (3) [__op0_high_*__op1_high_] 758 759 760 This gives a 4-word result. Since we are only interested in the 761 lower 2 words, partial result (3) and the upper words of (2a) and 762 (2b) don't need to be calculated. Hence (2a) and (2b) can be 763 calculated using non-widening multiplication. 764 765 (1), however, needs to be calculated with an unsigned widening 766 multiplication. If this operation is not directly supported we 767 try using a signed widening multiplication and adjust the result. 768 This adjustment works as follows: 769 770 If both operands are positive then no adjustment is needed. 771 772 If the operands have different signs, for example op0_low < 0 and 773 op1_low >= 0, the instruction treats the most significant bit of 774 op0_low as a sign bit instead of a bit with significance 775 2**(BITS_PER_WORD-1), i.e. the instruction multiplies op1_low 776 with 2**BITS_PER_WORD - op0_low, and two's complements the 777 result. Conclusion: We need to add op1_low * 2**BITS_PER_WORD to 778 the result. 779 780 Similarly, if both operands are negative, we need to add 781 (op0_low + op1_low) * 2**BITS_PER_WORD. 782 783 We use a trick to adjust quickly. We logically shift op0_low right 784 (op1_low) BITS_PER_WORD-1 steps to get 0 or 1, and add this to 785 op0_high (op1_high) before it is used to calculate 2b (2a). If no 786 logical shift exists, we do an arithmetic right shift and subtract 787 the 0 or -1. */ 788 789 static rtx 790 expand_doubleword_mult (machine_mode mode, rtx op0, rtx op1, rtx target, 791 bool umulp, enum optab_methods methods) 792 { 793 int low = (WORDS_BIG_ENDIAN ? 1 : 0); 794 int high = (WORDS_BIG_ENDIAN ? 0 : 1); 795 rtx wordm1 = (umulp ? NULL_RTX 796 : gen_int_shift_amount (word_mode, BITS_PER_WORD - 1)); 797 rtx product, adjust, product_high, temp; 798 799 rtx op0_high = operand_subword_force (op0, high, mode); 800 rtx op0_low = operand_subword_force (op0, low, mode); 801 rtx op1_high = operand_subword_force (op1, high, mode); 802 rtx op1_low = operand_subword_force (op1, low, mode); 803 804 /* If we're using an unsigned multiply to directly compute the product 805 of the low-order words of the operands and perform any required 806 adjustments of the operands, we begin by trying two more multiplications 807 and then computing the appropriate sum. 808 809 We have checked above that the required addition is provided. 810 Full-word addition will normally always succeed, especially if 811 it is provided at all, so we don't worry about its failure. The 812 multiplication may well fail, however, so we do handle that. */ 813 814 if (!umulp) 815 { 816 /* ??? This could be done with emit_store_flag where available. */ 817 temp = expand_binop (word_mode, lshr_optab, op0_low, wordm1, 818 NULL_RTX, 1, methods); 819 if (temp) 820 op0_high = expand_binop (word_mode, add_optab, op0_high, temp, 821 NULL_RTX, 0, OPTAB_DIRECT); 822 else 823 { 824 temp = expand_binop (word_mode, ashr_optab, op0_low, wordm1, 825 NULL_RTX, 0, methods); 826 if (!temp) 827 return NULL_RTX; 828 op0_high = expand_binop (word_mode, sub_optab, op0_high, temp, 829 NULL_RTX, 0, OPTAB_DIRECT); 830 } 831 832 if (!op0_high) 833 return NULL_RTX; 834 } 835 836 adjust = expand_binop (word_mode, smul_optab, op0_high, op1_low, 837 NULL_RTX, 0, OPTAB_DIRECT); 838 if (!adjust) 839 return NULL_RTX; 840 841 /* OP0_HIGH should now be dead. */ 842 843 if (!umulp) 844 { 845 /* ??? This could be done with emit_store_flag where available. */ 846 temp = expand_binop (word_mode, lshr_optab, op1_low, wordm1, 847 NULL_RTX, 1, methods); 848 if (temp) 849 op1_high = expand_binop (word_mode, add_optab, op1_high, temp, 850 NULL_RTX, 0, OPTAB_DIRECT); 851 else 852 { 853 temp = expand_binop (word_mode, ashr_optab, op1_low, wordm1, 854 NULL_RTX, 0, methods); 855 if (!temp) 856 return NULL_RTX; 857 op1_high = expand_binop (word_mode, sub_optab, op1_high, temp, 858 NULL_RTX, 0, OPTAB_DIRECT); 859 } 860 861 if (!op1_high) 862 return NULL_RTX; 863 } 864 865 temp = expand_binop (word_mode, smul_optab, op1_high, op0_low, 866 NULL_RTX, 0, OPTAB_DIRECT); 867 if (!temp) 868 return NULL_RTX; 869 870 /* OP1_HIGH should now be dead. */ 871 872 adjust = expand_binop (word_mode, add_optab, adjust, temp, 873 NULL_RTX, 0, OPTAB_DIRECT); 874 875 if (target && !REG_P (target)) 876 target = NULL_RTX; 877 878 /* *_widen_optab needs to determine operand mode, make sure at least 879 one operand has non-VOID mode. */ 880 if (GET_MODE (op0_low) == VOIDmode && GET_MODE (op1_low) == VOIDmode) 881 op0_low = force_reg (word_mode, op0_low); 882 883 if (umulp) 884 product = expand_binop (mode, umul_widen_optab, op0_low, op1_low, 885 target, 1, OPTAB_DIRECT); 886 else 887 product = expand_binop (mode, smul_widen_optab, op0_low, op1_low, 888 target, 1, OPTAB_DIRECT); 889 890 if (!product) 891 return NULL_RTX; 892 893 product_high = operand_subword (product, high, 1, mode); 894 adjust = expand_binop (word_mode, add_optab, product_high, adjust, 895 NULL_RTX, 0, OPTAB_DIRECT); 896 emit_move_insn (product_high, adjust); 897 return product; 898 } 899 900 /* Wrapper around expand_binop which takes an rtx code to specify 901 the operation to perform, not an optab pointer. All other 902 arguments are the same. */ 903 rtx 904 expand_simple_binop (machine_mode mode, enum rtx_code code, rtx op0, 905 rtx op1, rtx target, int unsignedp, 906 enum optab_methods methods) 907 { 908 optab binop = code_to_optab (code); 909 gcc_assert (binop); 910 911 return expand_binop (mode, binop, op0, op1, target, unsignedp, methods); 912 } 913 914 /* Return whether OP0 and OP1 should be swapped when expanding a commutative 915 binop. Order them according to commutative_operand_precedence and, if 916 possible, try to put TARGET or a pseudo first. */ 917 static bool 918 swap_commutative_operands_with_target (rtx target, rtx op0, rtx op1) 919 { 920 int op0_prec = commutative_operand_precedence (op0); 921 int op1_prec = commutative_operand_precedence (op1); 922 923 if (op0_prec < op1_prec) 924 return true; 925 926 if (op0_prec > op1_prec) 927 return false; 928 929 /* With equal precedence, both orders are ok, but it is better if the 930 first operand is TARGET, or if both TARGET and OP0 are pseudos. */ 931 if (target == 0 || REG_P (target)) 932 return (REG_P (op1) && !REG_P (op0)) || target == op1; 933 else 934 return rtx_equal_p (op1, target); 935 } 936 937 /* Return true if BINOPTAB implements a shift operation. */ 938 939 static bool 940 shift_optab_p (optab binoptab) 941 { 942 switch (optab_to_code (binoptab)) 943 { 944 case ASHIFT: 945 case SS_ASHIFT: 946 case US_ASHIFT: 947 case ASHIFTRT: 948 case LSHIFTRT: 949 case ROTATE: 950 case ROTATERT: 951 return true; 952 953 default: 954 return false; 955 } 956 } 957 958 /* Return true if BINOPTAB implements a commutative binary operation. */ 959 960 static bool 961 commutative_optab_p (optab binoptab) 962 { 963 return (GET_RTX_CLASS (optab_to_code (binoptab)) == RTX_COMM_ARITH 964 || binoptab == smul_widen_optab 965 || binoptab == umul_widen_optab 966 || binoptab == smul_highpart_optab 967 || binoptab == umul_highpart_optab); 968 } 969 970 /* X is to be used in mode MODE as operand OPN to BINOPTAB. If we're 971 optimizing, and if the operand is a constant that costs more than 972 1 instruction, force the constant into a register and return that 973 register. Return X otherwise. UNSIGNEDP says whether X is unsigned. */ 974 975 static rtx 976 avoid_expensive_constant (machine_mode mode, optab binoptab, 977 int opn, rtx x, bool unsignedp) 978 { 979 bool speed = optimize_insn_for_speed_p (); 980 981 if (mode != VOIDmode 982 && optimize 983 && CONSTANT_P (x) 984 && (rtx_cost (x, mode, optab_to_code (binoptab), opn, speed) 985 > set_src_cost (x, mode, speed))) 986 { 987 if (CONST_INT_P (x)) 988 { 989 HOST_WIDE_INT intval = trunc_int_for_mode (INTVAL (x), mode); 990 if (intval != INTVAL (x)) 991 x = GEN_INT (intval); 992 } 993 else 994 x = convert_modes (mode, VOIDmode, x, unsignedp); 995 x = force_reg (mode, x); 996 } 997 return x; 998 } 999 1000 /* Helper function for expand_binop: handle the case where there 1001 is an insn ICODE that directly implements the indicated operation. 1002 Returns null if this is not possible. */ 1003 static rtx 1004 expand_binop_directly (enum insn_code icode, machine_mode mode, optab binoptab, 1005 rtx op0, rtx op1, 1006 rtx target, int unsignedp, enum optab_methods methods, 1007 rtx_insn *last) 1008 { 1009 machine_mode xmode0 = insn_data[(int) icode].operand[1].mode; 1010 machine_mode xmode1 = insn_data[(int) icode].operand[2].mode; 1011 machine_mode mode0, mode1, tmp_mode; 1012 struct expand_operand ops[3]; 1013 bool commutative_p; 1014 rtx_insn *pat; 1015 rtx xop0 = op0, xop1 = op1; 1016 bool canonicalize_op1 = false; 1017 1018 /* If it is a commutative operator and the modes would match 1019 if we would swap the operands, we can save the conversions. */ 1020 commutative_p = commutative_optab_p (binoptab); 1021 if (commutative_p 1022 && GET_MODE (xop0) != xmode0 && GET_MODE (xop1) != xmode1 1023 && GET_MODE (xop0) == xmode1 && GET_MODE (xop1) == xmode1) 1024 std::swap (xop0, xop1); 1025 1026 /* If we are optimizing, force expensive constants into a register. */ 1027 xop0 = avoid_expensive_constant (xmode0, binoptab, 0, xop0, unsignedp); 1028 if (!shift_optab_p (binoptab)) 1029 xop1 = avoid_expensive_constant (xmode1, binoptab, 1, xop1, unsignedp); 1030 else 1031 /* Shifts and rotates often use a different mode for op1 from op0; 1032 for VOIDmode constants we don't know the mode, so force it 1033 to be canonicalized using convert_modes. */ 1034 canonicalize_op1 = true; 1035 1036 /* In case the insn wants input operands in modes different from 1037 those of the actual operands, convert the operands. It would 1038 seem that we don't need to convert CONST_INTs, but we do, so 1039 that they're properly zero-extended, sign-extended or truncated 1040 for their mode. */ 1041 1042 mode0 = GET_MODE (xop0) != VOIDmode ? GET_MODE (xop0) : mode; 1043 if (xmode0 != VOIDmode && xmode0 != mode0) 1044 { 1045 xop0 = convert_modes (xmode0, mode0, xop0, unsignedp); 1046 mode0 = xmode0; 1047 } 1048 1049 mode1 = ((GET_MODE (xop1) != VOIDmode || canonicalize_op1) 1050 ? GET_MODE (xop1) : mode); 1051 if (xmode1 != VOIDmode && xmode1 != mode1) 1052 { 1053 xop1 = convert_modes (xmode1, mode1, xop1, unsignedp); 1054 mode1 = xmode1; 1055 } 1056 1057 /* If operation is commutative, 1058 try to make the first operand a register. 1059 Even better, try to make it the same as the target. 1060 Also try to make the last operand a constant. */ 1061 if (commutative_p 1062 && swap_commutative_operands_with_target (target, xop0, xop1)) 1063 std::swap (xop0, xop1); 1064 1065 /* Now, if insn's predicates don't allow our operands, put them into 1066 pseudo regs. */ 1067 1068 if (binoptab == vec_pack_trunc_optab 1069 || binoptab == vec_pack_usat_optab 1070 || binoptab == vec_pack_ssat_optab 1071 || binoptab == vec_pack_ufix_trunc_optab 1072 || binoptab == vec_pack_sfix_trunc_optab) 1073 { 1074 /* The mode of the result is different then the mode of the 1075 arguments. */ 1076 tmp_mode = insn_data[(int) icode].operand[0].mode; 1077 if (VECTOR_MODE_P (mode) 1078 && maybe_ne (GET_MODE_NUNITS (tmp_mode), 2 * GET_MODE_NUNITS (mode))) 1079 { 1080 delete_insns_since (last); 1081 return NULL_RTX; 1082 } 1083 } 1084 else 1085 tmp_mode = mode; 1086 1087 create_output_operand (&ops[0], target, tmp_mode); 1088 create_input_operand (&ops[1], xop0, mode0); 1089 create_input_operand (&ops[2], xop1, mode1); 1090 pat = maybe_gen_insn (icode, 3, ops); 1091 if (pat) 1092 { 1093 /* If PAT is composed of more than one insn, try to add an appropriate 1094 REG_EQUAL note to it. If we can't because TEMP conflicts with an 1095 operand, call expand_binop again, this time without a target. */ 1096 if (INSN_P (pat) && NEXT_INSN (pat) != NULL_RTX 1097 && ! add_equal_note (pat, ops[0].value, 1098 optab_to_code (binoptab), 1099 ops[1].value, ops[2].value, mode0)) 1100 { 1101 delete_insns_since (last); 1102 return expand_binop (mode, binoptab, op0, op1, NULL_RTX, 1103 unsignedp, methods); 1104 } 1105 1106 emit_insn (pat); 1107 return ops[0].value; 1108 } 1109 delete_insns_since (last); 1110 return NULL_RTX; 1111 } 1112 1113 /* Generate code to perform an operation specified by BINOPTAB 1114 on operands OP0 and OP1, with result having machine-mode MODE. 1115 1116 UNSIGNEDP is for the case where we have to widen the operands 1117 to perform the operation. It says to use zero-extension. 1118 1119 If TARGET is nonzero, the value 1120 is generated there, if it is convenient to do so. 1121 In all cases an rtx is returned for the locus of the value; 1122 this may or may not be TARGET. */ 1123 1124 rtx 1125 expand_binop (machine_mode mode, optab binoptab, rtx op0, rtx op1, 1126 rtx target, int unsignedp, enum optab_methods methods) 1127 { 1128 enum optab_methods next_methods 1129 = (methods == OPTAB_LIB || methods == OPTAB_LIB_WIDEN 1130 ? OPTAB_WIDEN : methods); 1131 enum mode_class mclass; 1132 enum insn_code icode; 1133 machine_mode wider_mode; 1134 scalar_int_mode int_mode; 1135 rtx libfunc; 1136 rtx temp; 1137 rtx_insn *entry_last = get_last_insn (); 1138 rtx_insn *last; 1139 1140 mclass = GET_MODE_CLASS (mode); 1141 1142 /* If subtracting an integer constant, convert this into an addition of 1143 the negated constant. */ 1144 1145 if (binoptab == sub_optab && CONST_INT_P (op1)) 1146 { 1147 op1 = negate_rtx (mode, op1); 1148 binoptab = add_optab; 1149 } 1150 /* For shifts, constant invalid op1 might be expanded from different 1151 mode than MODE. As those are invalid, force them to a register 1152 to avoid further problems during expansion. */ 1153 else if (CONST_INT_P (op1) 1154 && shift_optab_p (binoptab) 1155 && UINTVAL (op1) >= GET_MODE_BITSIZE (GET_MODE_INNER (mode))) 1156 { 1157 op1 = gen_int_mode (INTVAL (op1), GET_MODE_INNER (mode)); 1158 op1 = force_reg (GET_MODE_INNER (mode), op1); 1159 } 1160 1161 /* Record where to delete back to if we backtrack. */ 1162 last = get_last_insn (); 1163 1164 /* If we can do it with a three-operand insn, do so. */ 1165 1166 if (methods != OPTAB_MUST_WIDEN) 1167 { 1168 if (convert_optab_p (binoptab)) 1169 { 1170 machine_mode from_mode = widened_mode (mode, op0, op1); 1171 icode = find_widening_optab_handler (binoptab, mode, from_mode); 1172 } 1173 else 1174 icode = optab_handler (binoptab, mode); 1175 if (icode != CODE_FOR_nothing) 1176 { 1177 temp = expand_binop_directly (icode, mode, binoptab, op0, op1, 1178 target, unsignedp, methods, last); 1179 if (temp) 1180 return temp; 1181 } 1182 } 1183 1184 /* If we were trying to rotate, and that didn't work, try rotating 1185 the other direction before falling back to shifts and bitwise-or. */ 1186 if (((binoptab == rotl_optab 1187 && (icode = optab_handler (rotr_optab, mode)) != CODE_FOR_nothing) 1188 || (binoptab == rotr_optab 1189 && (icode = optab_handler (rotl_optab, mode)) != CODE_FOR_nothing)) 1190 && is_int_mode (mode, &int_mode)) 1191 { 1192 optab otheroptab = (binoptab == rotl_optab ? rotr_optab : rotl_optab); 1193 rtx newop1; 1194 unsigned int bits = GET_MODE_PRECISION (int_mode); 1195 1196 if (CONST_INT_P (op1)) 1197 newop1 = gen_int_shift_amount (int_mode, bits - INTVAL (op1)); 1198 else if (targetm.shift_truncation_mask (int_mode) == bits - 1) 1199 newop1 = negate_rtx (GET_MODE (op1), op1); 1200 else 1201 newop1 = expand_binop (GET_MODE (op1), sub_optab, 1202 gen_int_mode (bits, GET_MODE (op1)), op1, 1203 NULL_RTX, unsignedp, OPTAB_DIRECT); 1204 1205 temp = expand_binop_directly (icode, int_mode, otheroptab, op0, newop1, 1206 target, unsignedp, methods, last); 1207 if (temp) 1208 return temp; 1209 } 1210 1211 /* If this is a multiply, see if we can do a widening operation that 1212 takes operands of this mode and makes a wider mode. */ 1213 1214 if (binoptab == smul_optab 1215 && GET_MODE_2XWIDER_MODE (mode).exists (&wider_mode) 1216 && (convert_optab_handler ((unsignedp 1217 ? umul_widen_optab 1218 : smul_widen_optab), 1219 wider_mode, mode) != CODE_FOR_nothing)) 1220 { 1221 /* *_widen_optab needs to determine operand mode, make sure at least 1222 one operand has non-VOID mode. */ 1223 if (GET_MODE (op0) == VOIDmode && GET_MODE (op1) == VOIDmode) 1224 op0 = force_reg (mode, op0); 1225 temp = expand_binop (wider_mode, 1226 unsignedp ? umul_widen_optab : smul_widen_optab, 1227 op0, op1, NULL_RTX, unsignedp, OPTAB_DIRECT); 1228 1229 if (temp != 0) 1230 { 1231 if (GET_MODE_CLASS (mode) == MODE_INT 1232 && TRULY_NOOP_TRUNCATION_MODES_P (mode, GET_MODE (temp))) 1233 return gen_lowpart (mode, temp); 1234 else 1235 return convert_to_mode (mode, temp, unsignedp); 1236 } 1237 } 1238 1239 /* If this is a vector shift by a scalar, see if we can do a vector 1240 shift by a vector. If so, broadcast the scalar into a vector. */ 1241 if (mclass == MODE_VECTOR_INT) 1242 { 1243 optab otheroptab = unknown_optab; 1244 1245 if (binoptab == ashl_optab) 1246 otheroptab = vashl_optab; 1247 else if (binoptab == ashr_optab) 1248 otheroptab = vashr_optab; 1249 else if (binoptab == lshr_optab) 1250 otheroptab = vlshr_optab; 1251 else if (binoptab == rotl_optab) 1252 otheroptab = vrotl_optab; 1253 else if (binoptab == rotr_optab) 1254 otheroptab = vrotr_optab; 1255 1256 if (otheroptab 1257 && (icode = optab_handler (otheroptab, mode)) != CODE_FOR_nothing) 1258 { 1259 /* The scalar may have been extended to be too wide. Truncate 1260 it back to the proper size to fit in the broadcast vector. */ 1261 scalar_mode inner_mode = GET_MODE_INNER (mode); 1262 if (!CONST_INT_P (op1) 1263 && (GET_MODE_BITSIZE (as_a <scalar_int_mode> (GET_MODE (op1))) 1264 > GET_MODE_BITSIZE (inner_mode))) 1265 op1 = force_reg (inner_mode, 1266 simplify_gen_unary (TRUNCATE, inner_mode, op1, 1267 GET_MODE (op1))); 1268 rtx vop1 = expand_vector_broadcast (mode, op1); 1269 if (vop1) 1270 { 1271 temp = expand_binop_directly (icode, mode, otheroptab, op0, vop1, 1272 target, unsignedp, methods, last); 1273 if (temp) 1274 return temp; 1275 } 1276 } 1277 } 1278 1279 /* Look for a wider mode of the same class for which we think we 1280 can open-code the operation. Check for a widening multiply at the 1281 wider mode as well. */ 1282 1283 if (CLASS_HAS_WIDER_MODES_P (mclass) 1284 && methods != OPTAB_DIRECT && methods != OPTAB_LIB) 1285 FOR_EACH_WIDER_MODE (wider_mode, mode) 1286 { 1287 machine_mode next_mode; 1288 if (optab_handler (binoptab, wider_mode) != CODE_FOR_nothing 1289 || (binoptab == smul_optab 1290 && GET_MODE_WIDER_MODE (wider_mode).exists (&next_mode) 1291 && (find_widening_optab_handler ((unsignedp 1292 ? umul_widen_optab 1293 : smul_widen_optab), 1294 next_mode, mode) 1295 != CODE_FOR_nothing))) 1296 { 1297 rtx xop0 = op0, xop1 = op1; 1298 int no_extend = 0; 1299 1300 /* For certain integer operations, we need not actually extend 1301 the narrow operands, as long as we will truncate 1302 the results to the same narrowness. */ 1303 1304 if ((binoptab == ior_optab || binoptab == and_optab 1305 || binoptab == xor_optab 1306 || binoptab == add_optab || binoptab == sub_optab 1307 || binoptab == smul_optab || binoptab == ashl_optab) 1308 && mclass == MODE_INT) 1309 { 1310 no_extend = 1; 1311 xop0 = avoid_expensive_constant (mode, binoptab, 0, 1312 xop0, unsignedp); 1313 if (binoptab != ashl_optab) 1314 xop1 = avoid_expensive_constant (mode, binoptab, 1, 1315 xop1, unsignedp); 1316 } 1317 1318 xop0 = widen_operand (xop0, wider_mode, mode, unsignedp, no_extend); 1319 1320 /* The second operand of a shift must always be extended. */ 1321 xop1 = widen_operand (xop1, wider_mode, mode, unsignedp, 1322 no_extend && binoptab != ashl_optab); 1323 1324 temp = expand_binop (wider_mode, binoptab, xop0, xop1, NULL_RTX, 1325 unsignedp, OPTAB_DIRECT); 1326 if (temp) 1327 { 1328 if (mclass != MODE_INT 1329 || !TRULY_NOOP_TRUNCATION_MODES_P (mode, wider_mode)) 1330 { 1331 if (target == 0) 1332 target = gen_reg_rtx (mode); 1333 convert_move (target, temp, 0); 1334 return target; 1335 } 1336 else 1337 return gen_lowpart (mode, temp); 1338 } 1339 else 1340 delete_insns_since (last); 1341 } 1342 } 1343 1344 /* If operation is commutative, 1345 try to make the first operand a register. 1346 Even better, try to make it the same as the target. 1347 Also try to make the last operand a constant. */ 1348 if (commutative_optab_p (binoptab) 1349 && swap_commutative_operands_with_target (target, op0, op1)) 1350 std::swap (op0, op1); 1351 1352 /* These can be done a word at a time. */ 1353 if ((binoptab == and_optab || binoptab == ior_optab || binoptab == xor_optab) 1354 && is_int_mode (mode, &int_mode) 1355 && GET_MODE_SIZE (int_mode) > UNITS_PER_WORD 1356 && optab_handler (binoptab, word_mode) != CODE_FOR_nothing) 1357 { 1358 int i; 1359 rtx_insn *insns; 1360 1361 /* If TARGET is the same as one of the operands, the REG_EQUAL note 1362 won't be accurate, so use a new target. */ 1363 if (target == 0 1364 || target == op0 1365 || target == op1 1366 || !valid_multiword_target_p (target)) 1367 target = gen_reg_rtx (int_mode); 1368 1369 start_sequence (); 1370 1371 /* Do the actual arithmetic. */ 1372 for (i = 0; i < GET_MODE_BITSIZE (int_mode) / BITS_PER_WORD; i++) 1373 { 1374 rtx target_piece = operand_subword (target, i, 1, int_mode); 1375 rtx x = expand_binop (word_mode, binoptab, 1376 operand_subword_force (op0, i, int_mode), 1377 operand_subword_force (op1, i, int_mode), 1378 target_piece, unsignedp, next_methods); 1379 1380 if (x == 0) 1381 break; 1382 1383 if (target_piece != x) 1384 emit_move_insn (target_piece, x); 1385 } 1386 1387 insns = get_insns (); 1388 end_sequence (); 1389 1390 if (i == GET_MODE_BITSIZE (int_mode) / BITS_PER_WORD) 1391 { 1392 emit_insn (insns); 1393 return target; 1394 } 1395 } 1396 1397 /* Synthesize double word shifts from single word shifts. */ 1398 if ((binoptab == lshr_optab || binoptab == ashl_optab 1399 || binoptab == ashr_optab) 1400 && is_int_mode (mode, &int_mode) 1401 && (CONST_INT_P (op1) || optimize_insn_for_speed_p ()) 1402 && GET_MODE_SIZE (int_mode) == 2 * UNITS_PER_WORD 1403 && GET_MODE_PRECISION (int_mode) == GET_MODE_BITSIZE (int_mode) 1404 && optab_handler (binoptab, word_mode) != CODE_FOR_nothing 1405 && optab_handler (ashl_optab, word_mode) != CODE_FOR_nothing 1406 && optab_handler (lshr_optab, word_mode) != CODE_FOR_nothing) 1407 { 1408 unsigned HOST_WIDE_INT shift_mask, double_shift_mask; 1409 scalar_int_mode op1_mode; 1410 1411 double_shift_mask = targetm.shift_truncation_mask (int_mode); 1412 shift_mask = targetm.shift_truncation_mask (word_mode); 1413 op1_mode = (GET_MODE (op1) != VOIDmode 1414 ? as_a <scalar_int_mode> (GET_MODE (op1)) 1415 : word_mode); 1416 1417 /* Apply the truncation to constant shifts. */ 1418 if (double_shift_mask > 0 && CONST_INT_P (op1)) 1419 op1 = gen_int_mode (INTVAL (op1) & double_shift_mask, op1_mode); 1420 1421 if (op1 == CONST0_RTX (op1_mode)) 1422 return op0; 1423 1424 /* Make sure that this is a combination that expand_doubleword_shift 1425 can handle. See the comments there for details. */ 1426 if (double_shift_mask == 0 1427 || (shift_mask == BITS_PER_WORD - 1 1428 && double_shift_mask == BITS_PER_WORD * 2 - 1)) 1429 { 1430 rtx_insn *insns; 1431 rtx into_target, outof_target; 1432 rtx into_input, outof_input; 1433 int left_shift, outof_word; 1434 1435 /* If TARGET is the same as one of the operands, the REG_EQUAL note 1436 won't be accurate, so use a new target. */ 1437 if (target == 0 1438 || target == op0 1439 || target == op1 1440 || !valid_multiword_target_p (target)) 1441 target = gen_reg_rtx (int_mode); 1442 1443 start_sequence (); 1444 1445 /* OUTOF_* is the word we are shifting bits away from, and 1446 INTO_* is the word that we are shifting bits towards, thus 1447 they differ depending on the direction of the shift and 1448 WORDS_BIG_ENDIAN. */ 1449 1450 left_shift = binoptab == ashl_optab; 1451 outof_word = left_shift ^ ! WORDS_BIG_ENDIAN; 1452 1453 outof_target = operand_subword (target, outof_word, 1, int_mode); 1454 into_target = operand_subword (target, 1 - outof_word, 1, int_mode); 1455 1456 outof_input = operand_subword_force (op0, outof_word, int_mode); 1457 into_input = operand_subword_force (op0, 1 - outof_word, int_mode); 1458 1459 if (expand_doubleword_shift (op1_mode, binoptab, 1460 outof_input, into_input, op1, 1461 outof_target, into_target, 1462 unsignedp, next_methods, shift_mask)) 1463 { 1464 insns = get_insns (); 1465 end_sequence (); 1466 1467 emit_insn (insns); 1468 return target; 1469 } 1470 end_sequence (); 1471 } 1472 } 1473 1474 /* Synthesize double word rotates from single word shifts. */ 1475 if ((binoptab == rotl_optab || binoptab == rotr_optab) 1476 && is_int_mode (mode, &int_mode) 1477 && CONST_INT_P (op1) 1478 && GET_MODE_PRECISION (int_mode) == 2 * BITS_PER_WORD 1479 && optab_handler (ashl_optab, word_mode) != CODE_FOR_nothing 1480 && optab_handler (lshr_optab, word_mode) != CODE_FOR_nothing) 1481 { 1482 rtx_insn *insns; 1483 rtx into_target, outof_target; 1484 rtx into_input, outof_input; 1485 rtx inter; 1486 int shift_count, left_shift, outof_word; 1487 1488 /* If TARGET is the same as one of the operands, the REG_EQUAL note 1489 won't be accurate, so use a new target. Do this also if target is not 1490 a REG, first because having a register instead may open optimization 1491 opportunities, and second because if target and op0 happen to be MEMs 1492 designating the same location, we would risk clobbering it too early 1493 in the code sequence we generate below. */ 1494 if (target == 0 1495 || target == op0 1496 || target == op1 1497 || !REG_P (target) 1498 || !valid_multiword_target_p (target)) 1499 target = gen_reg_rtx (int_mode); 1500 1501 start_sequence (); 1502 1503 shift_count = INTVAL (op1); 1504 1505 /* OUTOF_* is the word we are shifting bits away from, and 1506 INTO_* is the word that we are shifting bits towards, thus 1507 they differ depending on the direction of the shift and 1508 WORDS_BIG_ENDIAN. */ 1509 1510 left_shift = (binoptab == rotl_optab); 1511 outof_word = left_shift ^ ! WORDS_BIG_ENDIAN; 1512 1513 outof_target = operand_subword (target, outof_word, 1, int_mode); 1514 into_target = operand_subword (target, 1 - outof_word, 1, int_mode); 1515 1516 outof_input = operand_subword_force (op0, outof_word, int_mode); 1517 into_input = operand_subword_force (op0, 1 - outof_word, int_mode); 1518 1519 if (shift_count == BITS_PER_WORD) 1520 { 1521 /* This is just a word swap. */ 1522 emit_move_insn (outof_target, into_input); 1523 emit_move_insn (into_target, outof_input); 1524 inter = const0_rtx; 1525 } 1526 else 1527 { 1528 rtx into_temp1, into_temp2, outof_temp1, outof_temp2; 1529 HOST_WIDE_INT first_shift_count, second_shift_count; 1530 optab reverse_unsigned_shift, unsigned_shift; 1531 1532 reverse_unsigned_shift = (left_shift ^ (shift_count < BITS_PER_WORD) 1533 ? lshr_optab : ashl_optab); 1534 1535 unsigned_shift = (left_shift ^ (shift_count < BITS_PER_WORD) 1536 ? ashl_optab : lshr_optab); 1537 1538 if (shift_count > BITS_PER_WORD) 1539 { 1540 first_shift_count = shift_count - BITS_PER_WORD; 1541 second_shift_count = 2 * BITS_PER_WORD - shift_count; 1542 } 1543 else 1544 { 1545 first_shift_count = BITS_PER_WORD - shift_count; 1546 second_shift_count = shift_count; 1547 } 1548 rtx first_shift_count_rtx 1549 = gen_int_shift_amount (word_mode, first_shift_count); 1550 rtx second_shift_count_rtx 1551 = gen_int_shift_amount (word_mode, second_shift_count); 1552 1553 into_temp1 = expand_binop (word_mode, unsigned_shift, 1554 outof_input, first_shift_count_rtx, 1555 NULL_RTX, unsignedp, next_methods); 1556 into_temp2 = expand_binop (word_mode, reverse_unsigned_shift, 1557 into_input, second_shift_count_rtx, 1558 NULL_RTX, unsignedp, next_methods); 1559 1560 if (into_temp1 != 0 && into_temp2 != 0) 1561 inter = expand_binop (word_mode, ior_optab, into_temp1, into_temp2, 1562 into_target, unsignedp, next_methods); 1563 else 1564 inter = 0; 1565 1566 if (inter != 0 && inter != into_target) 1567 emit_move_insn (into_target, inter); 1568 1569 outof_temp1 = expand_binop (word_mode, unsigned_shift, 1570 into_input, first_shift_count_rtx, 1571 NULL_RTX, unsignedp, next_methods); 1572 outof_temp2 = expand_binop (word_mode, reverse_unsigned_shift, 1573 outof_input, second_shift_count_rtx, 1574 NULL_RTX, unsignedp, next_methods); 1575 1576 if (inter != 0 && outof_temp1 != 0 && outof_temp2 != 0) 1577 inter = expand_binop (word_mode, ior_optab, 1578 outof_temp1, outof_temp2, 1579 outof_target, unsignedp, next_methods); 1580 1581 if (inter != 0 && inter != outof_target) 1582 emit_move_insn (outof_target, inter); 1583 } 1584 1585 insns = get_insns (); 1586 end_sequence (); 1587 1588 if (inter != 0) 1589 { 1590 emit_insn (insns); 1591 return target; 1592 } 1593 } 1594 1595 /* These can be done a word at a time by propagating carries. */ 1596 if ((binoptab == add_optab || binoptab == sub_optab) 1597 && is_int_mode (mode, &int_mode) 1598 && GET_MODE_SIZE (int_mode) >= 2 * UNITS_PER_WORD 1599 && optab_handler (binoptab, word_mode) != CODE_FOR_nothing) 1600 { 1601 unsigned int i; 1602 optab otheroptab = binoptab == add_optab ? sub_optab : add_optab; 1603 const unsigned int nwords = GET_MODE_BITSIZE (int_mode) / BITS_PER_WORD; 1604 rtx carry_in = NULL_RTX, carry_out = NULL_RTX; 1605 rtx xop0, xop1, xtarget; 1606 1607 /* We can handle either a 1 or -1 value for the carry. If STORE_FLAG 1608 value is one of those, use it. Otherwise, use 1 since it is the 1609 one easiest to get. */ 1610 #if STORE_FLAG_VALUE == 1 || STORE_FLAG_VALUE == -1 1611 int normalizep = STORE_FLAG_VALUE; 1612 #else 1613 int normalizep = 1; 1614 #endif 1615 1616 /* Prepare the operands. */ 1617 xop0 = force_reg (int_mode, op0); 1618 xop1 = force_reg (int_mode, op1); 1619 1620 xtarget = gen_reg_rtx (int_mode); 1621 1622 if (target == 0 || !REG_P (target) || !valid_multiword_target_p (target)) 1623 target = xtarget; 1624 1625 /* Indicate for flow that the entire target reg is being set. */ 1626 if (REG_P (target)) 1627 emit_clobber (xtarget); 1628 1629 /* Do the actual arithmetic. */ 1630 for (i = 0; i < nwords; i++) 1631 { 1632 int index = (WORDS_BIG_ENDIAN ? nwords - i - 1 : i); 1633 rtx target_piece = operand_subword (xtarget, index, 1, int_mode); 1634 rtx op0_piece = operand_subword_force (xop0, index, int_mode); 1635 rtx op1_piece = operand_subword_force (xop1, index, int_mode); 1636 rtx x; 1637 1638 /* Main add/subtract of the input operands. */ 1639 x = expand_binop (word_mode, binoptab, 1640 op0_piece, op1_piece, 1641 target_piece, unsignedp, next_methods); 1642 if (x == 0) 1643 break; 1644 1645 if (i + 1 < nwords) 1646 { 1647 /* Store carry from main add/subtract. */ 1648 carry_out = gen_reg_rtx (word_mode); 1649 carry_out = emit_store_flag_force (carry_out, 1650 (binoptab == add_optab 1651 ? LT : GT), 1652 x, op0_piece, 1653 word_mode, 1, normalizep); 1654 } 1655 1656 if (i > 0) 1657 { 1658 rtx newx; 1659 1660 /* Add/subtract previous carry to main result. */ 1661 newx = expand_binop (word_mode, 1662 normalizep == 1 ? binoptab : otheroptab, 1663 x, carry_in, 1664 NULL_RTX, 1, next_methods); 1665 1666 if (i + 1 < nwords) 1667 { 1668 /* Get out carry from adding/subtracting carry in. */ 1669 rtx carry_tmp = gen_reg_rtx (word_mode); 1670 carry_tmp = emit_store_flag_force (carry_tmp, 1671 (binoptab == add_optab 1672 ? LT : GT), 1673 newx, x, 1674 word_mode, 1, normalizep); 1675 1676 /* Logical-ior the two poss. carry together. */ 1677 carry_out = expand_binop (word_mode, ior_optab, 1678 carry_out, carry_tmp, 1679 carry_out, 0, next_methods); 1680 if (carry_out == 0) 1681 break; 1682 } 1683 emit_move_insn (target_piece, newx); 1684 } 1685 else 1686 { 1687 if (x != target_piece) 1688 emit_move_insn (target_piece, x); 1689 } 1690 1691 carry_in = carry_out; 1692 } 1693 1694 if (i == GET_MODE_BITSIZE (int_mode) / (unsigned) BITS_PER_WORD) 1695 { 1696 if (optab_handler (mov_optab, int_mode) != CODE_FOR_nothing 1697 || ! rtx_equal_p (target, xtarget)) 1698 { 1699 rtx_insn *temp = emit_move_insn (target, xtarget); 1700 1701 set_dst_reg_note (temp, REG_EQUAL, 1702 gen_rtx_fmt_ee (optab_to_code (binoptab), 1703 int_mode, copy_rtx (xop0), 1704 copy_rtx (xop1)), 1705 target); 1706 } 1707 else 1708 target = xtarget; 1709 1710 return target; 1711 } 1712 1713 else 1714 delete_insns_since (last); 1715 } 1716 1717 /* Attempt to synthesize double word multiplies using a sequence of word 1718 mode multiplications. We first attempt to generate a sequence using a 1719 more efficient unsigned widening multiply, and if that fails we then 1720 try using a signed widening multiply. */ 1721 1722 if (binoptab == smul_optab 1723 && is_int_mode (mode, &int_mode) 1724 && GET_MODE_SIZE (int_mode) == 2 * UNITS_PER_WORD 1725 && optab_handler (smul_optab, word_mode) != CODE_FOR_nothing 1726 && optab_handler (add_optab, word_mode) != CODE_FOR_nothing) 1727 { 1728 rtx product = NULL_RTX; 1729 if (convert_optab_handler (umul_widen_optab, int_mode, word_mode) 1730 != CODE_FOR_nothing) 1731 { 1732 product = expand_doubleword_mult (int_mode, op0, op1, target, 1733 true, methods); 1734 if (!product) 1735 delete_insns_since (last); 1736 } 1737 1738 if (product == NULL_RTX 1739 && (convert_optab_handler (smul_widen_optab, int_mode, word_mode) 1740 != CODE_FOR_nothing)) 1741 { 1742 product = expand_doubleword_mult (int_mode, op0, op1, target, 1743 false, methods); 1744 if (!product) 1745 delete_insns_since (last); 1746 } 1747 1748 if (product != NULL_RTX) 1749 { 1750 if (optab_handler (mov_optab, int_mode) != CODE_FOR_nothing) 1751 { 1752 rtx_insn *move = emit_move_insn (target ? target : product, 1753 product); 1754 set_dst_reg_note (move, 1755 REG_EQUAL, 1756 gen_rtx_fmt_ee (MULT, int_mode, 1757 copy_rtx (op0), 1758 copy_rtx (op1)), 1759 target ? target : product); 1760 } 1761 return product; 1762 } 1763 } 1764 1765 /* It can't be open-coded in this mode. 1766 Use a library call if one is available and caller says that's ok. */ 1767 1768 libfunc = optab_libfunc (binoptab, mode); 1769 if (libfunc 1770 && (methods == OPTAB_LIB || methods == OPTAB_LIB_WIDEN)) 1771 { 1772 rtx_insn *insns; 1773 rtx op1x = op1; 1774 machine_mode op1_mode = mode; 1775 rtx value; 1776 1777 start_sequence (); 1778 1779 if (shift_optab_p (binoptab)) 1780 { 1781 op1_mode = targetm.libgcc_shift_count_mode (); 1782 /* Specify unsigned here, 1783 since negative shift counts are meaningless. */ 1784 op1x = convert_to_mode (op1_mode, op1, 1); 1785 } 1786 1787 if (GET_MODE (op0) != VOIDmode 1788 && GET_MODE (op0) != mode) 1789 op0 = convert_to_mode (mode, op0, unsignedp); 1790 1791 /* Pass 1 for NO_QUEUE so we don't lose any increments 1792 if the libcall is cse'd or moved. */ 1793 value = emit_library_call_value (libfunc, 1794 NULL_RTX, LCT_CONST, mode, 1795 op0, mode, op1x, op1_mode); 1796 1797 insns = get_insns (); 1798 end_sequence (); 1799 1800 bool trapv = trapv_binoptab_p (binoptab); 1801 target = gen_reg_rtx (mode); 1802 emit_libcall_block_1 (insns, target, value, 1803 trapv ? NULL_RTX 1804 : gen_rtx_fmt_ee (optab_to_code (binoptab), 1805 mode, op0, op1), trapv); 1806 1807 return target; 1808 } 1809 1810 delete_insns_since (last); 1811 1812 /* It can't be done in this mode. Can we do it in a wider mode? */ 1813 1814 if (! (methods == OPTAB_WIDEN || methods == OPTAB_LIB_WIDEN 1815 || methods == OPTAB_MUST_WIDEN)) 1816 { 1817 /* Caller says, don't even try. */ 1818 delete_insns_since (entry_last); 1819 return 0; 1820 } 1821 1822 /* Compute the value of METHODS to pass to recursive calls. 1823 Don't allow widening to be tried recursively. */ 1824 1825 methods = (methods == OPTAB_LIB_WIDEN ? OPTAB_LIB : OPTAB_DIRECT); 1826 1827 /* Look for a wider mode of the same class for which it appears we can do 1828 the operation. */ 1829 1830 if (CLASS_HAS_WIDER_MODES_P (mclass)) 1831 { 1832 /* This code doesn't make sense for conversion optabs, since we 1833 wouldn't then want to extend the operands to be the same size 1834 as the result. */ 1835 gcc_assert (!convert_optab_p (binoptab)); 1836 FOR_EACH_WIDER_MODE (wider_mode, mode) 1837 { 1838 if (optab_handler (binoptab, wider_mode) 1839 || (methods == OPTAB_LIB 1840 && optab_libfunc (binoptab, wider_mode))) 1841 { 1842 rtx xop0 = op0, xop1 = op1; 1843 int no_extend = 0; 1844 1845 /* For certain integer operations, we need not actually extend 1846 the narrow operands, as long as we will truncate 1847 the results to the same narrowness. */ 1848 1849 if ((binoptab == ior_optab || binoptab == and_optab 1850 || binoptab == xor_optab 1851 || binoptab == add_optab || binoptab == sub_optab 1852 || binoptab == smul_optab || binoptab == ashl_optab) 1853 && mclass == MODE_INT) 1854 no_extend = 1; 1855 1856 xop0 = widen_operand (xop0, wider_mode, mode, 1857 unsignedp, no_extend); 1858 1859 /* The second operand of a shift must always be extended. */ 1860 xop1 = widen_operand (xop1, wider_mode, mode, unsignedp, 1861 no_extend && binoptab != ashl_optab); 1862 1863 temp = expand_binop (wider_mode, binoptab, xop0, xop1, NULL_RTX, 1864 unsignedp, methods); 1865 if (temp) 1866 { 1867 if (mclass != MODE_INT 1868 || !TRULY_NOOP_TRUNCATION_MODES_P (mode, wider_mode)) 1869 { 1870 if (target == 0) 1871 target = gen_reg_rtx (mode); 1872 convert_move (target, temp, 0); 1873 return target; 1874 } 1875 else 1876 return gen_lowpart (mode, temp); 1877 } 1878 else 1879 delete_insns_since (last); 1880 } 1881 } 1882 } 1883 1884 delete_insns_since (entry_last); 1885 return 0; 1886 } 1887 1888 /* Expand a binary operator which has both signed and unsigned forms. 1889 UOPTAB is the optab for unsigned operations, and SOPTAB is for 1890 signed operations. 1891 1892 If we widen unsigned operands, we may use a signed wider operation instead 1893 of an unsigned wider operation, since the result would be the same. */ 1894 1895 rtx 1896 sign_expand_binop (machine_mode mode, optab uoptab, optab soptab, 1897 rtx op0, rtx op1, rtx target, int unsignedp, 1898 enum optab_methods methods) 1899 { 1900 rtx temp; 1901 optab direct_optab = unsignedp ? uoptab : soptab; 1902 bool save_enable; 1903 1904 /* Do it without widening, if possible. */ 1905 temp = expand_binop (mode, direct_optab, op0, op1, target, 1906 unsignedp, OPTAB_DIRECT); 1907 if (temp || methods == OPTAB_DIRECT) 1908 return temp; 1909 1910 /* Try widening to a signed int. Disable any direct use of any 1911 signed insn in the current mode. */ 1912 save_enable = swap_optab_enable (soptab, mode, false); 1913 1914 temp = expand_binop (mode, soptab, op0, op1, target, 1915 unsignedp, OPTAB_WIDEN); 1916 1917 /* For unsigned operands, try widening to an unsigned int. */ 1918 if (!temp && unsignedp) 1919 temp = expand_binop (mode, uoptab, op0, op1, target, 1920 unsignedp, OPTAB_WIDEN); 1921 if (temp || methods == OPTAB_WIDEN) 1922 goto egress; 1923 1924 /* Use the right width libcall if that exists. */ 1925 temp = expand_binop (mode, direct_optab, op0, op1, target, 1926 unsignedp, OPTAB_LIB); 1927 if (temp || methods == OPTAB_LIB) 1928 goto egress; 1929 1930 /* Must widen and use a libcall, use either signed or unsigned. */ 1931 temp = expand_binop (mode, soptab, op0, op1, target, 1932 unsignedp, methods); 1933 if (!temp && unsignedp) 1934 temp = expand_binop (mode, uoptab, op0, op1, target, 1935 unsignedp, methods); 1936 1937 egress: 1938 /* Undo the fiddling above. */ 1939 if (save_enable) 1940 swap_optab_enable (soptab, mode, true); 1941 return temp; 1942 } 1943 1944 /* Generate code to perform an operation specified by UNOPPTAB 1945 on operand OP0, with two results to TARG0 and TARG1. 1946 We assume that the order of the operands for the instruction 1947 is TARG0, TARG1, OP0. 1948 1949 Either TARG0 or TARG1 may be zero, but what that means is that 1950 the result is not actually wanted. We will generate it into 1951 a dummy pseudo-reg and discard it. They may not both be zero. 1952 1953 Returns 1 if this operation can be performed; 0 if not. */ 1954 1955 int 1956 expand_twoval_unop (optab unoptab, rtx op0, rtx targ0, rtx targ1, 1957 int unsignedp) 1958 { 1959 machine_mode mode = GET_MODE (targ0 ? targ0 : targ1); 1960 enum mode_class mclass; 1961 machine_mode wider_mode; 1962 rtx_insn *entry_last = get_last_insn (); 1963 rtx_insn *last; 1964 1965 mclass = GET_MODE_CLASS (mode); 1966 1967 if (!targ0) 1968 targ0 = gen_reg_rtx (mode); 1969 if (!targ1) 1970 targ1 = gen_reg_rtx (mode); 1971 1972 /* Record where to go back to if we fail. */ 1973 last = get_last_insn (); 1974 1975 if (optab_handler (unoptab, mode) != CODE_FOR_nothing) 1976 { 1977 struct expand_operand ops[3]; 1978 enum insn_code icode = optab_handler (unoptab, mode); 1979 1980 create_fixed_operand (&ops[0], targ0); 1981 create_fixed_operand (&ops[1], targ1); 1982 create_convert_operand_from (&ops[2], op0, mode, unsignedp); 1983 if (maybe_expand_insn (icode, 3, ops)) 1984 return 1; 1985 } 1986 1987 /* It can't be done in this mode. Can we do it in a wider mode? */ 1988 1989 if (CLASS_HAS_WIDER_MODES_P (mclass)) 1990 { 1991 FOR_EACH_WIDER_MODE (wider_mode, mode) 1992 { 1993 if (optab_handler (unoptab, wider_mode) != CODE_FOR_nothing) 1994 { 1995 rtx t0 = gen_reg_rtx (wider_mode); 1996 rtx t1 = gen_reg_rtx (wider_mode); 1997 rtx cop0 = convert_modes (wider_mode, mode, op0, unsignedp); 1998 1999 if (expand_twoval_unop (unoptab, cop0, t0, t1, unsignedp)) 2000 { 2001 convert_move (targ0, t0, unsignedp); 2002 convert_move (targ1, t1, unsignedp); 2003 return 1; 2004 } 2005 else 2006 delete_insns_since (last); 2007 } 2008 } 2009 } 2010 2011 delete_insns_since (entry_last); 2012 return 0; 2013 } 2014 2015 /* Generate code to perform an operation specified by BINOPTAB 2016 on operands OP0 and OP1, with two results to TARG1 and TARG2. 2017 We assume that the order of the operands for the instruction 2018 is TARG0, OP0, OP1, TARG1, which would fit a pattern like 2019 [(set TARG0 (operate OP0 OP1)) (set TARG1 (operate ...))]. 2020 2021 Either TARG0 or TARG1 may be zero, but what that means is that 2022 the result is not actually wanted. We will generate it into 2023 a dummy pseudo-reg and discard it. They may not both be zero. 2024 2025 Returns 1 if this operation can be performed; 0 if not. */ 2026 2027 int 2028 expand_twoval_binop (optab binoptab, rtx op0, rtx op1, rtx targ0, rtx targ1, 2029 int unsignedp) 2030 { 2031 machine_mode mode = GET_MODE (targ0 ? targ0 : targ1); 2032 enum mode_class mclass; 2033 machine_mode wider_mode; 2034 rtx_insn *entry_last = get_last_insn (); 2035 rtx_insn *last; 2036 2037 mclass = GET_MODE_CLASS (mode); 2038 2039 if (!targ0) 2040 targ0 = gen_reg_rtx (mode); 2041 if (!targ1) 2042 targ1 = gen_reg_rtx (mode); 2043 2044 /* Record where to go back to if we fail. */ 2045 last = get_last_insn (); 2046 2047 if (optab_handler (binoptab, mode) != CODE_FOR_nothing) 2048 { 2049 struct expand_operand ops[4]; 2050 enum insn_code icode = optab_handler (binoptab, mode); 2051 machine_mode mode0 = insn_data[icode].operand[1].mode; 2052 machine_mode mode1 = insn_data[icode].operand[2].mode; 2053 rtx xop0 = op0, xop1 = op1; 2054 2055 /* If we are optimizing, force expensive constants into a register. */ 2056 xop0 = avoid_expensive_constant (mode0, binoptab, 0, xop0, unsignedp); 2057 xop1 = avoid_expensive_constant (mode1, binoptab, 1, xop1, unsignedp); 2058 2059 create_fixed_operand (&ops[0], targ0); 2060 create_convert_operand_from (&ops[1], op0, mode, unsignedp); 2061 create_convert_operand_from (&ops[2], op1, mode, unsignedp); 2062 create_fixed_operand (&ops[3], targ1); 2063 if (maybe_expand_insn (icode, 4, ops)) 2064 return 1; 2065 delete_insns_since (last); 2066 } 2067 2068 /* It can't be done in this mode. Can we do it in a wider mode? */ 2069 2070 if (CLASS_HAS_WIDER_MODES_P (mclass)) 2071 { 2072 FOR_EACH_WIDER_MODE (wider_mode, mode) 2073 { 2074 if (optab_handler (binoptab, wider_mode) != CODE_FOR_nothing) 2075 { 2076 rtx t0 = gen_reg_rtx (wider_mode); 2077 rtx t1 = gen_reg_rtx (wider_mode); 2078 rtx cop0 = convert_modes (wider_mode, mode, op0, unsignedp); 2079 rtx cop1 = convert_modes (wider_mode, mode, op1, unsignedp); 2080 2081 if (expand_twoval_binop (binoptab, cop0, cop1, 2082 t0, t1, unsignedp)) 2083 { 2084 convert_move (targ0, t0, unsignedp); 2085 convert_move (targ1, t1, unsignedp); 2086 return 1; 2087 } 2088 else 2089 delete_insns_since (last); 2090 } 2091 } 2092 } 2093 2094 delete_insns_since (entry_last); 2095 return 0; 2096 } 2097 2098 /* Expand the two-valued library call indicated by BINOPTAB, but 2099 preserve only one of the values. If TARG0 is non-NULL, the first 2100 value is placed into TARG0; otherwise the second value is placed 2101 into TARG1. Exactly one of TARG0 and TARG1 must be non-NULL. The 2102 value stored into TARG0 or TARG1 is equivalent to (CODE OP0 OP1). 2103 This routine assumes that the value returned by the library call is 2104 as if the return value was of an integral mode twice as wide as the 2105 mode of OP0. Returns 1 if the call was successful. */ 2106 2107 bool 2108 expand_twoval_binop_libfunc (optab binoptab, rtx op0, rtx op1, 2109 rtx targ0, rtx targ1, enum rtx_code code) 2110 { 2111 machine_mode mode; 2112 machine_mode libval_mode; 2113 rtx libval; 2114 rtx_insn *insns; 2115 rtx libfunc; 2116 2117 /* Exactly one of TARG0 or TARG1 should be non-NULL. */ 2118 gcc_assert (!targ0 != !targ1); 2119 2120 mode = GET_MODE (op0); 2121 libfunc = optab_libfunc (binoptab, mode); 2122 if (!libfunc) 2123 return false; 2124 2125 /* The value returned by the library function will have twice as 2126 many bits as the nominal MODE. */ 2127 libval_mode = smallest_int_mode_for_size (2 * GET_MODE_BITSIZE (mode)); 2128 start_sequence (); 2129 libval = emit_library_call_value (libfunc, NULL_RTX, LCT_CONST, 2130 libval_mode, 2131 op0, mode, 2132 op1, mode); 2133 /* Get the part of VAL containing the value that we want. */ 2134 libval = simplify_gen_subreg (mode, libval, libval_mode, 2135 targ0 ? 0 : GET_MODE_SIZE (mode)); 2136 insns = get_insns (); 2137 end_sequence (); 2138 /* Move the into the desired location. */ 2139 emit_libcall_block (insns, targ0 ? targ0 : targ1, libval, 2140 gen_rtx_fmt_ee (code, mode, op0, op1)); 2141 2142 return true; 2143 } 2144 2145 2146 /* Wrapper around expand_unop which takes an rtx code to specify 2147 the operation to perform, not an optab pointer. All other 2148 arguments are the same. */ 2149 rtx 2150 expand_simple_unop (machine_mode mode, enum rtx_code code, rtx op0, 2151 rtx target, int unsignedp) 2152 { 2153 optab unop = code_to_optab (code); 2154 gcc_assert (unop); 2155 2156 return expand_unop (mode, unop, op0, target, unsignedp); 2157 } 2158 2159 /* Try calculating 2160 (clz:narrow x) 2161 as 2162 (clz:wide (zero_extend:wide x)) - ((width wide) - (width narrow)). 2163 2164 A similar operation can be used for clrsb. UNOPTAB says which operation 2165 we are trying to expand. */ 2166 static rtx 2167 widen_leading (scalar_int_mode mode, rtx op0, rtx target, optab unoptab) 2168 { 2169 opt_scalar_int_mode wider_mode_iter; 2170 FOR_EACH_WIDER_MODE (wider_mode_iter, mode) 2171 { 2172 scalar_int_mode wider_mode = wider_mode_iter.require (); 2173 if (optab_handler (unoptab, wider_mode) != CODE_FOR_nothing) 2174 { 2175 rtx xop0, temp; 2176 rtx_insn *last; 2177 2178 last = get_last_insn (); 2179 2180 if (target == 0) 2181 target = gen_reg_rtx (mode); 2182 xop0 = widen_operand (op0, wider_mode, mode, 2183 unoptab != clrsb_optab, false); 2184 temp = expand_unop (wider_mode, unoptab, xop0, NULL_RTX, 2185 unoptab != clrsb_optab); 2186 if (temp != 0) 2187 temp = expand_binop 2188 (wider_mode, sub_optab, temp, 2189 gen_int_mode (GET_MODE_PRECISION (wider_mode) 2190 - GET_MODE_PRECISION (mode), 2191 wider_mode), 2192 target, true, OPTAB_DIRECT); 2193 if (temp == 0) 2194 delete_insns_since (last); 2195 2196 return temp; 2197 } 2198 } 2199 return 0; 2200 } 2201 2202 /* Try calculating clz of a double-word quantity as two clz's of word-sized 2203 quantities, choosing which based on whether the high word is nonzero. */ 2204 static rtx 2205 expand_doubleword_clz (scalar_int_mode mode, rtx op0, rtx target) 2206 { 2207 rtx xop0 = force_reg (mode, op0); 2208 rtx subhi = gen_highpart (word_mode, xop0); 2209 rtx sublo = gen_lowpart (word_mode, xop0); 2210 rtx_code_label *hi0_label = gen_label_rtx (); 2211 rtx_code_label *after_label = gen_label_rtx (); 2212 rtx_insn *seq; 2213 rtx temp, result; 2214 2215 /* If we were not given a target, use a word_mode register, not a 2216 'mode' register. The result will fit, and nobody is expecting 2217 anything bigger (the return type of __builtin_clz* is int). */ 2218 if (!target) 2219 target = gen_reg_rtx (word_mode); 2220 2221 /* In any case, write to a word_mode scratch in both branches of the 2222 conditional, so we can ensure there is a single move insn setting 2223 'target' to tag a REG_EQUAL note on. */ 2224 result = gen_reg_rtx (word_mode); 2225 2226 start_sequence (); 2227 2228 /* If the high word is not equal to zero, 2229 then clz of the full value is clz of the high word. */ 2230 emit_cmp_and_jump_insns (subhi, CONST0_RTX (word_mode), EQ, 0, 2231 word_mode, true, hi0_label); 2232 2233 temp = expand_unop_direct (word_mode, clz_optab, subhi, result, true); 2234 if (!temp) 2235 goto fail; 2236 2237 if (temp != result) 2238 convert_move (result, temp, true); 2239 2240 emit_jump_insn (targetm.gen_jump (after_label)); 2241 emit_barrier (); 2242 2243 /* Else clz of the full value is clz of the low word plus the number 2244 of bits in the high word. */ 2245 emit_label (hi0_label); 2246 2247 temp = expand_unop_direct (word_mode, clz_optab, sublo, 0, true); 2248 if (!temp) 2249 goto fail; 2250 temp = expand_binop (word_mode, add_optab, temp, 2251 gen_int_mode (GET_MODE_BITSIZE (word_mode), word_mode), 2252 result, true, OPTAB_DIRECT); 2253 if (!temp) 2254 goto fail; 2255 if (temp != result) 2256 convert_move (result, temp, true); 2257 2258 emit_label (after_label); 2259 convert_move (target, result, true); 2260 2261 seq = get_insns (); 2262 end_sequence (); 2263 2264 add_equal_note (seq, target, CLZ, xop0, NULL_RTX, mode); 2265 emit_insn (seq); 2266 return target; 2267 2268 fail: 2269 end_sequence (); 2270 return 0; 2271 } 2272 2273 /* Try calculating popcount of a double-word quantity as two popcount's of 2274 word-sized quantities and summing up the results. */ 2275 static rtx 2276 expand_doubleword_popcount (scalar_int_mode mode, rtx op0, rtx target) 2277 { 2278 rtx t0, t1, t; 2279 rtx_insn *seq; 2280 2281 start_sequence (); 2282 2283 t0 = expand_unop_direct (word_mode, popcount_optab, 2284 operand_subword_force (op0, 0, mode), NULL_RTX, 2285 true); 2286 t1 = expand_unop_direct (word_mode, popcount_optab, 2287 operand_subword_force (op0, 1, mode), NULL_RTX, 2288 true); 2289 if (!t0 || !t1) 2290 { 2291 end_sequence (); 2292 return NULL_RTX; 2293 } 2294 2295 /* If we were not given a target, use a word_mode register, not a 2296 'mode' register. The result will fit, and nobody is expecting 2297 anything bigger (the return type of __builtin_popcount* is int). */ 2298 if (!target) 2299 target = gen_reg_rtx (word_mode); 2300 2301 t = expand_binop (word_mode, add_optab, t0, t1, target, 0, OPTAB_DIRECT); 2302 2303 seq = get_insns (); 2304 end_sequence (); 2305 2306 add_equal_note (seq, t, POPCOUNT, op0, NULL_RTX, mode); 2307 emit_insn (seq); 2308 return t; 2309 } 2310 2311 /* Try calculating 2312 (parity:wide x) 2313 as 2314 (parity:narrow (low (x) ^ high (x))) */ 2315 static rtx 2316 expand_doubleword_parity (scalar_int_mode mode, rtx op0, rtx target) 2317 { 2318 rtx t = expand_binop (word_mode, xor_optab, 2319 operand_subword_force (op0, 0, mode), 2320 operand_subword_force (op0, 1, mode), 2321 NULL_RTX, 0, OPTAB_DIRECT); 2322 return expand_unop (word_mode, parity_optab, t, target, true); 2323 } 2324 2325 /* Try calculating 2326 (bswap:narrow x) 2327 as 2328 (lshiftrt:wide (bswap:wide x) ((width wide) - (width narrow))). */ 2329 static rtx 2330 widen_bswap (scalar_int_mode mode, rtx op0, rtx target) 2331 { 2332 rtx x; 2333 rtx_insn *last; 2334 opt_scalar_int_mode wider_mode_iter; 2335 2336 FOR_EACH_WIDER_MODE (wider_mode_iter, mode) 2337 if (optab_handler (bswap_optab, wider_mode_iter.require ()) 2338 != CODE_FOR_nothing) 2339 break; 2340 2341 if (!wider_mode_iter.exists ()) 2342 return NULL_RTX; 2343 2344 scalar_int_mode wider_mode = wider_mode_iter.require (); 2345 last = get_last_insn (); 2346 2347 x = widen_operand (op0, wider_mode, mode, true, true); 2348 x = expand_unop (wider_mode, bswap_optab, x, NULL_RTX, true); 2349 2350 gcc_assert (GET_MODE_PRECISION (wider_mode) == GET_MODE_BITSIZE (wider_mode) 2351 && GET_MODE_PRECISION (mode) == GET_MODE_BITSIZE (mode)); 2352 if (x != 0) 2353 x = expand_shift (RSHIFT_EXPR, wider_mode, x, 2354 GET_MODE_BITSIZE (wider_mode) 2355 - GET_MODE_BITSIZE (mode), 2356 NULL_RTX, true); 2357 2358 if (x != 0) 2359 { 2360 if (target == 0) 2361 target = gen_reg_rtx (mode); 2362 emit_move_insn (target, gen_lowpart (mode, x)); 2363 } 2364 else 2365 delete_insns_since (last); 2366 2367 return target; 2368 } 2369 2370 /* Try calculating bswap as two bswaps of two word-sized operands. */ 2371 2372 static rtx 2373 expand_doubleword_bswap (machine_mode mode, rtx op, rtx target) 2374 { 2375 rtx t0, t1; 2376 2377 t1 = expand_unop (word_mode, bswap_optab, 2378 operand_subword_force (op, 0, mode), NULL_RTX, true); 2379 t0 = expand_unop (word_mode, bswap_optab, 2380 operand_subword_force (op, 1, mode), NULL_RTX, true); 2381 2382 if (target == 0 || !valid_multiword_target_p (target)) 2383 target = gen_reg_rtx (mode); 2384 if (REG_P (target)) 2385 emit_clobber (target); 2386 emit_move_insn (operand_subword (target, 0, 1, mode), t0); 2387 emit_move_insn (operand_subword (target, 1, 1, mode), t1); 2388 2389 return target; 2390 } 2391 2392 /* Try calculating (parity x) as (and (popcount x) 1), where 2393 popcount can also be done in a wider mode. */ 2394 static rtx 2395 expand_parity (scalar_int_mode mode, rtx op0, rtx target) 2396 { 2397 enum mode_class mclass = GET_MODE_CLASS (mode); 2398 opt_scalar_int_mode wider_mode_iter; 2399 FOR_EACH_MODE_FROM (wider_mode_iter, mode) 2400 { 2401 scalar_int_mode wider_mode = wider_mode_iter.require (); 2402 if (optab_handler (popcount_optab, wider_mode) != CODE_FOR_nothing) 2403 { 2404 rtx xop0, temp; 2405 rtx_insn *last; 2406 2407 last = get_last_insn (); 2408 2409 if (target == 0 || GET_MODE (target) != wider_mode) 2410 target = gen_reg_rtx (wider_mode); 2411 2412 xop0 = widen_operand (op0, wider_mode, mode, true, false); 2413 temp = expand_unop (wider_mode, popcount_optab, xop0, NULL_RTX, 2414 true); 2415 if (temp != 0) 2416 temp = expand_binop (wider_mode, and_optab, temp, const1_rtx, 2417 target, true, OPTAB_DIRECT); 2418 2419 if (temp) 2420 { 2421 if (mclass != MODE_INT 2422 || !TRULY_NOOP_TRUNCATION_MODES_P (mode, wider_mode)) 2423 return convert_to_mode (mode, temp, 0); 2424 else 2425 return gen_lowpart (mode, temp); 2426 } 2427 else 2428 delete_insns_since (last); 2429 } 2430 } 2431 return 0; 2432 } 2433 2434 /* Try calculating ctz(x) as K - clz(x & -x) , 2435 where K is GET_MODE_PRECISION(mode) - 1. 2436 2437 Both __builtin_ctz and __builtin_clz are undefined at zero, so we 2438 don't have to worry about what the hardware does in that case. (If 2439 the clz instruction produces the usual value at 0, which is K, the 2440 result of this code sequence will be -1; expand_ffs, below, relies 2441 on this. It might be nice to have it be K instead, for consistency 2442 with the (very few) processors that provide a ctz with a defined 2443 value, but that would take one more instruction, and it would be 2444 less convenient for expand_ffs anyway. */ 2445 2446 static rtx 2447 expand_ctz (scalar_int_mode mode, rtx op0, rtx target) 2448 { 2449 rtx_insn *seq; 2450 rtx temp; 2451 2452 if (optab_handler (clz_optab, mode) == CODE_FOR_nothing) 2453 return 0; 2454 2455 start_sequence (); 2456 2457 temp = expand_unop_direct (mode, neg_optab, op0, NULL_RTX, true); 2458 if (temp) 2459 temp = expand_binop (mode, and_optab, op0, temp, NULL_RTX, 2460 true, OPTAB_DIRECT); 2461 if (temp) 2462 temp = expand_unop_direct (mode, clz_optab, temp, NULL_RTX, true); 2463 if (temp) 2464 temp = expand_binop (mode, sub_optab, 2465 gen_int_mode (GET_MODE_PRECISION (mode) - 1, mode), 2466 temp, target, 2467 true, OPTAB_DIRECT); 2468 if (temp == 0) 2469 { 2470 end_sequence (); 2471 return 0; 2472 } 2473 2474 seq = get_insns (); 2475 end_sequence (); 2476 2477 add_equal_note (seq, temp, CTZ, op0, NULL_RTX, mode); 2478 emit_insn (seq); 2479 return temp; 2480 } 2481 2482 2483 /* Try calculating ffs(x) using ctz(x) if we have that instruction, or 2484 else with the sequence used by expand_clz. 2485 2486 The ffs builtin promises to return zero for a zero value and ctz/clz 2487 may have an undefined value in that case. If they do not give us a 2488 convenient value, we have to generate a test and branch. */ 2489 static rtx 2490 expand_ffs (scalar_int_mode mode, rtx op0, rtx target) 2491 { 2492 HOST_WIDE_INT val = 0; 2493 bool defined_at_zero = false; 2494 rtx temp; 2495 rtx_insn *seq; 2496 2497 if (optab_handler (ctz_optab, mode) != CODE_FOR_nothing) 2498 { 2499 start_sequence (); 2500 2501 temp = expand_unop_direct (mode, ctz_optab, op0, 0, true); 2502 if (!temp) 2503 goto fail; 2504 2505 defined_at_zero = (CTZ_DEFINED_VALUE_AT_ZERO (mode, val) == 2); 2506 } 2507 else if (optab_handler (clz_optab, mode) != CODE_FOR_nothing) 2508 { 2509 start_sequence (); 2510 temp = expand_ctz (mode, op0, 0); 2511 if (!temp) 2512 goto fail; 2513 2514 if (CLZ_DEFINED_VALUE_AT_ZERO (mode, val) == 2) 2515 { 2516 defined_at_zero = true; 2517 val = (GET_MODE_PRECISION (mode) - 1) - val; 2518 } 2519 } 2520 else 2521 return 0; 2522 2523 if (defined_at_zero && val == -1) 2524 /* No correction needed at zero. */; 2525 else 2526 { 2527 /* We don't try to do anything clever with the situation found 2528 on some processors (eg Alpha) where ctz(0:mode) == 2529 bitsize(mode). If someone can think of a way to send N to -1 2530 and leave alone all values in the range 0..N-1 (where N is a 2531 power of two), cheaper than this test-and-branch, please add it. 2532 2533 The test-and-branch is done after the operation itself, in case 2534 the operation sets condition codes that can be recycled for this. 2535 (This is true on i386, for instance.) */ 2536 2537 rtx_code_label *nonzero_label = gen_label_rtx (); 2538 emit_cmp_and_jump_insns (op0, CONST0_RTX (mode), NE, 0, 2539 mode, true, nonzero_label); 2540 2541 convert_move (temp, GEN_INT (-1), false); 2542 emit_label (nonzero_label); 2543 } 2544 2545 /* temp now has a value in the range -1..bitsize-1. ffs is supposed 2546 to produce a value in the range 0..bitsize. */ 2547 temp = expand_binop (mode, add_optab, temp, gen_int_mode (1, mode), 2548 target, false, OPTAB_DIRECT); 2549 if (!temp) 2550 goto fail; 2551 2552 seq = get_insns (); 2553 end_sequence (); 2554 2555 add_equal_note (seq, temp, FFS, op0, NULL_RTX, mode); 2556 emit_insn (seq); 2557 return temp; 2558 2559 fail: 2560 end_sequence (); 2561 return 0; 2562 } 2563 2564 /* Extract the OMODE lowpart from VAL, which has IMODE. Under certain 2565 conditions, VAL may already be a SUBREG against which we cannot generate 2566 a further SUBREG. In this case, we expect forcing the value into a 2567 register will work around the situation. */ 2568 2569 static rtx 2570 lowpart_subreg_maybe_copy (machine_mode omode, rtx val, 2571 machine_mode imode) 2572 { 2573 rtx ret; 2574 ret = lowpart_subreg (omode, val, imode); 2575 if (ret == NULL) 2576 { 2577 val = force_reg (imode, val); 2578 ret = lowpart_subreg (omode, val, imode); 2579 gcc_assert (ret != NULL); 2580 } 2581 return ret; 2582 } 2583 2584 /* Expand a floating point absolute value or negation operation via a 2585 logical operation on the sign bit. */ 2586 2587 static rtx 2588 expand_absneg_bit (enum rtx_code code, scalar_float_mode mode, 2589 rtx op0, rtx target) 2590 { 2591 const struct real_format *fmt; 2592 int bitpos, word, nwords, i; 2593 scalar_int_mode imode; 2594 rtx temp; 2595 rtx_insn *insns; 2596 2597 /* The format has to have a simple sign bit. */ 2598 fmt = REAL_MODE_FORMAT (mode); 2599 if (fmt == NULL) 2600 return NULL_RTX; 2601 2602 bitpos = fmt->signbit_rw; 2603 if (bitpos < 0) 2604 return NULL_RTX; 2605 2606 /* Don't create negative zeros if the format doesn't support them. */ 2607 if (code == NEG && !fmt->has_signed_zero) 2608 return NULL_RTX; 2609 2610 if (GET_MODE_SIZE (mode) <= UNITS_PER_WORD) 2611 { 2612 if (!int_mode_for_mode (mode).exists (&imode)) 2613 return NULL_RTX; 2614 word = 0; 2615 nwords = 1; 2616 } 2617 else 2618 { 2619 imode = word_mode; 2620 2621 if (FLOAT_WORDS_BIG_ENDIAN) 2622 word = (GET_MODE_BITSIZE (mode) - bitpos) / BITS_PER_WORD; 2623 else 2624 word = bitpos / BITS_PER_WORD; 2625 bitpos = bitpos % BITS_PER_WORD; 2626 nwords = (GET_MODE_BITSIZE (mode) + BITS_PER_WORD - 1) / BITS_PER_WORD; 2627 } 2628 2629 wide_int mask = wi::set_bit_in_zero (bitpos, GET_MODE_PRECISION (imode)); 2630 if (code == ABS) 2631 mask = ~mask; 2632 2633 if (target == 0 2634 || target == op0 2635 || (nwords > 1 && !valid_multiword_target_p (target))) 2636 target = gen_reg_rtx (mode); 2637 2638 if (nwords > 1) 2639 { 2640 start_sequence (); 2641 2642 for (i = 0; i < nwords; ++i) 2643 { 2644 rtx targ_piece = operand_subword (target, i, 1, mode); 2645 rtx op0_piece = operand_subword_force (op0, i, mode); 2646 2647 if (i == word) 2648 { 2649 temp = expand_binop (imode, code == ABS ? and_optab : xor_optab, 2650 op0_piece, 2651 immed_wide_int_const (mask, imode), 2652 targ_piece, 1, OPTAB_LIB_WIDEN); 2653 if (temp != targ_piece) 2654 emit_move_insn (targ_piece, temp); 2655 } 2656 else 2657 emit_move_insn (targ_piece, op0_piece); 2658 } 2659 2660 insns = get_insns (); 2661 end_sequence (); 2662 2663 emit_insn (insns); 2664 } 2665 else 2666 { 2667 temp = expand_binop (imode, code == ABS ? and_optab : xor_optab, 2668 gen_lowpart (imode, op0), 2669 immed_wide_int_const (mask, imode), 2670 gen_lowpart (imode, target), 1, OPTAB_LIB_WIDEN); 2671 target = lowpart_subreg_maybe_copy (mode, temp, imode); 2672 2673 set_dst_reg_note (get_last_insn (), REG_EQUAL, 2674 gen_rtx_fmt_e (code, mode, copy_rtx (op0)), 2675 target); 2676 } 2677 2678 return target; 2679 } 2680 2681 /* As expand_unop, but will fail rather than attempt the operation in a 2682 different mode or with a libcall. */ 2683 static rtx 2684 expand_unop_direct (machine_mode mode, optab unoptab, rtx op0, rtx target, 2685 int unsignedp) 2686 { 2687 if (optab_handler (unoptab, mode) != CODE_FOR_nothing) 2688 { 2689 struct expand_operand ops[2]; 2690 enum insn_code icode = optab_handler (unoptab, mode); 2691 rtx_insn *last = get_last_insn (); 2692 rtx_insn *pat; 2693 2694 create_output_operand (&ops[0], target, mode); 2695 create_convert_operand_from (&ops[1], op0, mode, unsignedp); 2696 pat = maybe_gen_insn (icode, 2, ops); 2697 if (pat) 2698 { 2699 if (INSN_P (pat) && NEXT_INSN (pat) != NULL_RTX 2700 && ! add_equal_note (pat, ops[0].value, 2701 optab_to_code (unoptab), 2702 ops[1].value, NULL_RTX, mode)) 2703 { 2704 delete_insns_since (last); 2705 return expand_unop (mode, unoptab, op0, NULL_RTX, unsignedp); 2706 } 2707 2708 emit_insn (pat); 2709 2710 return ops[0].value; 2711 } 2712 } 2713 return 0; 2714 } 2715 2716 /* Generate code to perform an operation specified by UNOPTAB 2717 on operand OP0, with result having machine-mode MODE. 2718 2719 UNSIGNEDP is for the case where we have to widen the operands 2720 to perform the operation. It says to use zero-extension. 2721 2722 If TARGET is nonzero, the value 2723 is generated there, if it is convenient to do so. 2724 In all cases an rtx is returned for the locus of the value; 2725 this may or may not be TARGET. */ 2726 2727 rtx 2728 expand_unop (machine_mode mode, optab unoptab, rtx op0, rtx target, 2729 int unsignedp) 2730 { 2731 enum mode_class mclass = GET_MODE_CLASS (mode); 2732 machine_mode wider_mode; 2733 scalar_int_mode int_mode; 2734 scalar_float_mode float_mode; 2735 rtx temp; 2736 rtx libfunc; 2737 2738 temp = expand_unop_direct (mode, unoptab, op0, target, unsignedp); 2739 if (temp) 2740 return temp; 2741 2742 /* It can't be done in this mode. Can we open-code it in a wider mode? */ 2743 2744 /* Widening (or narrowing) clz needs special treatment. */ 2745 if (unoptab == clz_optab) 2746 { 2747 if (is_a <scalar_int_mode> (mode, &int_mode)) 2748 { 2749 temp = widen_leading (int_mode, op0, target, unoptab); 2750 if (temp) 2751 return temp; 2752 2753 if (GET_MODE_SIZE (int_mode) == 2 * UNITS_PER_WORD 2754 && optab_handler (unoptab, word_mode) != CODE_FOR_nothing) 2755 { 2756 temp = expand_doubleword_clz (int_mode, op0, target); 2757 if (temp) 2758 return temp; 2759 } 2760 } 2761 2762 goto try_libcall; 2763 } 2764 2765 if (unoptab == clrsb_optab) 2766 { 2767 if (is_a <scalar_int_mode> (mode, &int_mode)) 2768 { 2769 temp = widen_leading (int_mode, op0, target, unoptab); 2770 if (temp) 2771 return temp; 2772 } 2773 goto try_libcall; 2774 } 2775 2776 if (unoptab == popcount_optab 2777 && is_a <scalar_int_mode> (mode, &int_mode) 2778 && GET_MODE_SIZE (int_mode) == 2 * UNITS_PER_WORD 2779 && optab_handler (unoptab, word_mode) != CODE_FOR_nothing 2780 && optimize_insn_for_speed_p ()) 2781 { 2782 temp = expand_doubleword_popcount (int_mode, op0, target); 2783 if (temp) 2784 return temp; 2785 } 2786 2787 if (unoptab == parity_optab 2788 && is_a <scalar_int_mode> (mode, &int_mode) 2789 && GET_MODE_SIZE (int_mode) == 2 * UNITS_PER_WORD 2790 && (optab_handler (unoptab, word_mode) != CODE_FOR_nothing 2791 || optab_handler (popcount_optab, word_mode) != CODE_FOR_nothing) 2792 && optimize_insn_for_speed_p ()) 2793 { 2794 temp = expand_doubleword_parity (int_mode, op0, target); 2795 if (temp) 2796 return temp; 2797 } 2798 2799 /* Widening (or narrowing) bswap needs special treatment. */ 2800 if (unoptab == bswap_optab) 2801 { 2802 /* HImode is special because in this mode BSWAP is equivalent to ROTATE 2803 or ROTATERT. First try these directly; if this fails, then try the 2804 obvious pair of shifts with allowed widening, as this will probably 2805 be always more efficient than the other fallback methods. */ 2806 if (mode == HImode) 2807 { 2808 rtx_insn *last; 2809 rtx temp1, temp2; 2810 2811 if (optab_handler (rotl_optab, mode) != CODE_FOR_nothing) 2812 { 2813 temp = expand_binop (mode, rotl_optab, op0, 2814 gen_int_shift_amount (mode, 8), 2815 target, unsignedp, OPTAB_DIRECT); 2816 if (temp) 2817 return temp; 2818 } 2819 2820 if (optab_handler (rotr_optab, mode) != CODE_FOR_nothing) 2821 { 2822 temp = expand_binop (mode, rotr_optab, op0, 2823 gen_int_shift_amount (mode, 8), 2824 target, unsignedp, OPTAB_DIRECT); 2825 if (temp) 2826 return temp; 2827 } 2828 2829 last = get_last_insn (); 2830 2831 temp1 = expand_binop (mode, ashl_optab, op0, 2832 gen_int_shift_amount (mode, 8), NULL_RTX, 2833 unsignedp, OPTAB_WIDEN); 2834 temp2 = expand_binop (mode, lshr_optab, op0, 2835 gen_int_shift_amount (mode, 8), NULL_RTX, 2836 unsignedp, OPTAB_WIDEN); 2837 if (temp1 && temp2) 2838 { 2839 temp = expand_binop (mode, ior_optab, temp1, temp2, target, 2840 unsignedp, OPTAB_WIDEN); 2841 if (temp) 2842 return temp; 2843 } 2844 2845 delete_insns_since (last); 2846 } 2847 2848 if (is_a <scalar_int_mode> (mode, &int_mode)) 2849 { 2850 temp = widen_bswap (int_mode, op0, target); 2851 if (temp) 2852 return temp; 2853 2854 if (GET_MODE_SIZE (int_mode) == 2 * UNITS_PER_WORD 2855 && optab_handler (unoptab, word_mode) != CODE_FOR_nothing) 2856 { 2857 temp = expand_doubleword_bswap (mode, op0, target); 2858 if (temp) 2859 return temp; 2860 } 2861 } 2862 2863 goto try_libcall; 2864 } 2865 2866 if (CLASS_HAS_WIDER_MODES_P (mclass)) 2867 FOR_EACH_WIDER_MODE (wider_mode, mode) 2868 { 2869 if (optab_handler (unoptab, wider_mode) != CODE_FOR_nothing) 2870 { 2871 rtx xop0 = op0; 2872 rtx_insn *last = get_last_insn (); 2873 2874 /* For certain operations, we need not actually extend 2875 the narrow operand, as long as we will truncate the 2876 results to the same narrowness. */ 2877 2878 xop0 = widen_operand (xop0, wider_mode, mode, unsignedp, 2879 (unoptab == neg_optab 2880 || unoptab == one_cmpl_optab) 2881 && mclass == MODE_INT); 2882 2883 temp = expand_unop (wider_mode, unoptab, xop0, NULL_RTX, 2884 unsignedp); 2885 2886 if (temp) 2887 { 2888 if (mclass != MODE_INT 2889 || !TRULY_NOOP_TRUNCATION_MODES_P (mode, wider_mode)) 2890 { 2891 if (target == 0) 2892 target = gen_reg_rtx (mode); 2893 convert_move (target, temp, 0); 2894 return target; 2895 } 2896 else 2897 return gen_lowpart (mode, temp); 2898 } 2899 else 2900 delete_insns_since (last); 2901 } 2902 } 2903 2904 /* These can be done a word at a time. */ 2905 if (unoptab == one_cmpl_optab 2906 && is_int_mode (mode, &int_mode) 2907 && GET_MODE_SIZE (int_mode) > UNITS_PER_WORD 2908 && optab_handler (unoptab, word_mode) != CODE_FOR_nothing) 2909 { 2910 int i; 2911 rtx_insn *insns; 2912 2913 if (target == 0 || target == op0 || !valid_multiword_target_p (target)) 2914 target = gen_reg_rtx (int_mode); 2915 2916 start_sequence (); 2917 2918 /* Do the actual arithmetic. */ 2919 for (i = 0; i < GET_MODE_BITSIZE (int_mode) / BITS_PER_WORD; i++) 2920 { 2921 rtx target_piece = operand_subword (target, i, 1, int_mode); 2922 rtx x = expand_unop (word_mode, unoptab, 2923 operand_subword_force (op0, i, int_mode), 2924 target_piece, unsignedp); 2925 2926 if (target_piece != x) 2927 emit_move_insn (target_piece, x); 2928 } 2929 2930 insns = get_insns (); 2931 end_sequence (); 2932 2933 emit_insn (insns); 2934 return target; 2935 } 2936 2937 if (optab_to_code (unoptab) == NEG) 2938 { 2939 /* Try negating floating point values by flipping the sign bit. */ 2940 if (is_a <scalar_float_mode> (mode, &float_mode)) 2941 { 2942 temp = expand_absneg_bit (NEG, float_mode, op0, target); 2943 if (temp) 2944 return temp; 2945 } 2946 2947 /* If there is no negation pattern, and we have no negative zero, 2948 try subtracting from zero. */ 2949 if (!HONOR_SIGNED_ZEROS (mode)) 2950 { 2951 temp = expand_binop (mode, (unoptab == negv_optab 2952 ? subv_optab : sub_optab), 2953 CONST0_RTX (mode), op0, target, 2954 unsignedp, OPTAB_DIRECT); 2955 if (temp) 2956 return temp; 2957 } 2958 } 2959 2960 /* Try calculating parity (x) as popcount (x) % 2. */ 2961 if (unoptab == parity_optab && is_a <scalar_int_mode> (mode, &int_mode)) 2962 { 2963 temp = expand_parity (int_mode, op0, target); 2964 if (temp) 2965 return temp; 2966 } 2967 2968 /* Try implementing ffs (x) in terms of clz (x). */ 2969 if (unoptab == ffs_optab && is_a <scalar_int_mode> (mode, &int_mode)) 2970 { 2971 temp = expand_ffs (int_mode, op0, target); 2972 if (temp) 2973 return temp; 2974 } 2975 2976 /* Try implementing ctz (x) in terms of clz (x). */ 2977 if (unoptab == ctz_optab && is_a <scalar_int_mode> (mode, &int_mode)) 2978 { 2979 temp = expand_ctz (int_mode, op0, target); 2980 if (temp) 2981 return temp; 2982 } 2983 2984 try_libcall: 2985 /* Now try a library call in this mode. */ 2986 libfunc = optab_libfunc (unoptab, mode); 2987 if (libfunc) 2988 { 2989 rtx_insn *insns; 2990 rtx value; 2991 rtx eq_value; 2992 machine_mode outmode = mode; 2993 2994 /* All of these functions return small values. Thus we choose to 2995 have them return something that isn't a double-word. */ 2996 if (unoptab == ffs_optab || unoptab == clz_optab || unoptab == ctz_optab 2997 || unoptab == clrsb_optab || unoptab == popcount_optab 2998 || unoptab == parity_optab) 2999 outmode 3000 = GET_MODE (hard_libcall_value (TYPE_MODE (integer_type_node), 3001 optab_libfunc (unoptab, mode))); 3002 3003 start_sequence (); 3004 3005 /* Pass 1 for NO_QUEUE so we don't lose any increments 3006 if the libcall is cse'd or moved. */ 3007 value = emit_library_call_value (libfunc, NULL_RTX, LCT_CONST, outmode, 3008 op0, mode); 3009 insns = get_insns (); 3010 end_sequence (); 3011 3012 target = gen_reg_rtx (outmode); 3013 bool trapv = trapv_unoptab_p (unoptab); 3014 if (trapv) 3015 eq_value = NULL_RTX; 3016 else 3017 { 3018 eq_value = gen_rtx_fmt_e (optab_to_code (unoptab), mode, op0); 3019 if (GET_MODE_UNIT_SIZE (outmode) < GET_MODE_UNIT_SIZE (mode)) 3020 eq_value = simplify_gen_unary (TRUNCATE, outmode, eq_value, mode); 3021 else if (GET_MODE_UNIT_SIZE (outmode) > GET_MODE_UNIT_SIZE (mode)) 3022 eq_value = simplify_gen_unary (ZERO_EXTEND, 3023 outmode, eq_value, mode); 3024 } 3025 emit_libcall_block_1 (insns, target, value, eq_value, trapv); 3026 3027 return target; 3028 } 3029 3030 /* It can't be done in this mode. Can we do it in a wider mode? */ 3031 3032 if (CLASS_HAS_WIDER_MODES_P (mclass)) 3033 { 3034 FOR_EACH_WIDER_MODE (wider_mode, mode) 3035 { 3036 if (optab_handler (unoptab, wider_mode) != CODE_FOR_nothing 3037 || optab_libfunc (unoptab, wider_mode)) 3038 { 3039 rtx xop0 = op0; 3040 rtx_insn *last = get_last_insn (); 3041 3042 /* For certain operations, we need not actually extend 3043 the narrow operand, as long as we will truncate the 3044 results to the same narrowness. */ 3045 xop0 = widen_operand (xop0, wider_mode, mode, unsignedp, 3046 (unoptab == neg_optab 3047 || unoptab == one_cmpl_optab 3048 || unoptab == bswap_optab) 3049 && mclass == MODE_INT); 3050 3051 temp = expand_unop (wider_mode, unoptab, xop0, NULL_RTX, 3052 unsignedp); 3053 3054 /* If we are generating clz using wider mode, adjust the 3055 result. Similarly for clrsb. */ 3056 if ((unoptab == clz_optab || unoptab == clrsb_optab) 3057 && temp != 0) 3058 { 3059 scalar_int_mode wider_int_mode 3060 = as_a <scalar_int_mode> (wider_mode); 3061 int_mode = as_a <scalar_int_mode> (mode); 3062 temp = expand_binop 3063 (wider_mode, sub_optab, temp, 3064 gen_int_mode (GET_MODE_PRECISION (wider_int_mode) 3065 - GET_MODE_PRECISION (int_mode), 3066 wider_int_mode), 3067 target, true, OPTAB_DIRECT); 3068 } 3069 3070 /* Likewise for bswap. */ 3071 if (unoptab == bswap_optab && temp != 0) 3072 { 3073 scalar_int_mode wider_int_mode 3074 = as_a <scalar_int_mode> (wider_mode); 3075 int_mode = as_a <scalar_int_mode> (mode); 3076 gcc_assert (GET_MODE_PRECISION (wider_int_mode) 3077 == GET_MODE_BITSIZE (wider_int_mode) 3078 && GET_MODE_PRECISION (int_mode) 3079 == GET_MODE_BITSIZE (int_mode)); 3080 3081 temp = expand_shift (RSHIFT_EXPR, wider_int_mode, temp, 3082 GET_MODE_BITSIZE (wider_int_mode) 3083 - GET_MODE_BITSIZE (int_mode), 3084 NULL_RTX, true); 3085 } 3086 3087 if (temp) 3088 { 3089 if (mclass != MODE_INT) 3090 { 3091 if (target == 0) 3092 target = gen_reg_rtx (mode); 3093 convert_move (target, temp, 0); 3094 return target; 3095 } 3096 else 3097 return gen_lowpart (mode, temp); 3098 } 3099 else 3100 delete_insns_since (last); 3101 } 3102 } 3103 } 3104 3105 /* One final attempt at implementing negation via subtraction, 3106 this time allowing widening of the operand. */ 3107 if (optab_to_code (unoptab) == NEG && !HONOR_SIGNED_ZEROS (mode)) 3108 { 3109 rtx temp; 3110 temp = expand_binop (mode, 3111 unoptab == negv_optab ? subv_optab : sub_optab, 3112 CONST0_RTX (mode), op0, 3113 target, unsignedp, OPTAB_LIB_WIDEN); 3114 if (temp) 3115 return temp; 3116 } 3117 3118 return 0; 3119 } 3120 3121 /* Emit code to compute the absolute value of OP0, with result to 3122 TARGET if convenient. (TARGET may be 0.) The return value says 3123 where the result actually is to be found. 3124 3125 MODE is the mode of the operand; the mode of the result is 3126 different but can be deduced from MODE. 3127 3128 */ 3129 3130 rtx 3131 expand_abs_nojump (machine_mode mode, rtx op0, rtx target, 3132 int result_unsignedp) 3133 { 3134 rtx temp; 3135 3136 if (GET_MODE_CLASS (mode) != MODE_INT 3137 || ! flag_trapv) 3138 result_unsignedp = 1; 3139 3140 /* First try to do it with a special abs instruction. */ 3141 temp = expand_unop (mode, result_unsignedp ? abs_optab : absv_optab, 3142 op0, target, 0); 3143 if (temp != 0) 3144 return temp; 3145 3146 /* For floating point modes, try clearing the sign bit. */ 3147 scalar_float_mode float_mode; 3148 if (is_a <scalar_float_mode> (mode, &float_mode)) 3149 { 3150 temp = expand_absneg_bit (ABS, float_mode, op0, target); 3151 if (temp) 3152 return temp; 3153 } 3154 3155 /* If we have a MAX insn, we can do this as MAX (x, -x). */ 3156 if (optab_handler (smax_optab, mode) != CODE_FOR_nothing 3157 && !HONOR_SIGNED_ZEROS (mode)) 3158 { 3159 rtx_insn *last = get_last_insn (); 3160 3161 temp = expand_unop (mode, result_unsignedp ? neg_optab : negv_optab, 3162 op0, NULL_RTX, 0); 3163 if (temp != 0) 3164 temp = expand_binop (mode, smax_optab, op0, temp, target, 0, 3165 OPTAB_WIDEN); 3166 3167 if (temp != 0) 3168 return temp; 3169 3170 delete_insns_since (last); 3171 } 3172 3173 /* If this machine has expensive jumps, we can do integer absolute 3174 value of X as (((signed) x >> (W-1)) ^ x) - ((signed) x >> (W-1)), 3175 where W is the width of MODE. */ 3176 3177 scalar_int_mode int_mode; 3178 if (is_int_mode (mode, &int_mode) 3179 && BRANCH_COST (optimize_insn_for_speed_p (), 3180 false) >= 2) 3181 { 3182 rtx extended = expand_shift (RSHIFT_EXPR, int_mode, op0, 3183 GET_MODE_PRECISION (int_mode) - 1, 3184 NULL_RTX, 0); 3185 3186 temp = expand_binop (int_mode, xor_optab, extended, op0, target, 0, 3187 OPTAB_LIB_WIDEN); 3188 if (temp != 0) 3189 temp = expand_binop (int_mode, 3190 result_unsignedp ? sub_optab : subv_optab, 3191 temp, extended, target, 0, OPTAB_LIB_WIDEN); 3192 3193 if (temp != 0) 3194 return temp; 3195 } 3196 3197 return NULL_RTX; 3198 } 3199 3200 rtx 3201 expand_abs (machine_mode mode, rtx op0, rtx target, 3202 int result_unsignedp, int safe) 3203 { 3204 rtx temp; 3205 rtx_code_label *op1; 3206 3207 if (GET_MODE_CLASS (mode) != MODE_INT 3208 || ! flag_trapv) 3209 result_unsignedp = 1; 3210 3211 temp = expand_abs_nojump (mode, op0, target, result_unsignedp); 3212 if (temp != 0) 3213 return temp; 3214 3215 /* If that does not win, use conditional jump and negate. */ 3216 3217 /* It is safe to use the target if it is the same 3218 as the source if this is also a pseudo register */ 3219 if (op0 == target && REG_P (op0) 3220 && REGNO (op0) >= FIRST_PSEUDO_REGISTER) 3221 safe = 1; 3222 3223 op1 = gen_label_rtx (); 3224 if (target == 0 || ! safe 3225 || GET_MODE (target) != mode 3226 || (MEM_P (target) && MEM_VOLATILE_P (target)) 3227 || (REG_P (target) 3228 && REGNO (target) < FIRST_PSEUDO_REGISTER)) 3229 target = gen_reg_rtx (mode); 3230 3231 emit_move_insn (target, op0); 3232 NO_DEFER_POP; 3233 3234 do_compare_rtx_and_jump (target, CONST0_RTX (mode), GE, 0, mode, 3235 NULL_RTX, NULL, op1, 3236 profile_probability::uninitialized ()); 3237 3238 op0 = expand_unop (mode, result_unsignedp ? neg_optab : negv_optab, 3239 target, target, 0); 3240 if (op0 != target) 3241 emit_move_insn (target, op0); 3242 emit_label (op1); 3243 OK_DEFER_POP; 3244 return target; 3245 } 3246 3247 /* Emit code to compute the one's complement absolute value of OP0 3248 (if (OP0 < 0) OP0 = ~OP0), with result to TARGET if convenient. 3249 (TARGET may be NULL_RTX.) The return value says where the result 3250 actually is to be found. 3251 3252 MODE is the mode of the operand; the mode of the result is 3253 different but can be deduced from MODE. */ 3254 3255 rtx 3256 expand_one_cmpl_abs_nojump (machine_mode mode, rtx op0, rtx target) 3257 { 3258 rtx temp; 3259 3260 /* Not applicable for floating point modes. */ 3261 if (FLOAT_MODE_P (mode)) 3262 return NULL_RTX; 3263 3264 /* If we have a MAX insn, we can do this as MAX (x, ~x). */ 3265 if (optab_handler (smax_optab, mode) != CODE_FOR_nothing) 3266 { 3267 rtx_insn *last = get_last_insn (); 3268 3269 temp = expand_unop (mode, one_cmpl_optab, op0, NULL_RTX, 0); 3270 if (temp != 0) 3271 temp = expand_binop (mode, smax_optab, op0, temp, target, 0, 3272 OPTAB_WIDEN); 3273 3274 if (temp != 0) 3275 return temp; 3276 3277 delete_insns_since (last); 3278 } 3279 3280 /* If this machine has expensive jumps, we can do one's complement 3281 absolute value of X as (((signed) x >> (W-1)) ^ x). */ 3282 3283 scalar_int_mode int_mode; 3284 if (is_int_mode (mode, &int_mode) 3285 && BRANCH_COST (optimize_insn_for_speed_p (), 3286 false) >= 2) 3287 { 3288 rtx extended = expand_shift (RSHIFT_EXPR, int_mode, op0, 3289 GET_MODE_PRECISION (int_mode) - 1, 3290 NULL_RTX, 0); 3291 3292 temp = expand_binop (int_mode, xor_optab, extended, op0, target, 0, 3293 OPTAB_LIB_WIDEN); 3294 3295 if (temp != 0) 3296 return temp; 3297 } 3298 3299 return NULL_RTX; 3300 } 3301 3302 /* A subroutine of expand_copysign, perform the copysign operation using the 3303 abs and neg primitives advertised to exist on the target. The assumption 3304 is that we have a split register file, and leaving op0 in fp registers, 3305 and not playing with subregs so much, will help the register allocator. */ 3306 3307 static rtx 3308 expand_copysign_absneg (scalar_float_mode mode, rtx op0, rtx op1, rtx target, 3309 int bitpos, bool op0_is_abs) 3310 { 3311 scalar_int_mode imode; 3312 enum insn_code icode; 3313 rtx sign; 3314 rtx_code_label *label; 3315 3316 if (target == op1) 3317 target = NULL_RTX; 3318 3319 /* Check if the back end provides an insn that handles signbit for the 3320 argument's mode. */ 3321 icode = optab_handler (signbit_optab, mode); 3322 if (icode != CODE_FOR_nothing) 3323 { 3324 imode = as_a <scalar_int_mode> (insn_data[(int) icode].operand[0].mode); 3325 sign = gen_reg_rtx (imode); 3326 emit_unop_insn (icode, sign, op1, UNKNOWN); 3327 } 3328 else 3329 { 3330 if (GET_MODE_SIZE (mode) <= UNITS_PER_WORD) 3331 { 3332 if (!int_mode_for_mode (mode).exists (&imode)) 3333 return NULL_RTX; 3334 op1 = gen_lowpart (imode, op1); 3335 } 3336 else 3337 { 3338 int word; 3339 3340 imode = word_mode; 3341 if (FLOAT_WORDS_BIG_ENDIAN) 3342 word = (GET_MODE_BITSIZE (mode) - bitpos) / BITS_PER_WORD; 3343 else 3344 word = bitpos / BITS_PER_WORD; 3345 bitpos = bitpos % BITS_PER_WORD; 3346 op1 = operand_subword_force (op1, word, mode); 3347 } 3348 3349 wide_int mask = wi::set_bit_in_zero (bitpos, GET_MODE_PRECISION (imode)); 3350 sign = expand_binop (imode, and_optab, op1, 3351 immed_wide_int_const (mask, imode), 3352 NULL_RTX, 1, OPTAB_LIB_WIDEN); 3353 } 3354 3355 if (!op0_is_abs) 3356 { 3357 op0 = expand_unop (mode, abs_optab, op0, target, 0); 3358 if (op0 == NULL) 3359 return NULL_RTX; 3360 target = op0; 3361 } 3362 else 3363 { 3364 if (target == NULL_RTX) 3365 target = copy_to_reg (op0); 3366 else 3367 emit_move_insn (target, op0); 3368 } 3369 3370 label = gen_label_rtx (); 3371 emit_cmp_and_jump_insns (sign, const0_rtx, EQ, NULL_RTX, imode, 1, label); 3372 3373 if (CONST_DOUBLE_AS_FLOAT_P (op0)) 3374 op0 = simplify_unary_operation (NEG, mode, op0, mode); 3375 else 3376 op0 = expand_unop (mode, neg_optab, op0, target, 0); 3377 if (op0 != target) 3378 emit_move_insn (target, op0); 3379 3380 emit_label (label); 3381 3382 return target; 3383 } 3384 3385 3386 /* A subroutine of expand_copysign, perform the entire copysign operation 3387 with integer bitmasks. BITPOS is the position of the sign bit; OP0_IS_ABS 3388 is true if op0 is known to have its sign bit clear. */ 3389 3390 static rtx 3391 expand_copysign_bit (scalar_float_mode mode, rtx op0, rtx op1, rtx target, 3392 int bitpos, bool op0_is_abs) 3393 { 3394 scalar_int_mode imode; 3395 int word, nwords, i; 3396 rtx temp; 3397 rtx_insn *insns; 3398 3399 if (GET_MODE_SIZE (mode) <= UNITS_PER_WORD) 3400 { 3401 if (!int_mode_for_mode (mode).exists (&imode)) 3402 return NULL_RTX; 3403 word = 0; 3404 nwords = 1; 3405 } 3406 else 3407 { 3408 imode = word_mode; 3409 3410 if (FLOAT_WORDS_BIG_ENDIAN) 3411 word = (GET_MODE_BITSIZE (mode) - bitpos) / BITS_PER_WORD; 3412 else 3413 word = bitpos / BITS_PER_WORD; 3414 bitpos = bitpos % BITS_PER_WORD; 3415 nwords = (GET_MODE_BITSIZE (mode) + BITS_PER_WORD - 1) / BITS_PER_WORD; 3416 } 3417 3418 wide_int mask = wi::set_bit_in_zero (bitpos, GET_MODE_PRECISION (imode)); 3419 3420 if (target == 0 3421 || target == op0 3422 || target == op1 3423 || (nwords > 1 && !valid_multiword_target_p (target))) 3424 target = gen_reg_rtx (mode); 3425 3426 if (nwords > 1) 3427 { 3428 start_sequence (); 3429 3430 for (i = 0; i < nwords; ++i) 3431 { 3432 rtx targ_piece = operand_subword (target, i, 1, mode); 3433 rtx op0_piece = operand_subword_force (op0, i, mode); 3434 3435 if (i == word) 3436 { 3437 if (!op0_is_abs) 3438 op0_piece 3439 = expand_binop (imode, and_optab, op0_piece, 3440 immed_wide_int_const (~mask, imode), 3441 NULL_RTX, 1, OPTAB_LIB_WIDEN); 3442 op1 = expand_binop (imode, and_optab, 3443 operand_subword_force (op1, i, mode), 3444 immed_wide_int_const (mask, imode), 3445 NULL_RTX, 1, OPTAB_LIB_WIDEN); 3446 3447 temp = expand_binop (imode, ior_optab, op0_piece, op1, 3448 targ_piece, 1, OPTAB_LIB_WIDEN); 3449 if (temp != targ_piece) 3450 emit_move_insn (targ_piece, temp); 3451 } 3452 else 3453 emit_move_insn (targ_piece, op0_piece); 3454 } 3455 3456 insns = get_insns (); 3457 end_sequence (); 3458 3459 emit_insn (insns); 3460 } 3461 else 3462 { 3463 op1 = expand_binop (imode, and_optab, gen_lowpart (imode, op1), 3464 immed_wide_int_const (mask, imode), 3465 NULL_RTX, 1, OPTAB_LIB_WIDEN); 3466 3467 op0 = gen_lowpart (imode, op0); 3468 if (!op0_is_abs) 3469 op0 = expand_binop (imode, and_optab, op0, 3470 immed_wide_int_const (~mask, imode), 3471 NULL_RTX, 1, OPTAB_LIB_WIDEN); 3472 3473 temp = expand_binop (imode, ior_optab, op0, op1, 3474 gen_lowpart (imode, target), 1, OPTAB_LIB_WIDEN); 3475 target = lowpart_subreg_maybe_copy (mode, temp, imode); 3476 } 3477 3478 return target; 3479 } 3480 3481 /* Expand the C99 copysign operation. OP0 and OP1 must be the same 3482 scalar floating point mode. Return NULL if we do not know how to 3483 expand the operation inline. */ 3484 3485 rtx 3486 expand_copysign (rtx op0, rtx op1, rtx target) 3487 { 3488 scalar_float_mode mode; 3489 const struct real_format *fmt; 3490 bool op0_is_abs; 3491 rtx temp; 3492 3493 mode = as_a <scalar_float_mode> (GET_MODE (op0)); 3494 gcc_assert (GET_MODE (op1) == mode); 3495 3496 /* First try to do it with a special instruction. */ 3497 temp = expand_binop (mode, copysign_optab, op0, op1, 3498 target, 0, OPTAB_DIRECT); 3499 if (temp) 3500 return temp; 3501 3502 fmt = REAL_MODE_FORMAT (mode); 3503 if (fmt == NULL || !fmt->has_signed_zero) 3504 return NULL_RTX; 3505 3506 op0_is_abs = false; 3507 if (CONST_DOUBLE_AS_FLOAT_P (op0)) 3508 { 3509 if (real_isneg (CONST_DOUBLE_REAL_VALUE (op0))) 3510 op0 = simplify_unary_operation (ABS, mode, op0, mode); 3511 op0_is_abs = true; 3512 } 3513 3514 if (fmt->signbit_ro >= 0 3515 && (CONST_DOUBLE_AS_FLOAT_P (op0) 3516 || (optab_handler (neg_optab, mode) != CODE_FOR_nothing 3517 && optab_handler (abs_optab, mode) != CODE_FOR_nothing))) 3518 { 3519 temp = expand_copysign_absneg (mode, op0, op1, target, 3520 fmt->signbit_ro, op0_is_abs); 3521 if (temp) 3522 return temp; 3523 } 3524 3525 if (fmt->signbit_rw < 0) 3526 return NULL_RTX; 3527 return expand_copysign_bit (mode, op0, op1, target, 3528 fmt->signbit_rw, op0_is_abs); 3529 } 3530 3531 /* Generate an instruction whose insn-code is INSN_CODE, 3532 with two operands: an output TARGET and an input OP0. 3533 TARGET *must* be nonzero, and the output is always stored there. 3534 CODE is an rtx code such that (CODE OP0) is an rtx that describes 3535 the value that is stored into TARGET. 3536 3537 Return false if expansion failed. */ 3538 3539 bool 3540 maybe_emit_unop_insn (enum insn_code icode, rtx target, rtx op0, 3541 enum rtx_code code) 3542 { 3543 struct expand_operand ops[2]; 3544 rtx_insn *pat; 3545 3546 create_output_operand (&ops[0], target, GET_MODE (target)); 3547 create_input_operand (&ops[1], op0, GET_MODE (op0)); 3548 pat = maybe_gen_insn (icode, 2, ops); 3549 if (!pat) 3550 return false; 3551 3552 if (INSN_P (pat) && NEXT_INSN (pat) != NULL_RTX 3553 && code != UNKNOWN) 3554 add_equal_note (pat, ops[0].value, code, ops[1].value, NULL_RTX, 3555 GET_MODE (op0)); 3556 3557 emit_insn (pat); 3558 3559 if (ops[0].value != target) 3560 emit_move_insn (target, ops[0].value); 3561 return true; 3562 } 3563 /* Generate an instruction whose insn-code is INSN_CODE, 3564 with two operands: an output TARGET and an input OP0. 3565 TARGET *must* be nonzero, and the output is always stored there. 3566 CODE is an rtx code such that (CODE OP0) is an rtx that describes 3567 the value that is stored into TARGET. */ 3568 3569 void 3570 emit_unop_insn (enum insn_code icode, rtx target, rtx op0, enum rtx_code code) 3571 { 3572 bool ok = maybe_emit_unop_insn (icode, target, op0, code); 3573 gcc_assert (ok); 3574 } 3575 3576 struct no_conflict_data 3577 { 3578 rtx target; 3579 rtx_insn *first, *insn; 3580 bool must_stay; 3581 }; 3582 3583 /* Called via note_stores by emit_libcall_block. Set P->must_stay if 3584 the currently examined clobber / store has to stay in the list of 3585 insns that constitute the actual libcall block. */ 3586 static void 3587 no_conflict_move_test (rtx dest, const_rtx set, void *p0) 3588 { 3589 struct no_conflict_data *p= (struct no_conflict_data *) p0; 3590 3591 /* If this inns directly contributes to setting the target, it must stay. */ 3592 if (reg_overlap_mentioned_p (p->target, dest)) 3593 p->must_stay = true; 3594 /* If we haven't committed to keeping any other insns in the list yet, 3595 there is nothing more to check. */ 3596 else if (p->insn == p->first) 3597 return; 3598 /* If this insn sets / clobbers a register that feeds one of the insns 3599 already in the list, this insn has to stay too. */ 3600 else if (reg_overlap_mentioned_p (dest, PATTERN (p->first)) 3601 || (CALL_P (p->first) && (find_reg_fusage (p->first, USE, dest))) 3602 || reg_used_between_p (dest, p->first, p->insn) 3603 /* Likewise if this insn depends on a register set by a previous 3604 insn in the list, or if it sets a result (presumably a hard 3605 register) that is set or clobbered by a previous insn. 3606 N.B. the modified_*_p (SET_DEST...) tests applied to a MEM 3607 SET_DEST perform the former check on the address, and the latter 3608 check on the MEM. */ 3609 || (GET_CODE (set) == SET 3610 && (modified_in_p (SET_SRC (set), p->first) 3611 || modified_in_p (SET_DEST (set), p->first) 3612 || modified_between_p (SET_SRC (set), p->first, p->insn) 3613 || modified_between_p (SET_DEST (set), p->first, p->insn)))) 3614 p->must_stay = true; 3615 } 3616 3617 3618 /* Emit code to make a call to a constant function or a library call. 3619 3620 INSNS is a list containing all insns emitted in the call. 3621 These insns leave the result in RESULT. Our block is to copy RESULT 3622 to TARGET, which is logically equivalent to EQUIV. 3623 3624 We first emit any insns that set a pseudo on the assumption that these are 3625 loading constants into registers; doing so allows them to be safely cse'ed 3626 between blocks. Then we emit all the other insns in the block, followed by 3627 an insn to move RESULT to TARGET. This last insn will have a REQ_EQUAL 3628 note with an operand of EQUIV. */ 3629 3630 static void 3631 emit_libcall_block_1 (rtx_insn *insns, rtx target, rtx result, rtx equiv, 3632 bool equiv_may_trap) 3633 { 3634 rtx final_dest = target; 3635 rtx_insn *next, *last, *insn; 3636 3637 /* If this is a reg with REG_USERVAR_P set, then it could possibly turn 3638 into a MEM later. Protect the libcall block from this change. */ 3639 if (! REG_P (target) || REG_USERVAR_P (target)) 3640 target = gen_reg_rtx (GET_MODE (target)); 3641 3642 /* If we're using non-call exceptions, a libcall corresponding to an 3643 operation that may trap may also trap. */ 3644 /* ??? See the comment in front of make_reg_eh_region_note. */ 3645 if (cfun->can_throw_non_call_exceptions 3646 && (equiv_may_trap || may_trap_p (equiv))) 3647 { 3648 for (insn = insns; insn; insn = NEXT_INSN (insn)) 3649 if (CALL_P (insn)) 3650 { 3651 rtx note = find_reg_note (insn, REG_EH_REGION, NULL_RTX); 3652 if (note) 3653 { 3654 int lp_nr = INTVAL (XEXP (note, 0)); 3655 if (lp_nr == 0 || lp_nr == INT_MIN) 3656 remove_note (insn, note); 3657 } 3658 } 3659 } 3660 else 3661 { 3662 /* Look for any CALL_INSNs in this sequence, and attach a REG_EH_REGION 3663 reg note to indicate that this call cannot throw or execute a nonlocal 3664 goto (unless there is already a REG_EH_REGION note, in which case 3665 we update it). */ 3666 for (insn = insns; insn; insn = NEXT_INSN (insn)) 3667 if (CALL_P (insn)) 3668 make_reg_eh_region_note_nothrow_nononlocal (insn); 3669 } 3670 3671 /* First emit all insns that set pseudos. Remove them from the list as 3672 we go. Avoid insns that set pseudos which were referenced in previous 3673 insns. These can be generated by move_by_pieces, for example, 3674 to update an address. Similarly, avoid insns that reference things 3675 set in previous insns. */ 3676 3677 for (insn = insns; insn; insn = next) 3678 { 3679 rtx set = single_set (insn); 3680 3681 next = NEXT_INSN (insn); 3682 3683 if (set != 0 && REG_P (SET_DEST (set)) 3684 && REGNO (SET_DEST (set)) >= FIRST_PSEUDO_REGISTER) 3685 { 3686 struct no_conflict_data data; 3687 3688 data.target = const0_rtx; 3689 data.first = insns; 3690 data.insn = insn; 3691 data.must_stay = 0; 3692 note_stores (PATTERN (insn), no_conflict_move_test, &data); 3693 if (! data.must_stay) 3694 { 3695 if (PREV_INSN (insn)) 3696 SET_NEXT_INSN (PREV_INSN (insn)) = next; 3697 else 3698 insns = next; 3699 3700 if (next) 3701 SET_PREV_INSN (next) = PREV_INSN (insn); 3702 3703 add_insn (insn); 3704 } 3705 } 3706 3707 /* Some ports use a loop to copy large arguments onto the stack. 3708 Don't move anything outside such a loop. */ 3709 if (LABEL_P (insn)) 3710 break; 3711 } 3712 3713 /* Write the remaining insns followed by the final copy. */ 3714 for (insn = insns; insn; insn = next) 3715 { 3716 next = NEXT_INSN (insn); 3717 3718 add_insn (insn); 3719 } 3720 3721 last = emit_move_insn (target, result); 3722 if (equiv) 3723 set_dst_reg_note (last, REG_EQUAL, copy_rtx (equiv), target); 3724 3725 if (final_dest != target) 3726 emit_move_insn (final_dest, target); 3727 } 3728 3729 void 3730 emit_libcall_block (rtx_insn *insns, rtx target, rtx result, rtx equiv) 3731 { 3732 emit_libcall_block_1 (insns, target, result, equiv, false); 3733 } 3734 3735 /* Nonzero if we can perform a comparison of mode MODE straightforwardly. 3736 PURPOSE describes how this comparison will be used. CODE is the rtx 3737 comparison code we will be using. 3738 3739 ??? Actually, CODE is slightly weaker than that. A target is still 3740 required to implement all of the normal bcc operations, but not 3741 required to implement all (or any) of the unordered bcc operations. */ 3742 3743 int 3744 can_compare_p (enum rtx_code code, machine_mode mode, 3745 enum can_compare_purpose purpose) 3746 { 3747 rtx test; 3748 test = gen_rtx_fmt_ee (code, mode, const0_rtx, const0_rtx); 3749 do 3750 { 3751 enum insn_code icode; 3752 3753 if (purpose == ccp_jump 3754 && (icode = optab_handler (cbranch_optab, mode)) != CODE_FOR_nothing 3755 && insn_operand_matches (icode, 0, test)) 3756 return 1; 3757 if (purpose == ccp_store_flag 3758 && (icode = optab_handler (cstore_optab, mode)) != CODE_FOR_nothing 3759 && insn_operand_matches (icode, 1, test)) 3760 return 1; 3761 if (purpose == ccp_cmov 3762 && optab_handler (cmov_optab, mode) != CODE_FOR_nothing) 3763 return 1; 3764 3765 mode = GET_MODE_WIDER_MODE (mode).else_void (); 3766 PUT_MODE (test, mode); 3767 } 3768 while (mode != VOIDmode); 3769 3770 return 0; 3771 } 3772 3773 /* This function is called when we are going to emit a compare instruction that 3774 compares the values found in X and Y, using the rtl operator COMPARISON. 3775 3776 If they have mode BLKmode, then SIZE specifies the size of both operands. 3777 3778 UNSIGNEDP nonzero says that the operands are unsigned; 3779 this matters if they need to be widened (as given by METHODS). 3780 3781 *PTEST is where the resulting comparison RTX is returned or NULL_RTX 3782 if we failed to produce one. 3783 3784 *PMODE is the mode of the inputs (in case they are const_int). 3785 3786 This function performs all the setup necessary so that the caller only has 3787 to emit a single comparison insn. This setup can involve doing a BLKmode 3788 comparison or emitting a library call to perform the comparison if no insn 3789 is available to handle it. 3790 The values which are passed in through pointers can be modified; the caller 3791 should perform the comparison on the modified values. Constant 3792 comparisons must have already been folded. */ 3793 3794 static void 3795 prepare_cmp_insn (rtx x, rtx y, enum rtx_code comparison, rtx size, 3796 int unsignedp, enum optab_methods methods, 3797 rtx *ptest, machine_mode *pmode) 3798 { 3799 machine_mode mode = *pmode; 3800 rtx libfunc, test; 3801 machine_mode cmp_mode; 3802 enum mode_class mclass; 3803 3804 /* The other methods are not needed. */ 3805 gcc_assert (methods == OPTAB_DIRECT || methods == OPTAB_WIDEN 3806 || methods == OPTAB_LIB_WIDEN); 3807 3808 /* If we are optimizing, force expensive constants into a register. */ 3809 if (CONSTANT_P (x) && optimize 3810 && (rtx_cost (x, mode, COMPARE, 0, optimize_insn_for_speed_p ()) 3811 > COSTS_N_INSNS (1))) 3812 x = force_reg (mode, x); 3813 3814 if (CONSTANT_P (y) && optimize 3815 && (rtx_cost (y, mode, COMPARE, 1, optimize_insn_for_speed_p ()) 3816 > COSTS_N_INSNS (1))) 3817 y = force_reg (mode, y); 3818 3819 #if HAVE_cc0 3820 /* Make sure if we have a canonical comparison. The RTL 3821 documentation states that canonical comparisons are required only 3822 for targets which have cc0. */ 3823 gcc_assert (!CONSTANT_P (x) || CONSTANT_P (y)); 3824 #endif 3825 3826 /* Don't let both operands fail to indicate the mode. */ 3827 if (GET_MODE (x) == VOIDmode && GET_MODE (y) == VOIDmode) 3828 x = force_reg (mode, x); 3829 if (mode == VOIDmode) 3830 mode = GET_MODE (x) != VOIDmode ? GET_MODE (x) : GET_MODE (y); 3831 3832 /* Handle all BLKmode compares. */ 3833 3834 if (mode == BLKmode) 3835 { 3836 machine_mode result_mode; 3837 enum insn_code cmp_code; 3838 rtx result; 3839 rtx opalign 3840 = GEN_INT (MIN (MEM_ALIGN (x), MEM_ALIGN (y)) / BITS_PER_UNIT); 3841 3842 gcc_assert (size); 3843 3844 /* Try to use a memory block compare insn - either cmpstr 3845 or cmpmem will do. */ 3846 opt_scalar_int_mode cmp_mode_iter; 3847 FOR_EACH_MODE_IN_CLASS (cmp_mode_iter, MODE_INT) 3848 { 3849 scalar_int_mode cmp_mode = cmp_mode_iter.require (); 3850 cmp_code = direct_optab_handler (cmpmem_optab, cmp_mode); 3851 if (cmp_code == CODE_FOR_nothing) 3852 cmp_code = direct_optab_handler (cmpstr_optab, cmp_mode); 3853 if (cmp_code == CODE_FOR_nothing) 3854 cmp_code = direct_optab_handler (cmpstrn_optab, cmp_mode); 3855 if (cmp_code == CODE_FOR_nothing) 3856 continue; 3857 3858 /* Must make sure the size fits the insn's mode. */ 3859 if (CONST_INT_P (size) 3860 ? UINTVAL (size) > GET_MODE_MASK (cmp_mode) 3861 : (GET_MODE_BITSIZE (as_a <scalar_int_mode> (GET_MODE (size))) 3862 > GET_MODE_BITSIZE (cmp_mode))) 3863 continue; 3864 3865 result_mode = insn_data[cmp_code].operand[0].mode; 3866 result = gen_reg_rtx (result_mode); 3867 size = convert_to_mode (cmp_mode, size, 1); 3868 emit_insn (GEN_FCN (cmp_code) (result, x, y, size, opalign)); 3869 3870 *ptest = gen_rtx_fmt_ee (comparison, VOIDmode, result, const0_rtx); 3871 *pmode = result_mode; 3872 return; 3873 } 3874 3875 if (methods != OPTAB_LIB && methods != OPTAB_LIB_WIDEN) 3876 goto fail; 3877 3878 /* Otherwise call a library function. */ 3879 result = emit_block_comp_via_libcall (x, y, size); 3880 3881 x = result; 3882 y = const0_rtx; 3883 mode = TYPE_MODE (integer_type_node); 3884 methods = OPTAB_LIB_WIDEN; 3885 unsignedp = false; 3886 } 3887 3888 /* Don't allow operands to the compare to trap, as that can put the 3889 compare and branch in different basic blocks. */ 3890 if (cfun->can_throw_non_call_exceptions) 3891 { 3892 if (may_trap_p (x)) 3893 x = copy_to_reg (x); 3894 if (may_trap_p (y)) 3895 y = copy_to_reg (y); 3896 } 3897 3898 if (GET_MODE_CLASS (mode) == MODE_CC) 3899 { 3900 enum insn_code icode = optab_handler (cbranch_optab, CCmode); 3901 test = gen_rtx_fmt_ee (comparison, VOIDmode, x, y); 3902 gcc_assert (icode != CODE_FOR_nothing 3903 && insn_operand_matches (icode, 0, test)); 3904 *ptest = test; 3905 return; 3906 } 3907 3908 mclass = GET_MODE_CLASS (mode); 3909 test = gen_rtx_fmt_ee (comparison, VOIDmode, x, y); 3910 FOR_EACH_MODE_FROM (cmp_mode, mode) 3911 { 3912 enum insn_code icode; 3913 icode = optab_handler (cbranch_optab, cmp_mode); 3914 if (icode != CODE_FOR_nothing 3915 && insn_operand_matches (icode, 0, test)) 3916 { 3917 rtx_insn *last = get_last_insn (); 3918 rtx op0 = prepare_operand (icode, x, 1, mode, cmp_mode, unsignedp); 3919 rtx op1 = prepare_operand (icode, y, 2, mode, cmp_mode, unsignedp); 3920 if (op0 && op1 3921 && insn_operand_matches (icode, 1, op0) 3922 && insn_operand_matches (icode, 2, op1)) 3923 { 3924 XEXP (test, 0) = op0; 3925 XEXP (test, 1) = op1; 3926 *ptest = test; 3927 *pmode = cmp_mode; 3928 return; 3929 } 3930 delete_insns_since (last); 3931 } 3932 3933 if (methods == OPTAB_DIRECT || !CLASS_HAS_WIDER_MODES_P (mclass)) 3934 break; 3935 } 3936 3937 if (methods != OPTAB_LIB_WIDEN) 3938 goto fail; 3939 3940 if (SCALAR_FLOAT_MODE_P (mode)) 3941 { 3942 /* Small trick if UNORDERED isn't implemented by the hardware. */ 3943 if (comparison == UNORDERED && rtx_equal_p (x, y)) 3944 { 3945 prepare_cmp_insn (x, y, UNLT, NULL_RTX, unsignedp, OPTAB_WIDEN, 3946 ptest, pmode); 3947 if (*ptest) 3948 return; 3949 } 3950 3951 prepare_float_lib_cmp (x, y, comparison, ptest, pmode); 3952 } 3953 else 3954 { 3955 rtx result; 3956 machine_mode ret_mode; 3957 3958 /* Handle a libcall just for the mode we are using. */ 3959 libfunc = optab_libfunc (cmp_optab, mode); 3960 gcc_assert (libfunc); 3961 3962 /* If we want unsigned, and this mode has a distinct unsigned 3963 comparison routine, use that. */ 3964 if (unsignedp) 3965 { 3966 rtx ulibfunc = optab_libfunc (ucmp_optab, mode); 3967 if (ulibfunc) 3968 libfunc = ulibfunc; 3969 } 3970 3971 ret_mode = targetm.libgcc_cmp_return_mode (); 3972 result = emit_library_call_value (libfunc, NULL_RTX, LCT_CONST, 3973 ret_mode, x, mode, y, mode); 3974 3975 /* There are two kinds of comparison routines. Biased routines 3976 return 0/1/2, and unbiased routines return -1/0/1. Other parts 3977 of gcc expect that the comparison operation is equivalent 3978 to the modified comparison. For signed comparisons compare the 3979 result against 1 in the biased case, and zero in the unbiased 3980 case. For unsigned comparisons always compare against 1 after 3981 biasing the unbiased result by adding 1. This gives us a way to 3982 represent LTU. 3983 The comparisons in the fixed-point helper library are always 3984 biased. */ 3985 x = result; 3986 y = const1_rtx; 3987 3988 if (!TARGET_LIB_INT_CMP_BIASED && !ALL_FIXED_POINT_MODE_P (mode)) 3989 { 3990 if (unsignedp) 3991 x = plus_constant (ret_mode, result, 1); 3992 else 3993 y = const0_rtx; 3994 } 3995 3996 *pmode = ret_mode; 3997 prepare_cmp_insn (x, y, comparison, NULL_RTX, unsignedp, methods, 3998 ptest, pmode); 3999 } 4000 4001 return; 4002 4003 fail: 4004 *ptest = NULL_RTX; 4005 } 4006 4007 /* Before emitting an insn with code ICODE, make sure that X, which is going 4008 to be used for operand OPNUM of the insn, is converted from mode MODE to 4009 WIDER_MODE (UNSIGNEDP determines whether it is an unsigned conversion), and 4010 that it is accepted by the operand predicate. Return the new value. */ 4011 4012 rtx 4013 prepare_operand (enum insn_code icode, rtx x, int opnum, machine_mode mode, 4014 machine_mode wider_mode, int unsignedp) 4015 { 4016 if (mode != wider_mode) 4017 x = convert_modes (wider_mode, mode, x, unsignedp); 4018 4019 if (!insn_operand_matches (icode, opnum, x)) 4020 { 4021 machine_mode op_mode = insn_data[(int) icode].operand[opnum].mode; 4022 if (reload_completed) 4023 return NULL_RTX; 4024 if (GET_MODE (x) != op_mode && GET_MODE (x) != VOIDmode) 4025 return NULL_RTX; 4026 x = copy_to_mode_reg (op_mode, x); 4027 } 4028 4029 return x; 4030 } 4031 4032 /* Subroutine of emit_cmp_and_jump_insns; this function is called when we know 4033 we can do the branch. */ 4034 4035 static void 4036 emit_cmp_and_jump_insn_1 (rtx test, machine_mode mode, rtx label, 4037 profile_probability prob) 4038 { 4039 machine_mode optab_mode; 4040 enum mode_class mclass; 4041 enum insn_code icode; 4042 rtx_insn *insn; 4043 4044 mclass = GET_MODE_CLASS (mode); 4045 optab_mode = (mclass == MODE_CC) ? CCmode : mode; 4046 icode = optab_handler (cbranch_optab, optab_mode); 4047 4048 gcc_assert (icode != CODE_FOR_nothing); 4049 gcc_assert (insn_operand_matches (icode, 0, test)); 4050 insn = emit_jump_insn (GEN_FCN (icode) (test, XEXP (test, 0), 4051 XEXP (test, 1), label)); 4052 if (prob.initialized_p () 4053 && profile_status_for_fn (cfun) != PROFILE_ABSENT 4054 && insn 4055 && JUMP_P (insn) 4056 && any_condjump_p (insn) 4057 && !find_reg_note (insn, REG_BR_PROB, 0)) 4058 add_reg_br_prob_note (insn, prob); 4059 } 4060 4061 /* Generate code to compare X with Y so that the condition codes are 4062 set and to jump to LABEL if the condition is true. If X is a 4063 constant and Y is not a constant, then the comparison is swapped to 4064 ensure that the comparison RTL has the canonical form. 4065 4066 UNSIGNEDP nonzero says that X and Y are unsigned; this matters if they 4067 need to be widened. UNSIGNEDP is also used to select the proper 4068 branch condition code. 4069 4070 If X and Y have mode BLKmode, then SIZE specifies the size of both X and Y. 4071 4072 MODE is the mode of the inputs (in case they are const_int). 4073 4074 COMPARISON is the rtl operator to compare with (EQ, NE, GT, etc.). 4075 It will be potentially converted into an unsigned variant based on 4076 UNSIGNEDP to select a proper jump instruction. 4077 4078 PROB is the probability of jumping to LABEL. */ 4079 4080 void 4081 emit_cmp_and_jump_insns (rtx x, rtx y, enum rtx_code comparison, rtx size, 4082 machine_mode mode, int unsignedp, rtx label, 4083 profile_probability prob) 4084 { 4085 rtx op0 = x, op1 = y; 4086 rtx test; 4087 4088 /* Swap operands and condition to ensure canonical RTL. */ 4089 if (swap_commutative_operands_p (x, y) 4090 && can_compare_p (swap_condition (comparison), mode, ccp_jump)) 4091 { 4092 op0 = y, op1 = x; 4093 comparison = swap_condition (comparison); 4094 } 4095 4096 /* If OP0 is still a constant, then both X and Y must be constants 4097 or the opposite comparison is not supported. Force X into a register 4098 to create canonical RTL. */ 4099 if (CONSTANT_P (op0)) 4100 op0 = force_reg (mode, op0); 4101 4102 if (unsignedp) 4103 comparison = unsigned_condition (comparison); 4104 4105 prepare_cmp_insn (op0, op1, comparison, size, unsignedp, OPTAB_LIB_WIDEN, 4106 &test, &mode); 4107 emit_cmp_and_jump_insn_1 (test, mode, label, prob); 4108 } 4109 4110 4111 /* Emit a library call comparison between floating point X and Y. 4112 COMPARISON is the rtl operator to compare with (EQ, NE, GT, etc.). */ 4113 4114 static void 4115 prepare_float_lib_cmp (rtx x, rtx y, enum rtx_code comparison, 4116 rtx *ptest, machine_mode *pmode) 4117 { 4118 enum rtx_code swapped = swap_condition (comparison); 4119 enum rtx_code reversed = reverse_condition_maybe_unordered (comparison); 4120 machine_mode orig_mode = GET_MODE (x); 4121 machine_mode mode; 4122 rtx true_rtx, false_rtx; 4123 rtx value, target, equiv; 4124 rtx_insn *insns; 4125 rtx libfunc = 0; 4126 bool reversed_p = false; 4127 scalar_int_mode cmp_mode = targetm.libgcc_cmp_return_mode (); 4128 4129 FOR_EACH_MODE_FROM (mode, orig_mode) 4130 { 4131 if (code_to_optab (comparison) 4132 && (libfunc = optab_libfunc (code_to_optab (comparison), mode))) 4133 break; 4134 4135 if (code_to_optab (swapped) 4136 && (libfunc = optab_libfunc (code_to_optab (swapped), mode))) 4137 { 4138 std::swap (x, y); 4139 comparison = swapped; 4140 break; 4141 } 4142 4143 if (code_to_optab (reversed) 4144 && (libfunc = optab_libfunc (code_to_optab (reversed), mode))) 4145 { 4146 comparison = reversed; 4147 reversed_p = true; 4148 break; 4149 } 4150 } 4151 4152 gcc_assert (mode != VOIDmode); 4153 4154 if (mode != orig_mode) 4155 { 4156 x = convert_to_mode (mode, x, 0); 4157 y = convert_to_mode (mode, y, 0); 4158 } 4159 4160 /* Attach a REG_EQUAL note describing the semantics of the libcall to 4161 the RTL. The allows the RTL optimizers to delete the libcall if the 4162 condition can be determined at compile-time. */ 4163 if (comparison == UNORDERED 4164 || FLOAT_LIB_COMPARE_RETURNS_BOOL (mode, comparison)) 4165 { 4166 true_rtx = const_true_rtx; 4167 false_rtx = const0_rtx; 4168 } 4169 else 4170 { 4171 switch (comparison) 4172 { 4173 case EQ: 4174 true_rtx = const0_rtx; 4175 false_rtx = const_true_rtx; 4176 break; 4177 4178 case NE: 4179 true_rtx = const_true_rtx; 4180 false_rtx = const0_rtx; 4181 break; 4182 4183 case GT: 4184 true_rtx = const1_rtx; 4185 false_rtx = const0_rtx; 4186 break; 4187 4188 case GE: 4189 true_rtx = const0_rtx; 4190 false_rtx = constm1_rtx; 4191 break; 4192 4193 case LT: 4194 true_rtx = constm1_rtx; 4195 false_rtx = const0_rtx; 4196 break; 4197 4198 case LE: 4199 true_rtx = const0_rtx; 4200 false_rtx = const1_rtx; 4201 break; 4202 4203 default: 4204 gcc_unreachable (); 4205 } 4206 } 4207 4208 if (comparison == UNORDERED) 4209 { 4210 rtx temp = simplify_gen_relational (NE, cmp_mode, mode, x, x); 4211 equiv = simplify_gen_relational (NE, cmp_mode, mode, y, y); 4212 equiv = simplify_gen_ternary (IF_THEN_ELSE, cmp_mode, cmp_mode, 4213 temp, const_true_rtx, equiv); 4214 } 4215 else 4216 { 4217 equiv = simplify_gen_relational (comparison, cmp_mode, mode, x, y); 4218 if (! FLOAT_LIB_COMPARE_RETURNS_BOOL (mode, comparison)) 4219 equiv = simplify_gen_ternary (IF_THEN_ELSE, cmp_mode, cmp_mode, 4220 equiv, true_rtx, false_rtx); 4221 } 4222 4223 start_sequence (); 4224 value = emit_library_call_value (libfunc, NULL_RTX, LCT_CONST, 4225 cmp_mode, x, mode, y, mode); 4226 insns = get_insns (); 4227 end_sequence (); 4228 4229 target = gen_reg_rtx (cmp_mode); 4230 emit_libcall_block (insns, target, value, equiv); 4231 4232 if (comparison == UNORDERED 4233 || FLOAT_LIB_COMPARE_RETURNS_BOOL (mode, comparison) 4234 || reversed_p) 4235 *ptest = gen_rtx_fmt_ee (reversed_p ? EQ : NE, VOIDmode, target, false_rtx); 4236 else 4237 *ptest = gen_rtx_fmt_ee (comparison, VOIDmode, target, const0_rtx); 4238 4239 *pmode = cmp_mode; 4240 } 4241 4242 /* Generate code to indirectly jump to a location given in the rtx LOC. */ 4243 4244 void 4245 emit_indirect_jump (rtx loc) 4246 { 4247 if (!targetm.have_indirect_jump ()) 4248 sorry ("indirect jumps are not available on this target"); 4249 else 4250 { 4251 struct expand_operand ops[1]; 4252 create_address_operand (&ops[0], loc); 4253 expand_jump_insn (targetm.code_for_indirect_jump, 1, ops); 4254 emit_barrier (); 4255 } 4256 } 4257 4258 4259 /* Emit a conditional move instruction if the machine supports one for that 4260 condition and machine mode. 4261 4262 OP0 and OP1 are the operands that should be compared using CODE. CMODE is 4263 the mode to use should they be constants. If it is VOIDmode, they cannot 4264 both be constants. 4265 4266 OP2 should be stored in TARGET if the comparison is true, otherwise OP3 4267 should be stored there. MODE is the mode to use should they be constants. 4268 If it is VOIDmode, they cannot both be constants. 4269 4270 The result is either TARGET (perhaps modified) or NULL_RTX if the operation 4271 is not supported. */ 4272 4273 rtx 4274 emit_conditional_move (rtx target, enum rtx_code code, rtx op0, rtx op1, 4275 machine_mode cmode, rtx op2, rtx op3, 4276 machine_mode mode, int unsignedp) 4277 { 4278 rtx comparison; 4279 rtx_insn *last; 4280 enum insn_code icode; 4281 enum rtx_code reversed; 4282 4283 /* If the two source operands are identical, that's just a move. */ 4284 4285 if (rtx_equal_p (op2, op3)) 4286 { 4287 if (!target) 4288 target = gen_reg_rtx (mode); 4289 4290 emit_move_insn (target, op3); 4291 return target; 4292 } 4293 4294 /* If one operand is constant, make it the second one. Only do this 4295 if the other operand is not constant as well. */ 4296 4297 if (swap_commutative_operands_p (op0, op1)) 4298 { 4299 std::swap (op0, op1); 4300 code = swap_condition (code); 4301 } 4302 4303 /* get_condition will prefer to generate LT and GT even if the old 4304 comparison was against zero, so undo that canonicalization here since 4305 comparisons against zero are cheaper. */ 4306 if (code == LT && op1 == const1_rtx) 4307 code = LE, op1 = const0_rtx; 4308 else if (code == GT && op1 == constm1_rtx) 4309 code = GE, op1 = const0_rtx; 4310 4311 if (cmode == VOIDmode) 4312 cmode = GET_MODE (op0); 4313 4314 enum rtx_code orig_code = code; 4315 bool swapped = false; 4316 if (swap_commutative_operands_p (op2, op3) 4317 && ((reversed = reversed_comparison_code_parts (code, op0, op1, NULL)) 4318 != UNKNOWN)) 4319 { 4320 std::swap (op2, op3); 4321 code = reversed; 4322 swapped = true; 4323 } 4324 4325 if (mode == VOIDmode) 4326 mode = GET_MODE (op2); 4327 4328 icode = direct_optab_handler (movcc_optab, mode); 4329 4330 if (icode == CODE_FOR_nothing) 4331 return NULL_RTX; 4332 4333 if (!target) 4334 target = gen_reg_rtx (mode); 4335 4336 for (int pass = 0; ; pass++) 4337 { 4338 code = unsignedp ? unsigned_condition (code) : code; 4339 comparison = simplify_gen_relational (code, VOIDmode, cmode, op0, op1); 4340 4341 /* We can get const0_rtx or const_true_rtx in some circumstances. Just 4342 punt and let the caller figure out how best to deal with this 4343 situation. */ 4344 if (COMPARISON_P (comparison)) 4345 { 4346 saved_pending_stack_adjust save; 4347 save_pending_stack_adjust (&save); 4348 last = get_last_insn (); 4349 do_pending_stack_adjust (); 4350 machine_mode cmpmode = cmode; 4351 prepare_cmp_insn (XEXP (comparison, 0), XEXP (comparison, 1), 4352 GET_CODE (comparison), NULL_RTX, unsignedp, 4353 OPTAB_WIDEN, &comparison, &cmpmode); 4354 if (comparison) 4355 { 4356 struct expand_operand ops[4]; 4357 4358 create_output_operand (&ops[0], target, mode); 4359 create_fixed_operand (&ops[1], comparison); 4360 create_input_operand (&ops[2], op2, mode); 4361 create_input_operand (&ops[3], op3, mode); 4362 if (maybe_expand_insn (icode, 4, ops)) 4363 { 4364 if (ops[0].value != target) 4365 convert_move (target, ops[0].value, false); 4366 return target; 4367 } 4368 } 4369 delete_insns_since (last); 4370 restore_pending_stack_adjust (&save); 4371 } 4372 4373 if (pass == 1) 4374 return NULL_RTX; 4375 4376 /* If the preferred op2/op3 order is not usable, retry with other 4377 operand order, perhaps it will expand successfully. */ 4378 if (swapped) 4379 code = orig_code; 4380 else if ((reversed = reversed_comparison_code_parts (orig_code, op0, op1, 4381 NULL)) 4382 != UNKNOWN) 4383 code = reversed; 4384 else 4385 return NULL_RTX; 4386 std::swap (op2, op3); 4387 } 4388 } 4389 4390 4391 /* Emit a conditional negate or bitwise complement using the 4392 negcc or notcc optabs if available. Return NULL_RTX if such operations 4393 are not available. Otherwise return the RTX holding the result. 4394 TARGET is the desired destination of the result. COMP is the comparison 4395 on which to negate. If COND is true move into TARGET the negation 4396 or bitwise complement of OP1. Otherwise move OP2 into TARGET. 4397 CODE is either NEG or NOT. MODE is the machine mode in which the 4398 operation is performed. */ 4399 4400 rtx 4401 emit_conditional_neg_or_complement (rtx target, rtx_code code, 4402 machine_mode mode, rtx cond, rtx op1, 4403 rtx op2) 4404 { 4405 optab op = unknown_optab; 4406 if (code == NEG) 4407 op = negcc_optab; 4408 else if (code == NOT) 4409 op = notcc_optab; 4410 else 4411 gcc_unreachable (); 4412 4413 insn_code icode = direct_optab_handler (op, mode); 4414 4415 if (icode == CODE_FOR_nothing) 4416 return NULL_RTX; 4417 4418 if (!target) 4419 target = gen_reg_rtx (mode); 4420 4421 rtx_insn *last = get_last_insn (); 4422 struct expand_operand ops[4]; 4423 4424 create_output_operand (&ops[0], target, mode); 4425 create_fixed_operand (&ops[1], cond); 4426 create_input_operand (&ops[2], op1, mode); 4427 create_input_operand (&ops[3], op2, mode); 4428 4429 if (maybe_expand_insn (icode, 4, ops)) 4430 { 4431 if (ops[0].value != target) 4432 convert_move (target, ops[0].value, false); 4433 4434 return target; 4435 } 4436 delete_insns_since (last); 4437 return NULL_RTX; 4438 } 4439 4440 /* Emit a conditional addition instruction if the machine supports one for that 4441 condition and machine mode. 4442 4443 OP0 and OP1 are the operands that should be compared using CODE. CMODE is 4444 the mode to use should they be constants. If it is VOIDmode, they cannot 4445 both be constants. 4446 4447 OP2 should be stored in TARGET if the comparison is false, otherwise OP2+OP3 4448 should be stored there. MODE is the mode to use should they be constants. 4449 If it is VOIDmode, they cannot both be constants. 4450 4451 The result is either TARGET (perhaps modified) or NULL_RTX if the operation 4452 is not supported. */ 4453 4454 rtx 4455 emit_conditional_add (rtx target, enum rtx_code code, rtx op0, rtx op1, 4456 machine_mode cmode, rtx op2, rtx op3, 4457 machine_mode mode, int unsignedp) 4458 { 4459 rtx comparison; 4460 rtx_insn *last; 4461 enum insn_code icode; 4462 4463 /* If one operand is constant, make it the second one. Only do this 4464 if the other operand is not constant as well. */ 4465 4466 if (swap_commutative_operands_p (op0, op1)) 4467 { 4468 std::swap (op0, op1); 4469 code = swap_condition (code); 4470 } 4471 4472 /* get_condition will prefer to generate LT and GT even if the old 4473 comparison was against zero, so undo that canonicalization here since 4474 comparisons against zero are cheaper. */ 4475 if (code == LT && op1 == const1_rtx) 4476 code = LE, op1 = const0_rtx; 4477 else if (code == GT && op1 == constm1_rtx) 4478 code = GE, op1 = const0_rtx; 4479 4480 if (cmode == VOIDmode) 4481 cmode = GET_MODE (op0); 4482 4483 if (mode == VOIDmode) 4484 mode = GET_MODE (op2); 4485 4486 icode = optab_handler (addcc_optab, mode); 4487 4488 if (icode == CODE_FOR_nothing) 4489 return 0; 4490 4491 if (!target) 4492 target = gen_reg_rtx (mode); 4493 4494 code = unsignedp ? unsigned_condition (code) : code; 4495 comparison = simplify_gen_relational (code, VOIDmode, cmode, op0, op1); 4496 4497 /* We can get const0_rtx or const_true_rtx in some circumstances. Just 4498 return NULL and let the caller figure out how best to deal with this 4499 situation. */ 4500 if (!COMPARISON_P (comparison)) 4501 return NULL_RTX; 4502 4503 do_pending_stack_adjust (); 4504 last = get_last_insn (); 4505 prepare_cmp_insn (XEXP (comparison, 0), XEXP (comparison, 1), 4506 GET_CODE (comparison), NULL_RTX, unsignedp, OPTAB_WIDEN, 4507 &comparison, &cmode); 4508 if (comparison) 4509 { 4510 struct expand_operand ops[4]; 4511 4512 create_output_operand (&ops[0], target, mode); 4513 create_fixed_operand (&ops[1], comparison); 4514 create_input_operand (&ops[2], op2, mode); 4515 create_input_operand (&ops[3], op3, mode); 4516 if (maybe_expand_insn (icode, 4, ops)) 4517 { 4518 if (ops[0].value != target) 4519 convert_move (target, ops[0].value, false); 4520 return target; 4521 } 4522 } 4523 delete_insns_since (last); 4524 return NULL_RTX; 4525 } 4526 4527 /* These functions attempt to generate an insn body, rather than 4528 emitting the insn, but if the gen function already emits them, we 4529 make no attempt to turn them back into naked patterns. */ 4530 4531 /* Generate and return an insn body to add Y to X. */ 4532 4533 rtx_insn * 4534 gen_add2_insn (rtx x, rtx y) 4535 { 4536 enum insn_code icode = optab_handler (add_optab, GET_MODE (x)); 4537 4538 gcc_assert (insn_operand_matches (icode, 0, x)); 4539 gcc_assert (insn_operand_matches (icode, 1, x)); 4540 gcc_assert (insn_operand_matches (icode, 2, y)); 4541 4542 return GEN_FCN (icode) (x, x, y); 4543 } 4544 4545 /* Generate and return an insn body to add r1 and c, 4546 storing the result in r0. */ 4547 4548 rtx_insn * 4549 gen_add3_insn (rtx r0, rtx r1, rtx c) 4550 { 4551 enum insn_code icode = optab_handler (add_optab, GET_MODE (r0)); 4552 4553 if (icode == CODE_FOR_nothing 4554 || !insn_operand_matches (icode, 0, r0) 4555 || !insn_operand_matches (icode, 1, r1) 4556 || !insn_operand_matches (icode, 2, c)) 4557 return NULL; 4558 4559 return GEN_FCN (icode) (r0, r1, c); 4560 } 4561 4562 int 4563 have_add2_insn (rtx x, rtx y) 4564 { 4565 enum insn_code icode; 4566 4567 gcc_assert (GET_MODE (x) != VOIDmode); 4568 4569 icode = optab_handler (add_optab, GET_MODE (x)); 4570 4571 if (icode == CODE_FOR_nothing) 4572 return 0; 4573 4574 if (!insn_operand_matches (icode, 0, x) 4575 || !insn_operand_matches (icode, 1, x) 4576 || !insn_operand_matches (icode, 2, y)) 4577 return 0; 4578 4579 return 1; 4580 } 4581 4582 /* Generate and return an insn body to add Y to X. */ 4583 4584 rtx_insn * 4585 gen_addptr3_insn (rtx x, rtx y, rtx z) 4586 { 4587 enum insn_code icode = optab_handler (addptr3_optab, GET_MODE (x)); 4588 4589 gcc_assert (insn_operand_matches (icode, 0, x)); 4590 gcc_assert (insn_operand_matches (icode, 1, y)); 4591 gcc_assert (insn_operand_matches (icode, 2, z)); 4592 4593 return GEN_FCN (icode) (x, y, z); 4594 } 4595 4596 /* Return true if the target implements an addptr pattern and X, Y, 4597 and Z are valid for the pattern predicates. */ 4598 4599 int 4600 have_addptr3_insn (rtx x, rtx y, rtx z) 4601 { 4602 enum insn_code icode; 4603 4604 gcc_assert (GET_MODE (x) != VOIDmode); 4605 4606 icode = optab_handler (addptr3_optab, GET_MODE (x)); 4607 4608 if (icode == CODE_FOR_nothing) 4609 return 0; 4610 4611 if (!insn_operand_matches (icode, 0, x) 4612 || !insn_operand_matches (icode, 1, y) 4613 || !insn_operand_matches (icode, 2, z)) 4614 return 0; 4615 4616 return 1; 4617 } 4618 4619 /* Generate and return an insn body to subtract Y from X. */ 4620 4621 rtx_insn * 4622 gen_sub2_insn (rtx x, rtx y) 4623 { 4624 enum insn_code icode = optab_handler (sub_optab, GET_MODE (x)); 4625 4626 gcc_assert (insn_operand_matches (icode, 0, x)); 4627 gcc_assert (insn_operand_matches (icode, 1, x)); 4628 gcc_assert (insn_operand_matches (icode, 2, y)); 4629 4630 return GEN_FCN (icode) (x, x, y); 4631 } 4632 4633 /* Generate and return an insn body to subtract r1 and c, 4634 storing the result in r0. */ 4635 4636 rtx_insn * 4637 gen_sub3_insn (rtx r0, rtx r1, rtx c) 4638 { 4639 enum insn_code icode = optab_handler (sub_optab, GET_MODE (r0)); 4640 4641 if (icode == CODE_FOR_nothing 4642 || !insn_operand_matches (icode, 0, r0) 4643 || !insn_operand_matches (icode, 1, r1) 4644 || !insn_operand_matches (icode, 2, c)) 4645 return NULL; 4646 4647 return GEN_FCN (icode) (r0, r1, c); 4648 } 4649 4650 int 4651 have_sub2_insn (rtx x, rtx y) 4652 { 4653 enum insn_code icode; 4654 4655 gcc_assert (GET_MODE (x) != VOIDmode); 4656 4657 icode = optab_handler (sub_optab, GET_MODE (x)); 4658 4659 if (icode == CODE_FOR_nothing) 4660 return 0; 4661 4662 if (!insn_operand_matches (icode, 0, x) 4663 || !insn_operand_matches (icode, 1, x) 4664 || !insn_operand_matches (icode, 2, y)) 4665 return 0; 4666 4667 return 1; 4668 } 4669 4670 /* Generate the body of an insn to extend Y (with mode MFROM) 4671 into X (with mode MTO). Do zero-extension if UNSIGNEDP is nonzero. */ 4672 4673 rtx_insn * 4674 gen_extend_insn (rtx x, rtx y, machine_mode mto, 4675 machine_mode mfrom, int unsignedp) 4676 { 4677 enum insn_code icode = can_extend_p (mto, mfrom, unsignedp); 4678 return GEN_FCN (icode) (x, y); 4679 } 4680 4681 /* Generate code to convert FROM to floating point 4682 and store in TO. FROM must be fixed point and not VOIDmode. 4683 UNSIGNEDP nonzero means regard FROM as unsigned. 4684 Normally this is done by correcting the final value 4685 if it is negative. */ 4686 4687 void 4688 expand_float (rtx to, rtx from, int unsignedp) 4689 { 4690 enum insn_code icode; 4691 rtx target = to; 4692 scalar_mode from_mode, to_mode; 4693 machine_mode fmode, imode; 4694 bool can_do_signed = false; 4695 4696 /* Crash now, because we won't be able to decide which mode to use. */ 4697 gcc_assert (GET_MODE (from) != VOIDmode); 4698 4699 /* Look for an insn to do the conversion. Do it in the specified 4700 modes if possible; otherwise convert either input, output or both to 4701 wider mode. If the integer mode is wider than the mode of FROM, 4702 we can do the conversion signed even if the input is unsigned. */ 4703 4704 FOR_EACH_MODE_FROM (fmode, GET_MODE (to)) 4705 FOR_EACH_MODE_FROM (imode, GET_MODE (from)) 4706 { 4707 int doing_unsigned = unsignedp; 4708 4709 if (fmode != GET_MODE (to) 4710 && (significand_size (fmode) 4711 < GET_MODE_UNIT_PRECISION (GET_MODE (from)))) 4712 continue; 4713 4714 icode = can_float_p (fmode, imode, unsignedp); 4715 if (icode == CODE_FOR_nothing && unsignedp) 4716 { 4717 enum insn_code scode = can_float_p (fmode, imode, 0); 4718 if (scode != CODE_FOR_nothing) 4719 can_do_signed = true; 4720 if (imode != GET_MODE (from)) 4721 icode = scode, doing_unsigned = 0; 4722 } 4723 4724 if (icode != CODE_FOR_nothing) 4725 { 4726 if (imode != GET_MODE (from)) 4727 from = convert_to_mode (imode, from, unsignedp); 4728 4729 if (fmode != GET_MODE (to)) 4730 target = gen_reg_rtx (fmode); 4731 4732 emit_unop_insn (icode, target, from, 4733 doing_unsigned ? UNSIGNED_FLOAT : FLOAT); 4734 4735 if (target != to) 4736 convert_move (to, target, 0); 4737 return; 4738 } 4739 } 4740 4741 /* Unsigned integer, and no way to convert directly. Convert as signed, 4742 then unconditionally adjust the result. */ 4743 if (unsignedp 4744 && can_do_signed 4745 && is_a <scalar_mode> (GET_MODE (to), &to_mode) 4746 && is_a <scalar_mode> (GET_MODE (from), &from_mode)) 4747 { 4748 opt_scalar_mode fmode_iter; 4749 rtx_code_label *label = gen_label_rtx (); 4750 rtx temp; 4751 REAL_VALUE_TYPE offset; 4752 4753 /* Look for a usable floating mode FMODE wider than the source and at 4754 least as wide as the target. Using FMODE will avoid rounding woes 4755 with unsigned values greater than the signed maximum value. */ 4756 4757 FOR_EACH_MODE_FROM (fmode_iter, to_mode) 4758 { 4759 scalar_mode fmode = fmode_iter.require (); 4760 if (GET_MODE_PRECISION (from_mode) < GET_MODE_BITSIZE (fmode) 4761 && can_float_p (fmode, from_mode, 0) != CODE_FOR_nothing) 4762 break; 4763 } 4764 4765 if (!fmode_iter.exists (&fmode)) 4766 { 4767 /* There is no such mode. Pretend the target is wide enough. */ 4768 fmode = to_mode; 4769 4770 /* Avoid double-rounding when TO is narrower than FROM. */ 4771 if ((significand_size (fmode) + 1) 4772 < GET_MODE_PRECISION (from_mode)) 4773 { 4774 rtx temp1; 4775 rtx_code_label *neglabel = gen_label_rtx (); 4776 4777 /* Don't use TARGET if it isn't a register, is a hard register, 4778 or is the wrong mode. */ 4779 if (!REG_P (target) 4780 || REGNO (target) < FIRST_PSEUDO_REGISTER 4781 || GET_MODE (target) != fmode) 4782 target = gen_reg_rtx (fmode); 4783 4784 imode = from_mode; 4785 do_pending_stack_adjust (); 4786 4787 /* Test whether the sign bit is set. */ 4788 emit_cmp_and_jump_insns (from, const0_rtx, LT, NULL_RTX, imode, 4789 0, neglabel); 4790 4791 /* The sign bit is not set. Convert as signed. */ 4792 expand_float (target, from, 0); 4793 emit_jump_insn (targetm.gen_jump (label)); 4794 emit_barrier (); 4795 4796 /* The sign bit is set. 4797 Convert to a usable (positive signed) value by shifting right 4798 one bit, while remembering if a nonzero bit was shifted 4799 out; i.e., compute (from & 1) | (from >> 1). */ 4800 4801 emit_label (neglabel); 4802 temp = expand_binop (imode, and_optab, from, const1_rtx, 4803 NULL_RTX, 1, OPTAB_LIB_WIDEN); 4804 temp1 = expand_shift (RSHIFT_EXPR, imode, from, 1, NULL_RTX, 1); 4805 temp = expand_binop (imode, ior_optab, temp, temp1, temp, 1, 4806 OPTAB_LIB_WIDEN); 4807 expand_float (target, temp, 0); 4808 4809 /* Multiply by 2 to undo the shift above. */ 4810 temp = expand_binop (fmode, add_optab, target, target, 4811 target, 0, OPTAB_LIB_WIDEN); 4812 if (temp != target) 4813 emit_move_insn (target, temp); 4814 4815 do_pending_stack_adjust (); 4816 emit_label (label); 4817 goto done; 4818 } 4819 } 4820 4821 /* If we are about to do some arithmetic to correct for an 4822 unsigned operand, do it in a pseudo-register. */ 4823 4824 if (to_mode != fmode 4825 || !REG_P (to) || REGNO (to) < FIRST_PSEUDO_REGISTER) 4826 target = gen_reg_rtx (fmode); 4827 4828 /* Convert as signed integer to floating. */ 4829 expand_float (target, from, 0); 4830 4831 /* If FROM is negative (and therefore TO is negative), 4832 correct its value by 2**bitwidth. */ 4833 4834 do_pending_stack_adjust (); 4835 emit_cmp_and_jump_insns (from, const0_rtx, GE, NULL_RTX, from_mode, 4836 0, label); 4837 4838 4839 real_2expN (&offset, GET_MODE_PRECISION (from_mode), fmode); 4840 temp = expand_binop (fmode, add_optab, target, 4841 const_double_from_real_value (offset, fmode), 4842 target, 0, OPTAB_LIB_WIDEN); 4843 if (temp != target) 4844 emit_move_insn (target, temp); 4845 4846 do_pending_stack_adjust (); 4847 emit_label (label); 4848 goto done; 4849 } 4850 4851 /* No hardware instruction available; call a library routine. */ 4852 { 4853 rtx libfunc; 4854 rtx_insn *insns; 4855 rtx value; 4856 convert_optab tab = unsignedp ? ufloat_optab : sfloat_optab; 4857 4858 if (is_narrower_int_mode (GET_MODE (from), SImode)) 4859 from = convert_to_mode (SImode, from, unsignedp); 4860 4861 libfunc = convert_optab_libfunc (tab, GET_MODE (to), GET_MODE (from)); 4862 gcc_assert (libfunc); 4863 4864 start_sequence (); 4865 4866 value = emit_library_call_value (libfunc, NULL_RTX, LCT_CONST, 4867 GET_MODE (to), from, GET_MODE (from)); 4868 insns = get_insns (); 4869 end_sequence (); 4870 4871 emit_libcall_block (insns, target, value, 4872 gen_rtx_fmt_e (unsignedp ? UNSIGNED_FLOAT : FLOAT, 4873 GET_MODE (to), from)); 4874 } 4875 4876 done: 4877 4878 /* Copy result to requested destination 4879 if we have been computing in a temp location. */ 4880 4881 if (target != to) 4882 { 4883 if (GET_MODE (target) == GET_MODE (to)) 4884 emit_move_insn (to, target); 4885 else 4886 convert_move (to, target, 0); 4887 } 4888 } 4889 4890 /* Generate code to convert FROM to fixed point and store in TO. FROM 4891 must be floating point. */ 4892 4893 void 4894 expand_fix (rtx to, rtx from, int unsignedp) 4895 { 4896 enum insn_code icode; 4897 rtx target = to; 4898 machine_mode fmode, imode; 4899 opt_scalar_mode fmode_iter; 4900 bool must_trunc = false; 4901 4902 /* We first try to find a pair of modes, one real and one integer, at 4903 least as wide as FROM and TO, respectively, in which we can open-code 4904 this conversion. If the integer mode is wider than the mode of TO, 4905 we can do the conversion either signed or unsigned. */ 4906 4907 FOR_EACH_MODE_FROM (fmode, GET_MODE (from)) 4908 FOR_EACH_MODE_FROM (imode, GET_MODE (to)) 4909 { 4910 int doing_unsigned = unsignedp; 4911 4912 icode = can_fix_p (imode, fmode, unsignedp, &must_trunc); 4913 if (icode == CODE_FOR_nothing && imode != GET_MODE (to) && unsignedp) 4914 icode = can_fix_p (imode, fmode, 0, &must_trunc), doing_unsigned = 0; 4915 4916 if (icode != CODE_FOR_nothing) 4917 { 4918 rtx_insn *last = get_last_insn (); 4919 if (fmode != GET_MODE (from)) 4920 from = convert_to_mode (fmode, from, 0); 4921 4922 if (must_trunc) 4923 { 4924 rtx temp = gen_reg_rtx (GET_MODE (from)); 4925 from = expand_unop (GET_MODE (from), ftrunc_optab, from, 4926 temp, 0); 4927 } 4928 4929 if (imode != GET_MODE (to)) 4930 target = gen_reg_rtx (imode); 4931 4932 if (maybe_emit_unop_insn (icode, target, from, 4933 doing_unsigned ? UNSIGNED_FIX : FIX)) 4934 { 4935 if (target != to) 4936 convert_move (to, target, unsignedp); 4937 return; 4938 } 4939 delete_insns_since (last); 4940 } 4941 } 4942 4943 /* For an unsigned conversion, there is one more way to do it. 4944 If we have a signed conversion, we generate code that compares 4945 the real value to the largest representable positive number. If if 4946 is smaller, the conversion is done normally. Otherwise, subtract 4947 one plus the highest signed number, convert, and add it back. 4948 4949 We only need to check all real modes, since we know we didn't find 4950 anything with a wider integer mode. 4951 4952 This code used to extend FP value into mode wider than the destination. 4953 This is needed for decimal float modes which cannot accurately 4954 represent one plus the highest signed number of the same size, but 4955 not for binary modes. Consider, for instance conversion from SFmode 4956 into DImode. 4957 4958 The hot path through the code is dealing with inputs smaller than 2^63 4959 and doing just the conversion, so there is no bits to lose. 4960 4961 In the other path we know the value is positive in the range 2^63..2^64-1 4962 inclusive. (as for other input overflow happens and result is undefined) 4963 So we know that the most important bit set in mantissa corresponds to 4964 2^63. The subtraction of 2^63 should not generate any rounding as it 4965 simply clears out that bit. The rest is trivial. */ 4966 4967 scalar_int_mode to_mode; 4968 if (unsignedp 4969 && is_a <scalar_int_mode> (GET_MODE (to), &to_mode) 4970 && HWI_COMPUTABLE_MODE_P (to_mode)) 4971 FOR_EACH_MODE_FROM (fmode_iter, as_a <scalar_mode> (GET_MODE (from))) 4972 { 4973 scalar_mode fmode = fmode_iter.require (); 4974 if (CODE_FOR_nothing != can_fix_p (to_mode, fmode, 4975 0, &must_trunc) 4976 && (!DECIMAL_FLOAT_MODE_P (fmode) 4977 || (GET_MODE_BITSIZE (fmode) > GET_MODE_PRECISION (to_mode)))) 4978 { 4979 int bitsize; 4980 REAL_VALUE_TYPE offset; 4981 rtx limit; 4982 rtx_code_label *lab1, *lab2; 4983 rtx_insn *insn; 4984 4985 bitsize = GET_MODE_PRECISION (to_mode); 4986 real_2expN (&offset, bitsize - 1, fmode); 4987 limit = const_double_from_real_value (offset, fmode); 4988 lab1 = gen_label_rtx (); 4989 lab2 = gen_label_rtx (); 4990 4991 if (fmode != GET_MODE (from)) 4992 from = convert_to_mode (fmode, from, 0); 4993 4994 /* See if we need to do the subtraction. */ 4995 do_pending_stack_adjust (); 4996 emit_cmp_and_jump_insns (from, limit, GE, NULL_RTX, 4997 GET_MODE (from), 0, lab1); 4998 4999 /* If not, do the signed "fix" and branch around fixup code. */ 5000 expand_fix (to, from, 0); 5001 emit_jump_insn (targetm.gen_jump (lab2)); 5002 emit_barrier (); 5003 5004 /* Otherwise, subtract 2**(N-1), convert to signed number, 5005 then add 2**(N-1). Do the addition using XOR since this 5006 will often generate better code. */ 5007 emit_label (lab1); 5008 target = expand_binop (GET_MODE (from), sub_optab, from, limit, 5009 NULL_RTX, 0, OPTAB_LIB_WIDEN); 5010 expand_fix (to, target, 0); 5011 target = expand_binop (to_mode, xor_optab, to, 5012 gen_int_mode 5013 (HOST_WIDE_INT_1 << (bitsize - 1), 5014 to_mode), 5015 to, 1, OPTAB_LIB_WIDEN); 5016 5017 if (target != to) 5018 emit_move_insn (to, target); 5019 5020 emit_label (lab2); 5021 5022 if (optab_handler (mov_optab, to_mode) != CODE_FOR_nothing) 5023 { 5024 /* Make a place for a REG_NOTE and add it. */ 5025 insn = emit_move_insn (to, to); 5026 set_dst_reg_note (insn, REG_EQUAL, 5027 gen_rtx_fmt_e (UNSIGNED_FIX, to_mode, 5028 copy_rtx (from)), 5029 to); 5030 } 5031 5032 return; 5033 } 5034 } 5035 5036 /* We can't do it with an insn, so use a library call. But first ensure 5037 that the mode of TO is at least as wide as SImode, since those are the 5038 only library calls we know about. */ 5039 5040 if (is_narrower_int_mode (GET_MODE (to), SImode)) 5041 { 5042 target = gen_reg_rtx (SImode); 5043 5044 expand_fix (target, from, unsignedp); 5045 } 5046 else 5047 { 5048 rtx_insn *insns; 5049 rtx value; 5050 rtx libfunc; 5051 5052 convert_optab tab = unsignedp ? ufix_optab : sfix_optab; 5053 libfunc = convert_optab_libfunc (tab, GET_MODE (to), GET_MODE (from)); 5054 gcc_assert (libfunc); 5055 5056 start_sequence (); 5057 5058 value = emit_library_call_value (libfunc, NULL_RTX, LCT_CONST, 5059 GET_MODE (to), from, GET_MODE (from)); 5060 insns = get_insns (); 5061 end_sequence (); 5062 5063 emit_libcall_block (insns, target, value, 5064 gen_rtx_fmt_e (unsignedp ? UNSIGNED_FIX : FIX, 5065 GET_MODE (to), from)); 5066 } 5067 5068 if (target != to) 5069 { 5070 if (GET_MODE (to) == GET_MODE (target)) 5071 emit_move_insn (to, target); 5072 else 5073 convert_move (to, target, 0); 5074 } 5075 } 5076 5077 5078 /* Promote integer arguments for a libcall if necessary. 5079 emit_library_call_value cannot do the promotion because it does not 5080 know if it should do a signed or unsigned promotion. This is because 5081 there are no tree types defined for libcalls. */ 5082 5083 static rtx 5084 prepare_libcall_arg (rtx arg, int uintp) 5085 { 5086 scalar_int_mode mode; 5087 machine_mode arg_mode; 5088 if (is_a <scalar_int_mode> (GET_MODE (arg), &mode)) 5089 { 5090 /* If we need to promote the integer function argument we need to do 5091 it here instead of inside emit_library_call_value because in 5092 emit_library_call_value we don't know if we should do a signed or 5093 unsigned promotion. */ 5094 5095 int unsigned_p = 0; 5096 arg_mode = promote_function_mode (NULL_TREE, mode, 5097 &unsigned_p, NULL_TREE, 0); 5098 if (arg_mode != mode) 5099 return convert_to_mode (arg_mode, arg, uintp); 5100 } 5101 return arg; 5102 } 5103 5104 /* Generate code to convert FROM or TO a fixed-point. 5105 If UINTP is true, either TO or FROM is an unsigned integer. 5106 If SATP is true, we need to saturate the result. */ 5107 5108 void 5109 expand_fixed_convert (rtx to, rtx from, int uintp, int satp) 5110 { 5111 machine_mode to_mode = GET_MODE (to); 5112 machine_mode from_mode = GET_MODE (from); 5113 convert_optab tab; 5114 enum rtx_code this_code; 5115 enum insn_code code; 5116 rtx_insn *insns; 5117 rtx value; 5118 rtx libfunc; 5119 5120 if (to_mode == from_mode) 5121 { 5122 emit_move_insn (to, from); 5123 return; 5124 } 5125 5126 if (uintp) 5127 { 5128 tab = satp ? satfractuns_optab : fractuns_optab; 5129 this_code = satp ? UNSIGNED_SAT_FRACT : UNSIGNED_FRACT_CONVERT; 5130 } 5131 else 5132 { 5133 tab = satp ? satfract_optab : fract_optab; 5134 this_code = satp ? SAT_FRACT : FRACT_CONVERT; 5135 } 5136 code = convert_optab_handler (tab, to_mode, from_mode); 5137 if (code != CODE_FOR_nothing) 5138 { 5139 emit_unop_insn (code, to, from, this_code); 5140 return; 5141 } 5142 5143 libfunc = convert_optab_libfunc (tab, to_mode, from_mode); 5144 gcc_assert (libfunc); 5145 5146 from = prepare_libcall_arg (from, uintp); 5147 from_mode = GET_MODE (from); 5148 5149 start_sequence (); 5150 value = emit_library_call_value (libfunc, NULL_RTX, LCT_CONST, to_mode, 5151 from, from_mode); 5152 insns = get_insns (); 5153 end_sequence (); 5154 5155 emit_libcall_block (insns, to, value, 5156 gen_rtx_fmt_e (optab_to_code (tab), to_mode, from)); 5157 } 5158 5159 /* Generate code to convert FROM to fixed point and store in TO. FROM 5160 must be floating point, TO must be signed. Use the conversion optab 5161 TAB to do the conversion. */ 5162 5163 bool 5164 expand_sfix_optab (rtx to, rtx from, convert_optab tab) 5165 { 5166 enum insn_code icode; 5167 rtx target = to; 5168 machine_mode fmode, imode; 5169 5170 /* We first try to find a pair of modes, one real and one integer, at 5171 least as wide as FROM and TO, respectively, in which we can open-code 5172 this conversion. If the integer mode is wider than the mode of TO, 5173 we can do the conversion either signed or unsigned. */ 5174 5175 FOR_EACH_MODE_FROM (fmode, GET_MODE (from)) 5176 FOR_EACH_MODE_FROM (imode, GET_MODE (to)) 5177 { 5178 icode = convert_optab_handler (tab, imode, fmode); 5179 if (icode != CODE_FOR_nothing) 5180 { 5181 rtx_insn *last = get_last_insn (); 5182 if (fmode != GET_MODE (from)) 5183 from = convert_to_mode (fmode, from, 0); 5184 5185 if (imode != GET_MODE (to)) 5186 target = gen_reg_rtx (imode); 5187 5188 if (!maybe_emit_unop_insn (icode, target, from, UNKNOWN)) 5189 { 5190 delete_insns_since (last); 5191 continue; 5192 } 5193 if (target != to) 5194 convert_move (to, target, 0); 5195 return true; 5196 } 5197 } 5198 5199 return false; 5200 } 5201 5202 /* Report whether we have an instruction to perform the operation 5203 specified by CODE on operands of mode MODE. */ 5204 int 5205 have_insn_for (enum rtx_code code, machine_mode mode) 5206 { 5207 return (code_to_optab (code) 5208 && (optab_handler (code_to_optab (code), mode) 5209 != CODE_FOR_nothing)); 5210 } 5211 5212 /* Print information about the current contents of the optabs on 5213 STDERR. */ 5214 5215 DEBUG_FUNCTION void 5216 debug_optab_libfuncs (void) 5217 { 5218 int i, j, k; 5219 5220 /* Dump the arithmetic optabs. */ 5221 for (i = FIRST_NORM_OPTAB; i <= LAST_NORMLIB_OPTAB; ++i) 5222 for (j = 0; j < NUM_MACHINE_MODES; ++j) 5223 { 5224 rtx l = optab_libfunc ((optab) i, (machine_mode) j); 5225 if (l) 5226 { 5227 gcc_assert (GET_CODE (l) == SYMBOL_REF); 5228 fprintf (stderr, "%s\t%s:\t%s\n", 5229 GET_RTX_NAME (optab_to_code ((optab) i)), 5230 GET_MODE_NAME (j), 5231 XSTR (l, 0)); 5232 } 5233 } 5234 5235 /* Dump the conversion optabs. */ 5236 for (i = FIRST_CONV_OPTAB; i <= LAST_CONVLIB_OPTAB; ++i) 5237 for (j = 0; j < NUM_MACHINE_MODES; ++j) 5238 for (k = 0; k < NUM_MACHINE_MODES; ++k) 5239 { 5240 rtx l = convert_optab_libfunc ((optab) i, (machine_mode) j, 5241 (machine_mode) k); 5242 if (l) 5243 { 5244 gcc_assert (GET_CODE (l) == SYMBOL_REF); 5245 fprintf (stderr, "%s\t%s\t%s:\t%s\n", 5246 GET_RTX_NAME (optab_to_code ((optab) i)), 5247 GET_MODE_NAME (j), 5248 GET_MODE_NAME (k), 5249 XSTR (l, 0)); 5250 } 5251 } 5252 } 5253 5254 /* Generate insns to trap with code TCODE if OP1 and OP2 satisfy condition 5255 CODE. Return 0 on failure. */ 5256 5257 rtx_insn * 5258 gen_cond_trap (enum rtx_code code, rtx op1, rtx op2, rtx tcode) 5259 { 5260 machine_mode mode = GET_MODE (op1); 5261 enum insn_code icode; 5262 rtx_insn *insn; 5263 rtx trap_rtx; 5264 5265 if (mode == VOIDmode) 5266 return 0; 5267 5268 icode = optab_handler (ctrap_optab, mode); 5269 if (icode == CODE_FOR_nothing) 5270 return 0; 5271 5272 /* Some targets only accept a zero trap code. */ 5273 if (!insn_operand_matches (icode, 3, tcode)) 5274 return 0; 5275 5276 do_pending_stack_adjust (); 5277 start_sequence (); 5278 prepare_cmp_insn (op1, op2, code, NULL_RTX, false, OPTAB_DIRECT, 5279 &trap_rtx, &mode); 5280 if (!trap_rtx) 5281 insn = NULL; 5282 else 5283 insn = GEN_FCN (icode) (trap_rtx, XEXP (trap_rtx, 0), XEXP (trap_rtx, 1), 5284 tcode); 5285 5286 /* If that failed, then give up. */ 5287 if (insn == 0) 5288 { 5289 end_sequence (); 5290 return 0; 5291 } 5292 5293 emit_insn (insn); 5294 insn = get_insns (); 5295 end_sequence (); 5296 return insn; 5297 } 5298 5299 /* Return rtx code for TCODE. Use UNSIGNEDP to select signed 5300 or unsigned operation code. */ 5301 5302 enum rtx_code 5303 get_rtx_code (enum tree_code tcode, bool unsignedp) 5304 { 5305 enum rtx_code code; 5306 switch (tcode) 5307 { 5308 case EQ_EXPR: 5309 code = EQ; 5310 break; 5311 case NE_EXPR: 5312 code = NE; 5313 break; 5314 case LT_EXPR: 5315 code = unsignedp ? LTU : LT; 5316 break; 5317 case LE_EXPR: 5318 code = unsignedp ? LEU : LE; 5319 break; 5320 case GT_EXPR: 5321 code = unsignedp ? GTU : GT; 5322 break; 5323 case GE_EXPR: 5324 code = unsignedp ? GEU : GE; 5325 break; 5326 5327 case UNORDERED_EXPR: 5328 code = UNORDERED; 5329 break; 5330 case ORDERED_EXPR: 5331 code = ORDERED; 5332 break; 5333 case UNLT_EXPR: 5334 code = UNLT; 5335 break; 5336 case UNLE_EXPR: 5337 code = UNLE; 5338 break; 5339 case UNGT_EXPR: 5340 code = UNGT; 5341 break; 5342 case UNGE_EXPR: 5343 code = UNGE; 5344 break; 5345 case UNEQ_EXPR: 5346 code = UNEQ; 5347 break; 5348 case LTGT_EXPR: 5349 code = LTGT; 5350 break; 5351 5352 case BIT_AND_EXPR: 5353 code = AND; 5354 break; 5355 5356 case BIT_IOR_EXPR: 5357 code = IOR; 5358 break; 5359 5360 default: 5361 gcc_unreachable (); 5362 } 5363 return code; 5364 } 5365 5366 /* Return a comparison rtx of mode CMP_MODE for COND. Use UNSIGNEDP to 5367 select signed or unsigned operators. OPNO holds the index of the 5368 first comparison operand for insn ICODE. Do not generate the 5369 compare instruction itself. */ 5370 5371 static rtx 5372 vector_compare_rtx (machine_mode cmp_mode, enum tree_code tcode, 5373 tree t_op0, tree t_op1, bool unsignedp, 5374 enum insn_code icode, unsigned int opno) 5375 { 5376 struct expand_operand ops[2]; 5377 rtx rtx_op0, rtx_op1; 5378 machine_mode m0, m1; 5379 enum rtx_code rcode = get_rtx_code (tcode, unsignedp); 5380 5381 gcc_assert (TREE_CODE_CLASS (tcode) == tcc_comparison); 5382 5383 /* Expand operands. For vector types with scalar modes, e.g. where int64x1_t 5384 has mode DImode, this can produce a constant RTX of mode VOIDmode; in such 5385 cases, use the original mode. */ 5386 rtx_op0 = expand_expr (t_op0, NULL_RTX, TYPE_MODE (TREE_TYPE (t_op0)), 5387 EXPAND_STACK_PARM); 5388 m0 = GET_MODE (rtx_op0); 5389 if (m0 == VOIDmode) 5390 m0 = TYPE_MODE (TREE_TYPE (t_op0)); 5391 5392 rtx_op1 = expand_expr (t_op1, NULL_RTX, TYPE_MODE (TREE_TYPE (t_op1)), 5393 EXPAND_STACK_PARM); 5394 m1 = GET_MODE (rtx_op1); 5395 if (m1 == VOIDmode) 5396 m1 = TYPE_MODE (TREE_TYPE (t_op1)); 5397 5398 create_input_operand (&ops[0], rtx_op0, m0); 5399 create_input_operand (&ops[1], rtx_op1, m1); 5400 if (!maybe_legitimize_operands (icode, opno, 2, ops)) 5401 gcc_unreachable (); 5402 return gen_rtx_fmt_ee (rcode, cmp_mode, ops[0].value, ops[1].value); 5403 } 5404 5405 /* Check if vec_perm mask SEL is a constant equivalent to a shift of 5406 the first vec_perm operand, assuming the second operand is a constant 5407 vector of zeros. Return the shift distance in bits if so, or NULL_RTX 5408 if the vec_perm is not a shift. MODE is the mode of the value being 5409 shifted. */ 5410 static rtx 5411 shift_amt_for_vec_perm_mask (machine_mode mode, const vec_perm_indices &sel) 5412 { 5413 unsigned int bitsize = GET_MODE_UNIT_BITSIZE (mode); 5414 poly_int64 first = sel[0]; 5415 if (maybe_ge (sel[0], GET_MODE_NUNITS (mode))) 5416 return NULL_RTX; 5417 5418 if (!sel.series_p (0, 1, first, 1)) 5419 { 5420 unsigned int nelt; 5421 if (!GET_MODE_NUNITS (mode).is_constant (&nelt)) 5422 return NULL_RTX; 5423 for (unsigned int i = 1; i < nelt; i++) 5424 { 5425 poly_int64 expected = i + first; 5426 /* Indices into the second vector are all equivalent. */ 5427 if (maybe_lt (sel[i], nelt) 5428 ? maybe_ne (sel[i], expected) 5429 : maybe_lt (expected, nelt)) 5430 return NULL_RTX; 5431 } 5432 } 5433 5434 return gen_int_shift_amount (mode, first * bitsize); 5435 } 5436 5437 /* A subroutine of expand_vec_perm_var for expanding one vec_perm insn. */ 5438 5439 static rtx 5440 expand_vec_perm_1 (enum insn_code icode, rtx target, 5441 rtx v0, rtx v1, rtx sel) 5442 { 5443 machine_mode tmode = GET_MODE (target); 5444 machine_mode smode = GET_MODE (sel); 5445 struct expand_operand ops[4]; 5446 5447 gcc_assert (GET_MODE_CLASS (smode) == MODE_VECTOR_INT 5448 || mode_for_int_vector (tmode).require () == smode); 5449 create_output_operand (&ops[0], target, tmode); 5450 create_input_operand (&ops[3], sel, smode); 5451 5452 /* Make an effort to preserve v0 == v1. The target expander is able to 5453 rely on this to determine if we're permuting a single input operand. */ 5454 if (rtx_equal_p (v0, v1)) 5455 { 5456 if (!insn_operand_matches (icode, 1, v0)) 5457 v0 = force_reg (tmode, v0); 5458 gcc_checking_assert (insn_operand_matches (icode, 1, v0)); 5459 gcc_checking_assert (insn_operand_matches (icode, 2, v0)); 5460 5461 create_fixed_operand (&ops[1], v0); 5462 create_fixed_operand (&ops[2], v0); 5463 } 5464 else 5465 { 5466 create_input_operand (&ops[1], v0, tmode); 5467 create_input_operand (&ops[2], v1, tmode); 5468 } 5469 5470 if (maybe_expand_insn (icode, 4, ops)) 5471 return ops[0].value; 5472 return NULL_RTX; 5473 } 5474 5475 /* Implement a permutation of vectors v0 and v1 using the permutation 5476 vector in SEL and return the result. Use TARGET to hold the result 5477 if nonnull and convenient. 5478 5479 MODE is the mode of the vectors being permuted (V0 and V1). SEL_MODE 5480 is the TYPE_MODE associated with SEL, or BLKmode if SEL isn't known 5481 to have a particular mode. */ 5482 5483 rtx 5484 expand_vec_perm_const (machine_mode mode, rtx v0, rtx v1, 5485 const vec_perm_builder &sel, machine_mode sel_mode, 5486 rtx target) 5487 { 5488 if (!target || !register_operand (target, mode)) 5489 target = gen_reg_rtx (mode); 5490 5491 /* Set QIMODE to a different vector mode with byte elements. 5492 If no such mode, or if MODE already has byte elements, use VOIDmode. */ 5493 machine_mode qimode; 5494 if (!qimode_for_vec_perm (mode).exists (&qimode)) 5495 qimode = VOIDmode; 5496 5497 rtx_insn *last = get_last_insn (); 5498 5499 bool single_arg_p = rtx_equal_p (v0, v1); 5500 /* Always specify two input vectors here and leave the target to handle 5501 cases in which the inputs are equal. Not all backends can cope with 5502 the single-input representation when testing for a double-input 5503 target instruction. */ 5504 vec_perm_indices indices (sel, 2, GET_MODE_NUNITS (mode)); 5505 5506 /* See if this can be handled with a vec_shr. We only do this if the 5507 second vector is all zeroes. */ 5508 insn_code shift_code = optab_handler (vec_shr_optab, mode); 5509 insn_code shift_code_qi = ((qimode != VOIDmode && qimode != mode) 5510 ? optab_handler (vec_shr_optab, qimode) 5511 : CODE_FOR_nothing); 5512 5513 if (v1 == CONST0_RTX (GET_MODE (v1)) 5514 && (shift_code != CODE_FOR_nothing 5515 || shift_code_qi != CODE_FOR_nothing)) 5516 { 5517 rtx shift_amt = shift_amt_for_vec_perm_mask (mode, indices); 5518 if (shift_amt) 5519 { 5520 struct expand_operand ops[3]; 5521 if (shift_code != CODE_FOR_nothing) 5522 { 5523 create_output_operand (&ops[0], target, mode); 5524 create_input_operand (&ops[1], v0, mode); 5525 create_convert_operand_from_type (&ops[2], shift_amt, sizetype); 5526 if (maybe_expand_insn (shift_code, 3, ops)) 5527 return ops[0].value; 5528 } 5529 if (shift_code_qi != CODE_FOR_nothing) 5530 { 5531 rtx tmp = gen_reg_rtx (qimode); 5532 create_output_operand (&ops[0], tmp, qimode); 5533 create_input_operand (&ops[1], gen_lowpart (qimode, v0), qimode); 5534 create_convert_operand_from_type (&ops[2], shift_amt, sizetype); 5535 if (maybe_expand_insn (shift_code_qi, 3, ops)) 5536 return gen_lowpart (mode, ops[0].value); 5537 } 5538 } 5539 } 5540 5541 if (targetm.vectorize.vec_perm_const != NULL) 5542 { 5543 v0 = force_reg (mode, v0); 5544 if (single_arg_p) 5545 v1 = v0; 5546 else 5547 v1 = force_reg (mode, v1); 5548 5549 if (targetm.vectorize.vec_perm_const (mode, target, v0, v1, indices)) 5550 return target; 5551 } 5552 5553 /* Fall back to a constant byte-based permutation. */ 5554 vec_perm_indices qimode_indices; 5555 rtx target_qi = NULL_RTX, v0_qi = NULL_RTX, v1_qi = NULL_RTX; 5556 if (qimode != VOIDmode) 5557 { 5558 qimode_indices.new_expanded_vector (indices, GET_MODE_UNIT_SIZE (mode)); 5559 target_qi = gen_reg_rtx (qimode); 5560 v0_qi = gen_lowpart (qimode, v0); 5561 v1_qi = gen_lowpart (qimode, v1); 5562 if (targetm.vectorize.vec_perm_const != NULL 5563 && targetm.vectorize.vec_perm_const (qimode, target_qi, v0_qi, 5564 v1_qi, qimode_indices)) 5565 return gen_lowpart (mode, target_qi); 5566 } 5567 5568 /* Otherwise expand as a fully variable permuation. */ 5569 5570 /* The optabs are only defined for selectors with the same width 5571 as the values being permuted. */ 5572 machine_mode required_sel_mode; 5573 if (!mode_for_int_vector (mode).exists (&required_sel_mode) 5574 || !VECTOR_MODE_P (required_sel_mode)) 5575 { 5576 delete_insns_since (last); 5577 return NULL_RTX; 5578 } 5579 5580 /* We know that it is semantically valid to treat SEL as having SEL_MODE. 5581 If that isn't the mode we want then we need to prove that using 5582 REQUIRED_SEL_MODE is OK. */ 5583 if (sel_mode != required_sel_mode) 5584 { 5585 if (!selector_fits_mode_p (required_sel_mode, indices)) 5586 { 5587 delete_insns_since (last); 5588 return NULL_RTX; 5589 } 5590 sel_mode = required_sel_mode; 5591 } 5592 5593 insn_code icode = direct_optab_handler (vec_perm_optab, mode); 5594 if (icode != CODE_FOR_nothing) 5595 { 5596 rtx sel_rtx = vec_perm_indices_to_rtx (sel_mode, indices); 5597 rtx tmp = expand_vec_perm_1 (icode, target, v0, v1, sel_rtx); 5598 if (tmp) 5599 return tmp; 5600 } 5601 5602 if (qimode != VOIDmode 5603 && selector_fits_mode_p (qimode, qimode_indices)) 5604 { 5605 icode = direct_optab_handler (vec_perm_optab, qimode); 5606 if (icode != CODE_FOR_nothing) 5607 { 5608 rtx sel_qi = vec_perm_indices_to_rtx (qimode, qimode_indices); 5609 rtx tmp = expand_vec_perm_1 (icode, target_qi, v0_qi, v1_qi, sel_qi); 5610 if (tmp) 5611 return gen_lowpart (mode, tmp); 5612 } 5613 } 5614 5615 delete_insns_since (last); 5616 return NULL_RTX; 5617 } 5618 5619 /* Implement a permutation of vectors v0 and v1 using the permutation 5620 vector in SEL and return the result. Use TARGET to hold the result 5621 if nonnull and convenient. 5622 5623 MODE is the mode of the vectors being permuted (V0 and V1). 5624 SEL must have the integer equivalent of MODE and is known to be 5625 unsuitable for permutes with a constant permutation vector. */ 5626 5627 rtx 5628 expand_vec_perm_var (machine_mode mode, rtx v0, rtx v1, rtx sel, rtx target) 5629 { 5630 enum insn_code icode; 5631 unsigned int i, u; 5632 rtx tmp, sel_qi; 5633 5634 u = GET_MODE_UNIT_SIZE (mode); 5635 5636 if (!target || GET_MODE (target) != mode) 5637 target = gen_reg_rtx (mode); 5638 5639 icode = direct_optab_handler (vec_perm_optab, mode); 5640 if (icode != CODE_FOR_nothing) 5641 { 5642 tmp = expand_vec_perm_1 (icode, target, v0, v1, sel); 5643 if (tmp) 5644 return tmp; 5645 } 5646 5647 /* As a special case to aid several targets, lower the element-based 5648 permutation to a byte-based permutation and try again. */ 5649 machine_mode qimode; 5650 if (!qimode_for_vec_perm (mode).exists (&qimode) 5651 || maybe_gt (GET_MODE_NUNITS (qimode), GET_MODE_MASK (QImode) + 1)) 5652 return NULL_RTX; 5653 icode = direct_optab_handler (vec_perm_optab, qimode); 5654 if (icode == CODE_FOR_nothing) 5655 return NULL_RTX; 5656 5657 /* Multiply each element by its byte size. */ 5658 machine_mode selmode = GET_MODE (sel); 5659 if (u == 2) 5660 sel = expand_simple_binop (selmode, PLUS, sel, sel, 5661 NULL, 0, OPTAB_DIRECT); 5662 else 5663 sel = expand_simple_binop (selmode, ASHIFT, sel, 5664 gen_int_shift_amount (selmode, exact_log2 (u)), 5665 NULL, 0, OPTAB_DIRECT); 5666 gcc_assert (sel != NULL); 5667 5668 /* Broadcast the low byte each element into each of its bytes. 5669 The encoding has U interleaved stepped patterns, one for each 5670 byte of an element. */ 5671 vec_perm_builder const_sel (GET_MODE_SIZE (mode), u, 3); 5672 unsigned int low_byte_in_u = BYTES_BIG_ENDIAN ? u - 1 : 0; 5673 for (i = 0; i < 3; ++i) 5674 for (unsigned int j = 0; j < u; ++j) 5675 const_sel.quick_push (i * u + low_byte_in_u); 5676 sel = gen_lowpart (qimode, sel); 5677 sel = expand_vec_perm_const (qimode, sel, sel, const_sel, qimode, NULL); 5678 gcc_assert (sel != NULL); 5679 5680 /* Add the byte offset to each byte element. */ 5681 /* Note that the definition of the indicies here is memory ordering, 5682 so there should be no difference between big and little endian. */ 5683 rtx_vector_builder byte_indices (qimode, u, 1); 5684 for (i = 0; i < u; ++i) 5685 byte_indices.quick_push (GEN_INT (i)); 5686 tmp = byte_indices.build (); 5687 sel_qi = expand_simple_binop (qimode, PLUS, sel, tmp, 5688 sel, 0, OPTAB_DIRECT); 5689 gcc_assert (sel_qi != NULL); 5690 5691 tmp = mode != qimode ? gen_reg_rtx (qimode) : target; 5692 tmp = expand_vec_perm_1 (icode, tmp, gen_lowpart (qimode, v0), 5693 gen_lowpart (qimode, v1), sel_qi); 5694 if (tmp) 5695 tmp = gen_lowpart (mode, tmp); 5696 return tmp; 5697 } 5698 5699 /* Generate insns for a VEC_COND_EXPR with mask, given its TYPE and its 5700 three operands. */ 5701 5702 rtx 5703 expand_vec_cond_mask_expr (tree vec_cond_type, tree op0, tree op1, tree op2, 5704 rtx target) 5705 { 5706 struct expand_operand ops[4]; 5707 machine_mode mode = TYPE_MODE (vec_cond_type); 5708 machine_mode mask_mode = TYPE_MODE (TREE_TYPE (op0)); 5709 enum insn_code icode = get_vcond_mask_icode (mode, mask_mode); 5710 rtx mask, rtx_op1, rtx_op2; 5711 5712 if (icode == CODE_FOR_nothing) 5713 return 0; 5714 5715 mask = expand_normal (op0); 5716 rtx_op1 = expand_normal (op1); 5717 rtx_op2 = expand_normal (op2); 5718 5719 mask = force_reg (mask_mode, mask); 5720 rtx_op1 = force_reg (GET_MODE (rtx_op1), rtx_op1); 5721 5722 create_output_operand (&ops[0], target, mode); 5723 create_input_operand (&ops[1], rtx_op1, mode); 5724 create_input_operand (&ops[2], rtx_op2, mode); 5725 create_input_operand (&ops[3], mask, mask_mode); 5726 expand_insn (icode, 4, ops); 5727 5728 return ops[0].value; 5729 } 5730 5731 /* Generate insns for a VEC_COND_EXPR, given its TYPE and its 5732 three operands. */ 5733 5734 rtx 5735 expand_vec_cond_expr (tree vec_cond_type, tree op0, tree op1, tree op2, 5736 rtx target) 5737 { 5738 struct expand_operand ops[6]; 5739 enum insn_code icode; 5740 rtx comparison, rtx_op1, rtx_op2; 5741 machine_mode mode = TYPE_MODE (vec_cond_type); 5742 machine_mode cmp_op_mode; 5743 bool unsignedp; 5744 tree op0a, op0b; 5745 enum tree_code tcode; 5746 5747 if (COMPARISON_CLASS_P (op0)) 5748 { 5749 op0a = TREE_OPERAND (op0, 0); 5750 op0b = TREE_OPERAND (op0, 1); 5751 tcode = TREE_CODE (op0); 5752 } 5753 else 5754 { 5755 gcc_assert (VECTOR_BOOLEAN_TYPE_P (TREE_TYPE (op0))); 5756 if (get_vcond_mask_icode (mode, TYPE_MODE (TREE_TYPE (op0))) 5757 != CODE_FOR_nothing) 5758 return expand_vec_cond_mask_expr (vec_cond_type, op0, op1, 5759 op2, target); 5760 /* Fake op0 < 0. */ 5761 else 5762 { 5763 gcc_assert (GET_MODE_CLASS (TYPE_MODE (TREE_TYPE (op0))) 5764 == MODE_VECTOR_INT); 5765 op0a = op0; 5766 op0b = build_zero_cst (TREE_TYPE (op0)); 5767 tcode = LT_EXPR; 5768 } 5769 } 5770 cmp_op_mode = TYPE_MODE (TREE_TYPE (op0a)); 5771 unsignedp = TYPE_UNSIGNED (TREE_TYPE (op0a)); 5772 5773 5774 gcc_assert (known_eq (GET_MODE_SIZE (mode), GET_MODE_SIZE (cmp_op_mode)) 5775 && known_eq (GET_MODE_NUNITS (mode), 5776 GET_MODE_NUNITS (cmp_op_mode))); 5777 5778 icode = get_vcond_icode (mode, cmp_op_mode, unsignedp); 5779 if (icode == CODE_FOR_nothing) 5780 { 5781 if (tcode == EQ_EXPR || tcode == NE_EXPR) 5782 icode = get_vcond_eq_icode (mode, cmp_op_mode); 5783 if (icode == CODE_FOR_nothing) 5784 return 0; 5785 } 5786 5787 comparison = vector_compare_rtx (VOIDmode, tcode, op0a, op0b, unsignedp, 5788 icode, 4); 5789 rtx_op1 = expand_normal (op1); 5790 rtx_op2 = expand_normal (op2); 5791 5792 create_output_operand (&ops[0], target, mode); 5793 create_input_operand (&ops[1], rtx_op1, mode); 5794 create_input_operand (&ops[2], rtx_op2, mode); 5795 create_fixed_operand (&ops[3], comparison); 5796 create_fixed_operand (&ops[4], XEXP (comparison, 0)); 5797 create_fixed_operand (&ops[5], XEXP (comparison, 1)); 5798 expand_insn (icode, 6, ops); 5799 return ops[0].value; 5800 } 5801 5802 /* Generate VEC_SERIES_EXPR <OP0, OP1>, returning a value of mode VMODE. 5803 Use TARGET for the result if nonnull and convenient. */ 5804 5805 rtx 5806 expand_vec_series_expr (machine_mode vmode, rtx op0, rtx op1, rtx target) 5807 { 5808 struct expand_operand ops[3]; 5809 enum insn_code icode; 5810 machine_mode emode = GET_MODE_INNER (vmode); 5811 5812 icode = direct_optab_handler (vec_series_optab, vmode); 5813 gcc_assert (icode != CODE_FOR_nothing); 5814 5815 create_output_operand (&ops[0], target, vmode); 5816 create_input_operand (&ops[1], op0, emode); 5817 create_input_operand (&ops[2], op1, emode); 5818 5819 expand_insn (icode, 3, ops); 5820 return ops[0].value; 5821 } 5822 5823 /* Generate insns for a vector comparison into a mask. */ 5824 5825 rtx 5826 expand_vec_cmp_expr (tree type, tree exp, rtx target) 5827 { 5828 struct expand_operand ops[4]; 5829 enum insn_code icode; 5830 rtx comparison; 5831 machine_mode mask_mode = TYPE_MODE (type); 5832 machine_mode vmode; 5833 bool unsignedp; 5834 tree op0a, op0b; 5835 enum tree_code tcode; 5836 5837 op0a = TREE_OPERAND (exp, 0); 5838 op0b = TREE_OPERAND (exp, 1); 5839 tcode = TREE_CODE (exp); 5840 5841 unsignedp = TYPE_UNSIGNED (TREE_TYPE (op0a)); 5842 vmode = TYPE_MODE (TREE_TYPE (op0a)); 5843 5844 icode = get_vec_cmp_icode (vmode, mask_mode, unsignedp); 5845 if (icode == CODE_FOR_nothing) 5846 { 5847 if (tcode == EQ_EXPR || tcode == NE_EXPR) 5848 icode = get_vec_cmp_eq_icode (vmode, mask_mode); 5849 if (icode == CODE_FOR_nothing) 5850 return 0; 5851 } 5852 5853 comparison = vector_compare_rtx (mask_mode, tcode, op0a, op0b, 5854 unsignedp, icode, 2); 5855 create_output_operand (&ops[0], target, mask_mode); 5856 create_fixed_operand (&ops[1], comparison); 5857 create_fixed_operand (&ops[2], XEXP (comparison, 0)); 5858 create_fixed_operand (&ops[3], XEXP (comparison, 1)); 5859 expand_insn (icode, 4, ops); 5860 return ops[0].value; 5861 } 5862 5863 /* Expand a highpart multiply. */ 5864 5865 rtx 5866 expand_mult_highpart (machine_mode mode, rtx op0, rtx op1, 5867 rtx target, bool uns_p) 5868 { 5869 struct expand_operand eops[3]; 5870 enum insn_code icode; 5871 int method, i; 5872 machine_mode wmode; 5873 rtx m1, m2; 5874 optab tab1, tab2; 5875 5876 method = can_mult_highpart_p (mode, uns_p); 5877 switch (method) 5878 { 5879 case 0: 5880 return NULL_RTX; 5881 case 1: 5882 tab1 = uns_p ? umul_highpart_optab : smul_highpart_optab; 5883 return expand_binop (mode, tab1, op0, op1, target, uns_p, 5884 OPTAB_LIB_WIDEN); 5885 case 2: 5886 tab1 = uns_p ? vec_widen_umult_even_optab : vec_widen_smult_even_optab; 5887 tab2 = uns_p ? vec_widen_umult_odd_optab : vec_widen_smult_odd_optab; 5888 break; 5889 case 3: 5890 tab1 = uns_p ? vec_widen_umult_lo_optab : vec_widen_smult_lo_optab; 5891 tab2 = uns_p ? vec_widen_umult_hi_optab : vec_widen_smult_hi_optab; 5892 if (BYTES_BIG_ENDIAN) 5893 std::swap (tab1, tab2); 5894 break; 5895 default: 5896 gcc_unreachable (); 5897 } 5898 5899 icode = optab_handler (tab1, mode); 5900 wmode = insn_data[icode].operand[0].mode; 5901 gcc_checking_assert (known_eq (2 * GET_MODE_NUNITS (wmode), 5902 GET_MODE_NUNITS (mode))); 5903 gcc_checking_assert (known_eq (GET_MODE_SIZE (wmode), GET_MODE_SIZE (mode))); 5904 5905 create_output_operand (&eops[0], gen_reg_rtx (wmode), wmode); 5906 create_input_operand (&eops[1], op0, mode); 5907 create_input_operand (&eops[2], op1, mode); 5908 expand_insn (icode, 3, eops); 5909 m1 = gen_lowpart (mode, eops[0].value); 5910 5911 create_output_operand (&eops[0], gen_reg_rtx (wmode), wmode); 5912 create_input_operand (&eops[1], op0, mode); 5913 create_input_operand (&eops[2], op1, mode); 5914 expand_insn (optab_handler (tab2, mode), 3, eops); 5915 m2 = gen_lowpart (mode, eops[0].value); 5916 5917 vec_perm_builder sel; 5918 if (method == 2) 5919 { 5920 /* The encoding has 2 interleaved stepped patterns. */ 5921 sel.new_vector (GET_MODE_NUNITS (mode), 2, 3); 5922 for (i = 0; i < 6; ++i) 5923 sel.quick_push (!BYTES_BIG_ENDIAN + (i & ~1) 5924 + ((i & 1) ? GET_MODE_NUNITS (mode) : 0)); 5925 } 5926 else 5927 { 5928 /* The encoding has a single interleaved stepped pattern. */ 5929 sel.new_vector (GET_MODE_NUNITS (mode), 1, 3); 5930 for (i = 0; i < 3; ++i) 5931 sel.quick_push (2 * i + (BYTES_BIG_ENDIAN ? 0 : 1)); 5932 } 5933 5934 return expand_vec_perm_const (mode, m1, m2, sel, BLKmode, target); 5935 } 5936 5937 /* Helper function to find the MODE_CC set in a sync_compare_and_swap 5938 pattern. */ 5939 5940 static void 5941 find_cc_set (rtx x, const_rtx pat, void *data) 5942 { 5943 if (REG_P (x) && GET_MODE_CLASS (GET_MODE (x)) == MODE_CC 5944 && GET_CODE (pat) == SET) 5945 { 5946 rtx *p_cc_reg = (rtx *) data; 5947 gcc_assert (!*p_cc_reg); 5948 *p_cc_reg = x; 5949 } 5950 } 5951 5952 /* This is a helper function for the other atomic operations. This function 5953 emits a loop that contains SEQ that iterates until a compare-and-swap 5954 operation at the end succeeds. MEM is the memory to be modified. SEQ is 5955 a set of instructions that takes a value from OLD_REG as an input and 5956 produces a value in NEW_REG as an output. Before SEQ, OLD_REG will be 5957 set to the current contents of MEM. After SEQ, a compare-and-swap will 5958 attempt to update MEM with NEW_REG. The function returns true when the 5959 loop was generated successfully. */ 5960 5961 static bool 5962 expand_compare_and_swap_loop (rtx mem, rtx old_reg, rtx new_reg, rtx seq) 5963 { 5964 machine_mode mode = GET_MODE (mem); 5965 rtx_code_label *label; 5966 rtx cmp_reg, success, oldval; 5967 5968 /* The loop we want to generate looks like 5969 5970 cmp_reg = mem; 5971 label: 5972 old_reg = cmp_reg; 5973 seq; 5974 (success, cmp_reg) = compare-and-swap(mem, old_reg, new_reg) 5975 if (success) 5976 goto label; 5977 5978 Note that we only do the plain load from memory once. Subsequent 5979 iterations use the value loaded by the compare-and-swap pattern. */ 5980 5981 label = gen_label_rtx (); 5982 cmp_reg = gen_reg_rtx (mode); 5983 5984 emit_move_insn (cmp_reg, mem); 5985 emit_label (label); 5986 emit_move_insn (old_reg, cmp_reg); 5987 if (seq) 5988 emit_insn (seq); 5989 5990 success = NULL_RTX; 5991 oldval = cmp_reg; 5992 if (!expand_atomic_compare_and_swap (&success, &oldval, mem, old_reg, 5993 new_reg, false, MEMMODEL_SYNC_SEQ_CST, 5994 MEMMODEL_RELAXED)) 5995 return false; 5996 5997 if (oldval != cmp_reg) 5998 emit_move_insn (cmp_reg, oldval); 5999 6000 /* Mark this jump predicted not taken. */ 6001 emit_cmp_and_jump_insns (success, const0_rtx, EQ, const0_rtx, 6002 GET_MODE (success), 1, label, 6003 profile_probability::guessed_never ()); 6004 return true; 6005 } 6006 6007 6008 /* This function tries to emit an atomic_exchange intruction. VAL is written 6009 to *MEM using memory model MODEL. The previous contents of *MEM are returned, 6010 using TARGET if possible. */ 6011 6012 static rtx 6013 maybe_emit_atomic_exchange (rtx target, rtx mem, rtx val, enum memmodel model) 6014 { 6015 machine_mode mode = GET_MODE (mem); 6016 enum insn_code icode; 6017 6018 /* If the target supports the exchange directly, great. */ 6019 icode = direct_optab_handler (atomic_exchange_optab, mode); 6020 if (icode != CODE_FOR_nothing) 6021 { 6022 struct expand_operand ops[4]; 6023 6024 create_output_operand (&ops[0], target, mode); 6025 create_fixed_operand (&ops[1], mem); 6026 create_input_operand (&ops[2], val, mode); 6027 create_integer_operand (&ops[3], model); 6028 if (maybe_expand_insn (icode, 4, ops)) 6029 return ops[0].value; 6030 } 6031 6032 return NULL_RTX; 6033 } 6034 6035 /* This function tries to implement an atomic exchange operation using 6036 __sync_lock_test_and_set. VAL is written to *MEM using memory model MODEL. 6037 The previous contents of *MEM are returned, using TARGET if possible. 6038 Since this instructionn is an acquire barrier only, stronger memory 6039 models may require additional barriers to be emitted. */ 6040 6041 static rtx 6042 maybe_emit_sync_lock_test_and_set (rtx target, rtx mem, rtx val, 6043 enum memmodel model) 6044 { 6045 machine_mode mode = GET_MODE (mem); 6046 enum insn_code icode; 6047 rtx_insn *last_insn = get_last_insn (); 6048 6049 icode = optab_handler (sync_lock_test_and_set_optab, mode); 6050 6051 /* Legacy sync_lock_test_and_set is an acquire barrier. If the pattern 6052 exists, and the memory model is stronger than acquire, add a release 6053 barrier before the instruction. */ 6054 6055 if (is_mm_seq_cst (model) || is_mm_release (model) || is_mm_acq_rel (model)) 6056 expand_mem_thread_fence (model); 6057 6058 if (icode != CODE_FOR_nothing) 6059 { 6060 struct expand_operand ops[3]; 6061 create_output_operand (&ops[0], target, mode); 6062 create_fixed_operand (&ops[1], mem); 6063 create_input_operand (&ops[2], val, mode); 6064 if (maybe_expand_insn (icode, 3, ops)) 6065 return ops[0].value; 6066 } 6067 6068 /* If an external test-and-set libcall is provided, use that instead of 6069 any external compare-and-swap that we might get from the compare-and- 6070 swap-loop expansion later. */ 6071 if (!can_compare_and_swap_p (mode, false)) 6072 { 6073 rtx libfunc = optab_libfunc (sync_lock_test_and_set_optab, mode); 6074 if (libfunc != NULL) 6075 { 6076 rtx addr; 6077 6078 addr = convert_memory_address (ptr_mode, XEXP (mem, 0)); 6079 return emit_library_call_value (libfunc, NULL_RTX, LCT_NORMAL, 6080 mode, addr, ptr_mode, 6081 val, mode); 6082 } 6083 } 6084 6085 /* If the test_and_set can't be emitted, eliminate any barrier that might 6086 have been emitted. */ 6087 delete_insns_since (last_insn); 6088 return NULL_RTX; 6089 } 6090 6091 /* This function tries to implement an atomic exchange operation using a 6092 compare_and_swap loop. VAL is written to *MEM. The previous contents of 6093 *MEM are returned, using TARGET if possible. No memory model is required 6094 since a compare_and_swap loop is seq-cst. */ 6095 6096 static rtx 6097 maybe_emit_compare_and_swap_exchange_loop (rtx target, rtx mem, rtx val) 6098 { 6099 machine_mode mode = GET_MODE (mem); 6100 6101 if (can_compare_and_swap_p (mode, true)) 6102 { 6103 if (!target || !register_operand (target, mode)) 6104 target = gen_reg_rtx (mode); 6105 if (expand_compare_and_swap_loop (mem, target, val, NULL_RTX)) 6106 return target; 6107 } 6108 6109 return NULL_RTX; 6110 } 6111 6112 /* This function tries to implement an atomic test-and-set operation 6113 using the atomic_test_and_set instruction pattern. A boolean value 6114 is returned from the operation, using TARGET if possible. */ 6115 6116 static rtx 6117 maybe_emit_atomic_test_and_set (rtx target, rtx mem, enum memmodel model) 6118 { 6119 machine_mode pat_bool_mode; 6120 struct expand_operand ops[3]; 6121 6122 if (!targetm.have_atomic_test_and_set ()) 6123 return NULL_RTX; 6124 6125 /* While we always get QImode from __atomic_test_and_set, we get 6126 other memory modes from __sync_lock_test_and_set. Note that we 6127 use no endian adjustment here. This matches the 4.6 behavior 6128 in the Sparc backend. */ 6129 enum insn_code icode = targetm.code_for_atomic_test_and_set; 6130 gcc_checking_assert (insn_data[icode].operand[1].mode == QImode); 6131 if (GET_MODE (mem) != QImode) 6132 mem = adjust_address_nv (mem, QImode, 0); 6133 6134 pat_bool_mode = insn_data[icode].operand[0].mode; 6135 create_output_operand (&ops[0], target, pat_bool_mode); 6136 create_fixed_operand (&ops[1], mem); 6137 create_integer_operand (&ops[2], model); 6138 6139 if (maybe_expand_insn (icode, 3, ops)) 6140 return ops[0].value; 6141 return NULL_RTX; 6142 } 6143 6144 /* This function expands the legacy _sync_lock test_and_set operation which is 6145 generally an atomic exchange. Some limited targets only allow the 6146 constant 1 to be stored. This is an ACQUIRE operation. 6147 6148 TARGET is an optional place to stick the return value. 6149 MEM is where VAL is stored. */ 6150 6151 rtx 6152 expand_sync_lock_test_and_set (rtx target, rtx mem, rtx val) 6153 { 6154 rtx ret; 6155 6156 /* Try an atomic_exchange first. */ 6157 ret = maybe_emit_atomic_exchange (target, mem, val, MEMMODEL_SYNC_ACQUIRE); 6158 if (ret) 6159 return ret; 6160 6161 ret = maybe_emit_sync_lock_test_and_set (target, mem, val, 6162 MEMMODEL_SYNC_ACQUIRE); 6163 if (ret) 6164 return ret; 6165 6166 ret = maybe_emit_compare_and_swap_exchange_loop (target, mem, val); 6167 if (ret) 6168 return ret; 6169 6170 /* If there are no other options, try atomic_test_and_set if the value 6171 being stored is 1. */ 6172 if (val == const1_rtx) 6173 ret = maybe_emit_atomic_test_and_set (target, mem, MEMMODEL_SYNC_ACQUIRE); 6174 6175 return ret; 6176 } 6177 6178 /* This function expands the atomic test_and_set operation: 6179 atomically store a boolean TRUE into MEM and return the previous value. 6180 6181 MEMMODEL is the memory model variant to use. 6182 TARGET is an optional place to stick the return value. */ 6183 6184 rtx 6185 expand_atomic_test_and_set (rtx target, rtx mem, enum memmodel model) 6186 { 6187 machine_mode mode = GET_MODE (mem); 6188 rtx ret, trueval, subtarget; 6189 6190 ret = maybe_emit_atomic_test_and_set (target, mem, model); 6191 if (ret) 6192 return ret; 6193 6194 /* Be binary compatible with non-default settings of trueval, and different 6195 cpu revisions. E.g. one revision may have atomic-test-and-set, but 6196 another only has atomic-exchange. */ 6197 if (targetm.atomic_test_and_set_trueval == 1) 6198 { 6199 trueval = const1_rtx; 6200 subtarget = target ? target : gen_reg_rtx (mode); 6201 } 6202 else 6203 { 6204 trueval = gen_int_mode (targetm.atomic_test_and_set_trueval, mode); 6205 subtarget = gen_reg_rtx (mode); 6206 } 6207 6208 /* Try the atomic-exchange optab... */ 6209 ret = maybe_emit_atomic_exchange (subtarget, mem, trueval, model); 6210 6211 /* ... then an atomic-compare-and-swap loop ... */ 6212 if (!ret) 6213 ret = maybe_emit_compare_and_swap_exchange_loop (subtarget, mem, trueval); 6214 6215 /* ... before trying the vaguely defined legacy lock_test_and_set. */ 6216 if (!ret) 6217 ret = maybe_emit_sync_lock_test_and_set (subtarget, mem, trueval, model); 6218 6219 /* Recall that the legacy lock_test_and_set optab was allowed to do magic 6220 things with the value 1. Thus we try again without trueval. */ 6221 if (!ret && targetm.atomic_test_and_set_trueval != 1) 6222 ret = maybe_emit_sync_lock_test_and_set (subtarget, mem, const1_rtx, model); 6223 6224 /* Failing all else, assume a single threaded environment and simply 6225 perform the operation. */ 6226 if (!ret) 6227 { 6228 /* If the result is ignored skip the move to target. */ 6229 if (subtarget != const0_rtx) 6230 emit_move_insn (subtarget, mem); 6231 6232 emit_move_insn (mem, trueval); 6233 ret = subtarget; 6234 } 6235 6236 /* Recall that have to return a boolean value; rectify if trueval 6237 is not exactly one. */ 6238 if (targetm.atomic_test_and_set_trueval != 1) 6239 ret = emit_store_flag_force (target, NE, ret, const0_rtx, mode, 0, 1); 6240 6241 return ret; 6242 } 6243 6244 /* This function expands the atomic exchange operation: 6245 atomically store VAL in MEM and return the previous value in MEM. 6246 6247 MEMMODEL is the memory model variant to use. 6248 TARGET is an optional place to stick the return value. */ 6249 6250 rtx 6251 expand_atomic_exchange (rtx target, rtx mem, rtx val, enum memmodel model) 6252 { 6253 machine_mode mode = GET_MODE (mem); 6254 rtx ret; 6255 6256 /* If loads are not atomic for the required size and we are not called to 6257 provide a __sync builtin, do not do anything so that we stay consistent 6258 with atomic loads of the same size. */ 6259 if (!can_atomic_load_p (mode) && !is_mm_sync (model)) 6260 return NULL_RTX; 6261 6262 ret = maybe_emit_atomic_exchange (target, mem, val, model); 6263 6264 /* Next try a compare-and-swap loop for the exchange. */ 6265 if (!ret) 6266 ret = maybe_emit_compare_and_swap_exchange_loop (target, mem, val); 6267 6268 return ret; 6269 } 6270 6271 /* This function expands the atomic compare exchange operation: 6272 6273 *PTARGET_BOOL is an optional place to store the boolean success/failure. 6274 *PTARGET_OVAL is an optional place to store the old value from memory. 6275 Both target parameters may be NULL or const0_rtx to indicate that we do 6276 not care about that return value. Both target parameters are updated on 6277 success to the actual location of the corresponding result. 6278 6279 MEMMODEL is the memory model variant to use. 6280 6281 The return value of the function is true for success. */ 6282 6283 bool 6284 expand_atomic_compare_and_swap (rtx *ptarget_bool, rtx *ptarget_oval, 6285 rtx mem, rtx expected, rtx desired, 6286 bool is_weak, enum memmodel succ_model, 6287 enum memmodel fail_model) 6288 { 6289 machine_mode mode = GET_MODE (mem); 6290 struct expand_operand ops[8]; 6291 enum insn_code icode; 6292 rtx target_oval, target_bool = NULL_RTX; 6293 rtx libfunc; 6294 6295 /* If loads are not atomic for the required size and we are not called to 6296 provide a __sync builtin, do not do anything so that we stay consistent 6297 with atomic loads of the same size. */ 6298 if (!can_atomic_load_p (mode) && !is_mm_sync (succ_model)) 6299 return false; 6300 6301 /* Load expected into a register for the compare and swap. */ 6302 if (MEM_P (expected)) 6303 expected = copy_to_reg (expected); 6304 6305 /* Make sure we always have some place to put the return oldval. 6306 Further, make sure that place is distinct from the input expected, 6307 just in case we need that path down below. */ 6308 if (ptarget_oval && *ptarget_oval == const0_rtx) 6309 ptarget_oval = NULL; 6310 6311 if (ptarget_oval == NULL 6312 || (target_oval = *ptarget_oval) == NULL 6313 || reg_overlap_mentioned_p (expected, target_oval)) 6314 target_oval = gen_reg_rtx (mode); 6315 6316 icode = direct_optab_handler (atomic_compare_and_swap_optab, mode); 6317 if (icode != CODE_FOR_nothing) 6318 { 6319 machine_mode bool_mode = insn_data[icode].operand[0].mode; 6320 6321 if (ptarget_bool && *ptarget_bool == const0_rtx) 6322 ptarget_bool = NULL; 6323 6324 /* Make sure we always have a place for the bool operand. */ 6325 if (ptarget_bool == NULL 6326 || (target_bool = *ptarget_bool) == NULL 6327 || GET_MODE (target_bool) != bool_mode) 6328 target_bool = gen_reg_rtx (bool_mode); 6329 6330 /* Emit the compare_and_swap. */ 6331 create_output_operand (&ops[0], target_bool, bool_mode); 6332 create_output_operand (&ops[1], target_oval, mode); 6333 create_fixed_operand (&ops[2], mem); 6334 create_input_operand (&ops[3], expected, mode); 6335 create_input_operand (&ops[4], desired, mode); 6336 create_integer_operand (&ops[5], is_weak); 6337 create_integer_operand (&ops[6], succ_model); 6338 create_integer_operand (&ops[7], fail_model); 6339 if (maybe_expand_insn (icode, 8, ops)) 6340 { 6341 /* Return success/failure. */ 6342 target_bool = ops[0].value; 6343 target_oval = ops[1].value; 6344 goto success; 6345 } 6346 } 6347 6348 /* Otherwise fall back to the original __sync_val_compare_and_swap 6349 which is always seq-cst. */ 6350 icode = optab_handler (sync_compare_and_swap_optab, mode); 6351 if (icode != CODE_FOR_nothing) 6352 { 6353 rtx cc_reg; 6354 6355 create_output_operand (&ops[0], target_oval, mode); 6356 create_fixed_operand (&ops[1], mem); 6357 create_input_operand (&ops[2], expected, mode); 6358 create_input_operand (&ops[3], desired, mode); 6359 if (!maybe_expand_insn (icode, 4, ops)) 6360 return false; 6361 6362 target_oval = ops[0].value; 6363 6364 /* If the caller isn't interested in the boolean return value, 6365 skip the computation of it. */ 6366 if (ptarget_bool == NULL) 6367 goto success; 6368 6369 /* Otherwise, work out if the compare-and-swap succeeded. */ 6370 cc_reg = NULL_RTX; 6371 if (have_insn_for (COMPARE, CCmode)) 6372 note_stores (PATTERN (get_last_insn ()), find_cc_set, &cc_reg); 6373 if (cc_reg) 6374 { 6375 target_bool = emit_store_flag_force (target_bool, EQ, cc_reg, 6376 const0_rtx, VOIDmode, 0, 1); 6377 goto success; 6378 } 6379 goto success_bool_from_val; 6380 } 6381 6382 /* Also check for library support for __sync_val_compare_and_swap. */ 6383 libfunc = optab_libfunc (sync_compare_and_swap_optab, mode); 6384 if (libfunc != NULL) 6385 { 6386 rtx addr = convert_memory_address (ptr_mode, XEXP (mem, 0)); 6387 rtx target = emit_library_call_value (libfunc, NULL_RTX, LCT_NORMAL, 6388 mode, addr, ptr_mode, 6389 expected, mode, desired, mode); 6390 emit_move_insn (target_oval, target); 6391 6392 /* Compute the boolean return value only if requested. */ 6393 if (ptarget_bool) 6394 goto success_bool_from_val; 6395 else 6396 goto success; 6397 } 6398 6399 /* Failure. */ 6400 return false; 6401 6402 success_bool_from_val: 6403 target_bool = emit_store_flag_force (target_bool, EQ, target_oval, 6404 expected, VOIDmode, 1, 1); 6405 success: 6406 /* Make sure that the oval output winds up where the caller asked. */ 6407 if (ptarget_oval) 6408 *ptarget_oval = target_oval; 6409 if (ptarget_bool) 6410 *ptarget_bool = target_bool; 6411 return true; 6412 } 6413 6414 /* Generate asm volatile("" : : : "memory") as the memory blockage. */ 6415 6416 static void 6417 expand_asm_memory_blockage (void) 6418 { 6419 rtx asm_op, clob; 6420 6421 asm_op = gen_rtx_ASM_OPERANDS (VOIDmode, "", "", 0, 6422 rtvec_alloc (0), rtvec_alloc (0), 6423 rtvec_alloc (0), UNKNOWN_LOCATION); 6424 MEM_VOLATILE_P (asm_op) = 1; 6425 6426 clob = gen_rtx_SCRATCH (VOIDmode); 6427 clob = gen_rtx_MEM (BLKmode, clob); 6428 clob = gen_rtx_CLOBBER (VOIDmode, clob); 6429 6430 emit_insn (gen_rtx_PARALLEL (VOIDmode, gen_rtvec (2, asm_op, clob))); 6431 } 6432 6433 /* Do not propagate memory accesses across this point. */ 6434 6435 static void 6436 expand_memory_blockage (void) 6437 { 6438 if (targetm.have_memory_blockage ()) 6439 emit_insn (targetm.gen_memory_blockage ()); 6440 else 6441 expand_asm_memory_blockage (); 6442 } 6443 6444 /* This routine will either emit the mem_thread_fence pattern or issue a 6445 sync_synchronize to generate a fence for memory model MEMMODEL. */ 6446 6447 void 6448 expand_mem_thread_fence (enum memmodel model) 6449 { 6450 if (is_mm_relaxed (model)) 6451 return; 6452 if (targetm.have_mem_thread_fence ()) 6453 { 6454 emit_insn (targetm.gen_mem_thread_fence (GEN_INT (model))); 6455 expand_memory_blockage (); 6456 } 6457 else if (targetm.have_memory_barrier ()) 6458 emit_insn (targetm.gen_memory_barrier ()); 6459 else if (synchronize_libfunc != NULL_RTX) 6460 emit_library_call (synchronize_libfunc, LCT_NORMAL, VOIDmode); 6461 else 6462 expand_memory_blockage (); 6463 } 6464 6465 /* Emit a signal fence with given memory model. */ 6466 6467 void 6468 expand_mem_signal_fence (enum memmodel model) 6469 { 6470 /* No machine barrier is required to implement a signal fence, but 6471 a compiler memory barrier must be issued, except for relaxed MM. */ 6472 if (!is_mm_relaxed (model)) 6473 expand_memory_blockage (); 6474 } 6475 6476 /* This function expands the atomic load operation: 6477 return the atomically loaded value in MEM. 6478 6479 MEMMODEL is the memory model variant to use. 6480 TARGET is an option place to stick the return value. */ 6481 6482 rtx 6483 expand_atomic_load (rtx target, rtx mem, enum memmodel model) 6484 { 6485 machine_mode mode = GET_MODE (mem); 6486 enum insn_code icode; 6487 6488 /* If the target supports the load directly, great. */ 6489 icode = direct_optab_handler (atomic_load_optab, mode); 6490 if (icode != CODE_FOR_nothing) 6491 { 6492 struct expand_operand ops[3]; 6493 rtx_insn *last = get_last_insn (); 6494 if (is_mm_seq_cst (model)) 6495 expand_memory_blockage (); 6496 6497 create_output_operand (&ops[0], target, mode); 6498 create_fixed_operand (&ops[1], mem); 6499 create_integer_operand (&ops[2], model); 6500 if (maybe_expand_insn (icode, 3, ops)) 6501 { 6502 if (!is_mm_relaxed (model)) 6503 expand_memory_blockage (); 6504 return ops[0].value; 6505 } 6506 delete_insns_since (last); 6507 } 6508 6509 /* If the size of the object is greater than word size on this target, 6510 then we assume that a load will not be atomic. We could try to 6511 emulate a load with a compare-and-swap operation, but the store that 6512 doing this could result in would be incorrect if this is a volatile 6513 atomic load or targetting read-only-mapped memory. */ 6514 if (maybe_gt (GET_MODE_PRECISION (mode), BITS_PER_WORD)) 6515 /* If there is no atomic load, leave the library call. */ 6516 return NULL_RTX; 6517 6518 /* Otherwise assume loads are atomic, and emit the proper barriers. */ 6519 if (!target || target == const0_rtx) 6520 target = gen_reg_rtx (mode); 6521 6522 /* For SEQ_CST, emit a barrier before the load. */ 6523 if (is_mm_seq_cst (model)) 6524 expand_mem_thread_fence (model); 6525 6526 emit_move_insn (target, mem); 6527 6528 /* Emit the appropriate barrier after the load. */ 6529 expand_mem_thread_fence (model); 6530 6531 return target; 6532 } 6533 6534 /* This function expands the atomic store operation: 6535 Atomically store VAL in MEM. 6536 MEMMODEL is the memory model variant to use. 6537 USE_RELEASE is true if __sync_lock_release can be used as a fall back. 6538 function returns const0_rtx if a pattern was emitted. */ 6539 6540 rtx 6541 expand_atomic_store (rtx mem, rtx val, enum memmodel model, bool use_release) 6542 { 6543 machine_mode mode = GET_MODE (mem); 6544 enum insn_code icode; 6545 struct expand_operand ops[3]; 6546 6547 /* If the target supports the store directly, great. */ 6548 icode = direct_optab_handler (atomic_store_optab, mode); 6549 if (icode != CODE_FOR_nothing) 6550 { 6551 rtx_insn *last = get_last_insn (); 6552 if (!is_mm_relaxed (model)) 6553 expand_memory_blockage (); 6554 create_fixed_operand (&ops[0], mem); 6555 create_input_operand (&ops[1], val, mode); 6556 create_integer_operand (&ops[2], model); 6557 if (maybe_expand_insn (icode, 3, ops)) 6558 { 6559 if (is_mm_seq_cst (model)) 6560 expand_memory_blockage (); 6561 return const0_rtx; 6562 } 6563 delete_insns_since (last); 6564 } 6565 6566 /* If using __sync_lock_release is a viable alternative, try it. 6567 Note that this will not be set to true if we are expanding a generic 6568 __atomic_store_n. */ 6569 if (use_release) 6570 { 6571 icode = direct_optab_handler (sync_lock_release_optab, mode); 6572 if (icode != CODE_FOR_nothing) 6573 { 6574 create_fixed_operand (&ops[0], mem); 6575 create_input_operand (&ops[1], const0_rtx, mode); 6576 if (maybe_expand_insn (icode, 2, ops)) 6577 { 6578 /* lock_release is only a release barrier. */ 6579 if (is_mm_seq_cst (model)) 6580 expand_mem_thread_fence (model); 6581 return const0_rtx; 6582 } 6583 } 6584 } 6585 6586 /* If the size of the object is greater than word size on this target, 6587 a default store will not be atomic. */ 6588 if (maybe_gt (GET_MODE_PRECISION (mode), BITS_PER_WORD)) 6589 { 6590 /* If loads are atomic or we are called to provide a __sync builtin, 6591 we can try a atomic_exchange and throw away the result. Otherwise, 6592 don't do anything so that we do not create an inconsistency between 6593 loads and stores. */ 6594 if (can_atomic_load_p (mode) || is_mm_sync (model)) 6595 { 6596 rtx target = maybe_emit_atomic_exchange (NULL_RTX, mem, val, model); 6597 if (!target) 6598 target = maybe_emit_compare_and_swap_exchange_loop (NULL_RTX, mem, 6599 val); 6600 if (target) 6601 return const0_rtx; 6602 } 6603 return NULL_RTX; 6604 } 6605 6606 /* Otherwise assume stores are atomic, and emit the proper barriers. */ 6607 expand_mem_thread_fence (model); 6608 6609 emit_move_insn (mem, val); 6610 6611 /* For SEQ_CST, also emit a barrier after the store. */ 6612 if (is_mm_seq_cst (model)) 6613 expand_mem_thread_fence (model); 6614 6615 return const0_rtx; 6616 } 6617 6618 6619 /* Structure containing the pointers and values required to process the 6620 various forms of the atomic_fetch_op and atomic_op_fetch builtins. */ 6621 6622 struct atomic_op_functions 6623 { 6624 direct_optab mem_fetch_before; 6625 direct_optab mem_fetch_after; 6626 direct_optab mem_no_result; 6627 optab fetch_before; 6628 optab fetch_after; 6629 direct_optab no_result; 6630 enum rtx_code reverse_code; 6631 }; 6632 6633 6634 /* Fill in structure pointed to by OP with the various optab entries for an 6635 operation of type CODE. */ 6636 6637 static void 6638 get_atomic_op_for_code (struct atomic_op_functions *op, enum rtx_code code) 6639 { 6640 gcc_assert (op!= NULL); 6641 6642 /* If SWITCHABLE_TARGET is defined, then subtargets can be switched 6643 in the source code during compilation, and the optab entries are not 6644 computable until runtime. Fill in the values at runtime. */ 6645 switch (code) 6646 { 6647 case PLUS: 6648 op->mem_fetch_before = atomic_fetch_add_optab; 6649 op->mem_fetch_after = atomic_add_fetch_optab; 6650 op->mem_no_result = atomic_add_optab; 6651 op->fetch_before = sync_old_add_optab; 6652 op->fetch_after = sync_new_add_optab; 6653 op->no_result = sync_add_optab; 6654 op->reverse_code = MINUS; 6655 break; 6656 case MINUS: 6657 op->mem_fetch_before = atomic_fetch_sub_optab; 6658 op->mem_fetch_after = atomic_sub_fetch_optab; 6659 op->mem_no_result = atomic_sub_optab; 6660 op->fetch_before = sync_old_sub_optab; 6661 op->fetch_after = sync_new_sub_optab; 6662 op->no_result = sync_sub_optab; 6663 op->reverse_code = PLUS; 6664 break; 6665 case XOR: 6666 op->mem_fetch_before = atomic_fetch_xor_optab; 6667 op->mem_fetch_after = atomic_xor_fetch_optab; 6668 op->mem_no_result = atomic_xor_optab; 6669 op->fetch_before = sync_old_xor_optab; 6670 op->fetch_after = sync_new_xor_optab; 6671 op->no_result = sync_xor_optab; 6672 op->reverse_code = XOR; 6673 break; 6674 case AND: 6675 op->mem_fetch_before = atomic_fetch_and_optab; 6676 op->mem_fetch_after = atomic_and_fetch_optab; 6677 op->mem_no_result = atomic_and_optab; 6678 op->fetch_before = sync_old_and_optab; 6679 op->fetch_after = sync_new_and_optab; 6680 op->no_result = sync_and_optab; 6681 op->reverse_code = UNKNOWN; 6682 break; 6683 case IOR: 6684 op->mem_fetch_before = atomic_fetch_or_optab; 6685 op->mem_fetch_after = atomic_or_fetch_optab; 6686 op->mem_no_result = atomic_or_optab; 6687 op->fetch_before = sync_old_ior_optab; 6688 op->fetch_after = sync_new_ior_optab; 6689 op->no_result = sync_ior_optab; 6690 op->reverse_code = UNKNOWN; 6691 break; 6692 case NOT: 6693 op->mem_fetch_before = atomic_fetch_nand_optab; 6694 op->mem_fetch_after = atomic_nand_fetch_optab; 6695 op->mem_no_result = atomic_nand_optab; 6696 op->fetch_before = sync_old_nand_optab; 6697 op->fetch_after = sync_new_nand_optab; 6698 op->no_result = sync_nand_optab; 6699 op->reverse_code = UNKNOWN; 6700 break; 6701 default: 6702 gcc_unreachable (); 6703 } 6704 } 6705 6706 /* See if there is a more optimal way to implement the operation "*MEM CODE VAL" 6707 using memory order MODEL. If AFTER is true the operation needs to return 6708 the value of *MEM after the operation, otherwise the previous value. 6709 TARGET is an optional place to place the result. The result is unused if 6710 it is const0_rtx. 6711 Return the result if there is a better sequence, otherwise NULL_RTX. */ 6712 6713 static rtx 6714 maybe_optimize_fetch_op (rtx target, rtx mem, rtx val, enum rtx_code code, 6715 enum memmodel model, bool after) 6716 { 6717 /* If the value is prefetched, or not used, it may be possible to replace 6718 the sequence with a native exchange operation. */ 6719 if (!after || target == const0_rtx) 6720 { 6721 /* fetch_and (&x, 0, m) can be replaced with exchange (&x, 0, m). */ 6722 if (code == AND && val == const0_rtx) 6723 { 6724 if (target == const0_rtx) 6725 target = gen_reg_rtx (GET_MODE (mem)); 6726 return maybe_emit_atomic_exchange (target, mem, val, model); 6727 } 6728 6729 /* fetch_or (&x, -1, m) can be replaced with exchange (&x, -1, m). */ 6730 if (code == IOR && val == constm1_rtx) 6731 { 6732 if (target == const0_rtx) 6733 target = gen_reg_rtx (GET_MODE (mem)); 6734 return maybe_emit_atomic_exchange (target, mem, val, model); 6735 } 6736 } 6737 6738 return NULL_RTX; 6739 } 6740 6741 /* Try to emit an instruction for a specific operation varaition. 6742 OPTAB contains the OP functions. 6743 TARGET is an optional place to return the result. const0_rtx means unused. 6744 MEM is the memory location to operate on. 6745 VAL is the value to use in the operation. 6746 USE_MEMMODEL is TRUE if the variation with a memory model should be tried. 6747 MODEL is the memory model, if used. 6748 AFTER is true if the returned result is the value after the operation. */ 6749 6750 static rtx 6751 maybe_emit_op (const struct atomic_op_functions *optab, rtx target, rtx mem, 6752 rtx val, bool use_memmodel, enum memmodel model, bool after) 6753 { 6754 machine_mode mode = GET_MODE (mem); 6755 struct expand_operand ops[4]; 6756 enum insn_code icode; 6757 int op_counter = 0; 6758 int num_ops; 6759 6760 /* Check to see if there is a result returned. */ 6761 if (target == const0_rtx) 6762 { 6763 if (use_memmodel) 6764 { 6765 icode = direct_optab_handler (optab->mem_no_result, mode); 6766 create_integer_operand (&ops[2], model); 6767 num_ops = 3; 6768 } 6769 else 6770 { 6771 icode = direct_optab_handler (optab->no_result, mode); 6772 num_ops = 2; 6773 } 6774 } 6775 /* Otherwise, we need to generate a result. */ 6776 else 6777 { 6778 if (use_memmodel) 6779 { 6780 icode = direct_optab_handler (after ? optab->mem_fetch_after 6781 : optab->mem_fetch_before, mode); 6782 create_integer_operand (&ops[3], model); 6783 num_ops = 4; 6784 } 6785 else 6786 { 6787 icode = optab_handler (after ? optab->fetch_after 6788 : optab->fetch_before, mode); 6789 num_ops = 3; 6790 } 6791 create_output_operand (&ops[op_counter++], target, mode); 6792 } 6793 if (icode == CODE_FOR_nothing) 6794 return NULL_RTX; 6795 6796 create_fixed_operand (&ops[op_counter++], mem); 6797 /* VAL may have been promoted to a wider mode. Shrink it if so. */ 6798 create_convert_operand_to (&ops[op_counter++], val, mode, true); 6799 6800 if (maybe_expand_insn (icode, num_ops, ops)) 6801 return (target == const0_rtx ? const0_rtx : ops[0].value); 6802 6803 return NULL_RTX; 6804 } 6805 6806 6807 /* This function expands an atomic fetch_OP or OP_fetch operation: 6808 TARGET is an option place to stick the return value. const0_rtx indicates 6809 the result is unused. 6810 atomically fetch MEM, perform the operation with VAL and return it to MEM. 6811 CODE is the operation being performed (OP) 6812 MEMMODEL is the memory model variant to use. 6813 AFTER is true to return the result of the operation (OP_fetch). 6814 AFTER is false to return the value before the operation (fetch_OP). 6815 6816 This function will *only* generate instructions if there is a direct 6817 optab. No compare and swap loops or libcalls will be generated. */ 6818 6819 static rtx 6820 expand_atomic_fetch_op_no_fallback (rtx target, rtx mem, rtx val, 6821 enum rtx_code code, enum memmodel model, 6822 bool after) 6823 { 6824 machine_mode mode = GET_MODE (mem); 6825 struct atomic_op_functions optab; 6826 rtx result; 6827 bool unused_result = (target == const0_rtx); 6828 6829 get_atomic_op_for_code (&optab, code); 6830 6831 /* Check to see if there are any better instructions. */ 6832 result = maybe_optimize_fetch_op (target, mem, val, code, model, after); 6833 if (result) 6834 return result; 6835 6836 /* Check for the case where the result isn't used and try those patterns. */ 6837 if (unused_result) 6838 { 6839 /* Try the memory model variant first. */ 6840 result = maybe_emit_op (&optab, target, mem, val, true, model, true); 6841 if (result) 6842 return result; 6843 6844 /* Next try the old style withuot a memory model. */ 6845 result = maybe_emit_op (&optab, target, mem, val, false, model, true); 6846 if (result) 6847 return result; 6848 6849 /* There is no no-result pattern, so try patterns with a result. */ 6850 target = NULL_RTX; 6851 } 6852 6853 /* Try the __atomic version. */ 6854 result = maybe_emit_op (&optab, target, mem, val, true, model, after); 6855 if (result) 6856 return result; 6857 6858 /* Try the older __sync version. */ 6859 result = maybe_emit_op (&optab, target, mem, val, false, model, after); 6860 if (result) 6861 return result; 6862 6863 /* If the fetch value can be calculated from the other variation of fetch, 6864 try that operation. */ 6865 if (after || unused_result || optab.reverse_code != UNKNOWN) 6866 { 6867 /* Try the __atomic version, then the older __sync version. */ 6868 result = maybe_emit_op (&optab, target, mem, val, true, model, !after); 6869 if (!result) 6870 result = maybe_emit_op (&optab, target, mem, val, false, model, !after); 6871 6872 if (result) 6873 { 6874 /* If the result isn't used, no need to do compensation code. */ 6875 if (unused_result) 6876 return result; 6877 6878 /* Issue compensation code. Fetch_after == fetch_before OP val. 6879 Fetch_before == after REVERSE_OP val. */ 6880 if (!after) 6881 code = optab.reverse_code; 6882 if (code == NOT) 6883 { 6884 result = expand_simple_binop (mode, AND, result, val, NULL_RTX, 6885 true, OPTAB_LIB_WIDEN); 6886 result = expand_simple_unop (mode, NOT, result, target, true); 6887 } 6888 else 6889 result = expand_simple_binop (mode, code, result, val, target, 6890 true, OPTAB_LIB_WIDEN); 6891 return result; 6892 } 6893 } 6894 6895 /* No direct opcode can be generated. */ 6896 return NULL_RTX; 6897 } 6898 6899 6900 6901 /* This function expands an atomic fetch_OP or OP_fetch operation: 6902 TARGET is an option place to stick the return value. const0_rtx indicates 6903 the result is unused. 6904 atomically fetch MEM, perform the operation with VAL and return it to MEM. 6905 CODE is the operation being performed (OP) 6906 MEMMODEL is the memory model variant to use. 6907 AFTER is true to return the result of the operation (OP_fetch). 6908 AFTER is false to return the value before the operation (fetch_OP). */ 6909 rtx 6910 expand_atomic_fetch_op (rtx target, rtx mem, rtx val, enum rtx_code code, 6911 enum memmodel model, bool after) 6912 { 6913 machine_mode mode = GET_MODE (mem); 6914 rtx result; 6915 bool unused_result = (target == const0_rtx); 6916 6917 /* If loads are not atomic for the required size and we are not called to 6918 provide a __sync builtin, do not do anything so that we stay consistent 6919 with atomic loads of the same size. */ 6920 if (!can_atomic_load_p (mode) && !is_mm_sync (model)) 6921 return NULL_RTX; 6922 6923 result = expand_atomic_fetch_op_no_fallback (target, mem, val, code, model, 6924 after); 6925 6926 if (result) 6927 return result; 6928 6929 /* Add/sub can be implemented by doing the reverse operation with -(val). */ 6930 if (code == PLUS || code == MINUS) 6931 { 6932 rtx tmp; 6933 enum rtx_code reverse = (code == PLUS ? MINUS : PLUS); 6934 6935 start_sequence (); 6936 tmp = expand_simple_unop (mode, NEG, val, NULL_RTX, true); 6937 result = expand_atomic_fetch_op_no_fallback (target, mem, tmp, reverse, 6938 model, after); 6939 if (result) 6940 { 6941 /* PLUS worked so emit the insns and return. */ 6942 tmp = get_insns (); 6943 end_sequence (); 6944 emit_insn (tmp); 6945 return result; 6946 } 6947 6948 /* PLUS did not work, so throw away the negation code and continue. */ 6949 end_sequence (); 6950 } 6951 6952 /* Try the __sync libcalls only if we can't do compare-and-swap inline. */ 6953 if (!can_compare_and_swap_p (mode, false)) 6954 { 6955 rtx libfunc; 6956 bool fixup = false; 6957 enum rtx_code orig_code = code; 6958 struct atomic_op_functions optab; 6959 6960 get_atomic_op_for_code (&optab, code); 6961 libfunc = optab_libfunc (after ? optab.fetch_after 6962 : optab.fetch_before, mode); 6963 if (libfunc == NULL 6964 && (after || unused_result || optab.reverse_code != UNKNOWN)) 6965 { 6966 fixup = true; 6967 if (!after) 6968 code = optab.reverse_code; 6969 libfunc = optab_libfunc (after ? optab.fetch_before 6970 : optab.fetch_after, mode); 6971 } 6972 if (libfunc != NULL) 6973 { 6974 rtx addr = convert_memory_address (ptr_mode, XEXP (mem, 0)); 6975 result = emit_library_call_value (libfunc, NULL, LCT_NORMAL, mode, 6976 addr, ptr_mode, val, mode); 6977 6978 if (!unused_result && fixup) 6979 result = expand_simple_binop (mode, code, result, val, target, 6980 true, OPTAB_LIB_WIDEN); 6981 return result; 6982 } 6983 6984 /* We need the original code for any further attempts. */ 6985 code = orig_code; 6986 } 6987 6988 /* If nothing else has succeeded, default to a compare and swap loop. */ 6989 if (can_compare_and_swap_p (mode, true)) 6990 { 6991 rtx_insn *insn; 6992 rtx t0 = gen_reg_rtx (mode), t1; 6993 6994 start_sequence (); 6995 6996 /* If the result is used, get a register for it. */ 6997 if (!unused_result) 6998 { 6999 if (!target || !register_operand (target, mode)) 7000 target = gen_reg_rtx (mode); 7001 /* If fetch_before, copy the value now. */ 7002 if (!after) 7003 emit_move_insn (target, t0); 7004 } 7005 else 7006 target = const0_rtx; 7007 7008 t1 = t0; 7009 if (code == NOT) 7010 { 7011 t1 = expand_simple_binop (mode, AND, t1, val, NULL_RTX, 7012 true, OPTAB_LIB_WIDEN); 7013 t1 = expand_simple_unop (mode, code, t1, NULL_RTX, true); 7014 } 7015 else 7016 t1 = expand_simple_binop (mode, code, t1, val, NULL_RTX, true, 7017 OPTAB_LIB_WIDEN); 7018 7019 /* For after, copy the value now. */ 7020 if (!unused_result && after) 7021 emit_move_insn (target, t1); 7022 insn = get_insns (); 7023 end_sequence (); 7024 7025 if (t1 != NULL && expand_compare_and_swap_loop (mem, t0, t1, insn)) 7026 return target; 7027 } 7028 7029 return NULL_RTX; 7030 } 7031 7032 /* Return true if OPERAND is suitable for operand number OPNO of 7033 instruction ICODE. */ 7034 7035 bool 7036 insn_operand_matches (enum insn_code icode, unsigned int opno, rtx operand) 7037 { 7038 return (!insn_data[(int) icode].operand[opno].predicate 7039 || (insn_data[(int) icode].operand[opno].predicate 7040 (operand, insn_data[(int) icode].operand[opno].mode))); 7041 } 7042 7043 /* TARGET is a target of a multiword operation that we are going to 7044 implement as a series of word-mode operations. Return true if 7045 TARGET is suitable for this purpose. */ 7046 7047 bool 7048 valid_multiword_target_p (rtx target) 7049 { 7050 machine_mode mode; 7051 int i, size; 7052 7053 mode = GET_MODE (target); 7054 if (!GET_MODE_SIZE (mode).is_constant (&size)) 7055 return false; 7056 for (i = 0; i < size; i += UNITS_PER_WORD) 7057 if (!validate_subreg (word_mode, mode, target, i)) 7058 return false; 7059 return true; 7060 } 7061 7062 /* Make OP describe an input operand that has value INTVAL and that has 7063 no inherent mode. This function should only be used for operands that 7064 are always expand-time constants. The backend may request that INTVAL 7065 be copied into a different kind of rtx, but it must specify the mode 7066 of that rtx if so. */ 7067 7068 void 7069 create_integer_operand (struct expand_operand *op, poly_int64 intval) 7070 { 7071 create_expand_operand (op, EXPAND_INTEGER, 7072 gen_int_mode (intval, MAX_MODE_INT), 7073 VOIDmode, false, intval); 7074 } 7075 7076 /* Like maybe_legitimize_operand, but do not change the code of the 7077 current rtx value. */ 7078 7079 static bool 7080 maybe_legitimize_operand_same_code (enum insn_code icode, unsigned int opno, 7081 struct expand_operand *op) 7082 { 7083 /* See if the operand matches in its current form. */ 7084 if (insn_operand_matches (icode, opno, op->value)) 7085 return true; 7086 7087 /* If the operand is a memory whose address has no side effects, 7088 try forcing the address into a non-virtual pseudo register. 7089 The check for side effects is important because copy_to_mode_reg 7090 cannot handle things like auto-modified addresses. */ 7091 if (insn_data[(int) icode].operand[opno].allows_mem && MEM_P (op->value)) 7092 { 7093 rtx addr, mem; 7094 7095 mem = op->value; 7096 addr = XEXP (mem, 0); 7097 if (!(REG_P (addr) && REGNO (addr) > LAST_VIRTUAL_REGISTER) 7098 && !side_effects_p (addr)) 7099 { 7100 rtx_insn *last; 7101 machine_mode mode; 7102 7103 last = get_last_insn (); 7104 mode = get_address_mode (mem); 7105 mem = replace_equiv_address (mem, copy_to_mode_reg (mode, addr)); 7106 if (insn_operand_matches (icode, opno, mem)) 7107 { 7108 op->value = mem; 7109 return true; 7110 } 7111 delete_insns_since (last); 7112 } 7113 } 7114 7115 return false; 7116 } 7117 7118 /* Try to make OP match operand OPNO of instruction ICODE. Return true 7119 on success, storing the new operand value back in OP. */ 7120 7121 static bool 7122 maybe_legitimize_operand (enum insn_code icode, unsigned int opno, 7123 struct expand_operand *op) 7124 { 7125 machine_mode mode, imode; 7126 bool old_volatile_ok, result; 7127 7128 mode = op->mode; 7129 switch (op->type) 7130 { 7131 case EXPAND_FIXED: 7132 old_volatile_ok = volatile_ok; 7133 volatile_ok = true; 7134 result = maybe_legitimize_operand_same_code (icode, opno, op); 7135 volatile_ok = old_volatile_ok; 7136 return result; 7137 7138 case EXPAND_OUTPUT: 7139 gcc_assert (mode != VOIDmode); 7140 if (op->value 7141 && op->value != const0_rtx 7142 && GET_MODE (op->value) == mode 7143 && maybe_legitimize_operand_same_code (icode, opno, op)) 7144 return true; 7145 7146 op->value = gen_reg_rtx (mode); 7147 op->target = 0; 7148 break; 7149 7150 case EXPAND_INPUT: 7151 input: 7152 gcc_assert (mode != VOIDmode); 7153 gcc_assert (GET_MODE (op->value) == VOIDmode 7154 || GET_MODE (op->value) == mode); 7155 if (maybe_legitimize_operand_same_code (icode, opno, op)) 7156 return true; 7157 7158 op->value = copy_to_mode_reg (mode, op->value); 7159 break; 7160 7161 case EXPAND_CONVERT_TO: 7162 gcc_assert (mode != VOIDmode); 7163 op->value = convert_to_mode (mode, op->value, op->unsigned_p); 7164 goto input; 7165 7166 case EXPAND_CONVERT_FROM: 7167 if (GET_MODE (op->value) != VOIDmode) 7168 mode = GET_MODE (op->value); 7169 else 7170 /* The caller must tell us what mode this value has. */ 7171 gcc_assert (mode != VOIDmode); 7172 7173 imode = insn_data[(int) icode].operand[opno].mode; 7174 if (imode != VOIDmode && imode != mode) 7175 { 7176 op->value = convert_modes (imode, mode, op->value, op->unsigned_p); 7177 mode = imode; 7178 } 7179 goto input; 7180 7181 case EXPAND_ADDRESS: 7182 op->value = convert_memory_address (as_a <scalar_int_mode> (mode), 7183 op->value); 7184 goto input; 7185 7186 case EXPAND_INTEGER: 7187 mode = insn_data[(int) icode].operand[opno].mode; 7188 if (mode != VOIDmode 7189 && known_eq (trunc_int_for_mode (op->int_value, mode), 7190 op->int_value)) 7191 { 7192 op->value = gen_int_mode (op->int_value, mode); 7193 goto input; 7194 } 7195 break; 7196 } 7197 return insn_operand_matches (icode, opno, op->value); 7198 } 7199 7200 /* Make OP describe an input operand that should have the same value 7201 as VALUE, after any mode conversion that the target might request. 7202 TYPE is the type of VALUE. */ 7203 7204 void 7205 create_convert_operand_from_type (struct expand_operand *op, 7206 rtx value, tree type) 7207 { 7208 create_convert_operand_from (op, value, TYPE_MODE (type), 7209 TYPE_UNSIGNED (type)); 7210 } 7211 7212 /* Try to make operands [OPS, OPS + NOPS) match operands [OPNO, OPNO + NOPS) 7213 of instruction ICODE. Return true on success, leaving the new operand 7214 values in the OPS themselves. Emit no code on failure. */ 7215 7216 bool 7217 maybe_legitimize_operands (enum insn_code icode, unsigned int opno, 7218 unsigned int nops, struct expand_operand *ops) 7219 { 7220 rtx_insn *last; 7221 unsigned int i; 7222 7223 last = get_last_insn (); 7224 for (i = 0; i < nops; i++) 7225 if (!maybe_legitimize_operand (icode, opno + i, &ops[i])) 7226 { 7227 delete_insns_since (last); 7228 return false; 7229 } 7230 return true; 7231 } 7232 7233 /* Try to generate instruction ICODE, using operands [OPS, OPS + NOPS) 7234 as its operands. Return the instruction pattern on success, 7235 and emit any necessary set-up code. Return null and emit no 7236 code on failure. */ 7237 7238 rtx_insn * 7239 maybe_gen_insn (enum insn_code icode, unsigned int nops, 7240 struct expand_operand *ops) 7241 { 7242 gcc_assert (nops == (unsigned int) insn_data[(int) icode].n_generator_args); 7243 if (!maybe_legitimize_operands (icode, 0, nops, ops)) 7244 return NULL; 7245 7246 switch (nops) 7247 { 7248 case 1: 7249 return GEN_FCN (icode) (ops[0].value); 7250 case 2: 7251 return GEN_FCN (icode) (ops[0].value, ops[1].value); 7252 case 3: 7253 return GEN_FCN (icode) (ops[0].value, ops[1].value, ops[2].value); 7254 case 4: 7255 return GEN_FCN (icode) (ops[0].value, ops[1].value, ops[2].value, 7256 ops[3].value); 7257 case 5: 7258 return GEN_FCN (icode) (ops[0].value, ops[1].value, ops[2].value, 7259 ops[3].value, ops[4].value); 7260 case 6: 7261 return GEN_FCN (icode) (ops[0].value, ops[1].value, ops[2].value, 7262 ops[3].value, ops[4].value, ops[5].value); 7263 case 7: 7264 return GEN_FCN (icode) (ops[0].value, ops[1].value, ops[2].value, 7265 ops[3].value, ops[4].value, ops[5].value, 7266 ops[6].value); 7267 case 8: 7268 return GEN_FCN (icode) (ops[0].value, ops[1].value, ops[2].value, 7269 ops[3].value, ops[4].value, ops[5].value, 7270 ops[6].value, ops[7].value); 7271 case 9: 7272 return GEN_FCN (icode) (ops[0].value, ops[1].value, ops[2].value, 7273 ops[3].value, ops[4].value, ops[5].value, 7274 ops[6].value, ops[7].value, ops[8].value); 7275 } 7276 gcc_unreachable (); 7277 } 7278 7279 /* Try to emit instruction ICODE, using operands [OPS, OPS + NOPS) 7280 as its operands. Return true on success and emit no code on failure. */ 7281 7282 bool 7283 maybe_expand_insn (enum insn_code icode, unsigned int nops, 7284 struct expand_operand *ops) 7285 { 7286 rtx_insn *pat = maybe_gen_insn (icode, nops, ops); 7287 if (pat) 7288 { 7289 emit_insn (pat); 7290 return true; 7291 } 7292 return false; 7293 } 7294 7295 /* Like maybe_expand_insn, but for jumps. */ 7296 7297 bool 7298 maybe_expand_jump_insn (enum insn_code icode, unsigned int nops, 7299 struct expand_operand *ops) 7300 { 7301 rtx_insn *pat = maybe_gen_insn (icode, nops, ops); 7302 if (pat) 7303 { 7304 emit_jump_insn (pat); 7305 return true; 7306 } 7307 return false; 7308 } 7309 7310 /* Emit instruction ICODE, using operands [OPS, OPS + NOPS) 7311 as its operands. */ 7312 7313 void 7314 expand_insn (enum insn_code icode, unsigned int nops, 7315 struct expand_operand *ops) 7316 { 7317 if (!maybe_expand_insn (icode, nops, ops)) 7318 gcc_unreachable (); 7319 } 7320 7321 /* Like expand_insn, but for jumps. */ 7322 7323 void 7324 expand_jump_insn (enum insn_code icode, unsigned int nops, 7325 struct expand_operand *ops) 7326 { 7327 if (!maybe_expand_jump_insn (icode, nops, ops)) 7328 gcc_unreachable (); 7329 } 7330