1 /* Expand the basic unary and binary arithmetic operations, for GNU compiler.
2 Copyright (C) 1987-2018 Free Software Foundation, Inc.
3
4 This file is part of GCC.
5
6 GCC is free software; you can redistribute it and/or modify it under
7 the terms of the GNU General Public License as published by the Free
8 Software Foundation; either version 3, or (at your option) any later
9 version.
10
11 GCC is distributed in the hope that it will be useful, but WITHOUT ANY
12 WARRANTY; without even the implied warranty of MERCHANTABILITY or
13 FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
14 for more details.
15
16 You should have received a copy of the GNU General Public License
17 along with GCC; see the file COPYING3. If not see
18 <http://www.gnu.org/licenses/>. */
19
20
21 #include "config.h"
22 #include "system.h"
23 #include "coretypes.h"
24 #include "backend.h"
25 #include "target.h"
26 #include "rtl.h"
27 #include "tree.h"
28 #include "memmodel.h"
29 #include "predict.h"
30 #include "tm_p.h"
31 #include "expmed.h"
32 #include "optabs.h"
33 #include "emit-rtl.h"
34 #include "recog.h"
35 #include "diagnostic-core.h"
36 #include "rtx-vector-builder.h"
37
38 /* Include insn-config.h before expr.h so that HAVE_conditional_move
39 is properly defined. */
40 #include "stor-layout.h"
41 #include "except.h"
42 #include "dojump.h"
43 #include "explow.h"
44 #include "expr.h"
45 #include "optabs-tree.h"
46 #include "libfuncs.h"
47
48 static void prepare_float_lib_cmp (rtx, rtx, enum rtx_code, rtx *,
49 machine_mode *);
50 static rtx expand_unop_direct (machine_mode, optab, rtx, rtx, int);
51 static void emit_libcall_block_1 (rtx_insn *, rtx, rtx, rtx, bool);
52
53 /* Debug facility for use in GDB. */
54 void debug_optab_libfuncs (void);
55
56 /* Add a REG_EQUAL note to the last insn in INSNS. TARGET is being set to
57 the result of operation CODE applied to OP0 (and OP1 if it is a binary
58 operation). OP0_MODE is OP0's mode.
59
60 If the last insn does not set TARGET, don't do anything, but return 1.
61
62 If the last insn or a previous insn sets TARGET and TARGET is one of OP0
63 or OP1, don't add the REG_EQUAL note but return 0. Our caller can then
64 try again, ensuring that TARGET is not one of the operands. */
65
66 static int
add_equal_note(rtx_insn * insns,rtx target,enum rtx_code code,rtx op0,rtx op1,machine_mode op0_mode)67 add_equal_note (rtx_insn *insns, rtx target, enum rtx_code code, rtx op0,
68 rtx op1, machine_mode op0_mode)
69 {
70 rtx_insn *last_insn;
71 rtx set;
72 rtx note;
73
74 gcc_assert (insns && INSN_P (insns) && NEXT_INSN (insns));
75
76 if (GET_RTX_CLASS (code) != RTX_COMM_ARITH
77 && GET_RTX_CLASS (code) != RTX_BIN_ARITH
78 && GET_RTX_CLASS (code) != RTX_COMM_COMPARE
79 && GET_RTX_CLASS (code) != RTX_COMPARE
80 && GET_RTX_CLASS (code) != RTX_UNARY)
81 return 1;
82
83 if (GET_CODE (target) == ZERO_EXTRACT)
84 return 1;
85
86 for (last_insn = insns;
87 NEXT_INSN (last_insn) != NULL_RTX;
88 last_insn = NEXT_INSN (last_insn))
89 ;
90
91 /* If TARGET is in OP0 or OP1, punt. We'd end up with a note referencing
92 a value changing in the insn, so the note would be invalid for CSE. */
93 if (reg_overlap_mentioned_p (target, op0)
94 || (op1 && reg_overlap_mentioned_p (target, op1)))
95 {
96 if (MEM_P (target)
97 && (rtx_equal_p (target, op0)
98 || (op1 && rtx_equal_p (target, op1))))
99 {
100 /* For MEM target, with MEM = MEM op X, prefer no REG_EQUAL note
101 over expanding it as temp = MEM op X, MEM = temp. If the target
102 supports MEM = MEM op X instructions, it is sometimes too hard
103 to reconstruct that form later, especially if X is also a memory,
104 and due to multiple occurrences of addresses the address might
105 be forced into register unnecessarily.
106 Note that not emitting the REG_EQUIV note might inhibit
107 CSE in some cases. */
108 set = single_set (last_insn);
109 if (set
110 && GET_CODE (SET_SRC (set)) == code
111 && MEM_P (SET_DEST (set))
112 && (rtx_equal_p (SET_DEST (set), XEXP (SET_SRC (set), 0))
113 || (op1 && rtx_equal_p (SET_DEST (set),
114 XEXP (SET_SRC (set), 1)))))
115 return 1;
116 }
117 return 0;
118 }
119
120 set = set_for_reg_notes (last_insn);
121 if (set == NULL_RTX)
122 return 1;
123
124 if (! rtx_equal_p (SET_DEST (set), target)
125 /* For a STRICT_LOW_PART, the REG_NOTE applies to what is inside it. */
126 && (GET_CODE (SET_DEST (set)) != STRICT_LOW_PART
127 || ! rtx_equal_p (XEXP (SET_DEST (set), 0), target)))
128 return 1;
129
130 if (GET_RTX_CLASS (code) == RTX_UNARY)
131 switch (code)
132 {
133 case FFS:
134 case CLZ:
135 case CTZ:
136 case CLRSB:
137 case POPCOUNT:
138 case PARITY:
139 case BSWAP:
140 if (op0_mode != VOIDmode && GET_MODE (target) != op0_mode)
141 {
142 note = gen_rtx_fmt_e (code, op0_mode, copy_rtx (op0));
143 if (GET_MODE_UNIT_SIZE (op0_mode)
144 > GET_MODE_UNIT_SIZE (GET_MODE (target)))
145 note = simplify_gen_unary (TRUNCATE, GET_MODE (target),
146 note, op0_mode);
147 else
148 note = simplify_gen_unary (ZERO_EXTEND, GET_MODE (target),
149 note, op0_mode);
150 break;
151 }
152 /* FALLTHRU */
153 default:
154 note = gen_rtx_fmt_e (code, GET_MODE (target), copy_rtx (op0));
155 break;
156 }
157 else
158 note = gen_rtx_fmt_ee (code, GET_MODE (target), copy_rtx (op0), copy_rtx (op1));
159
160 set_unique_reg_note (last_insn, REG_EQUAL, note);
161
162 return 1;
163 }
164
165 /* Given two input operands, OP0 and OP1, determine what the correct from_mode
166 for a widening operation would be. In most cases this would be OP0, but if
167 that's a constant it'll be VOIDmode, which isn't useful. */
168
169 static machine_mode
widened_mode(machine_mode to_mode,rtx op0,rtx op1)170 widened_mode (machine_mode to_mode, rtx op0, rtx op1)
171 {
172 machine_mode m0 = GET_MODE (op0);
173 machine_mode m1 = GET_MODE (op1);
174 machine_mode result;
175
176 if (m0 == VOIDmode && m1 == VOIDmode)
177 return to_mode;
178 else if (m0 == VOIDmode || GET_MODE_UNIT_SIZE (m0) < GET_MODE_UNIT_SIZE (m1))
179 result = m1;
180 else
181 result = m0;
182
183 if (GET_MODE_UNIT_SIZE (result) > GET_MODE_UNIT_SIZE (to_mode))
184 return to_mode;
185
186 return result;
187 }
188
189 /* Widen OP to MODE and return the rtx for the widened operand. UNSIGNEDP
190 says whether OP is signed or unsigned. NO_EXTEND is nonzero if we need
191 not actually do a sign-extend or zero-extend, but can leave the
192 higher-order bits of the result rtx undefined, for example, in the case
193 of logical operations, but not right shifts. */
194
195 static rtx
widen_operand(rtx op,machine_mode mode,machine_mode oldmode,int unsignedp,int no_extend)196 widen_operand (rtx op, machine_mode mode, machine_mode oldmode,
197 int unsignedp, int no_extend)
198 {
199 rtx result;
200 scalar_int_mode int_mode;
201
202 /* If we don't have to extend and this is a constant, return it. */
203 if (no_extend && GET_MODE (op) == VOIDmode)
204 return op;
205
206 /* If we must extend do so. If OP is a SUBREG for a promoted object, also
207 extend since it will be more efficient to do so unless the signedness of
208 a promoted object differs from our extension. */
209 if (! no_extend
210 || !is_a <scalar_int_mode> (mode, &int_mode)
211 || (GET_CODE (op) == SUBREG && SUBREG_PROMOTED_VAR_P (op)
212 && SUBREG_CHECK_PROMOTED_SIGN (op, unsignedp)))
213 return convert_modes (mode, oldmode, op, unsignedp);
214
215 /* If MODE is no wider than a single word, we return a lowpart or paradoxical
216 SUBREG. */
217 if (GET_MODE_SIZE (int_mode) <= UNITS_PER_WORD)
218 return gen_lowpart (int_mode, force_reg (GET_MODE (op), op));
219
220 /* Otherwise, get an object of MODE, clobber it, and set the low-order
221 part to OP. */
222
223 result = gen_reg_rtx (int_mode);
224 emit_clobber (result);
225 emit_move_insn (gen_lowpart (GET_MODE (op), result), op);
226 return result;
227 }
228
229 /* Expand vector widening operations.
230
231 There are two different classes of operations handled here:
232 1) Operations whose result is wider than all the arguments to the operation.
233 Examples: VEC_UNPACK_HI/LO_EXPR, VEC_WIDEN_MULT_HI/LO_EXPR
234 In this case OP0 and optionally OP1 would be initialized,
235 but WIDE_OP wouldn't (not relevant for this case).
236 2) Operations whose result is of the same size as the last argument to the
237 operation, but wider than all the other arguments to the operation.
238 Examples: WIDEN_SUM_EXPR, VEC_DOT_PROD_EXPR.
239 In the case WIDE_OP, OP0 and optionally OP1 would be initialized.
240
241 E.g, when called to expand the following operations, this is how
242 the arguments will be initialized:
243 nops OP0 OP1 WIDE_OP
244 widening-sum 2 oprnd0 - oprnd1
245 widening-dot-product 3 oprnd0 oprnd1 oprnd2
246 widening-mult 2 oprnd0 oprnd1 -
247 type-promotion (vec-unpack) 1 oprnd0 - - */
248
249 rtx
expand_widen_pattern_expr(sepops ops,rtx op0,rtx op1,rtx wide_op,rtx target,int unsignedp)250 expand_widen_pattern_expr (sepops ops, rtx op0, rtx op1, rtx wide_op,
251 rtx target, int unsignedp)
252 {
253 struct expand_operand eops[4];
254 tree oprnd0, oprnd1, oprnd2;
255 machine_mode wmode = VOIDmode, tmode0, tmode1 = VOIDmode;
256 optab widen_pattern_optab;
257 enum insn_code icode;
258 int nops = TREE_CODE_LENGTH (ops->code);
259 int op;
260
261 oprnd0 = ops->op0;
262 tmode0 = TYPE_MODE (TREE_TYPE (oprnd0));
263 widen_pattern_optab =
264 optab_for_tree_code (ops->code, TREE_TYPE (oprnd0), optab_default);
265 if (ops->code == WIDEN_MULT_PLUS_EXPR
266 || ops->code == WIDEN_MULT_MINUS_EXPR)
267 icode = find_widening_optab_handler (widen_pattern_optab,
268 TYPE_MODE (TREE_TYPE (ops->op2)),
269 tmode0);
270 else
271 icode = optab_handler (widen_pattern_optab, tmode0);
272 gcc_assert (icode != CODE_FOR_nothing);
273
274 if (nops >= 2)
275 {
276 oprnd1 = ops->op1;
277 tmode1 = TYPE_MODE (TREE_TYPE (oprnd1));
278 }
279
280 /* The last operand is of a wider mode than the rest of the operands. */
281 if (nops == 2)
282 wmode = tmode1;
283 else if (nops == 3)
284 {
285 gcc_assert (tmode1 == tmode0);
286 gcc_assert (op1);
287 oprnd2 = ops->op2;
288 wmode = TYPE_MODE (TREE_TYPE (oprnd2));
289 }
290
291 op = 0;
292 create_output_operand (&eops[op++], target, TYPE_MODE (ops->type));
293 create_convert_operand_from (&eops[op++], op0, tmode0, unsignedp);
294 if (op1)
295 create_convert_operand_from (&eops[op++], op1, tmode1, unsignedp);
296 if (wide_op)
297 create_convert_operand_from (&eops[op++], wide_op, wmode, unsignedp);
298 expand_insn (icode, op, eops);
299 return eops[0].value;
300 }
301
302 /* Generate code to perform an operation specified by TERNARY_OPTAB
303 on operands OP0, OP1 and OP2, with result having machine-mode MODE.
304
305 UNSIGNEDP is for the case where we have to widen the operands
306 to perform the operation. It says to use zero-extension.
307
308 If TARGET is nonzero, the value
309 is generated there, if it is convenient to do so.
310 In all cases an rtx is returned for the locus of the value;
311 this may or may not be TARGET. */
312
313 rtx
expand_ternary_op(machine_mode mode,optab ternary_optab,rtx op0,rtx op1,rtx op2,rtx target,int unsignedp)314 expand_ternary_op (machine_mode mode, optab ternary_optab, rtx op0,
315 rtx op1, rtx op2, rtx target, int unsignedp)
316 {
317 struct expand_operand ops[4];
318 enum insn_code icode = optab_handler (ternary_optab, mode);
319
320 gcc_assert (optab_handler (ternary_optab, mode) != CODE_FOR_nothing);
321
322 create_output_operand (&ops[0], target, mode);
323 create_convert_operand_from (&ops[1], op0, mode, unsignedp);
324 create_convert_operand_from (&ops[2], op1, mode, unsignedp);
325 create_convert_operand_from (&ops[3], op2, mode, unsignedp);
326 expand_insn (icode, 4, ops);
327 return ops[0].value;
328 }
329
330
331 /* Like expand_binop, but return a constant rtx if the result can be
332 calculated at compile time. The arguments and return value are
333 otherwise the same as for expand_binop. */
334
335 rtx
simplify_expand_binop(machine_mode mode,optab binoptab,rtx op0,rtx op1,rtx target,int unsignedp,enum optab_methods methods)336 simplify_expand_binop (machine_mode mode, optab binoptab,
337 rtx op0, rtx op1, rtx target, int unsignedp,
338 enum optab_methods methods)
339 {
340 if (CONSTANT_P (op0) && CONSTANT_P (op1))
341 {
342 rtx x = simplify_binary_operation (optab_to_code (binoptab),
343 mode, op0, op1);
344 if (x)
345 return x;
346 }
347
348 return expand_binop (mode, binoptab, op0, op1, target, unsignedp, methods);
349 }
350
351 /* Like simplify_expand_binop, but always put the result in TARGET.
352 Return true if the expansion succeeded. */
353
354 bool
force_expand_binop(machine_mode mode,optab binoptab,rtx op0,rtx op1,rtx target,int unsignedp,enum optab_methods methods)355 force_expand_binop (machine_mode mode, optab binoptab,
356 rtx op0, rtx op1, rtx target, int unsignedp,
357 enum optab_methods methods)
358 {
359 rtx x = simplify_expand_binop (mode, binoptab, op0, op1,
360 target, unsignedp, methods);
361 if (x == 0)
362 return false;
363 if (x != target)
364 emit_move_insn (target, x);
365 return true;
366 }
367
368 /* Create a new vector value in VMODE with all elements set to OP. The
369 mode of OP must be the element mode of VMODE. If OP is a constant,
370 then the return value will be a constant. */
371
372 rtx
expand_vector_broadcast(machine_mode vmode,rtx op)373 expand_vector_broadcast (machine_mode vmode, rtx op)
374 {
375 int n;
376 rtvec vec;
377
378 gcc_checking_assert (VECTOR_MODE_P (vmode));
379
380 if (valid_for_const_vector_p (vmode, op))
381 return gen_const_vec_duplicate (vmode, op);
382
383 insn_code icode = optab_handler (vec_duplicate_optab, vmode);
384 if (icode != CODE_FOR_nothing)
385 {
386 struct expand_operand ops[2];
387 create_output_operand (&ops[0], NULL_RTX, vmode);
388 create_input_operand (&ops[1], op, GET_MODE (op));
389 expand_insn (icode, 2, ops);
390 return ops[0].value;
391 }
392
393 if (!GET_MODE_NUNITS (vmode).is_constant (&n))
394 return NULL;
395
396 /* ??? If the target doesn't have a vec_init, then we have no easy way
397 of performing this operation. Most of this sort of generic support
398 is hidden away in the vector lowering support in gimple. */
399 icode = convert_optab_handler (vec_init_optab, vmode,
400 GET_MODE_INNER (vmode));
401 if (icode == CODE_FOR_nothing)
402 return NULL;
403
404 vec = rtvec_alloc (n);
405 for (int i = 0; i < n; ++i)
406 RTVEC_ELT (vec, i) = op;
407 rtx ret = gen_reg_rtx (vmode);
408 emit_insn (GEN_FCN (icode) (ret, gen_rtx_PARALLEL (vmode, vec)));
409
410 return ret;
411 }
412
413 /* This subroutine of expand_doubleword_shift handles the cases in which
414 the effective shift value is >= BITS_PER_WORD. The arguments and return
415 value are the same as for the parent routine, except that SUPERWORD_OP1
416 is the shift count to use when shifting OUTOF_INPUT into INTO_TARGET.
417 INTO_TARGET may be null if the caller has decided to calculate it. */
418
419 static bool
expand_superword_shift(optab binoptab,rtx outof_input,rtx superword_op1,rtx outof_target,rtx into_target,int unsignedp,enum optab_methods methods)420 expand_superword_shift (optab binoptab, rtx outof_input, rtx superword_op1,
421 rtx outof_target, rtx into_target,
422 int unsignedp, enum optab_methods methods)
423 {
424 if (into_target != 0)
425 if (!force_expand_binop (word_mode, binoptab, outof_input, superword_op1,
426 into_target, unsignedp, methods))
427 return false;
428
429 if (outof_target != 0)
430 {
431 /* For a signed right shift, we must fill OUTOF_TARGET with copies
432 of the sign bit, otherwise we must fill it with zeros. */
433 if (binoptab != ashr_optab)
434 emit_move_insn (outof_target, CONST0_RTX (word_mode));
435 else
436 if (!force_expand_binop (word_mode, binoptab, outof_input,
437 gen_int_shift_amount (word_mode,
438 BITS_PER_WORD - 1),
439 outof_target, unsignedp, methods))
440 return false;
441 }
442 return true;
443 }
444
445 /* This subroutine of expand_doubleword_shift handles the cases in which
446 the effective shift value is < BITS_PER_WORD. The arguments and return
447 value are the same as for the parent routine. */
448
449 static bool
expand_subword_shift(scalar_int_mode op1_mode,optab binoptab,rtx outof_input,rtx into_input,rtx op1,rtx outof_target,rtx into_target,int unsignedp,enum optab_methods methods,unsigned HOST_WIDE_INT shift_mask)450 expand_subword_shift (scalar_int_mode op1_mode, optab binoptab,
451 rtx outof_input, rtx into_input, rtx op1,
452 rtx outof_target, rtx into_target,
453 int unsignedp, enum optab_methods methods,
454 unsigned HOST_WIDE_INT shift_mask)
455 {
456 optab reverse_unsigned_shift, unsigned_shift;
457 rtx tmp, carries;
458
459 reverse_unsigned_shift = (binoptab == ashl_optab ? lshr_optab : ashl_optab);
460 unsigned_shift = (binoptab == ashl_optab ? ashl_optab : lshr_optab);
461
462 /* The low OP1 bits of INTO_TARGET come from the high bits of OUTOF_INPUT.
463 We therefore need to shift OUTOF_INPUT by (BITS_PER_WORD - OP1) bits in
464 the opposite direction to BINOPTAB. */
465 if (CONSTANT_P (op1) || shift_mask >= BITS_PER_WORD)
466 {
467 carries = outof_input;
468 tmp = immed_wide_int_const (wi::shwi (BITS_PER_WORD,
469 op1_mode), op1_mode);
470 tmp = simplify_expand_binop (op1_mode, sub_optab, tmp, op1,
471 0, true, methods);
472 }
473 else
474 {
475 /* We must avoid shifting by BITS_PER_WORD bits since that is either
476 the same as a zero shift (if shift_mask == BITS_PER_WORD - 1) or
477 has unknown behavior. Do a single shift first, then shift by the
478 remainder. It's OK to use ~OP1 as the remainder if shift counts
479 are truncated to the mode size. */
480 carries = expand_binop (word_mode, reverse_unsigned_shift,
481 outof_input, const1_rtx, 0, unsignedp, methods);
482 if (shift_mask == BITS_PER_WORD - 1)
483 {
484 tmp = immed_wide_int_const
485 (wi::minus_one (GET_MODE_PRECISION (op1_mode)), op1_mode);
486 tmp = simplify_expand_binop (op1_mode, xor_optab, op1, tmp,
487 0, true, methods);
488 }
489 else
490 {
491 tmp = immed_wide_int_const (wi::shwi (BITS_PER_WORD - 1,
492 op1_mode), op1_mode);
493 tmp = simplify_expand_binop (op1_mode, sub_optab, tmp, op1,
494 0, true, methods);
495 }
496 }
497 if (tmp == 0 || carries == 0)
498 return false;
499 carries = expand_binop (word_mode, reverse_unsigned_shift,
500 carries, tmp, 0, unsignedp, methods);
501 if (carries == 0)
502 return false;
503
504 /* Shift INTO_INPUT logically by OP1. This is the last use of INTO_INPUT
505 so the result can go directly into INTO_TARGET if convenient. */
506 tmp = expand_binop (word_mode, unsigned_shift, into_input, op1,
507 into_target, unsignedp, methods);
508 if (tmp == 0)
509 return false;
510
511 /* Now OR in the bits carried over from OUTOF_INPUT. */
512 if (!force_expand_binop (word_mode, ior_optab, tmp, carries,
513 into_target, unsignedp, methods))
514 return false;
515
516 /* Use a standard word_mode shift for the out-of half. */
517 if (outof_target != 0)
518 if (!force_expand_binop (word_mode, binoptab, outof_input, op1,
519 outof_target, unsignedp, methods))
520 return false;
521
522 return true;
523 }
524
525
526 /* Try implementing expand_doubleword_shift using conditional moves.
527 The shift is by < BITS_PER_WORD if (CMP_CODE CMP1 CMP2) is true,
528 otherwise it is by >= BITS_PER_WORD. SUBWORD_OP1 and SUPERWORD_OP1
529 are the shift counts to use in the former and latter case. All other
530 arguments are the same as the parent routine. */
531
532 static bool
expand_doubleword_shift_condmove(scalar_int_mode op1_mode,optab binoptab,enum rtx_code cmp_code,rtx cmp1,rtx cmp2,rtx outof_input,rtx into_input,rtx subword_op1,rtx superword_op1,rtx outof_target,rtx into_target,int unsignedp,enum optab_methods methods,unsigned HOST_WIDE_INT shift_mask)533 expand_doubleword_shift_condmove (scalar_int_mode op1_mode, optab binoptab,
534 enum rtx_code cmp_code, rtx cmp1, rtx cmp2,
535 rtx outof_input, rtx into_input,
536 rtx subword_op1, rtx superword_op1,
537 rtx outof_target, rtx into_target,
538 int unsignedp, enum optab_methods methods,
539 unsigned HOST_WIDE_INT shift_mask)
540 {
541 rtx outof_superword, into_superword;
542
543 /* Put the superword version of the output into OUTOF_SUPERWORD and
544 INTO_SUPERWORD. */
545 outof_superword = outof_target != 0 ? gen_reg_rtx (word_mode) : 0;
546 if (outof_target != 0 && subword_op1 == superword_op1)
547 {
548 /* The value INTO_TARGET >> SUBWORD_OP1, which we later store in
549 OUTOF_TARGET, is the same as the value of INTO_SUPERWORD. */
550 into_superword = outof_target;
551 if (!expand_superword_shift (binoptab, outof_input, superword_op1,
552 outof_superword, 0, unsignedp, methods))
553 return false;
554 }
555 else
556 {
557 into_superword = gen_reg_rtx (word_mode);
558 if (!expand_superword_shift (binoptab, outof_input, superword_op1,
559 outof_superword, into_superword,
560 unsignedp, methods))
561 return false;
562 }
563
564 /* Put the subword version directly in OUTOF_TARGET and INTO_TARGET. */
565 if (!expand_subword_shift (op1_mode, binoptab,
566 outof_input, into_input, subword_op1,
567 outof_target, into_target,
568 unsignedp, methods, shift_mask))
569 return false;
570
571 /* Select between them. Do the INTO half first because INTO_SUPERWORD
572 might be the current value of OUTOF_TARGET. */
573 if (!emit_conditional_move (into_target, cmp_code, cmp1, cmp2, op1_mode,
574 into_target, into_superword, word_mode, false))
575 return false;
576
577 if (outof_target != 0)
578 if (!emit_conditional_move (outof_target, cmp_code, cmp1, cmp2, op1_mode,
579 outof_target, outof_superword,
580 word_mode, false))
581 return false;
582
583 return true;
584 }
585
586 /* Expand a doubleword shift (ashl, ashr or lshr) using word-mode shifts.
587 OUTOF_INPUT and INTO_INPUT are the two word-sized halves of the first
588 input operand; the shift moves bits in the direction OUTOF_INPUT->
589 INTO_TARGET. OUTOF_TARGET and INTO_TARGET are the equivalent words
590 of the target. OP1 is the shift count and OP1_MODE is its mode.
591 If OP1 is constant, it will have been truncated as appropriate
592 and is known to be nonzero.
593
594 If SHIFT_MASK is zero, the result of word shifts is undefined when the
595 shift count is outside the range [0, BITS_PER_WORD). This routine must
596 avoid generating such shifts for OP1s in the range [0, BITS_PER_WORD * 2).
597
598 If SHIFT_MASK is nonzero, all word-mode shift counts are effectively
599 masked by it and shifts in the range [BITS_PER_WORD, SHIFT_MASK) will
600 fill with zeros or sign bits as appropriate.
601
602 If SHIFT_MASK is BITS_PER_WORD - 1, this routine will synthesize
603 a doubleword shift whose equivalent mask is BITS_PER_WORD * 2 - 1.
604 Doing this preserves semantics required by SHIFT_COUNT_TRUNCATED.
605 In all other cases, shifts by values outside [0, BITS_PER_UNIT * 2)
606 are undefined.
607
608 BINOPTAB, UNSIGNEDP and METHODS are as for expand_binop. This function
609 may not use INTO_INPUT after modifying INTO_TARGET, and similarly for
610 OUTOF_INPUT and OUTOF_TARGET. OUTOF_TARGET can be null if the parent
611 function wants to calculate it itself.
612
613 Return true if the shift could be successfully synthesized. */
614
615 static bool
expand_doubleword_shift(scalar_int_mode op1_mode,optab binoptab,rtx outof_input,rtx into_input,rtx op1,rtx outof_target,rtx into_target,int unsignedp,enum optab_methods methods,unsigned HOST_WIDE_INT shift_mask)616 expand_doubleword_shift (scalar_int_mode op1_mode, optab binoptab,
617 rtx outof_input, rtx into_input, rtx op1,
618 rtx outof_target, rtx into_target,
619 int unsignedp, enum optab_methods methods,
620 unsigned HOST_WIDE_INT shift_mask)
621 {
622 rtx superword_op1, tmp, cmp1, cmp2;
623 enum rtx_code cmp_code;
624
625 /* See if word-mode shifts by BITS_PER_WORD...BITS_PER_WORD * 2 - 1 will
626 fill the result with sign or zero bits as appropriate. If so, the value
627 of OUTOF_TARGET will always be (SHIFT OUTOF_INPUT OP1). Recursively call
628 this routine to calculate INTO_TARGET (which depends on both OUTOF_INPUT
629 and INTO_INPUT), then emit code to set up OUTOF_TARGET.
630
631 This isn't worthwhile for constant shifts since the optimizers will
632 cope better with in-range shift counts. */
633 if (shift_mask >= BITS_PER_WORD
634 && outof_target != 0
635 && !CONSTANT_P (op1))
636 {
637 if (!expand_doubleword_shift (op1_mode, binoptab,
638 outof_input, into_input, op1,
639 0, into_target,
640 unsignedp, methods, shift_mask))
641 return false;
642 if (!force_expand_binop (word_mode, binoptab, outof_input, op1,
643 outof_target, unsignedp, methods))
644 return false;
645 return true;
646 }
647
648 /* Set CMP_CODE, CMP1 and CMP2 so that the rtx (CMP_CODE CMP1 CMP2)
649 is true when the effective shift value is less than BITS_PER_WORD.
650 Set SUPERWORD_OP1 to the shift count that should be used to shift
651 OUTOF_INPUT into INTO_TARGET when the condition is false. */
652 tmp = immed_wide_int_const (wi::shwi (BITS_PER_WORD, op1_mode), op1_mode);
653 if (!CONSTANT_P (op1) && shift_mask == BITS_PER_WORD - 1)
654 {
655 /* Set CMP1 to OP1 & BITS_PER_WORD. The result is zero iff OP1
656 is a subword shift count. */
657 cmp1 = simplify_expand_binop (op1_mode, and_optab, op1, tmp,
658 0, true, methods);
659 cmp2 = CONST0_RTX (op1_mode);
660 cmp_code = EQ;
661 superword_op1 = op1;
662 }
663 else
664 {
665 /* Set CMP1 to OP1 - BITS_PER_WORD. */
666 cmp1 = simplify_expand_binop (op1_mode, sub_optab, op1, tmp,
667 0, true, methods);
668 cmp2 = CONST0_RTX (op1_mode);
669 cmp_code = LT;
670 superword_op1 = cmp1;
671 }
672 if (cmp1 == 0)
673 return false;
674
675 /* If we can compute the condition at compile time, pick the
676 appropriate subroutine. */
677 tmp = simplify_relational_operation (cmp_code, SImode, op1_mode, cmp1, cmp2);
678 if (tmp != 0 && CONST_INT_P (tmp))
679 {
680 if (tmp == const0_rtx)
681 return expand_superword_shift (binoptab, outof_input, superword_op1,
682 outof_target, into_target,
683 unsignedp, methods);
684 else
685 return expand_subword_shift (op1_mode, binoptab,
686 outof_input, into_input, op1,
687 outof_target, into_target,
688 unsignedp, methods, shift_mask);
689 }
690
691 /* Try using conditional moves to generate straight-line code. */
692 if (HAVE_conditional_move)
693 {
694 rtx_insn *start = get_last_insn ();
695 if (expand_doubleword_shift_condmove (op1_mode, binoptab,
696 cmp_code, cmp1, cmp2,
697 outof_input, into_input,
698 op1, superword_op1,
699 outof_target, into_target,
700 unsignedp, methods, shift_mask))
701 return true;
702 delete_insns_since (start);
703 }
704
705 /* As a last resort, use branches to select the correct alternative. */
706 rtx_code_label *subword_label = gen_label_rtx ();
707 rtx_code_label *done_label = gen_label_rtx ();
708
709 NO_DEFER_POP;
710 do_compare_rtx_and_jump (cmp1, cmp2, cmp_code, false, op1_mode,
711 0, 0, subword_label,
712 profile_probability::uninitialized ());
713 OK_DEFER_POP;
714
715 if (!expand_superword_shift (binoptab, outof_input, superword_op1,
716 outof_target, into_target,
717 unsignedp, methods))
718 return false;
719
720 emit_jump_insn (targetm.gen_jump (done_label));
721 emit_barrier ();
722 emit_label (subword_label);
723
724 if (!expand_subword_shift (op1_mode, binoptab,
725 outof_input, into_input, op1,
726 outof_target, into_target,
727 unsignedp, methods, shift_mask))
728 return false;
729
730 emit_label (done_label);
731 return true;
732 }
733
734 /* Subroutine of expand_binop. Perform a double word multiplication of
735 operands OP0 and OP1 both of mode MODE, which is exactly twice as wide
736 as the target's word_mode. This function return NULL_RTX if anything
737 goes wrong, in which case it may have already emitted instructions
738 which need to be deleted.
739
740 If we want to multiply two two-word values and have normal and widening
741 multiplies of single-word values, we can do this with three smaller
742 multiplications.
743
744 The multiplication proceeds as follows:
745 _______________________
746 [__op0_high_|__op0_low__]
747 _______________________
748 * [__op1_high_|__op1_low__]
749 _______________________________________________
750 _______________________
751 (1) [__op0_low__*__op1_low__]
752 _______________________
753 (2a) [__op0_low__*__op1_high_]
754 _______________________
755 (2b) [__op0_high_*__op1_low__]
756 _______________________
757 (3) [__op0_high_*__op1_high_]
758
759
760 This gives a 4-word result. Since we are only interested in the
761 lower 2 words, partial result (3) and the upper words of (2a) and
762 (2b) don't need to be calculated. Hence (2a) and (2b) can be
763 calculated using non-widening multiplication.
764
765 (1), however, needs to be calculated with an unsigned widening
766 multiplication. If this operation is not directly supported we
767 try using a signed widening multiplication and adjust the result.
768 This adjustment works as follows:
769
770 If both operands are positive then no adjustment is needed.
771
772 If the operands have different signs, for example op0_low < 0 and
773 op1_low >= 0, the instruction treats the most significant bit of
774 op0_low as a sign bit instead of a bit with significance
775 2**(BITS_PER_WORD-1), i.e. the instruction multiplies op1_low
776 with 2**BITS_PER_WORD - op0_low, and two's complements the
777 result. Conclusion: We need to add op1_low * 2**BITS_PER_WORD to
778 the result.
779
780 Similarly, if both operands are negative, we need to add
781 (op0_low + op1_low) * 2**BITS_PER_WORD.
782
783 We use a trick to adjust quickly. We logically shift op0_low right
784 (op1_low) BITS_PER_WORD-1 steps to get 0 or 1, and add this to
785 op0_high (op1_high) before it is used to calculate 2b (2a). If no
786 logical shift exists, we do an arithmetic right shift and subtract
787 the 0 or -1. */
788
789 static rtx
expand_doubleword_mult(machine_mode mode,rtx op0,rtx op1,rtx target,bool umulp,enum optab_methods methods)790 expand_doubleword_mult (machine_mode mode, rtx op0, rtx op1, rtx target,
791 bool umulp, enum optab_methods methods)
792 {
793 int low = (WORDS_BIG_ENDIAN ? 1 : 0);
794 int high = (WORDS_BIG_ENDIAN ? 0 : 1);
795 rtx wordm1 = (umulp ? NULL_RTX
796 : gen_int_shift_amount (word_mode, BITS_PER_WORD - 1));
797 rtx product, adjust, product_high, temp;
798
799 rtx op0_high = operand_subword_force (op0, high, mode);
800 rtx op0_low = operand_subword_force (op0, low, mode);
801 rtx op1_high = operand_subword_force (op1, high, mode);
802 rtx op1_low = operand_subword_force (op1, low, mode);
803
804 /* If we're using an unsigned multiply to directly compute the product
805 of the low-order words of the operands and perform any required
806 adjustments of the operands, we begin by trying two more multiplications
807 and then computing the appropriate sum.
808
809 We have checked above that the required addition is provided.
810 Full-word addition will normally always succeed, especially if
811 it is provided at all, so we don't worry about its failure. The
812 multiplication may well fail, however, so we do handle that. */
813
814 if (!umulp)
815 {
816 /* ??? This could be done with emit_store_flag where available. */
817 temp = expand_binop (word_mode, lshr_optab, op0_low, wordm1,
818 NULL_RTX, 1, methods);
819 if (temp)
820 op0_high = expand_binop (word_mode, add_optab, op0_high, temp,
821 NULL_RTX, 0, OPTAB_DIRECT);
822 else
823 {
824 temp = expand_binop (word_mode, ashr_optab, op0_low, wordm1,
825 NULL_RTX, 0, methods);
826 if (!temp)
827 return NULL_RTX;
828 op0_high = expand_binop (word_mode, sub_optab, op0_high, temp,
829 NULL_RTX, 0, OPTAB_DIRECT);
830 }
831
832 if (!op0_high)
833 return NULL_RTX;
834 }
835
836 adjust = expand_binop (word_mode, smul_optab, op0_high, op1_low,
837 NULL_RTX, 0, OPTAB_DIRECT);
838 if (!adjust)
839 return NULL_RTX;
840
841 /* OP0_HIGH should now be dead. */
842
843 if (!umulp)
844 {
845 /* ??? This could be done with emit_store_flag where available. */
846 temp = expand_binop (word_mode, lshr_optab, op1_low, wordm1,
847 NULL_RTX, 1, methods);
848 if (temp)
849 op1_high = expand_binop (word_mode, add_optab, op1_high, temp,
850 NULL_RTX, 0, OPTAB_DIRECT);
851 else
852 {
853 temp = expand_binop (word_mode, ashr_optab, op1_low, wordm1,
854 NULL_RTX, 0, methods);
855 if (!temp)
856 return NULL_RTX;
857 op1_high = expand_binop (word_mode, sub_optab, op1_high, temp,
858 NULL_RTX, 0, OPTAB_DIRECT);
859 }
860
861 if (!op1_high)
862 return NULL_RTX;
863 }
864
865 temp = expand_binop (word_mode, smul_optab, op1_high, op0_low,
866 NULL_RTX, 0, OPTAB_DIRECT);
867 if (!temp)
868 return NULL_RTX;
869
870 /* OP1_HIGH should now be dead. */
871
872 adjust = expand_binop (word_mode, add_optab, adjust, temp,
873 NULL_RTX, 0, OPTAB_DIRECT);
874
875 if (target && !REG_P (target))
876 target = NULL_RTX;
877
878 /* *_widen_optab needs to determine operand mode, make sure at least
879 one operand has non-VOID mode. */
880 if (GET_MODE (op0_low) == VOIDmode && GET_MODE (op1_low) == VOIDmode)
881 op0_low = force_reg (word_mode, op0_low);
882
883 if (umulp)
884 product = expand_binop (mode, umul_widen_optab, op0_low, op1_low,
885 target, 1, OPTAB_DIRECT);
886 else
887 product = expand_binop (mode, smul_widen_optab, op0_low, op1_low,
888 target, 1, OPTAB_DIRECT);
889
890 if (!product)
891 return NULL_RTX;
892
893 product_high = operand_subword (product, high, 1, mode);
894 adjust = expand_binop (word_mode, add_optab, product_high, adjust,
895 NULL_RTX, 0, OPTAB_DIRECT);
896 emit_move_insn (product_high, adjust);
897 return product;
898 }
899
900 /* Wrapper around expand_binop which takes an rtx code to specify
901 the operation to perform, not an optab pointer. All other
902 arguments are the same. */
903 rtx
expand_simple_binop(machine_mode mode,enum rtx_code code,rtx op0,rtx op1,rtx target,int unsignedp,enum optab_methods methods)904 expand_simple_binop (machine_mode mode, enum rtx_code code, rtx op0,
905 rtx op1, rtx target, int unsignedp,
906 enum optab_methods methods)
907 {
908 optab binop = code_to_optab (code);
909 gcc_assert (binop);
910
911 return expand_binop (mode, binop, op0, op1, target, unsignedp, methods);
912 }
913
914 /* Return whether OP0 and OP1 should be swapped when expanding a commutative
915 binop. Order them according to commutative_operand_precedence and, if
916 possible, try to put TARGET or a pseudo first. */
917 static bool
swap_commutative_operands_with_target(rtx target,rtx op0,rtx op1)918 swap_commutative_operands_with_target (rtx target, rtx op0, rtx op1)
919 {
920 int op0_prec = commutative_operand_precedence (op0);
921 int op1_prec = commutative_operand_precedence (op1);
922
923 if (op0_prec < op1_prec)
924 return true;
925
926 if (op0_prec > op1_prec)
927 return false;
928
929 /* With equal precedence, both orders are ok, but it is better if the
930 first operand is TARGET, or if both TARGET and OP0 are pseudos. */
931 if (target == 0 || REG_P (target))
932 return (REG_P (op1) && !REG_P (op0)) || target == op1;
933 else
934 return rtx_equal_p (op1, target);
935 }
936
937 /* Return true if BINOPTAB implements a shift operation. */
938
939 static bool
shift_optab_p(optab binoptab)940 shift_optab_p (optab binoptab)
941 {
942 switch (optab_to_code (binoptab))
943 {
944 case ASHIFT:
945 case SS_ASHIFT:
946 case US_ASHIFT:
947 case ASHIFTRT:
948 case LSHIFTRT:
949 case ROTATE:
950 case ROTATERT:
951 return true;
952
953 default:
954 return false;
955 }
956 }
957
958 /* Return true if BINOPTAB implements a commutative binary operation. */
959
960 static bool
commutative_optab_p(optab binoptab)961 commutative_optab_p (optab binoptab)
962 {
963 return (GET_RTX_CLASS (optab_to_code (binoptab)) == RTX_COMM_ARITH
964 || binoptab == smul_widen_optab
965 || binoptab == umul_widen_optab
966 || binoptab == smul_highpart_optab
967 || binoptab == umul_highpart_optab);
968 }
969
970 /* X is to be used in mode MODE as operand OPN to BINOPTAB. If we're
971 optimizing, and if the operand is a constant that costs more than
972 1 instruction, force the constant into a register and return that
973 register. Return X otherwise. UNSIGNEDP says whether X is unsigned. */
974
975 static rtx
avoid_expensive_constant(machine_mode mode,optab binoptab,int opn,rtx x,bool unsignedp)976 avoid_expensive_constant (machine_mode mode, optab binoptab,
977 int opn, rtx x, bool unsignedp)
978 {
979 bool speed = optimize_insn_for_speed_p ();
980
981 if (mode != VOIDmode
982 && optimize
983 && CONSTANT_P (x)
984 && (rtx_cost (x, mode, optab_to_code (binoptab), opn, speed)
985 > set_src_cost (x, mode, speed)))
986 {
987 if (CONST_INT_P (x))
988 {
989 HOST_WIDE_INT intval = trunc_int_for_mode (INTVAL (x), mode);
990 if (intval != INTVAL (x))
991 x = GEN_INT (intval);
992 }
993 else
994 x = convert_modes (mode, VOIDmode, x, unsignedp);
995 x = force_reg (mode, x);
996 }
997 return x;
998 }
999
1000 /* Helper function for expand_binop: handle the case where there
1001 is an insn ICODE that directly implements the indicated operation.
1002 Returns null if this is not possible. */
1003 static rtx
expand_binop_directly(enum insn_code icode,machine_mode mode,optab binoptab,rtx op0,rtx op1,rtx target,int unsignedp,enum optab_methods methods,rtx_insn * last)1004 expand_binop_directly (enum insn_code icode, machine_mode mode, optab binoptab,
1005 rtx op0, rtx op1,
1006 rtx target, int unsignedp, enum optab_methods methods,
1007 rtx_insn *last)
1008 {
1009 machine_mode xmode0 = insn_data[(int) icode].operand[1].mode;
1010 machine_mode xmode1 = insn_data[(int) icode].operand[2].mode;
1011 machine_mode mode0, mode1, tmp_mode;
1012 struct expand_operand ops[3];
1013 bool commutative_p;
1014 rtx_insn *pat;
1015 rtx xop0 = op0, xop1 = op1;
1016 bool canonicalize_op1 = false;
1017
1018 /* If it is a commutative operator and the modes would match
1019 if we would swap the operands, we can save the conversions. */
1020 commutative_p = commutative_optab_p (binoptab);
1021 if (commutative_p
1022 && GET_MODE (xop0) != xmode0 && GET_MODE (xop1) != xmode1
1023 && GET_MODE (xop0) == xmode1 && GET_MODE (xop1) == xmode1)
1024 std::swap (xop0, xop1);
1025
1026 /* If we are optimizing, force expensive constants into a register. */
1027 xop0 = avoid_expensive_constant (xmode0, binoptab, 0, xop0, unsignedp);
1028 if (!shift_optab_p (binoptab))
1029 xop1 = avoid_expensive_constant (xmode1, binoptab, 1, xop1, unsignedp);
1030 else
1031 /* Shifts and rotates often use a different mode for op1 from op0;
1032 for VOIDmode constants we don't know the mode, so force it
1033 to be canonicalized using convert_modes. */
1034 canonicalize_op1 = true;
1035
1036 /* In case the insn wants input operands in modes different from
1037 those of the actual operands, convert the operands. It would
1038 seem that we don't need to convert CONST_INTs, but we do, so
1039 that they're properly zero-extended, sign-extended or truncated
1040 for their mode. */
1041
1042 mode0 = GET_MODE (xop0) != VOIDmode ? GET_MODE (xop0) : mode;
1043 if (xmode0 != VOIDmode && xmode0 != mode0)
1044 {
1045 xop0 = convert_modes (xmode0, mode0, xop0, unsignedp);
1046 mode0 = xmode0;
1047 }
1048
1049 mode1 = ((GET_MODE (xop1) != VOIDmode || canonicalize_op1)
1050 ? GET_MODE (xop1) : mode);
1051 if (xmode1 != VOIDmode && xmode1 != mode1)
1052 {
1053 xop1 = convert_modes (xmode1, mode1, xop1, unsignedp);
1054 mode1 = xmode1;
1055 }
1056
1057 /* If operation is commutative,
1058 try to make the first operand a register.
1059 Even better, try to make it the same as the target.
1060 Also try to make the last operand a constant. */
1061 if (commutative_p
1062 && swap_commutative_operands_with_target (target, xop0, xop1))
1063 std::swap (xop0, xop1);
1064
1065 /* Now, if insn's predicates don't allow our operands, put them into
1066 pseudo regs. */
1067
1068 if (binoptab == vec_pack_trunc_optab
1069 || binoptab == vec_pack_usat_optab
1070 || binoptab == vec_pack_ssat_optab
1071 || binoptab == vec_pack_ufix_trunc_optab
1072 || binoptab == vec_pack_sfix_trunc_optab)
1073 {
1074 /* The mode of the result is different then the mode of the
1075 arguments. */
1076 tmp_mode = insn_data[(int) icode].operand[0].mode;
1077 if (VECTOR_MODE_P (mode)
1078 && maybe_ne (GET_MODE_NUNITS (tmp_mode), 2 * GET_MODE_NUNITS (mode)))
1079 {
1080 delete_insns_since (last);
1081 return NULL_RTX;
1082 }
1083 }
1084 else
1085 tmp_mode = mode;
1086
1087 create_output_operand (&ops[0], target, tmp_mode);
1088 create_input_operand (&ops[1], xop0, mode0);
1089 create_input_operand (&ops[2], xop1, mode1);
1090 pat = maybe_gen_insn (icode, 3, ops);
1091 if (pat)
1092 {
1093 /* If PAT is composed of more than one insn, try to add an appropriate
1094 REG_EQUAL note to it. If we can't because TEMP conflicts with an
1095 operand, call expand_binop again, this time without a target. */
1096 if (INSN_P (pat) && NEXT_INSN (pat) != NULL_RTX
1097 && ! add_equal_note (pat, ops[0].value,
1098 optab_to_code (binoptab),
1099 ops[1].value, ops[2].value, mode0))
1100 {
1101 delete_insns_since (last);
1102 return expand_binop (mode, binoptab, op0, op1, NULL_RTX,
1103 unsignedp, methods);
1104 }
1105
1106 emit_insn (pat);
1107 return ops[0].value;
1108 }
1109 delete_insns_since (last);
1110 return NULL_RTX;
1111 }
1112
1113 /* Generate code to perform an operation specified by BINOPTAB
1114 on operands OP0 and OP1, with result having machine-mode MODE.
1115
1116 UNSIGNEDP is for the case where we have to widen the operands
1117 to perform the operation. It says to use zero-extension.
1118
1119 If TARGET is nonzero, the value
1120 is generated there, if it is convenient to do so.
1121 In all cases an rtx is returned for the locus of the value;
1122 this may or may not be TARGET. */
1123
1124 rtx
expand_binop(machine_mode mode,optab binoptab,rtx op0,rtx op1,rtx target,int unsignedp,enum optab_methods methods)1125 expand_binop (machine_mode mode, optab binoptab, rtx op0, rtx op1,
1126 rtx target, int unsignedp, enum optab_methods methods)
1127 {
1128 enum optab_methods next_methods
1129 = (methods == OPTAB_LIB || methods == OPTAB_LIB_WIDEN
1130 ? OPTAB_WIDEN : methods);
1131 enum mode_class mclass;
1132 enum insn_code icode;
1133 machine_mode wider_mode;
1134 scalar_int_mode int_mode;
1135 rtx libfunc;
1136 rtx temp;
1137 rtx_insn *entry_last = get_last_insn ();
1138 rtx_insn *last;
1139
1140 mclass = GET_MODE_CLASS (mode);
1141
1142 /* If subtracting an integer constant, convert this into an addition of
1143 the negated constant. */
1144
1145 if (binoptab == sub_optab && CONST_INT_P (op1))
1146 {
1147 op1 = negate_rtx (mode, op1);
1148 binoptab = add_optab;
1149 }
1150 /* For shifts, constant invalid op1 might be expanded from different
1151 mode than MODE. As those are invalid, force them to a register
1152 to avoid further problems during expansion. */
1153 else if (CONST_INT_P (op1)
1154 && shift_optab_p (binoptab)
1155 && UINTVAL (op1) >= GET_MODE_BITSIZE (GET_MODE_INNER (mode)))
1156 {
1157 op1 = gen_int_mode (INTVAL (op1), GET_MODE_INNER (mode));
1158 op1 = force_reg (GET_MODE_INNER (mode), op1);
1159 }
1160
1161 /* Record where to delete back to if we backtrack. */
1162 last = get_last_insn ();
1163
1164 /* If we can do it with a three-operand insn, do so. */
1165
1166 if (methods != OPTAB_MUST_WIDEN)
1167 {
1168 if (convert_optab_p (binoptab))
1169 {
1170 machine_mode from_mode = widened_mode (mode, op0, op1);
1171 icode = find_widening_optab_handler (binoptab, mode, from_mode);
1172 }
1173 else
1174 icode = optab_handler (binoptab, mode);
1175 if (icode != CODE_FOR_nothing)
1176 {
1177 temp = expand_binop_directly (icode, mode, binoptab, op0, op1,
1178 target, unsignedp, methods, last);
1179 if (temp)
1180 return temp;
1181 }
1182 }
1183
1184 /* If we were trying to rotate, and that didn't work, try rotating
1185 the other direction before falling back to shifts and bitwise-or. */
1186 if (((binoptab == rotl_optab
1187 && (icode = optab_handler (rotr_optab, mode)) != CODE_FOR_nothing)
1188 || (binoptab == rotr_optab
1189 && (icode = optab_handler (rotl_optab, mode)) != CODE_FOR_nothing))
1190 && is_int_mode (mode, &int_mode))
1191 {
1192 optab otheroptab = (binoptab == rotl_optab ? rotr_optab : rotl_optab);
1193 rtx newop1;
1194 unsigned int bits = GET_MODE_PRECISION (int_mode);
1195
1196 if (CONST_INT_P (op1))
1197 newop1 = gen_int_shift_amount (int_mode, bits - INTVAL (op1));
1198 else if (targetm.shift_truncation_mask (int_mode) == bits - 1)
1199 newop1 = negate_rtx (GET_MODE (op1), op1);
1200 else
1201 newop1 = expand_binop (GET_MODE (op1), sub_optab,
1202 gen_int_mode (bits, GET_MODE (op1)), op1,
1203 NULL_RTX, unsignedp, OPTAB_DIRECT);
1204
1205 temp = expand_binop_directly (icode, int_mode, otheroptab, op0, newop1,
1206 target, unsignedp, methods, last);
1207 if (temp)
1208 return temp;
1209 }
1210
1211 /* If this is a multiply, see if we can do a widening operation that
1212 takes operands of this mode and makes a wider mode. */
1213
1214 if (binoptab == smul_optab
1215 && GET_MODE_2XWIDER_MODE (mode).exists (&wider_mode)
1216 && (convert_optab_handler ((unsignedp
1217 ? umul_widen_optab
1218 : smul_widen_optab),
1219 wider_mode, mode) != CODE_FOR_nothing))
1220 {
1221 /* *_widen_optab needs to determine operand mode, make sure at least
1222 one operand has non-VOID mode. */
1223 if (GET_MODE (op0) == VOIDmode && GET_MODE (op1) == VOIDmode)
1224 op0 = force_reg (mode, op0);
1225 temp = expand_binop (wider_mode,
1226 unsignedp ? umul_widen_optab : smul_widen_optab,
1227 op0, op1, NULL_RTX, unsignedp, OPTAB_DIRECT);
1228
1229 if (temp != 0)
1230 {
1231 if (GET_MODE_CLASS (mode) == MODE_INT
1232 && TRULY_NOOP_TRUNCATION_MODES_P (mode, GET_MODE (temp)))
1233 return gen_lowpart (mode, temp);
1234 else
1235 return convert_to_mode (mode, temp, unsignedp);
1236 }
1237 }
1238
1239 /* If this is a vector shift by a scalar, see if we can do a vector
1240 shift by a vector. If so, broadcast the scalar into a vector. */
1241 if (mclass == MODE_VECTOR_INT)
1242 {
1243 optab otheroptab = unknown_optab;
1244
1245 if (binoptab == ashl_optab)
1246 otheroptab = vashl_optab;
1247 else if (binoptab == ashr_optab)
1248 otheroptab = vashr_optab;
1249 else if (binoptab == lshr_optab)
1250 otheroptab = vlshr_optab;
1251 else if (binoptab == rotl_optab)
1252 otheroptab = vrotl_optab;
1253 else if (binoptab == rotr_optab)
1254 otheroptab = vrotr_optab;
1255
1256 if (otheroptab
1257 && (icode = optab_handler (otheroptab, mode)) != CODE_FOR_nothing)
1258 {
1259 /* The scalar may have been extended to be too wide. Truncate
1260 it back to the proper size to fit in the broadcast vector. */
1261 scalar_mode inner_mode = GET_MODE_INNER (mode);
1262 if (!CONST_INT_P (op1)
1263 && (GET_MODE_BITSIZE (as_a <scalar_int_mode> (GET_MODE (op1)))
1264 > GET_MODE_BITSIZE (inner_mode)))
1265 op1 = force_reg (inner_mode,
1266 simplify_gen_unary (TRUNCATE, inner_mode, op1,
1267 GET_MODE (op1)));
1268 rtx vop1 = expand_vector_broadcast (mode, op1);
1269 if (vop1)
1270 {
1271 temp = expand_binop_directly (icode, mode, otheroptab, op0, vop1,
1272 target, unsignedp, methods, last);
1273 if (temp)
1274 return temp;
1275 }
1276 }
1277 }
1278
1279 /* Look for a wider mode of the same class for which we think we
1280 can open-code the operation. Check for a widening multiply at the
1281 wider mode as well. */
1282
1283 if (CLASS_HAS_WIDER_MODES_P (mclass)
1284 && methods != OPTAB_DIRECT && methods != OPTAB_LIB)
1285 FOR_EACH_WIDER_MODE (wider_mode, mode)
1286 {
1287 machine_mode next_mode;
1288 if (optab_handler (binoptab, wider_mode) != CODE_FOR_nothing
1289 || (binoptab == smul_optab
1290 && GET_MODE_WIDER_MODE (wider_mode).exists (&next_mode)
1291 && (find_widening_optab_handler ((unsignedp
1292 ? umul_widen_optab
1293 : smul_widen_optab),
1294 next_mode, mode)
1295 != CODE_FOR_nothing)))
1296 {
1297 rtx xop0 = op0, xop1 = op1;
1298 int no_extend = 0;
1299
1300 /* For certain integer operations, we need not actually extend
1301 the narrow operands, as long as we will truncate
1302 the results to the same narrowness. */
1303
1304 if ((binoptab == ior_optab || binoptab == and_optab
1305 || binoptab == xor_optab
1306 || binoptab == add_optab || binoptab == sub_optab
1307 || binoptab == smul_optab || binoptab == ashl_optab)
1308 && mclass == MODE_INT)
1309 {
1310 no_extend = 1;
1311 xop0 = avoid_expensive_constant (mode, binoptab, 0,
1312 xop0, unsignedp);
1313 if (binoptab != ashl_optab)
1314 xop1 = avoid_expensive_constant (mode, binoptab, 1,
1315 xop1, unsignedp);
1316 }
1317
1318 xop0 = widen_operand (xop0, wider_mode, mode, unsignedp, no_extend);
1319
1320 /* The second operand of a shift must always be extended. */
1321 xop1 = widen_operand (xop1, wider_mode, mode, unsignedp,
1322 no_extend && binoptab != ashl_optab);
1323
1324 temp = expand_binop (wider_mode, binoptab, xop0, xop1, NULL_RTX,
1325 unsignedp, OPTAB_DIRECT);
1326 if (temp)
1327 {
1328 if (mclass != MODE_INT
1329 || !TRULY_NOOP_TRUNCATION_MODES_P (mode, wider_mode))
1330 {
1331 if (target == 0)
1332 target = gen_reg_rtx (mode);
1333 convert_move (target, temp, 0);
1334 return target;
1335 }
1336 else
1337 return gen_lowpart (mode, temp);
1338 }
1339 else
1340 delete_insns_since (last);
1341 }
1342 }
1343
1344 /* If operation is commutative,
1345 try to make the first operand a register.
1346 Even better, try to make it the same as the target.
1347 Also try to make the last operand a constant. */
1348 if (commutative_optab_p (binoptab)
1349 && swap_commutative_operands_with_target (target, op0, op1))
1350 std::swap (op0, op1);
1351
1352 /* These can be done a word at a time. */
1353 if ((binoptab == and_optab || binoptab == ior_optab || binoptab == xor_optab)
1354 && is_int_mode (mode, &int_mode)
1355 && GET_MODE_SIZE (int_mode) > UNITS_PER_WORD
1356 && optab_handler (binoptab, word_mode) != CODE_FOR_nothing)
1357 {
1358 int i;
1359 rtx_insn *insns;
1360
1361 /* If TARGET is the same as one of the operands, the REG_EQUAL note
1362 won't be accurate, so use a new target. */
1363 if (target == 0
1364 || target == op0
1365 || target == op1
1366 || reg_overlap_mentioned_p (target, op0)
1367 || reg_overlap_mentioned_p (target, op1)
1368 || !valid_multiword_target_p (target))
1369 target = gen_reg_rtx (int_mode);
1370
1371 start_sequence ();
1372
1373 /* Do the actual arithmetic. */
1374 for (i = 0; i < GET_MODE_BITSIZE (int_mode) / BITS_PER_WORD; i++)
1375 {
1376 rtx target_piece = operand_subword (target, i, 1, int_mode);
1377 rtx x = expand_binop (word_mode, binoptab,
1378 operand_subword_force (op0, i, int_mode),
1379 operand_subword_force (op1, i, int_mode),
1380 target_piece, unsignedp, next_methods);
1381
1382 if (x == 0)
1383 break;
1384
1385 if (target_piece != x)
1386 emit_move_insn (target_piece, x);
1387 }
1388
1389 insns = get_insns ();
1390 end_sequence ();
1391
1392 if (i == GET_MODE_BITSIZE (int_mode) / BITS_PER_WORD)
1393 {
1394 emit_insn (insns);
1395 return target;
1396 }
1397 }
1398
1399 /* Synthesize double word shifts from single word shifts. */
1400 if ((binoptab == lshr_optab || binoptab == ashl_optab
1401 || binoptab == ashr_optab)
1402 && is_int_mode (mode, &int_mode)
1403 && (CONST_INT_P (op1) || optimize_insn_for_speed_p ())
1404 && GET_MODE_SIZE (int_mode) == 2 * UNITS_PER_WORD
1405 && GET_MODE_PRECISION (int_mode) == GET_MODE_BITSIZE (int_mode)
1406 && optab_handler (binoptab, word_mode) != CODE_FOR_nothing
1407 && optab_handler (ashl_optab, word_mode) != CODE_FOR_nothing
1408 && optab_handler (lshr_optab, word_mode) != CODE_FOR_nothing)
1409 {
1410 unsigned HOST_WIDE_INT shift_mask, double_shift_mask;
1411 scalar_int_mode op1_mode;
1412
1413 double_shift_mask = targetm.shift_truncation_mask (int_mode);
1414 shift_mask = targetm.shift_truncation_mask (word_mode);
1415 op1_mode = (GET_MODE (op1) != VOIDmode
1416 ? as_a <scalar_int_mode> (GET_MODE (op1))
1417 : word_mode);
1418
1419 /* Apply the truncation to constant shifts. */
1420 if (double_shift_mask > 0 && CONST_INT_P (op1))
1421 op1 = gen_int_mode (INTVAL (op1) & double_shift_mask, op1_mode);
1422
1423 if (op1 == CONST0_RTX (op1_mode))
1424 return op0;
1425
1426 /* Make sure that this is a combination that expand_doubleword_shift
1427 can handle. See the comments there for details. */
1428 if (double_shift_mask == 0
1429 || (shift_mask == BITS_PER_WORD - 1
1430 && double_shift_mask == BITS_PER_WORD * 2 - 1))
1431 {
1432 rtx_insn *insns;
1433 rtx into_target, outof_target;
1434 rtx into_input, outof_input;
1435 int left_shift, outof_word;
1436
1437 /* If TARGET is the same as one of the operands, the REG_EQUAL note
1438 won't be accurate, so use a new target. */
1439 if (target == 0
1440 || target == op0
1441 || target == op1
1442 || reg_overlap_mentioned_p (target, op0)
1443 || reg_overlap_mentioned_p (target, op1)
1444 || !valid_multiword_target_p (target))
1445 target = gen_reg_rtx (int_mode);
1446
1447 start_sequence ();
1448
1449 /* OUTOF_* is the word we are shifting bits away from, and
1450 INTO_* is the word that we are shifting bits towards, thus
1451 they differ depending on the direction of the shift and
1452 WORDS_BIG_ENDIAN. */
1453
1454 left_shift = binoptab == ashl_optab;
1455 outof_word = left_shift ^ ! WORDS_BIG_ENDIAN;
1456
1457 outof_target = operand_subword (target, outof_word, 1, int_mode);
1458 into_target = operand_subword (target, 1 - outof_word, 1, int_mode);
1459
1460 outof_input = operand_subword_force (op0, outof_word, int_mode);
1461 into_input = operand_subword_force (op0, 1 - outof_word, int_mode);
1462
1463 if (expand_doubleword_shift (op1_mode, binoptab,
1464 outof_input, into_input, op1,
1465 outof_target, into_target,
1466 unsignedp, next_methods, shift_mask))
1467 {
1468 insns = get_insns ();
1469 end_sequence ();
1470
1471 emit_insn (insns);
1472 return target;
1473 }
1474 end_sequence ();
1475 }
1476 }
1477
1478 /* Synthesize double word rotates from single word shifts. */
1479 if ((binoptab == rotl_optab || binoptab == rotr_optab)
1480 && is_int_mode (mode, &int_mode)
1481 && CONST_INT_P (op1)
1482 && GET_MODE_PRECISION (int_mode) == 2 * BITS_PER_WORD
1483 && optab_handler (ashl_optab, word_mode) != CODE_FOR_nothing
1484 && optab_handler (lshr_optab, word_mode) != CODE_FOR_nothing)
1485 {
1486 rtx_insn *insns;
1487 rtx into_target, outof_target;
1488 rtx into_input, outof_input;
1489 rtx inter;
1490 int shift_count, left_shift, outof_word;
1491
1492 /* If TARGET is the same as one of the operands, the REG_EQUAL note
1493 won't be accurate, so use a new target. Do this also if target is not
1494 a REG, first because having a register instead may open optimization
1495 opportunities, and second because if target and op0 happen to be MEMs
1496 designating the same location, we would risk clobbering it too early
1497 in the code sequence we generate below. */
1498 if (target == 0
1499 || target == op0
1500 || target == op1
1501 || !REG_P (target)
1502 || reg_overlap_mentioned_p (target, op0)
1503 || reg_overlap_mentioned_p (target, op1)
1504 || !valid_multiword_target_p (target))
1505 target = gen_reg_rtx (int_mode);
1506
1507 start_sequence ();
1508
1509 shift_count = INTVAL (op1);
1510
1511 /* OUTOF_* is the word we are shifting bits away from, and
1512 INTO_* is the word that we are shifting bits towards, thus
1513 they differ depending on the direction of the shift and
1514 WORDS_BIG_ENDIAN. */
1515
1516 left_shift = (binoptab == rotl_optab);
1517 outof_word = left_shift ^ ! WORDS_BIG_ENDIAN;
1518
1519 outof_target = operand_subword (target, outof_word, 1, int_mode);
1520 into_target = operand_subword (target, 1 - outof_word, 1, int_mode);
1521
1522 outof_input = operand_subword_force (op0, outof_word, int_mode);
1523 into_input = operand_subword_force (op0, 1 - outof_word, int_mode);
1524
1525 if (shift_count == BITS_PER_WORD)
1526 {
1527 /* This is just a word swap. */
1528 emit_move_insn (outof_target, into_input);
1529 emit_move_insn (into_target, outof_input);
1530 inter = const0_rtx;
1531 }
1532 else
1533 {
1534 rtx into_temp1, into_temp2, outof_temp1, outof_temp2;
1535 HOST_WIDE_INT first_shift_count, second_shift_count;
1536 optab reverse_unsigned_shift, unsigned_shift;
1537
1538 reverse_unsigned_shift = (left_shift ^ (shift_count < BITS_PER_WORD)
1539 ? lshr_optab : ashl_optab);
1540
1541 unsigned_shift = (left_shift ^ (shift_count < BITS_PER_WORD)
1542 ? ashl_optab : lshr_optab);
1543
1544 if (shift_count > BITS_PER_WORD)
1545 {
1546 first_shift_count = shift_count - BITS_PER_WORD;
1547 second_shift_count = 2 * BITS_PER_WORD - shift_count;
1548 }
1549 else
1550 {
1551 first_shift_count = BITS_PER_WORD - shift_count;
1552 second_shift_count = shift_count;
1553 }
1554 rtx first_shift_count_rtx
1555 = gen_int_shift_amount (word_mode, first_shift_count);
1556 rtx second_shift_count_rtx
1557 = gen_int_shift_amount (word_mode, second_shift_count);
1558
1559 into_temp1 = expand_binop (word_mode, unsigned_shift,
1560 outof_input, first_shift_count_rtx,
1561 NULL_RTX, unsignedp, next_methods);
1562 into_temp2 = expand_binop (word_mode, reverse_unsigned_shift,
1563 into_input, second_shift_count_rtx,
1564 NULL_RTX, unsignedp, next_methods);
1565
1566 if (into_temp1 != 0 && into_temp2 != 0)
1567 inter = expand_binop (word_mode, ior_optab, into_temp1, into_temp2,
1568 into_target, unsignedp, next_methods);
1569 else
1570 inter = 0;
1571
1572 if (inter != 0 && inter != into_target)
1573 emit_move_insn (into_target, inter);
1574
1575 outof_temp1 = expand_binop (word_mode, unsigned_shift,
1576 into_input, first_shift_count_rtx,
1577 NULL_RTX, unsignedp, next_methods);
1578 outof_temp2 = expand_binop (word_mode, reverse_unsigned_shift,
1579 outof_input, second_shift_count_rtx,
1580 NULL_RTX, unsignedp, next_methods);
1581
1582 if (inter != 0 && outof_temp1 != 0 && outof_temp2 != 0)
1583 inter = expand_binop (word_mode, ior_optab,
1584 outof_temp1, outof_temp2,
1585 outof_target, unsignedp, next_methods);
1586
1587 if (inter != 0 && inter != outof_target)
1588 emit_move_insn (outof_target, inter);
1589 }
1590
1591 insns = get_insns ();
1592 end_sequence ();
1593
1594 if (inter != 0)
1595 {
1596 emit_insn (insns);
1597 return target;
1598 }
1599 }
1600
1601 /* These can be done a word at a time by propagating carries. */
1602 if ((binoptab == add_optab || binoptab == sub_optab)
1603 && is_int_mode (mode, &int_mode)
1604 && GET_MODE_SIZE (int_mode) >= 2 * UNITS_PER_WORD
1605 && optab_handler (binoptab, word_mode) != CODE_FOR_nothing)
1606 {
1607 unsigned int i;
1608 optab otheroptab = binoptab == add_optab ? sub_optab : add_optab;
1609 const unsigned int nwords = GET_MODE_BITSIZE (int_mode) / BITS_PER_WORD;
1610 rtx carry_in = NULL_RTX, carry_out = NULL_RTX;
1611 rtx xop0, xop1, xtarget;
1612
1613 /* We can handle either a 1 or -1 value for the carry. If STORE_FLAG
1614 value is one of those, use it. Otherwise, use 1 since it is the
1615 one easiest to get. */
1616 #if STORE_FLAG_VALUE == 1 || STORE_FLAG_VALUE == -1
1617 int normalizep = STORE_FLAG_VALUE;
1618 #else
1619 int normalizep = 1;
1620 #endif
1621
1622 /* Prepare the operands. */
1623 xop0 = force_reg (int_mode, op0);
1624 xop1 = force_reg (int_mode, op1);
1625
1626 xtarget = gen_reg_rtx (int_mode);
1627
1628 if (target == 0 || !REG_P (target) || !valid_multiword_target_p (target))
1629 target = xtarget;
1630
1631 /* Indicate for flow that the entire target reg is being set. */
1632 if (REG_P (target))
1633 emit_clobber (xtarget);
1634
1635 /* Do the actual arithmetic. */
1636 for (i = 0; i < nwords; i++)
1637 {
1638 int index = (WORDS_BIG_ENDIAN ? nwords - i - 1 : i);
1639 rtx target_piece = operand_subword (xtarget, index, 1, int_mode);
1640 rtx op0_piece = operand_subword_force (xop0, index, int_mode);
1641 rtx op1_piece = operand_subword_force (xop1, index, int_mode);
1642 rtx x;
1643
1644 /* Main add/subtract of the input operands. */
1645 x = expand_binop (word_mode, binoptab,
1646 op0_piece, op1_piece,
1647 target_piece, unsignedp, next_methods);
1648 if (x == 0)
1649 break;
1650
1651 if (i + 1 < nwords)
1652 {
1653 /* Store carry from main add/subtract. */
1654 carry_out = gen_reg_rtx (word_mode);
1655 carry_out = emit_store_flag_force (carry_out,
1656 (binoptab == add_optab
1657 ? LT : GT),
1658 x, op0_piece,
1659 word_mode, 1, normalizep);
1660 }
1661
1662 if (i > 0)
1663 {
1664 rtx newx;
1665
1666 /* Add/subtract previous carry to main result. */
1667 newx = expand_binop (word_mode,
1668 normalizep == 1 ? binoptab : otheroptab,
1669 x, carry_in,
1670 NULL_RTX, 1, next_methods);
1671
1672 if (i + 1 < nwords)
1673 {
1674 /* Get out carry from adding/subtracting carry in. */
1675 rtx carry_tmp = gen_reg_rtx (word_mode);
1676 carry_tmp = emit_store_flag_force (carry_tmp,
1677 (binoptab == add_optab
1678 ? LT : GT),
1679 newx, x,
1680 word_mode, 1, normalizep);
1681
1682 /* Logical-ior the two poss. carry together. */
1683 carry_out = expand_binop (word_mode, ior_optab,
1684 carry_out, carry_tmp,
1685 carry_out, 0, next_methods);
1686 if (carry_out == 0)
1687 break;
1688 }
1689 emit_move_insn (target_piece, newx);
1690 }
1691 else
1692 {
1693 if (x != target_piece)
1694 emit_move_insn (target_piece, x);
1695 }
1696
1697 carry_in = carry_out;
1698 }
1699
1700 if (i == GET_MODE_BITSIZE (int_mode) / (unsigned) BITS_PER_WORD)
1701 {
1702 if (optab_handler (mov_optab, int_mode) != CODE_FOR_nothing
1703 || ! rtx_equal_p (target, xtarget))
1704 {
1705 rtx_insn *temp = emit_move_insn (target, xtarget);
1706
1707 set_dst_reg_note (temp, REG_EQUAL,
1708 gen_rtx_fmt_ee (optab_to_code (binoptab),
1709 int_mode, copy_rtx (xop0),
1710 copy_rtx (xop1)),
1711 target);
1712 }
1713 else
1714 target = xtarget;
1715
1716 return target;
1717 }
1718
1719 else
1720 delete_insns_since (last);
1721 }
1722
1723 /* Attempt to synthesize double word multiplies using a sequence of word
1724 mode multiplications. We first attempt to generate a sequence using a
1725 more efficient unsigned widening multiply, and if that fails we then
1726 try using a signed widening multiply. */
1727
1728 if (binoptab == smul_optab
1729 && is_int_mode (mode, &int_mode)
1730 && GET_MODE_SIZE (int_mode) == 2 * UNITS_PER_WORD
1731 && optab_handler (smul_optab, word_mode) != CODE_FOR_nothing
1732 && optab_handler (add_optab, word_mode) != CODE_FOR_nothing)
1733 {
1734 rtx product = NULL_RTX;
1735 if (convert_optab_handler (umul_widen_optab, int_mode, word_mode)
1736 != CODE_FOR_nothing)
1737 {
1738 product = expand_doubleword_mult (int_mode, op0, op1, target,
1739 true, methods);
1740 if (!product)
1741 delete_insns_since (last);
1742 }
1743
1744 if (product == NULL_RTX
1745 && (convert_optab_handler (smul_widen_optab, int_mode, word_mode)
1746 != CODE_FOR_nothing))
1747 {
1748 product = expand_doubleword_mult (int_mode, op0, op1, target,
1749 false, methods);
1750 if (!product)
1751 delete_insns_since (last);
1752 }
1753
1754 if (product != NULL_RTX)
1755 {
1756 if (optab_handler (mov_optab, int_mode) != CODE_FOR_nothing)
1757 {
1758 rtx_insn *move = emit_move_insn (target ? target : product,
1759 product);
1760 set_dst_reg_note (move,
1761 REG_EQUAL,
1762 gen_rtx_fmt_ee (MULT, int_mode,
1763 copy_rtx (op0),
1764 copy_rtx (op1)),
1765 target ? target : product);
1766 }
1767 return product;
1768 }
1769 }
1770
1771 /* It can't be open-coded in this mode.
1772 Use a library call if one is available and caller says that's ok. */
1773
1774 libfunc = optab_libfunc (binoptab, mode);
1775 if (libfunc
1776 && (methods == OPTAB_LIB || methods == OPTAB_LIB_WIDEN))
1777 {
1778 rtx_insn *insns;
1779 rtx op1x = op1;
1780 machine_mode op1_mode = mode;
1781 rtx value;
1782
1783 start_sequence ();
1784
1785 if (shift_optab_p (binoptab))
1786 {
1787 op1_mode = targetm.libgcc_shift_count_mode ();
1788 /* Specify unsigned here,
1789 since negative shift counts are meaningless. */
1790 op1x = convert_to_mode (op1_mode, op1, 1);
1791 }
1792
1793 if (GET_MODE (op0) != VOIDmode
1794 && GET_MODE (op0) != mode)
1795 op0 = convert_to_mode (mode, op0, unsignedp);
1796
1797 /* Pass 1 for NO_QUEUE so we don't lose any increments
1798 if the libcall is cse'd or moved. */
1799 value = emit_library_call_value (libfunc,
1800 NULL_RTX, LCT_CONST, mode,
1801 op0, mode, op1x, op1_mode);
1802
1803 insns = get_insns ();
1804 end_sequence ();
1805
1806 bool trapv = trapv_binoptab_p (binoptab);
1807 target = gen_reg_rtx (mode);
1808 emit_libcall_block_1 (insns, target, value,
1809 trapv ? NULL_RTX
1810 : gen_rtx_fmt_ee (optab_to_code (binoptab),
1811 mode, op0, op1), trapv);
1812
1813 return target;
1814 }
1815
1816 delete_insns_since (last);
1817
1818 /* It can't be done in this mode. Can we do it in a wider mode? */
1819
1820 if (! (methods == OPTAB_WIDEN || methods == OPTAB_LIB_WIDEN
1821 || methods == OPTAB_MUST_WIDEN))
1822 {
1823 /* Caller says, don't even try. */
1824 delete_insns_since (entry_last);
1825 return 0;
1826 }
1827
1828 /* Compute the value of METHODS to pass to recursive calls.
1829 Don't allow widening to be tried recursively. */
1830
1831 methods = (methods == OPTAB_LIB_WIDEN ? OPTAB_LIB : OPTAB_DIRECT);
1832
1833 /* Look for a wider mode of the same class for which it appears we can do
1834 the operation. */
1835
1836 if (CLASS_HAS_WIDER_MODES_P (mclass))
1837 {
1838 /* This code doesn't make sense for conversion optabs, since we
1839 wouldn't then want to extend the operands to be the same size
1840 as the result. */
1841 gcc_assert (!convert_optab_p (binoptab));
1842 FOR_EACH_WIDER_MODE (wider_mode, mode)
1843 {
1844 if (optab_handler (binoptab, wider_mode)
1845 || (methods == OPTAB_LIB
1846 && optab_libfunc (binoptab, wider_mode)))
1847 {
1848 rtx xop0 = op0, xop1 = op1;
1849 int no_extend = 0;
1850
1851 /* For certain integer operations, we need not actually extend
1852 the narrow operands, as long as we will truncate
1853 the results to the same narrowness. */
1854
1855 if ((binoptab == ior_optab || binoptab == and_optab
1856 || binoptab == xor_optab
1857 || binoptab == add_optab || binoptab == sub_optab
1858 || binoptab == smul_optab || binoptab == ashl_optab)
1859 && mclass == MODE_INT)
1860 no_extend = 1;
1861
1862 xop0 = widen_operand (xop0, wider_mode, mode,
1863 unsignedp, no_extend);
1864
1865 /* The second operand of a shift must always be extended. */
1866 xop1 = widen_operand (xop1, wider_mode, mode, unsignedp,
1867 no_extend && binoptab != ashl_optab);
1868
1869 temp = expand_binop (wider_mode, binoptab, xop0, xop1, NULL_RTX,
1870 unsignedp, methods);
1871 if (temp)
1872 {
1873 if (mclass != MODE_INT
1874 || !TRULY_NOOP_TRUNCATION_MODES_P (mode, wider_mode))
1875 {
1876 if (target == 0)
1877 target = gen_reg_rtx (mode);
1878 convert_move (target, temp, 0);
1879 return target;
1880 }
1881 else
1882 return gen_lowpart (mode, temp);
1883 }
1884 else
1885 delete_insns_since (last);
1886 }
1887 }
1888 }
1889
1890 delete_insns_since (entry_last);
1891 return 0;
1892 }
1893
1894 /* Expand a binary operator which has both signed and unsigned forms.
1895 UOPTAB is the optab for unsigned operations, and SOPTAB is for
1896 signed operations.
1897
1898 If we widen unsigned operands, we may use a signed wider operation instead
1899 of an unsigned wider operation, since the result would be the same. */
1900
1901 rtx
sign_expand_binop(machine_mode mode,optab uoptab,optab soptab,rtx op0,rtx op1,rtx target,int unsignedp,enum optab_methods methods)1902 sign_expand_binop (machine_mode mode, optab uoptab, optab soptab,
1903 rtx op0, rtx op1, rtx target, int unsignedp,
1904 enum optab_methods methods)
1905 {
1906 rtx temp;
1907 optab direct_optab = unsignedp ? uoptab : soptab;
1908 bool save_enable;
1909
1910 /* Do it without widening, if possible. */
1911 temp = expand_binop (mode, direct_optab, op0, op1, target,
1912 unsignedp, OPTAB_DIRECT);
1913 if (temp || methods == OPTAB_DIRECT)
1914 return temp;
1915
1916 /* Try widening to a signed int. Disable any direct use of any
1917 signed insn in the current mode. */
1918 save_enable = swap_optab_enable (soptab, mode, false);
1919
1920 temp = expand_binop (mode, soptab, op0, op1, target,
1921 unsignedp, OPTAB_WIDEN);
1922
1923 /* For unsigned operands, try widening to an unsigned int. */
1924 if (!temp && unsignedp)
1925 temp = expand_binop (mode, uoptab, op0, op1, target,
1926 unsignedp, OPTAB_WIDEN);
1927 if (temp || methods == OPTAB_WIDEN)
1928 goto egress;
1929
1930 /* Use the right width libcall if that exists. */
1931 temp = expand_binop (mode, direct_optab, op0, op1, target,
1932 unsignedp, OPTAB_LIB);
1933 if (temp || methods == OPTAB_LIB)
1934 goto egress;
1935
1936 /* Must widen and use a libcall, use either signed or unsigned. */
1937 temp = expand_binop (mode, soptab, op0, op1, target,
1938 unsignedp, methods);
1939 if (!temp && unsignedp)
1940 temp = expand_binop (mode, uoptab, op0, op1, target,
1941 unsignedp, methods);
1942
1943 egress:
1944 /* Undo the fiddling above. */
1945 if (save_enable)
1946 swap_optab_enable (soptab, mode, true);
1947 return temp;
1948 }
1949
1950 /* Generate code to perform an operation specified by UNOPPTAB
1951 on operand OP0, with two results to TARG0 and TARG1.
1952 We assume that the order of the operands for the instruction
1953 is TARG0, TARG1, OP0.
1954
1955 Either TARG0 or TARG1 may be zero, but what that means is that
1956 the result is not actually wanted. We will generate it into
1957 a dummy pseudo-reg and discard it. They may not both be zero.
1958
1959 Returns 1 if this operation can be performed; 0 if not. */
1960
1961 int
expand_twoval_unop(optab unoptab,rtx op0,rtx targ0,rtx targ1,int unsignedp)1962 expand_twoval_unop (optab unoptab, rtx op0, rtx targ0, rtx targ1,
1963 int unsignedp)
1964 {
1965 machine_mode mode = GET_MODE (targ0 ? targ0 : targ1);
1966 enum mode_class mclass;
1967 machine_mode wider_mode;
1968 rtx_insn *entry_last = get_last_insn ();
1969 rtx_insn *last;
1970
1971 mclass = GET_MODE_CLASS (mode);
1972
1973 if (!targ0)
1974 targ0 = gen_reg_rtx (mode);
1975 if (!targ1)
1976 targ1 = gen_reg_rtx (mode);
1977
1978 /* Record where to go back to if we fail. */
1979 last = get_last_insn ();
1980
1981 if (optab_handler (unoptab, mode) != CODE_FOR_nothing)
1982 {
1983 struct expand_operand ops[3];
1984 enum insn_code icode = optab_handler (unoptab, mode);
1985
1986 create_fixed_operand (&ops[0], targ0);
1987 create_fixed_operand (&ops[1], targ1);
1988 create_convert_operand_from (&ops[2], op0, mode, unsignedp);
1989 if (maybe_expand_insn (icode, 3, ops))
1990 return 1;
1991 }
1992
1993 /* It can't be done in this mode. Can we do it in a wider mode? */
1994
1995 if (CLASS_HAS_WIDER_MODES_P (mclass))
1996 {
1997 FOR_EACH_WIDER_MODE (wider_mode, mode)
1998 {
1999 if (optab_handler (unoptab, wider_mode) != CODE_FOR_nothing)
2000 {
2001 rtx t0 = gen_reg_rtx (wider_mode);
2002 rtx t1 = gen_reg_rtx (wider_mode);
2003 rtx cop0 = convert_modes (wider_mode, mode, op0, unsignedp);
2004
2005 if (expand_twoval_unop (unoptab, cop0, t0, t1, unsignedp))
2006 {
2007 convert_move (targ0, t0, unsignedp);
2008 convert_move (targ1, t1, unsignedp);
2009 return 1;
2010 }
2011 else
2012 delete_insns_since (last);
2013 }
2014 }
2015 }
2016
2017 delete_insns_since (entry_last);
2018 return 0;
2019 }
2020
2021 /* Generate code to perform an operation specified by BINOPTAB
2022 on operands OP0 and OP1, with two results to TARG1 and TARG2.
2023 We assume that the order of the operands for the instruction
2024 is TARG0, OP0, OP1, TARG1, which would fit a pattern like
2025 [(set TARG0 (operate OP0 OP1)) (set TARG1 (operate ...))].
2026
2027 Either TARG0 or TARG1 may be zero, but what that means is that
2028 the result is not actually wanted. We will generate it into
2029 a dummy pseudo-reg and discard it. They may not both be zero.
2030
2031 Returns 1 if this operation can be performed; 0 if not. */
2032
2033 int
expand_twoval_binop(optab binoptab,rtx op0,rtx op1,rtx targ0,rtx targ1,int unsignedp)2034 expand_twoval_binop (optab binoptab, rtx op0, rtx op1, rtx targ0, rtx targ1,
2035 int unsignedp)
2036 {
2037 machine_mode mode = GET_MODE (targ0 ? targ0 : targ1);
2038 enum mode_class mclass;
2039 machine_mode wider_mode;
2040 rtx_insn *entry_last = get_last_insn ();
2041 rtx_insn *last;
2042
2043 mclass = GET_MODE_CLASS (mode);
2044
2045 if (!targ0)
2046 targ0 = gen_reg_rtx (mode);
2047 if (!targ1)
2048 targ1 = gen_reg_rtx (mode);
2049
2050 /* Record where to go back to if we fail. */
2051 last = get_last_insn ();
2052
2053 if (optab_handler (binoptab, mode) != CODE_FOR_nothing)
2054 {
2055 struct expand_operand ops[4];
2056 enum insn_code icode = optab_handler (binoptab, mode);
2057 machine_mode mode0 = insn_data[icode].operand[1].mode;
2058 machine_mode mode1 = insn_data[icode].operand[2].mode;
2059 rtx xop0 = op0, xop1 = op1;
2060
2061 /* If we are optimizing, force expensive constants into a register. */
2062 xop0 = avoid_expensive_constant (mode0, binoptab, 0, xop0, unsignedp);
2063 xop1 = avoid_expensive_constant (mode1, binoptab, 1, xop1, unsignedp);
2064
2065 create_fixed_operand (&ops[0], targ0);
2066 create_convert_operand_from (&ops[1], op0, mode, unsignedp);
2067 create_convert_operand_from (&ops[2], op1, mode, unsignedp);
2068 create_fixed_operand (&ops[3], targ1);
2069 if (maybe_expand_insn (icode, 4, ops))
2070 return 1;
2071 delete_insns_since (last);
2072 }
2073
2074 /* It can't be done in this mode. Can we do it in a wider mode? */
2075
2076 if (CLASS_HAS_WIDER_MODES_P (mclass))
2077 {
2078 FOR_EACH_WIDER_MODE (wider_mode, mode)
2079 {
2080 if (optab_handler (binoptab, wider_mode) != CODE_FOR_nothing)
2081 {
2082 rtx t0 = gen_reg_rtx (wider_mode);
2083 rtx t1 = gen_reg_rtx (wider_mode);
2084 rtx cop0 = convert_modes (wider_mode, mode, op0, unsignedp);
2085 rtx cop1 = convert_modes (wider_mode, mode, op1, unsignedp);
2086
2087 if (expand_twoval_binop (binoptab, cop0, cop1,
2088 t0, t1, unsignedp))
2089 {
2090 convert_move (targ0, t0, unsignedp);
2091 convert_move (targ1, t1, unsignedp);
2092 return 1;
2093 }
2094 else
2095 delete_insns_since (last);
2096 }
2097 }
2098 }
2099
2100 delete_insns_since (entry_last);
2101 return 0;
2102 }
2103
2104 /* Expand the two-valued library call indicated by BINOPTAB, but
2105 preserve only one of the values. If TARG0 is non-NULL, the first
2106 value is placed into TARG0; otherwise the second value is placed
2107 into TARG1. Exactly one of TARG0 and TARG1 must be non-NULL. The
2108 value stored into TARG0 or TARG1 is equivalent to (CODE OP0 OP1).
2109 This routine assumes that the value returned by the library call is
2110 as if the return value was of an integral mode twice as wide as the
2111 mode of OP0. Returns 1 if the call was successful. */
2112
2113 bool
expand_twoval_binop_libfunc(optab binoptab,rtx op0,rtx op1,rtx targ0,rtx targ1,enum rtx_code code)2114 expand_twoval_binop_libfunc (optab binoptab, rtx op0, rtx op1,
2115 rtx targ0, rtx targ1, enum rtx_code code)
2116 {
2117 machine_mode mode;
2118 machine_mode libval_mode;
2119 rtx libval;
2120 rtx_insn *insns;
2121 rtx libfunc;
2122
2123 /* Exactly one of TARG0 or TARG1 should be non-NULL. */
2124 gcc_assert (!targ0 != !targ1);
2125
2126 mode = GET_MODE (op0);
2127 libfunc = optab_libfunc (binoptab, mode);
2128 if (!libfunc)
2129 return false;
2130
2131 /* The value returned by the library function will have twice as
2132 many bits as the nominal MODE. */
2133 libval_mode = smallest_int_mode_for_size (2 * GET_MODE_BITSIZE (mode));
2134 start_sequence ();
2135 libval = emit_library_call_value (libfunc, NULL_RTX, LCT_CONST,
2136 libval_mode,
2137 op0, mode,
2138 op1, mode);
2139 /* Get the part of VAL containing the value that we want. */
2140 libval = simplify_gen_subreg (mode, libval, libval_mode,
2141 targ0 ? 0 : GET_MODE_SIZE (mode));
2142 insns = get_insns ();
2143 end_sequence ();
2144 /* Move the into the desired location. */
2145 emit_libcall_block (insns, targ0 ? targ0 : targ1, libval,
2146 gen_rtx_fmt_ee (code, mode, op0, op1));
2147
2148 return true;
2149 }
2150
2151
2152 /* Wrapper around expand_unop which takes an rtx code to specify
2153 the operation to perform, not an optab pointer. All other
2154 arguments are the same. */
2155 rtx
expand_simple_unop(machine_mode mode,enum rtx_code code,rtx op0,rtx target,int unsignedp)2156 expand_simple_unop (machine_mode mode, enum rtx_code code, rtx op0,
2157 rtx target, int unsignedp)
2158 {
2159 optab unop = code_to_optab (code);
2160 gcc_assert (unop);
2161
2162 return expand_unop (mode, unop, op0, target, unsignedp);
2163 }
2164
2165 /* Try calculating
2166 (clz:narrow x)
2167 as
2168 (clz:wide (zero_extend:wide x)) - ((width wide) - (width narrow)).
2169
2170 A similar operation can be used for clrsb. UNOPTAB says which operation
2171 we are trying to expand. */
2172 static rtx
widen_leading(scalar_int_mode mode,rtx op0,rtx target,optab unoptab)2173 widen_leading (scalar_int_mode mode, rtx op0, rtx target, optab unoptab)
2174 {
2175 opt_scalar_int_mode wider_mode_iter;
2176 FOR_EACH_WIDER_MODE (wider_mode_iter, mode)
2177 {
2178 scalar_int_mode wider_mode = wider_mode_iter.require ();
2179 if (optab_handler (unoptab, wider_mode) != CODE_FOR_nothing)
2180 {
2181 rtx xop0, temp;
2182 rtx_insn *last;
2183
2184 last = get_last_insn ();
2185
2186 if (target == 0)
2187 target = gen_reg_rtx (mode);
2188 xop0 = widen_operand (op0, wider_mode, mode,
2189 unoptab != clrsb_optab, false);
2190 temp = expand_unop (wider_mode, unoptab, xop0, NULL_RTX,
2191 unoptab != clrsb_optab);
2192 if (temp != 0)
2193 temp = expand_binop
2194 (wider_mode, sub_optab, temp,
2195 gen_int_mode (GET_MODE_PRECISION (wider_mode)
2196 - GET_MODE_PRECISION (mode),
2197 wider_mode),
2198 target, true, OPTAB_DIRECT);
2199 if (temp == 0)
2200 delete_insns_since (last);
2201
2202 return temp;
2203 }
2204 }
2205 return 0;
2206 }
2207
2208 /* Try calculating clz of a double-word quantity as two clz's of word-sized
2209 quantities, choosing which based on whether the high word is nonzero. */
2210 static rtx
expand_doubleword_clz(scalar_int_mode mode,rtx op0,rtx target)2211 expand_doubleword_clz (scalar_int_mode mode, rtx op0, rtx target)
2212 {
2213 rtx xop0 = force_reg (mode, op0);
2214 rtx subhi = gen_highpart (word_mode, xop0);
2215 rtx sublo = gen_lowpart (word_mode, xop0);
2216 rtx_code_label *hi0_label = gen_label_rtx ();
2217 rtx_code_label *after_label = gen_label_rtx ();
2218 rtx_insn *seq;
2219 rtx temp, result;
2220
2221 /* If we were not given a target, use a word_mode register, not a
2222 'mode' register. The result will fit, and nobody is expecting
2223 anything bigger (the return type of __builtin_clz* is int). */
2224 if (!target)
2225 target = gen_reg_rtx (word_mode);
2226
2227 /* In any case, write to a word_mode scratch in both branches of the
2228 conditional, so we can ensure there is a single move insn setting
2229 'target' to tag a REG_EQUAL note on. */
2230 result = gen_reg_rtx (word_mode);
2231
2232 start_sequence ();
2233
2234 /* If the high word is not equal to zero,
2235 then clz of the full value is clz of the high word. */
2236 emit_cmp_and_jump_insns (subhi, CONST0_RTX (word_mode), EQ, 0,
2237 word_mode, true, hi0_label);
2238
2239 temp = expand_unop_direct (word_mode, clz_optab, subhi, result, true);
2240 if (!temp)
2241 goto fail;
2242
2243 if (temp != result)
2244 convert_move (result, temp, true);
2245
2246 emit_jump_insn (targetm.gen_jump (after_label));
2247 emit_barrier ();
2248
2249 /* Else clz of the full value is clz of the low word plus the number
2250 of bits in the high word. */
2251 emit_label (hi0_label);
2252
2253 temp = expand_unop_direct (word_mode, clz_optab, sublo, 0, true);
2254 if (!temp)
2255 goto fail;
2256 temp = expand_binop (word_mode, add_optab, temp,
2257 gen_int_mode (GET_MODE_BITSIZE (word_mode), word_mode),
2258 result, true, OPTAB_DIRECT);
2259 if (!temp)
2260 goto fail;
2261 if (temp != result)
2262 convert_move (result, temp, true);
2263
2264 emit_label (after_label);
2265 convert_move (target, result, true);
2266
2267 seq = get_insns ();
2268 end_sequence ();
2269
2270 add_equal_note (seq, target, CLZ, xop0, NULL_RTX, mode);
2271 emit_insn (seq);
2272 return target;
2273
2274 fail:
2275 end_sequence ();
2276 return 0;
2277 }
2278
2279 /* Try calculating popcount of a double-word quantity as two popcount's of
2280 word-sized quantities and summing up the results. */
2281 static rtx
expand_doubleword_popcount(scalar_int_mode mode,rtx op0,rtx target)2282 expand_doubleword_popcount (scalar_int_mode mode, rtx op0, rtx target)
2283 {
2284 rtx t0, t1, t;
2285 rtx_insn *seq;
2286
2287 start_sequence ();
2288
2289 t0 = expand_unop_direct (word_mode, popcount_optab,
2290 operand_subword_force (op0, 0, mode), NULL_RTX,
2291 true);
2292 t1 = expand_unop_direct (word_mode, popcount_optab,
2293 operand_subword_force (op0, 1, mode), NULL_RTX,
2294 true);
2295 if (!t0 || !t1)
2296 {
2297 end_sequence ();
2298 return NULL_RTX;
2299 }
2300
2301 /* If we were not given a target, use a word_mode register, not a
2302 'mode' register. The result will fit, and nobody is expecting
2303 anything bigger (the return type of __builtin_popcount* is int). */
2304 if (!target)
2305 target = gen_reg_rtx (word_mode);
2306
2307 t = expand_binop (word_mode, add_optab, t0, t1, target, 0, OPTAB_DIRECT);
2308
2309 seq = get_insns ();
2310 end_sequence ();
2311
2312 add_equal_note (seq, t, POPCOUNT, op0, NULL_RTX, mode);
2313 emit_insn (seq);
2314 return t;
2315 }
2316
2317 /* Try calculating
2318 (parity:wide x)
2319 as
2320 (parity:narrow (low (x) ^ high (x))) */
2321 static rtx
expand_doubleword_parity(scalar_int_mode mode,rtx op0,rtx target)2322 expand_doubleword_parity (scalar_int_mode mode, rtx op0, rtx target)
2323 {
2324 rtx t = expand_binop (word_mode, xor_optab,
2325 operand_subword_force (op0, 0, mode),
2326 operand_subword_force (op0, 1, mode),
2327 NULL_RTX, 0, OPTAB_DIRECT);
2328 return expand_unop (word_mode, parity_optab, t, target, true);
2329 }
2330
2331 /* Try calculating
2332 (bswap:narrow x)
2333 as
2334 (lshiftrt:wide (bswap:wide x) ((width wide) - (width narrow))). */
2335 static rtx
widen_bswap(scalar_int_mode mode,rtx op0,rtx target)2336 widen_bswap (scalar_int_mode mode, rtx op0, rtx target)
2337 {
2338 rtx x;
2339 rtx_insn *last;
2340 opt_scalar_int_mode wider_mode_iter;
2341
2342 FOR_EACH_WIDER_MODE (wider_mode_iter, mode)
2343 if (optab_handler (bswap_optab, wider_mode_iter.require ())
2344 != CODE_FOR_nothing)
2345 break;
2346
2347 if (!wider_mode_iter.exists ())
2348 return NULL_RTX;
2349
2350 scalar_int_mode wider_mode = wider_mode_iter.require ();
2351 last = get_last_insn ();
2352
2353 x = widen_operand (op0, wider_mode, mode, true, true);
2354 x = expand_unop (wider_mode, bswap_optab, x, NULL_RTX, true);
2355
2356 gcc_assert (GET_MODE_PRECISION (wider_mode) == GET_MODE_BITSIZE (wider_mode)
2357 && GET_MODE_PRECISION (mode) == GET_MODE_BITSIZE (mode));
2358 if (x != 0)
2359 x = expand_shift (RSHIFT_EXPR, wider_mode, x,
2360 GET_MODE_BITSIZE (wider_mode)
2361 - GET_MODE_BITSIZE (mode),
2362 NULL_RTX, true);
2363
2364 if (x != 0)
2365 {
2366 if (target == 0)
2367 target = gen_reg_rtx (mode);
2368 emit_move_insn (target, gen_lowpart (mode, x));
2369 }
2370 else
2371 delete_insns_since (last);
2372
2373 return target;
2374 }
2375
2376 /* Try calculating bswap as two bswaps of two word-sized operands. */
2377
2378 static rtx
expand_doubleword_bswap(machine_mode mode,rtx op,rtx target)2379 expand_doubleword_bswap (machine_mode mode, rtx op, rtx target)
2380 {
2381 rtx t0, t1;
2382
2383 t1 = expand_unop (word_mode, bswap_optab,
2384 operand_subword_force (op, 0, mode), NULL_RTX, true);
2385 t0 = expand_unop (word_mode, bswap_optab,
2386 operand_subword_force (op, 1, mode), NULL_RTX, true);
2387
2388 if (target == 0 || !valid_multiword_target_p (target))
2389 target = gen_reg_rtx (mode);
2390 if (REG_P (target))
2391 emit_clobber (target);
2392 emit_move_insn (operand_subword (target, 0, 1, mode), t0);
2393 emit_move_insn (operand_subword (target, 1, 1, mode), t1);
2394
2395 return target;
2396 }
2397
2398 /* Try calculating (parity x) as (and (popcount x) 1), where
2399 popcount can also be done in a wider mode. */
2400 static rtx
expand_parity(scalar_int_mode mode,rtx op0,rtx target)2401 expand_parity (scalar_int_mode mode, rtx op0, rtx target)
2402 {
2403 enum mode_class mclass = GET_MODE_CLASS (mode);
2404 opt_scalar_int_mode wider_mode_iter;
2405 FOR_EACH_MODE_FROM (wider_mode_iter, mode)
2406 {
2407 scalar_int_mode wider_mode = wider_mode_iter.require ();
2408 if (optab_handler (popcount_optab, wider_mode) != CODE_FOR_nothing)
2409 {
2410 rtx xop0, temp;
2411 rtx_insn *last;
2412
2413 last = get_last_insn ();
2414
2415 if (target == 0 || GET_MODE (target) != wider_mode)
2416 target = gen_reg_rtx (wider_mode);
2417
2418 xop0 = widen_operand (op0, wider_mode, mode, true, false);
2419 temp = expand_unop (wider_mode, popcount_optab, xop0, NULL_RTX,
2420 true);
2421 if (temp != 0)
2422 temp = expand_binop (wider_mode, and_optab, temp, const1_rtx,
2423 target, true, OPTAB_DIRECT);
2424
2425 if (temp)
2426 {
2427 if (mclass != MODE_INT
2428 || !TRULY_NOOP_TRUNCATION_MODES_P (mode, wider_mode))
2429 return convert_to_mode (mode, temp, 0);
2430 else
2431 return gen_lowpart (mode, temp);
2432 }
2433 else
2434 delete_insns_since (last);
2435 }
2436 }
2437 return 0;
2438 }
2439
2440 /* Try calculating ctz(x) as K - clz(x & -x) ,
2441 where K is GET_MODE_PRECISION(mode) - 1.
2442
2443 Both __builtin_ctz and __builtin_clz are undefined at zero, so we
2444 don't have to worry about what the hardware does in that case. (If
2445 the clz instruction produces the usual value at 0, which is K, the
2446 result of this code sequence will be -1; expand_ffs, below, relies
2447 on this. It might be nice to have it be K instead, for consistency
2448 with the (very few) processors that provide a ctz with a defined
2449 value, but that would take one more instruction, and it would be
2450 less convenient for expand_ffs anyway. */
2451
2452 static rtx
expand_ctz(scalar_int_mode mode,rtx op0,rtx target)2453 expand_ctz (scalar_int_mode mode, rtx op0, rtx target)
2454 {
2455 rtx_insn *seq;
2456 rtx temp;
2457
2458 if (optab_handler (clz_optab, mode) == CODE_FOR_nothing)
2459 return 0;
2460
2461 start_sequence ();
2462
2463 temp = expand_unop_direct (mode, neg_optab, op0, NULL_RTX, true);
2464 if (temp)
2465 temp = expand_binop (mode, and_optab, op0, temp, NULL_RTX,
2466 true, OPTAB_DIRECT);
2467 if (temp)
2468 temp = expand_unop_direct (mode, clz_optab, temp, NULL_RTX, true);
2469 if (temp)
2470 temp = expand_binop (mode, sub_optab,
2471 gen_int_mode (GET_MODE_PRECISION (mode) - 1, mode),
2472 temp, target,
2473 true, OPTAB_DIRECT);
2474 if (temp == 0)
2475 {
2476 end_sequence ();
2477 return 0;
2478 }
2479
2480 seq = get_insns ();
2481 end_sequence ();
2482
2483 add_equal_note (seq, temp, CTZ, op0, NULL_RTX, mode);
2484 emit_insn (seq);
2485 return temp;
2486 }
2487
2488
2489 /* Try calculating ffs(x) using ctz(x) if we have that instruction, or
2490 else with the sequence used by expand_clz.
2491
2492 The ffs builtin promises to return zero for a zero value and ctz/clz
2493 may have an undefined value in that case. If they do not give us a
2494 convenient value, we have to generate a test and branch. */
2495 static rtx
expand_ffs(scalar_int_mode mode,rtx op0,rtx target)2496 expand_ffs (scalar_int_mode mode, rtx op0, rtx target)
2497 {
2498 HOST_WIDE_INT val = 0;
2499 bool defined_at_zero = false;
2500 rtx temp;
2501 rtx_insn *seq;
2502
2503 if (optab_handler (ctz_optab, mode) != CODE_FOR_nothing)
2504 {
2505 start_sequence ();
2506
2507 temp = expand_unop_direct (mode, ctz_optab, op0, 0, true);
2508 if (!temp)
2509 goto fail;
2510
2511 defined_at_zero = (CTZ_DEFINED_VALUE_AT_ZERO (mode, val) == 2);
2512 }
2513 else if (optab_handler (clz_optab, mode) != CODE_FOR_nothing)
2514 {
2515 start_sequence ();
2516 temp = expand_ctz (mode, op0, 0);
2517 if (!temp)
2518 goto fail;
2519
2520 if (CLZ_DEFINED_VALUE_AT_ZERO (mode, val) == 2)
2521 {
2522 defined_at_zero = true;
2523 val = (GET_MODE_PRECISION (mode) - 1) - val;
2524 }
2525 }
2526 else
2527 return 0;
2528
2529 if (defined_at_zero && val == -1)
2530 /* No correction needed at zero. */;
2531 else
2532 {
2533 /* We don't try to do anything clever with the situation found
2534 on some processors (eg Alpha) where ctz(0:mode) ==
2535 bitsize(mode). If someone can think of a way to send N to -1
2536 and leave alone all values in the range 0..N-1 (where N is a
2537 power of two), cheaper than this test-and-branch, please add it.
2538
2539 The test-and-branch is done after the operation itself, in case
2540 the operation sets condition codes that can be recycled for this.
2541 (This is true on i386, for instance.) */
2542
2543 rtx_code_label *nonzero_label = gen_label_rtx ();
2544 emit_cmp_and_jump_insns (op0, CONST0_RTX (mode), NE, 0,
2545 mode, true, nonzero_label);
2546
2547 convert_move (temp, GEN_INT (-1), false);
2548 emit_label (nonzero_label);
2549 }
2550
2551 /* temp now has a value in the range -1..bitsize-1. ffs is supposed
2552 to produce a value in the range 0..bitsize. */
2553 temp = expand_binop (mode, add_optab, temp, gen_int_mode (1, mode),
2554 target, false, OPTAB_DIRECT);
2555 if (!temp)
2556 goto fail;
2557
2558 seq = get_insns ();
2559 end_sequence ();
2560
2561 add_equal_note (seq, temp, FFS, op0, NULL_RTX, mode);
2562 emit_insn (seq);
2563 return temp;
2564
2565 fail:
2566 end_sequence ();
2567 return 0;
2568 }
2569
2570 /* Extract the OMODE lowpart from VAL, which has IMODE. Under certain
2571 conditions, VAL may already be a SUBREG against which we cannot generate
2572 a further SUBREG. In this case, we expect forcing the value into a
2573 register will work around the situation. */
2574
2575 static rtx
lowpart_subreg_maybe_copy(machine_mode omode,rtx val,machine_mode imode)2576 lowpart_subreg_maybe_copy (machine_mode omode, rtx val,
2577 machine_mode imode)
2578 {
2579 rtx ret;
2580 ret = lowpart_subreg (omode, val, imode);
2581 if (ret == NULL)
2582 {
2583 val = force_reg (imode, val);
2584 ret = lowpart_subreg (omode, val, imode);
2585 gcc_assert (ret != NULL);
2586 }
2587 return ret;
2588 }
2589
2590 /* Expand a floating point absolute value or negation operation via a
2591 logical operation on the sign bit. */
2592
2593 static rtx
expand_absneg_bit(enum rtx_code code,scalar_float_mode mode,rtx op0,rtx target)2594 expand_absneg_bit (enum rtx_code code, scalar_float_mode mode,
2595 rtx op0, rtx target)
2596 {
2597 const struct real_format *fmt;
2598 int bitpos, word, nwords, i;
2599 scalar_int_mode imode;
2600 rtx temp;
2601 rtx_insn *insns;
2602
2603 /* The format has to have a simple sign bit. */
2604 fmt = REAL_MODE_FORMAT (mode);
2605 if (fmt == NULL)
2606 return NULL_RTX;
2607
2608 bitpos = fmt->signbit_rw;
2609 if (bitpos < 0)
2610 return NULL_RTX;
2611
2612 /* Don't create negative zeros if the format doesn't support them. */
2613 if (code == NEG && !fmt->has_signed_zero)
2614 return NULL_RTX;
2615
2616 if (GET_MODE_SIZE (mode) <= UNITS_PER_WORD)
2617 {
2618 if (!int_mode_for_mode (mode).exists (&imode))
2619 return NULL_RTX;
2620 word = 0;
2621 nwords = 1;
2622 }
2623 else
2624 {
2625 imode = word_mode;
2626
2627 if (FLOAT_WORDS_BIG_ENDIAN)
2628 word = (GET_MODE_BITSIZE (mode) - bitpos) / BITS_PER_WORD;
2629 else
2630 word = bitpos / BITS_PER_WORD;
2631 bitpos = bitpos % BITS_PER_WORD;
2632 nwords = (GET_MODE_BITSIZE (mode) + BITS_PER_WORD - 1) / BITS_PER_WORD;
2633 }
2634
2635 wide_int mask = wi::set_bit_in_zero (bitpos, GET_MODE_PRECISION (imode));
2636 if (code == ABS)
2637 mask = ~mask;
2638
2639 if (target == 0
2640 || target == op0
2641 || reg_overlap_mentioned_p (target, op0)
2642 || (nwords > 1 && !valid_multiword_target_p (target)))
2643 target = gen_reg_rtx (mode);
2644
2645 if (nwords > 1)
2646 {
2647 start_sequence ();
2648
2649 for (i = 0; i < nwords; ++i)
2650 {
2651 rtx targ_piece = operand_subword (target, i, 1, mode);
2652 rtx op0_piece = operand_subword_force (op0, i, mode);
2653
2654 if (i == word)
2655 {
2656 temp = expand_binop (imode, code == ABS ? and_optab : xor_optab,
2657 op0_piece,
2658 immed_wide_int_const (mask, imode),
2659 targ_piece, 1, OPTAB_LIB_WIDEN);
2660 if (temp != targ_piece)
2661 emit_move_insn (targ_piece, temp);
2662 }
2663 else
2664 emit_move_insn (targ_piece, op0_piece);
2665 }
2666
2667 insns = get_insns ();
2668 end_sequence ();
2669
2670 emit_insn (insns);
2671 }
2672 else
2673 {
2674 temp = expand_binop (imode, code == ABS ? and_optab : xor_optab,
2675 gen_lowpart (imode, op0),
2676 immed_wide_int_const (mask, imode),
2677 gen_lowpart (imode, target), 1, OPTAB_LIB_WIDEN);
2678 target = lowpart_subreg_maybe_copy (mode, temp, imode);
2679
2680 set_dst_reg_note (get_last_insn (), REG_EQUAL,
2681 gen_rtx_fmt_e (code, mode, copy_rtx (op0)),
2682 target);
2683 }
2684
2685 return target;
2686 }
2687
2688 /* As expand_unop, but will fail rather than attempt the operation in a
2689 different mode or with a libcall. */
2690 static rtx
expand_unop_direct(machine_mode mode,optab unoptab,rtx op0,rtx target,int unsignedp)2691 expand_unop_direct (machine_mode mode, optab unoptab, rtx op0, rtx target,
2692 int unsignedp)
2693 {
2694 if (optab_handler (unoptab, mode) != CODE_FOR_nothing)
2695 {
2696 struct expand_operand ops[2];
2697 enum insn_code icode = optab_handler (unoptab, mode);
2698 rtx_insn *last = get_last_insn ();
2699 rtx_insn *pat;
2700
2701 create_output_operand (&ops[0], target, mode);
2702 create_convert_operand_from (&ops[1], op0, mode, unsignedp);
2703 pat = maybe_gen_insn (icode, 2, ops);
2704 if (pat)
2705 {
2706 if (INSN_P (pat) && NEXT_INSN (pat) != NULL_RTX
2707 && ! add_equal_note (pat, ops[0].value,
2708 optab_to_code (unoptab),
2709 ops[1].value, NULL_RTX, mode))
2710 {
2711 delete_insns_since (last);
2712 return expand_unop (mode, unoptab, op0, NULL_RTX, unsignedp);
2713 }
2714
2715 emit_insn (pat);
2716
2717 return ops[0].value;
2718 }
2719 }
2720 return 0;
2721 }
2722
2723 /* Generate code to perform an operation specified by UNOPTAB
2724 on operand OP0, with result having machine-mode MODE.
2725
2726 UNSIGNEDP is for the case where we have to widen the operands
2727 to perform the operation. It says to use zero-extension.
2728
2729 If TARGET is nonzero, the value
2730 is generated there, if it is convenient to do so.
2731 In all cases an rtx is returned for the locus of the value;
2732 this may or may not be TARGET. */
2733
2734 rtx
expand_unop(machine_mode mode,optab unoptab,rtx op0,rtx target,int unsignedp)2735 expand_unop (machine_mode mode, optab unoptab, rtx op0, rtx target,
2736 int unsignedp)
2737 {
2738 enum mode_class mclass = GET_MODE_CLASS (mode);
2739 machine_mode wider_mode;
2740 scalar_int_mode int_mode;
2741 scalar_float_mode float_mode;
2742 rtx temp;
2743 rtx libfunc;
2744
2745 temp = expand_unop_direct (mode, unoptab, op0, target, unsignedp);
2746 if (temp)
2747 return temp;
2748
2749 /* It can't be done in this mode. Can we open-code it in a wider mode? */
2750
2751 /* Widening (or narrowing) clz needs special treatment. */
2752 if (unoptab == clz_optab)
2753 {
2754 if (is_a <scalar_int_mode> (mode, &int_mode))
2755 {
2756 temp = widen_leading (int_mode, op0, target, unoptab);
2757 if (temp)
2758 return temp;
2759
2760 if (GET_MODE_SIZE (int_mode) == 2 * UNITS_PER_WORD
2761 && optab_handler (unoptab, word_mode) != CODE_FOR_nothing)
2762 {
2763 temp = expand_doubleword_clz (int_mode, op0, target);
2764 if (temp)
2765 return temp;
2766 }
2767 }
2768
2769 goto try_libcall;
2770 }
2771
2772 if (unoptab == clrsb_optab)
2773 {
2774 if (is_a <scalar_int_mode> (mode, &int_mode))
2775 {
2776 temp = widen_leading (int_mode, op0, target, unoptab);
2777 if (temp)
2778 return temp;
2779 }
2780 goto try_libcall;
2781 }
2782
2783 if (unoptab == popcount_optab
2784 && is_a <scalar_int_mode> (mode, &int_mode)
2785 && GET_MODE_SIZE (int_mode) == 2 * UNITS_PER_WORD
2786 && optab_handler (unoptab, word_mode) != CODE_FOR_nothing
2787 && optimize_insn_for_speed_p ())
2788 {
2789 temp = expand_doubleword_popcount (int_mode, op0, target);
2790 if (temp)
2791 return temp;
2792 }
2793
2794 if (unoptab == parity_optab
2795 && is_a <scalar_int_mode> (mode, &int_mode)
2796 && GET_MODE_SIZE (int_mode) == 2 * UNITS_PER_WORD
2797 && (optab_handler (unoptab, word_mode) != CODE_FOR_nothing
2798 || optab_handler (popcount_optab, word_mode) != CODE_FOR_nothing)
2799 && optimize_insn_for_speed_p ())
2800 {
2801 temp = expand_doubleword_parity (int_mode, op0, target);
2802 if (temp)
2803 return temp;
2804 }
2805
2806 /* Widening (or narrowing) bswap needs special treatment. */
2807 if (unoptab == bswap_optab)
2808 {
2809 /* HImode is special because in this mode BSWAP is equivalent to ROTATE
2810 or ROTATERT. First try these directly; if this fails, then try the
2811 obvious pair of shifts with allowed widening, as this will probably
2812 be always more efficient than the other fallback methods. */
2813 if (mode == HImode)
2814 {
2815 rtx_insn *last;
2816 rtx temp1, temp2;
2817
2818 if (optab_handler (rotl_optab, mode) != CODE_FOR_nothing)
2819 {
2820 temp = expand_binop (mode, rotl_optab, op0,
2821 gen_int_shift_amount (mode, 8),
2822 target, unsignedp, OPTAB_DIRECT);
2823 if (temp)
2824 return temp;
2825 }
2826
2827 if (optab_handler (rotr_optab, mode) != CODE_FOR_nothing)
2828 {
2829 temp = expand_binop (mode, rotr_optab, op0,
2830 gen_int_shift_amount (mode, 8),
2831 target, unsignedp, OPTAB_DIRECT);
2832 if (temp)
2833 return temp;
2834 }
2835
2836 last = get_last_insn ();
2837
2838 temp1 = expand_binop (mode, ashl_optab, op0,
2839 gen_int_shift_amount (mode, 8), NULL_RTX,
2840 unsignedp, OPTAB_WIDEN);
2841 temp2 = expand_binop (mode, lshr_optab, op0,
2842 gen_int_shift_amount (mode, 8), NULL_RTX,
2843 unsignedp, OPTAB_WIDEN);
2844 if (temp1 && temp2)
2845 {
2846 temp = expand_binop (mode, ior_optab, temp1, temp2, target,
2847 unsignedp, OPTAB_WIDEN);
2848 if (temp)
2849 return temp;
2850 }
2851
2852 delete_insns_since (last);
2853 }
2854
2855 if (is_a <scalar_int_mode> (mode, &int_mode))
2856 {
2857 temp = widen_bswap (int_mode, op0, target);
2858 if (temp)
2859 return temp;
2860
2861 if (GET_MODE_SIZE (int_mode) == 2 * UNITS_PER_WORD
2862 && optab_handler (unoptab, word_mode) != CODE_FOR_nothing)
2863 {
2864 temp = expand_doubleword_bswap (mode, op0, target);
2865 if (temp)
2866 return temp;
2867 }
2868 }
2869
2870 goto try_libcall;
2871 }
2872
2873 if (CLASS_HAS_WIDER_MODES_P (mclass))
2874 FOR_EACH_WIDER_MODE (wider_mode, mode)
2875 {
2876 if (optab_handler (unoptab, wider_mode) != CODE_FOR_nothing)
2877 {
2878 rtx xop0 = op0;
2879 rtx_insn *last = get_last_insn ();
2880
2881 /* For certain operations, we need not actually extend
2882 the narrow operand, as long as we will truncate the
2883 results to the same narrowness. */
2884
2885 xop0 = widen_operand (xop0, wider_mode, mode, unsignedp,
2886 (unoptab == neg_optab
2887 || unoptab == one_cmpl_optab)
2888 && mclass == MODE_INT);
2889
2890 temp = expand_unop (wider_mode, unoptab, xop0, NULL_RTX,
2891 unsignedp);
2892
2893 if (temp)
2894 {
2895 if (mclass != MODE_INT
2896 || !TRULY_NOOP_TRUNCATION_MODES_P (mode, wider_mode))
2897 {
2898 if (target == 0)
2899 target = gen_reg_rtx (mode);
2900 convert_move (target, temp, 0);
2901 return target;
2902 }
2903 else
2904 return gen_lowpart (mode, temp);
2905 }
2906 else
2907 delete_insns_since (last);
2908 }
2909 }
2910
2911 /* These can be done a word at a time. */
2912 if (unoptab == one_cmpl_optab
2913 && is_int_mode (mode, &int_mode)
2914 && GET_MODE_SIZE (int_mode) > UNITS_PER_WORD
2915 && optab_handler (unoptab, word_mode) != CODE_FOR_nothing)
2916 {
2917 int i;
2918 rtx_insn *insns;
2919
2920 if (target == 0
2921 || target == op0
2922 || reg_overlap_mentioned_p (target, op0)
2923 || !valid_multiword_target_p (target))
2924 target = gen_reg_rtx (int_mode);
2925
2926 start_sequence ();
2927
2928 /* Do the actual arithmetic. */
2929 for (i = 0; i < GET_MODE_BITSIZE (int_mode) / BITS_PER_WORD; i++)
2930 {
2931 rtx target_piece = operand_subword (target, i, 1, int_mode);
2932 rtx x = expand_unop (word_mode, unoptab,
2933 operand_subword_force (op0, i, int_mode),
2934 target_piece, unsignedp);
2935
2936 if (target_piece != x)
2937 emit_move_insn (target_piece, x);
2938 }
2939
2940 insns = get_insns ();
2941 end_sequence ();
2942
2943 emit_insn (insns);
2944 return target;
2945 }
2946
2947 if (optab_to_code (unoptab) == NEG)
2948 {
2949 /* Try negating floating point values by flipping the sign bit. */
2950 if (is_a <scalar_float_mode> (mode, &float_mode))
2951 {
2952 temp = expand_absneg_bit (NEG, float_mode, op0, target);
2953 if (temp)
2954 return temp;
2955 }
2956
2957 /* If there is no negation pattern, and we have no negative zero,
2958 try subtracting from zero. */
2959 if (!HONOR_SIGNED_ZEROS (mode))
2960 {
2961 temp = expand_binop (mode, (unoptab == negv_optab
2962 ? subv_optab : sub_optab),
2963 CONST0_RTX (mode), op0, target,
2964 unsignedp, OPTAB_DIRECT);
2965 if (temp)
2966 return temp;
2967 }
2968 }
2969
2970 /* Try calculating parity (x) as popcount (x) % 2. */
2971 if (unoptab == parity_optab && is_a <scalar_int_mode> (mode, &int_mode))
2972 {
2973 temp = expand_parity (int_mode, op0, target);
2974 if (temp)
2975 return temp;
2976 }
2977
2978 /* Try implementing ffs (x) in terms of clz (x). */
2979 if (unoptab == ffs_optab && is_a <scalar_int_mode> (mode, &int_mode))
2980 {
2981 temp = expand_ffs (int_mode, op0, target);
2982 if (temp)
2983 return temp;
2984 }
2985
2986 /* Try implementing ctz (x) in terms of clz (x). */
2987 if (unoptab == ctz_optab && is_a <scalar_int_mode> (mode, &int_mode))
2988 {
2989 temp = expand_ctz (int_mode, op0, target);
2990 if (temp)
2991 return temp;
2992 }
2993
2994 try_libcall:
2995 /* Now try a library call in this mode. */
2996 libfunc = optab_libfunc (unoptab, mode);
2997 if (libfunc)
2998 {
2999 rtx_insn *insns;
3000 rtx value;
3001 rtx eq_value;
3002 machine_mode outmode = mode;
3003
3004 /* All of these functions return small values. Thus we choose to
3005 have them return something that isn't a double-word. */
3006 if (unoptab == ffs_optab || unoptab == clz_optab || unoptab == ctz_optab
3007 || unoptab == clrsb_optab || unoptab == popcount_optab
3008 || unoptab == parity_optab)
3009 outmode
3010 = GET_MODE (hard_libcall_value (TYPE_MODE (integer_type_node),
3011 optab_libfunc (unoptab, mode)));
3012
3013 start_sequence ();
3014
3015 /* Pass 1 for NO_QUEUE so we don't lose any increments
3016 if the libcall is cse'd or moved. */
3017 value = emit_library_call_value (libfunc, NULL_RTX, LCT_CONST, outmode,
3018 op0, mode);
3019 insns = get_insns ();
3020 end_sequence ();
3021
3022 target = gen_reg_rtx (outmode);
3023 bool trapv = trapv_unoptab_p (unoptab);
3024 if (trapv)
3025 eq_value = NULL_RTX;
3026 else
3027 {
3028 eq_value = gen_rtx_fmt_e (optab_to_code (unoptab), mode, op0);
3029 if (GET_MODE_UNIT_SIZE (outmode) < GET_MODE_UNIT_SIZE (mode))
3030 eq_value = simplify_gen_unary (TRUNCATE, outmode, eq_value, mode);
3031 else if (GET_MODE_UNIT_SIZE (outmode) > GET_MODE_UNIT_SIZE (mode))
3032 eq_value = simplify_gen_unary (ZERO_EXTEND,
3033 outmode, eq_value, mode);
3034 }
3035 emit_libcall_block_1 (insns, target, value, eq_value, trapv);
3036
3037 return target;
3038 }
3039
3040 /* It can't be done in this mode. Can we do it in a wider mode? */
3041
3042 if (CLASS_HAS_WIDER_MODES_P (mclass))
3043 {
3044 FOR_EACH_WIDER_MODE (wider_mode, mode)
3045 {
3046 if (optab_handler (unoptab, wider_mode) != CODE_FOR_nothing
3047 || optab_libfunc (unoptab, wider_mode))
3048 {
3049 rtx xop0 = op0;
3050 rtx_insn *last = get_last_insn ();
3051
3052 /* For certain operations, we need not actually extend
3053 the narrow operand, as long as we will truncate the
3054 results to the same narrowness. */
3055 xop0 = widen_operand (xop0, wider_mode, mode, unsignedp,
3056 (unoptab == neg_optab
3057 || unoptab == one_cmpl_optab
3058 || unoptab == bswap_optab)
3059 && mclass == MODE_INT);
3060
3061 temp = expand_unop (wider_mode, unoptab, xop0, NULL_RTX,
3062 unsignedp);
3063
3064 /* If we are generating clz using wider mode, adjust the
3065 result. Similarly for clrsb. */
3066 if ((unoptab == clz_optab || unoptab == clrsb_optab)
3067 && temp != 0)
3068 {
3069 scalar_int_mode wider_int_mode
3070 = as_a <scalar_int_mode> (wider_mode);
3071 int_mode = as_a <scalar_int_mode> (mode);
3072 temp = expand_binop
3073 (wider_mode, sub_optab, temp,
3074 gen_int_mode (GET_MODE_PRECISION (wider_int_mode)
3075 - GET_MODE_PRECISION (int_mode),
3076 wider_int_mode),
3077 target, true, OPTAB_DIRECT);
3078 }
3079
3080 /* Likewise for bswap. */
3081 if (unoptab == bswap_optab && temp != 0)
3082 {
3083 scalar_int_mode wider_int_mode
3084 = as_a <scalar_int_mode> (wider_mode);
3085 int_mode = as_a <scalar_int_mode> (mode);
3086 gcc_assert (GET_MODE_PRECISION (wider_int_mode)
3087 == GET_MODE_BITSIZE (wider_int_mode)
3088 && GET_MODE_PRECISION (int_mode)
3089 == GET_MODE_BITSIZE (int_mode));
3090
3091 temp = expand_shift (RSHIFT_EXPR, wider_int_mode, temp,
3092 GET_MODE_BITSIZE (wider_int_mode)
3093 - GET_MODE_BITSIZE (int_mode),
3094 NULL_RTX, true);
3095 }
3096
3097 if (temp)
3098 {
3099 if (mclass != MODE_INT)
3100 {
3101 if (target == 0)
3102 target = gen_reg_rtx (mode);
3103 convert_move (target, temp, 0);
3104 return target;
3105 }
3106 else
3107 return gen_lowpart (mode, temp);
3108 }
3109 else
3110 delete_insns_since (last);
3111 }
3112 }
3113 }
3114
3115 /* One final attempt at implementing negation via subtraction,
3116 this time allowing widening of the operand. */
3117 if (optab_to_code (unoptab) == NEG && !HONOR_SIGNED_ZEROS (mode))
3118 {
3119 rtx temp;
3120 temp = expand_binop (mode,
3121 unoptab == negv_optab ? subv_optab : sub_optab,
3122 CONST0_RTX (mode), op0,
3123 target, unsignedp, OPTAB_LIB_WIDEN);
3124 if (temp)
3125 return temp;
3126 }
3127
3128 return 0;
3129 }
3130
3131 /* Emit code to compute the absolute value of OP0, with result to
3132 TARGET if convenient. (TARGET may be 0.) The return value says
3133 where the result actually is to be found.
3134
3135 MODE is the mode of the operand; the mode of the result is
3136 different but can be deduced from MODE.
3137
3138 */
3139
3140 rtx
expand_abs_nojump(machine_mode mode,rtx op0,rtx target,int result_unsignedp)3141 expand_abs_nojump (machine_mode mode, rtx op0, rtx target,
3142 int result_unsignedp)
3143 {
3144 rtx temp;
3145
3146 if (GET_MODE_CLASS (mode) != MODE_INT
3147 || ! flag_trapv)
3148 result_unsignedp = 1;
3149
3150 /* First try to do it with a special abs instruction. */
3151 temp = expand_unop (mode, result_unsignedp ? abs_optab : absv_optab,
3152 op0, target, 0);
3153 if (temp != 0)
3154 return temp;
3155
3156 /* For floating point modes, try clearing the sign bit. */
3157 scalar_float_mode float_mode;
3158 if (is_a <scalar_float_mode> (mode, &float_mode))
3159 {
3160 temp = expand_absneg_bit (ABS, float_mode, op0, target);
3161 if (temp)
3162 return temp;
3163 }
3164
3165 /* If we have a MAX insn, we can do this as MAX (x, -x). */
3166 if (optab_handler (smax_optab, mode) != CODE_FOR_nothing
3167 && !HONOR_SIGNED_ZEROS (mode))
3168 {
3169 rtx_insn *last = get_last_insn ();
3170
3171 temp = expand_unop (mode, result_unsignedp ? neg_optab : negv_optab,
3172 op0, NULL_RTX, 0);
3173 if (temp != 0)
3174 temp = expand_binop (mode, smax_optab, op0, temp, target, 0,
3175 OPTAB_WIDEN);
3176
3177 if (temp != 0)
3178 return temp;
3179
3180 delete_insns_since (last);
3181 }
3182
3183 /* If this machine has expensive jumps, we can do integer absolute
3184 value of X as (((signed) x >> (W-1)) ^ x) - ((signed) x >> (W-1)),
3185 where W is the width of MODE. */
3186
3187 scalar_int_mode int_mode;
3188 if (is_int_mode (mode, &int_mode)
3189 && BRANCH_COST (optimize_insn_for_speed_p (),
3190 false) >= 2)
3191 {
3192 rtx extended = expand_shift (RSHIFT_EXPR, int_mode, op0,
3193 GET_MODE_PRECISION (int_mode) - 1,
3194 NULL_RTX, 0);
3195
3196 temp = expand_binop (int_mode, xor_optab, extended, op0, target, 0,
3197 OPTAB_LIB_WIDEN);
3198 if (temp != 0)
3199 temp = expand_binop (int_mode,
3200 result_unsignedp ? sub_optab : subv_optab,
3201 temp, extended, target, 0, OPTAB_LIB_WIDEN);
3202
3203 if (temp != 0)
3204 return temp;
3205 }
3206
3207 return NULL_RTX;
3208 }
3209
3210 rtx
expand_abs(machine_mode mode,rtx op0,rtx target,int result_unsignedp,int safe)3211 expand_abs (machine_mode mode, rtx op0, rtx target,
3212 int result_unsignedp, int safe)
3213 {
3214 rtx temp;
3215 rtx_code_label *op1;
3216
3217 if (GET_MODE_CLASS (mode) != MODE_INT
3218 || ! flag_trapv)
3219 result_unsignedp = 1;
3220
3221 temp = expand_abs_nojump (mode, op0, target, result_unsignedp);
3222 if (temp != 0)
3223 return temp;
3224
3225 /* If that does not win, use conditional jump and negate. */
3226
3227 /* It is safe to use the target if it is the same
3228 as the source if this is also a pseudo register */
3229 if (op0 == target && REG_P (op0)
3230 && REGNO (op0) >= FIRST_PSEUDO_REGISTER)
3231 safe = 1;
3232
3233 op1 = gen_label_rtx ();
3234 if (target == 0 || ! safe
3235 || GET_MODE (target) != mode
3236 || (MEM_P (target) && MEM_VOLATILE_P (target))
3237 || (REG_P (target)
3238 && REGNO (target) < FIRST_PSEUDO_REGISTER))
3239 target = gen_reg_rtx (mode);
3240
3241 emit_move_insn (target, op0);
3242 NO_DEFER_POP;
3243
3244 do_compare_rtx_and_jump (target, CONST0_RTX (mode), GE, 0, mode,
3245 NULL_RTX, NULL, op1,
3246 profile_probability::uninitialized ());
3247
3248 op0 = expand_unop (mode, result_unsignedp ? neg_optab : negv_optab,
3249 target, target, 0);
3250 if (op0 != target)
3251 emit_move_insn (target, op0);
3252 emit_label (op1);
3253 OK_DEFER_POP;
3254 return target;
3255 }
3256
3257 /* Emit code to compute the one's complement absolute value of OP0
3258 (if (OP0 < 0) OP0 = ~OP0), with result to TARGET if convenient.
3259 (TARGET may be NULL_RTX.) The return value says where the result
3260 actually is to be found.
3261
3262 MODE is the mode of the operand; the mode of the result is
3263 different but can be deduced from MODE. */
3264
3265 rtx
expand_one_cmpl_abs_nojump(machine_mode mode,rtx op0,rtx target)3266 expand_one_cmpl_abs_nojump (machine_mode mode, rtx op0, rtx target)
3267 {
3268 rtx temp;
3269
3270 /* Not applicable for floating point modes. */
3271 if (FLOAT_MODE_P (mode))
3272 return NULL_RTX;
3273
3274 /* If we have a MAX insn, we can do this as MAX (x, ~x). */
3275 if (optab_handler (smax_optab, mode) != CODE_FOR_nothing)
3276 {
3277 rtx_insn *last = get_last_insn ();
3278
3279 temp = expand_unop (mode, one_cmpl_optab, op0, NULL_RTX, 0);
3280 if (temp != 0)
3281 temp = expand_binop (mode, smax_optab, op0, temp, target, 0,
3282 OPTAB_WIDEN);
3283
3284 if (temp != 0)
3285 return temp;
3286
3287 delete_insns_since (last);
3288 }
3289
3290 /* If this machine has expensive jumps, we can do one's complement
3291 absolute value of X as (((signed) x >> (W-1)) ^ x). */
3292
3293 scalar_int_mode int_mode;
3294 if (is_int_mode (mode, &int_mode)
3295 && BRANCH_COST (optimize_insn_for_speed_p (),
3296 false) >= 2)
3297 {
3298 rtx extended = expand_shift (RSHIFT_EXPR, int_mode, op0,
3299 GET_MODE_PRECISION (int_mode) - 1,
3300 NULL_RTX, 0);
3301
3302 temp = expand_binop (int_mode, xor_optab, extended, op0, target, 0,
3303 OPTAB_LIB_WIDEN);
3304
3305 if (temp != 0)
3306 return temp;
3307 }
3308
3309 return NULL_RTX;
3310 }
3311
3312 /* A subroutine of expand_copysign, perform the copysign operation using the
3313 abs and neg primitives advertised to exist on the target. The assumption
3314 is that we have a split register file, and leaving op0 in fp registers,
3315 and not playing with subregs so much, will help the register allocator. */
3316
3317 static rtx
expand_copysign_absneg(scalar_float_mode mode,rtx op0,rtx op1,rtx target,int bitpos,bool op0_is_abs)3318 expand_copysign_absneg (scalar_float_mode mode, rtx op0, rtx op1, rtx target,
3319 int bitpos, bool op0_is_abs)
3320 {
3321 scalar_int_mode imode;
3322 enum insn_code icode;
3323 rtx sign;
3324 rtx_code_label *label;
3325
3326 if (target == op1)
3327 target = NULL_RTX;
3328
3329 /* Check if the back end provides an insn that handles signbit for the
3330 argument's mode. */
3331 icode = optab_handler (signbit_optab, mode);
3332 if (icode != CODE_FOR_nothing)
3333 {
3334 imode = as_a <scalar_int_mode> (insn_data[(int) icode].operand[0].mode);
3335 sign = gen_reg_rtx (imode);
3336 emit_unop_insn (icode, sign, op1, UNKNOWN);
3337 }
3338 else
3339 {
3340 if (GET_MODE_SIZE (mode) <= UNITS_PER_WORD)
3341 {
3342 if (!int_mode_for_mode (mode).exists (&imode))
3343 return NULL_RTX;
3344 op1 = gen_lowpart (imode, op1);
3345 }
3346 else
3347 {
3348 int word;
3349
3350 imode = word_mode;
3351 if (FLOAT_WORDS_BIG_ENDIAN)
3352 word = (GET_MODE_BITSIZE (mode) - bitpos) / BITS_PER_WORD;
3353 else
3354 word = bitpos / BITS_PER_WORD;
3355 bitpos = bitpos % BITS_PER_WORD;
3356 op1 = operand_subword_force (op1, word, mode);
3357 }
3358
3359 wide_int mask = wi::set_bit_in_zero (bitpos, GET_MODE_PRECISION (imode));
3360 sign = expand_binop (imode, and_optab, op1,
3361 immed_wide_int_const (mask, imode),
3362 NULL_RTX, 1, OPTAB_LIB_WIDEN);
3363 }
3364
3365 if (!op0_is_abs)
3366 {
3367 op0 = expand_unop (mode, abs_optab, op0, target, 0);
3368 if (op0 == NULL)
3369 return NULL_RTX;
3370 target = op0;
3371 }
3372 else
3373 {
3374 if (target == NULL_RTX)
3375 target = copy_to_reg (op0);
3376 else
3377 emit_move_insn (target, op0);
3378 }
3379
3380 label = gen_label_rtx ();
3381 emit_cmp_and_jump_insns (sign, const0_rtx, EQ, NULL_RTX, imode, 1, label);
3382
3383 if (CONST_DOUBLE_AS_FLOAT_P (op0))
3384 op0 = simplify_unary_operation (NEG, mode, op0, mode);
3385 else
3386 op0 = expand_unop (mode, neg_optab, op0, target, 0);
3387 if (op0 != target)
3388 emit_move_insn (target, op0);
3389
3390 emit_label (label);
3391
3392 return target;
3393 }
3394
3395
3396 /* A subroutine of expand_copysign, perform the entire copysign operation
3397 with integer bitmasks. BITPOS is the position of the sign bit; OP0_IS_ABS
3398 is true if op0 is known to have its sign bit clear. */
3399
3400 static rtx
expand_copysign_bit(scalar_float_mode mode,rtx op0,rtx op1,rtx target,int bitpos,bool op0_is_abs)3401 expand_copysign_bit (scalar_float_mode mode, rtx op0, rtx op1, rtx target,
3402 int bitpos, bool op0_is_abs)
3403 {
3404 scalar_int_mode imode;
3405 int word, nwords, i;
3406 rtx temp;
3407 rtx_insn *insns;
3408
3409 if (GET_MODE_SIZE (mode) <= UNITS_PER_WORD)
3410 {
3411 if (!int_mode_for_mode (mode).exists (&imode))
3412 return NULL_RTX;
3413 word = 0;
3414 nwords = 1;
3415 }
3416 else
3417 {
3418 imode = word_mode;
3419
3420 if (FLOAT_WORDS_BIG_ENDIAN)
3421 word = (GET_MODE_BITSIZE (mode) - bitpos) / BITS_PER_WORD;
3422 else
3423 word = bitpos / BITS_PER_WORD;
3424 bitpos = bitpos % BITS_PER_WORD;
3425 nwords = (GET_MODE_BITSIZE (mode) + BITS_PER_WORD - 1) / BITS_PER_WORD;
3426 }
3427
3428 wide_int mask = wi::set_bit_in_zero (bitpos, GET_MODE_PRECISION (imode));
3429
3430 if (target == 0
3431 || target == op0
3432 || target == op1
3433 || reg_overlap_mentioned_p (target, op0)
3434 || reg_overlap_mentioned_p (target, op1)
3435 || (nwords > 1 && !valid_multiword_target_p (target)))
3436 target = gen_reg_rtx (mode);
3437
3438 if (nwords > 1)
3439 {
3440 start_sequence ();
3441
3442 for (i = 0; i < nwords; ++i)
3443 {
3444 rtx targ_piece = operand_subword (target, i, 1, mode);
3445 rtx op0_piece = operand_subword_force (op0, i, mode);
3446
3447 if (i == word)
3448 {
3449 if (!op0_is_abs)
3450 op0_piece
3451 = expand_binop (imode, and_optab, op0_piece,
3452 immed_wide_int_const (~mask, imode),
3453 NULL_RTX, 1, OPTAB_LIB_WIDEN);
3454 op1 = expand_binop (imode, and_optab,
3455 operand_subword_force (op1, i, mode),
3456 immed_wide_int_const (mask, imode),
3457 NULL_RTX, 1, OPTAB_LIB_WIDEN);
3458
3459 temp = expand_binop (imode, ior_optab, op0_piece, op1,
3460 targ_piece, 1, OPTAB_LIB_WIDEN);
3461 if (temp != targ_piece)
3462 emit_move_insn (targ_piece, temp);
3463 }
3464 else
3465 emit_move_insn (targ_piece, op0_piece);
3466 }
3467
3468 insns = get_insns ();
3469 end_sequence ();
3470
3471 emit_insn (insns);
3472 }
3473 else
3474 {
3475 op1 = expand_binop (imode, and_optab, gen_lowpart (imode, op1),
3476 immed_wide_int_const (mask, imode),
3477 NULL_RTX, 1, OPTAB_LIB_WIDEN);
3478
3479 op0 = gen_lowpart (imode, op0);
3480 if (!op0_is_abs)
3481 op0 = expand_binop (imode, and_optab, op0,
3482 immed_wide_int_const (~mask, imode),
3483 NULL_RTX, 1, OPTAB_LIB_WIDEN);
3484
3485 temp = expand_binop (imode, ior_optab, op0, op1,
3486 gen_lowpart (imode, target), 1, OPTAB_LIB_WIDEN);
3487 target = lowpart_subreg_maybe_copy (mode, temp, imode);
3488 }
3489
3490 return target;
3491 }
3492
3493 /* Expand the C99 copysign operation. OP0 and OP1 must be the same
3494 scalar floating point mode. Return NULL if we do not know how to
3495 expand the operation inline. */
3496
3497 rtx
expand_copysign(rtx op0,rtx op1,rtx target)3498 expand_copysign (rtx op0, rtx op1, rtx target)
3499 {
3500 scalar_float_mode mode;
3501 const struct real_format *fmt;
3502 bool op0_is_abs;
3503 rtx temp;
3504
3505 mode = as_a <scalar_float_mode> (GET_MODE (op0));
3506 gcc_assert (GET_MODE (op1) == mode);
3507
3508 /* First try to do it with a special instruction. */
3509 temp = expand_binop (mode, copysign_optab, op0, op1,
3510 target, 0, OPTAB_DIRECT);
3511 if (temp)
3512 return temp;
3513
3514 fmt = REAL_MODE_FORMAT (mode);
3515 if (fmt == NULL || !fmt->has_signed_zero)
3516 return NULL_RTX;
3517
3518 op0_is_abs = false;
3519 if (CONST_DOUBLE_AS_FLOAT_P (op0))
3520 {
3521 if (real_isneg (CONST_DOUBLE_REAL_VALUE (op0)))
3522 op0 = simplify_unary_operation (ABS, mode, op0, mode);
3523 op0_is_abs = true;
3524 }
3525
3526 if (fmt->signbit_ro >= 0
3527 && (CONST_DOUBLE_AS_FLOAT_P (op0)
3528 || (optab_handler (neg_optab, mode) != CODE_FOR_nothing
3529 && optab_handler (abs_optab, mode) != CODE_FOR_nothing)))
3530 {
3531 temp = expand_copysign_absneg (mode, op0, op1, target,
3532 fmt->signbit_ro, op0_is_abs);
3533 if (temp)
3534 return temp;
3535 }
3536
3537 if (fmt->signbit_rw < 0)
3538 return NULL_RTX;
3539 return expand_copysign_bit (mode, op0, op1, target,
3540 fmt->signbit_rw, op0_is_abs);
3541 }
3542
3543 /* Generate an instruction whose insn-code is INSN_CODE,
3544 with two operands: an output TARGET and an input OP0.
3545 TARGET *must* be nonzero, and the output is always stored there.
3546 CODE is an rtx code such that (CODE OP0) is an rtx that describes
3547 the value that is stored into TARGET.
3548
3549 Return false if expansion failed. */
3550
3551 bool
maybe_emit_unop_insn(enum insn_code icode,rtx target,rtx op0,enum rtx_code code)3552 maybe_emit_unop_insn (enum insn_code icode, rtx target, rtx op0,
3553 enum rtx_code code)
3554 {
3555 struct expand_operand ops[2];
3556 rtx_insn *pat;
3557
3558 create_output_operand (&ops[0], target, GET_MODE (target));
3559 create_input_operand (&ops[1], op0, GET_MODE (op0));
3560 pat = maybe_gen_insn (icode, 2, ops);
3561 if (!pat)
3562 return false;
3563
3564 if (INSN_P (pat) && NEXT_INSN (pat) != NULL_RTX
3565 && code != UNKNOWN)
3566 add_equal_note (pat, ops[0].value, code, ops[1].value, NULL_RTX,
3567 GET_MODE (op0));
3568
3569 emit_insn (pat);
3570
3571 if (ops[0].value != target)
3572 emit_move_insn (target, ops[0].value);
3573 return true;
3574 }
3575 /* Generate an instruction whose insn-code is INSN_CODE,
3576 with two operands: an output TARGET and an input OP0.
3577 TARGET *must* be nonzero, and the output is always stored there.
3578 CODE is an rtx code such that (CODE OP0) is an rtx that describes
3579 the value that is stored into TARGET. */
3580
3581 void
emit_unop_insn(enum insn_code icode,rtx target,rtx op0,enum rtx_code code)3582 emit_unop_insn (enum insn_code icode, rtx target, rtx op0, enum rtx_code code)
3583 {
3584 bool ok = maybe_emit_unop_insn (icode, target, op0, code);
3585 gcc_assert (ok);
3586 }
3587
3588 struct no_conflict_data
3589 {
3590 rtx target;
3591 rtx_insn *first, *insn;
3592 bool must_stay;
3593 };
3594
3595 /* Called via note_stores by emit_libcall_block. Set P->must_stay if
3596 the currently examined clobber / store has to stay in the list of
3597 insns that constitute the actual libcall block. */
3598 static void
no_conflict_move_test(rtx dest,const_rtx set,void * p0)3599 no_conflict_move_test (rtx dest, const_rtx set, void *p0)
3600 {
3601 struct no_conflict_data *p= (struct no_conflict_data *) p0;
3602
3603 /* If this inns directly contributes to setting the target, it must stay. */
3604 if (reg_overlap_mentioned_p (p->target, dest))
3605 p->must_stay = true;
3606 /* If we haven't committed to keeping any other insns in the list yet,
3607 there is nothing more to check. */
3608 else if (p->insn == p->first)
3609 return;
3610 /* If this insn sets / clobbers a register that feeds one of the insns
3611 already in the list, this insn has to stay too. */
3612 else if (reg_overlap_mentioned_p (dest, PATTERN (p->first))
3613 || (CALL_P (p->first) && (find_reg_fusage (p->first, USE, dest)))
3614 || reg_used_between_p (dest, p->first, p->insn)
3615 /* Likewise if this insn depends on a register set by a previous
3616 insn in the list, or if it sets a result (presumably a hard
3617 register) that is set or clobbered by a previous insn.
3618 N.B. the modified_*_p (SET_DEST...) tests applied to a MEM
3619 SET_DEST perform the former check on the address, and the latter
3620 check on the MEM. */
3621 || (GET_CODE (set) == SET
3622 && (modified_in_p (SET_SRC (set), p->first)
3623 || modified_in_p (SET_DEST (set), p->first)
3624 || modified_between_p (SET_SRC (set), p->first, p->insn)
3625 || modified_between_p (SET_DEST (set), p->first, p->insn))))
3626 p->must_stay = true;
3627 }
3628
3629
3630 /* Emit code to make a call to a constant function or a library call.
3631
3632 INSNS is a list containing all insns emitted in the call.
3633 These insns leave the result in RESULT. Our block is to copy RESULT
3634 to TARGET, which is logically equivalent to EQUIV.
3635
3636 We first emit any insns that set a pseudo on the assumption that these are
3637 loading constants into registers; doing so allows them to be safely cse'ed
3638 between blocks. Then we emit all the other insns in the block, followed by
3639 an insn to move RESULT to TARGET. This last insn will have a REQ_EQUAL
3640 note with an operand of EQUIV. */
3641
3642 static void
emit_libcall_block_1(rtx_insn * insns,rtx target,rtx result,rtx equiv,bool equiv_may_trap)3643 emit_libcall_block_1 (rtx_insn *insns, rtx target, rtx result, rtx equiv,
3644 bool equiv_may_trap)
3645 {
3646 rtx final_dest = target;
3647 rtx_insn *next, *last, *insn;
3648
3649 /* If this is a reg with REG_USERVAR_P set, then it could possibly turn
3650 into a MEM later. Protect the libcall block from this change. */
3651 if (! REG_P (target) || REG_USERVAR_P (target))
3652 target = gen_reg_rtx (GET_MODE (target));
3653
3654 /* If we're using non-call exceptions, a libcall corresponding to an
3655 operation that may trap may also trap. */
3656 /* ??? See the comment in front of make_reg_eh_region_note. */
3657 if (cfun->can_throw_non_call_exceptions
3658 && (equiv_may_trap || may_trap_p (equiv)))
3659 {
3660 for (insn = insns; insn; insn = NEXT_INSN (insn))
3661 if (CALL_P (insn))
3662 {
3663 rtx note = find_reg_note (insn, REG_EH_REGION, NULL_RTX);
3664 if (note)
3665 {
3666 int lp_nr = INTVAL (XEXP (note, 0));
3667 if (lp_nr == 0 || lp_nr == INT_MIN)
3668 remove_note (insn, note);
3669 }
3670 }
3671 }
3672 else
3673 {
3674 /* Look for any CALL_INSNs in this sequence, and attach a REG_EH_REGION
3675 reg note to indicate that this call cannot throw or execute a nonlocal
3676 goto (unless there is already a REG_EH_REGION note, in which case
3677 we update it). */
3678 for (insn = insns; insn; insn = NEXT_INSN (insn))
3679 if (CALL_P (insn))
3680 make_reg_eh_region_note_nothrow_nononlocal (insn);
3681 }
3682
3683 /* First emit all insns that set pseudos. Remove them from the list as
3684 we go. Avoid insns that set pseudos which were referenced in previous
3685 insns. These can be generated by move_by_pieces, for example,
3686 to update an address. Similarly, avoid insns that reference things
3687 set in previous insns. */
3688
3689 for (insn = insns; insn; insn = next)
3690 {
3691 rtx set = single_set (insn);
3692
3693 next = NEXT_INSN (insn);
3694
3695 if (set != 0 && REG_P (SET_DEST (set))
3696 && REGNO (SET_DEST (set)) >= FIRST_PSEUDO_REGISTER)
3697 {
3698 struct no_conflict_data data;
3699
3700 data.target = const0_rtx;
3701 data.first = insns;
3702 data.insn = insn;
3703 data.must_stay = 0;
3704 note_stores (PATTERN (insn), no_conflict_move_test, &data);
3705 if (! data.must_stay)
3706 {
3707 if (PREV_INSN (insn))
3708 SET_NEXT_INSN (PREV_INSN (insn)) = next;
3709 else
3710 insns = next;
3711
3712 if (next)
3713 SET_PREV_INSN (next) = PREV_INSN (insn);
3714
3715 add_insn (insn);
3716 }
3717 }
3718
3719 /* Some ports use a loop to copy large arguments onto the stack.
3720 Don't move anything outside such a loop. */
3721 if (LABEL_P (insn))
3722 break;
3723 }
3724
3725 /* Write the remaining insns followed by the final copy. */
3726 for (insn = insns; insn; insn = next)
3727 {
3728 next = NEXT_INSN (insn);
3729
3730 add_insn (insn);
3731 }
3732
3733 last = emit_move_insn (target, result);
3734 if (equiv)
3735 set_dst_reg_note (last, REG_EQUAL, copy_rtx (equiv), target);
3736
3737 if (final_dest != target)
3738 emit_move_insn (final_dest, target);
3739 }
3740
3741 void
emit_libcall_block(rtx_insn * insns,rtx target,rtx result,rtx equiv)3742 emit_libcall_block (rtx_insn *insns, rtx target, rtx result, rtx equiv)
3743 {
3744 emit_libcall_block_1 (insns, target, result, equiv, false);
3745 }
3746
3747 /* Nonzero if we can perform a comparison of mode MODE straightforwardly.
3748 PURPOSE describes how this comparison will be used. CODE is the rtx
3749 comparison code we will be using.
3750
3751 ??? Actually, CODE is slightly weaker than that. A target is still
3752 required to implement all of the normal bcc operations, but not
3753 required to implement all (or any) of the unordered bcc operations. */
3754
3755 int
can_compare_p(enum rtx_code code,machine_mode mode,enum can_compare_purpose purpose)3756 can_compare_p (enum rtx_code code, machine_mode mode,
3757 enum can_compare_purpose purpose)
3758 {
3759 rtx test;
3760 test = gen_rtx_fmt_ee (code, mode, const0_rtx, const0_rtx);
3761 do
3762 {
3763 enum insn_code icode;
3764
3765 if (purpose == ccp_jump
3766 && (icode = optab_handler (cbranch_optab, mode)) != CODE_FOR_nothing
3767 && insn_operand_matches (icode, 0, test))
3768 return 1;
3769 if (purpose == ccp_store_flag
3770 && (icode = optab_handler (cstore_optab, mode)) != CODE_FOR_nothing
3771 && insn_operand_matches (icode, 1, test))
3772 return 1;
3773 if (purpose == ccp_cmov
3774 && optab_handler (cmov_optab, mode) != CODE_FOR_nothing)
3775 return 1;
3776
3777 mode = GET_MODE_WIDER_MODE (mode).else_void ();
3778 PUT_MODE (test, mode);
3779 }
3780 while (mode != VOIDmode);
3781
3782 return 0;
3783 }
3784
3785 /* This function is called when we are going to emit a compare instruction that
3786 compares the values found in X and Y, using the rtl operator COMPARISON.
3787
3788 If they have mode BLKmode, then SIZE specifies the size of both operands.
3789
3790 UNSIGNEDP nonzero says that the operands are unsigned;
3791 this matters if they need to be widened (as given by METHODS).
3792
3793 *PTEST is where the resulting comparison RTX is returned or NULL_RTX
3794 if we failed to produce one.
3795
3796 *PMODE is the mode of the inputs (in case they are const_int).
3797
3798 This function performs all the setup necessary so that the caller only has
3799 to emit a single comparison insn. This setup can involve doing a BLKmode
3800 comparison or emitting a library call to perform the comparison if no insn
3801 is available to handle it.
3802 The values which are passed in through pointers can be modified; the caller
3803 should perform the comparison on the modified values. Constant
3804 comparisons must have already been folded. */
3805
3806 static void
prepare_cmp_insn(rtx x,rtx y,enum rtx_code comparison,rtx size,int unsignedp,enum optab_methods methods,rtx * ptest,machine_mode * pmode)3807 prepare_cmp_insn (rtx x, rtx y, enum rtx_code comparison, rtx size,
3808 int unsignedp, enum optab_methods methods,
3809 rtx *ptest, machine_mode *pmode)
3810 {
3811 machine_mode mode = *pmode;
3812 rtx libfunc, test;
3813 machine_mode cmp_mode;
3814 enum mode_class mclass;
3815
3816 /* The other methods are not needed. */
3817 gcc_assert (methods == OPTAB_DIRECT || methods == OPTAB_WIDEN
3818 || methods == OPTAB_LIB_WIDEN);
3819
3820 /* If we are optimizing, force expensive constants into a register. */
3821 if (CONSTANT_P (x) && optimize
3822 && (rtx_cost (x, mode, COMPARE, 0, optimize_insn_for_speed_p ())
3823 > COSTS_N_INSNS (1)))
3824 x = force_reg (mode, x);
3825
3826 if (CONSTANT_P (y) && optimize
3827 && (rtx_cost (y, mode, COMPARE, 1, optimize_insn_for_speed_p ())
3828 > COSTS_N_INSNS (1)))
3829 y = force_reg (mode, y);
3830
3831 #if HAVE_cc0
3832 /* Make sure if we have a canonical comparison. The RTL
3833 documentation states that canonical comparisons are required only
3834 for targets which have cc0. */
3835 gcc_assert (!CONSTANT_P (x) || CONSTANT_P (y));
3836 #endif
3837
3838 /* Don't let both operands fail to indicate the mode. */
3839 if (GET_MODE (x) == VOIDmode && GET_MODE (y) == VOIDmode)
3840 x = force_reg (mode, x);
3841 if (mode == VOIDmode)
3842 mode = GET_MODE (x) != VOIDmode ? GET_MODE (x) : GET_MODE (y);
3843
3844 /* Handle all BLKmode compares. */
3845
3846 if (mode == BLKmode)
3847 {
3848 machine_mode result_mode;
3849 enum insn_code cmp_code;
3850 rtx result;
3851 rtx opalign
3852 = GEN_INT (MIN (MEM_ALIGN (x), MEM_ALIGN (y)) / BITS_PER_UNIT);
3853
3854 gcc_assert (size);
3855
3856 /* Try to use a memory block compare insn - either cmpstr
3857 or cmpmem will do. */
3858 opt_scalar_int_mode cmp_mode_iter;
3859 FOR_EACH_MODE_IN_CLASS (cmp_mode_iter, MODE_INT)
3860 {
3861 scalar_int_mode cmp_mode = cmp_mode_iter.require ();
3862 cmp_code = direct_optab_handler (cmpmem_optab, cmp_mode);
3863 if (cmp_code == CODE_FOR_nothing)
3864 cmp_code = direct_optab_handler (cmpstr_optab, cmp_mode);
3865 if (cmp_code == CODE_FOR_nothing)
3866 cmp_code = direct_optab_handler (cmpstrn_optab, cmp_mode);
3867 if (cmp_code == CODE_FOR_nothing)
3868 continue;
3869
3870 /* Must make sure the size fits the insn's mode. */
3871 if (CONST_INT_P (size)
3872 ? UINTVAL (size) > GET_MODE_MASK (cmp_mode)
3873 : (GET_MODE_BITSIZE (as_a <scalar_int_mode> (GET_MODE (size)))
3874 > GET_MODE_BITSIZE (cmp_mode)))
3875 continue;
3876
3877 result_mode = insn_data[cmp_code].operand[0].mode;
3878 result = gen_reg_rtx (result_mode);
3879 size = convert_to_mode (cmp_mode, size, 1);
3880 emit_insn (GEN_FCN (cmp_code) (result, x, y, size, opalign));
3881
3882 *ptest = gen_rtx_fmt_ee (comparison, VOIDmode, result, const0_rtx);
3883 *pmode = result_mode;
3884 return;
3885 }
3886
3887 if (methods != OPTAB_LIB && methods != OPTAB_LIB_WIDEN)
3888 goto fail;
3889
3890 /* Otherwise call a library function. */
3891 result = emit_block_comp_via_libcall (x, y, size);
3892
3893 x = result;
3894 y = const0_rtx;
3895 mode = TYPE_MODE (integer_type_node);
3896 methods = OPTAB_LIB_WIDEN;
3897 unsignedp = false;
3898 }
3899
3900 /* Don't allow operands to the compare to trap, as that can put the
3901 compare and branch in different basic blocks. */
3902 if (cfun->can_throw_non_call_exceptions)
3903 {
3904 if (may_trap_p (x))
3905 x = copy_to_reg (x);
3906 if (may_trap_p (y))
3907 y = copy_to_reg (y);
3908 }
3909
3910 if (GET_MODE_CLASS (mode) == MODE_CC)
3911 {
3912 enum insn_code icode = optab_handler (cbranch_optab, CCmode);
3913 test = gen_rtx_fmt_ee (comparison, VOIDmode, x, y);
3914 gcc_assert (icode != CODE_FOR_nothing
3915 && insn_operand_matches (icode, 0, test));
3916 *ptest = test;
3917 return;
3918 }
3919
3920 mclass = GET_MODE_CLASS (mode);
3921 test = gen_rtx_fmt_ee (comparison, VOIDmode, x, y);
3922 FOR_EACH_MODE_FROM (cmp_mode, mode)
3923 {
3924 enum insn_code icode;
3925 icode = optab_handler (cbranch_optab, cmp_mode);
3926 if (icode != CODE_FOR_nothing
3927 && insn_operand_matches (icode, 0, test))
3928 {
3929 rtx_insn *last = get_last_insn ();
3930 rtx op0 = prepare_operand (icode, x, 1, mode, cmp_mode, unsignedp);
3931 rtx op1 = prepare_operand (icode, y, 2, mode, cmp_mode, unsignedp);
3932 if (op0 && op1
3933 && insn_operand_matches (icode, 1, op0)
3934 && insn_operand_matches (icode, 2, op1))
3935 {
3936 XEXP (test, 0) = op0;
3937 XEXP (test, 1) = op1;
3938 *ptest = test;
3939 *pmode = cmp_mode;
3940 return;
3941 }
3942 delete_insns_since (last);
3943 }
3944
3945 if (methods == OPTAB_DIRECT || !CLASS_HAS_WIDER_MODES_P (mclass))
3946 break;
3947 }
3948
3949 if (methods != OPTAB_LIB_WIDEN)
3950 goto fail;
3951
3952 if (SCALAR_FLOAT_MODE_P (mode))
3953 {
3954 /* Small trick if UNORDERED isn't implemented by the hardware. */
3955 if (comparison == UNORDERED && rtx_equal_p (x, y))
3956 {
3957 prepare_cmp_insn (x, y, UNLT, NULL_RTX, unsignedp, OPTAB_WIDEN,
3958 ptest, pmode);
3959 if (*ptest)
3960 return;
3961 }
3962
3963 prepare_float_lib_cmp (x, y, comparison, ptest, pmode);
3964 }
3965 else
3966 {
3967 rtx result;
3968 machine_mode ret_mode;
3969
3970 /* Handle a libcall just for the mode we are using. */
3971 libfunc = optab_libfunc (cmp_optab, mode);
3972 gcc_assert (libfunc);
3973
3974 /* If we want unsigned, and this mode has a distinct unsigned
3975 comparison routine, use that. */
3976 if (unsignedp)
3977 {
3978 rtx ulibfunc = optab_libfunc (ucmp_optab, mode);
3979 if (ulibfunc)
3980 libfunc = ulibfunc;
3981 }
3982
3983 ret_mode = targetm.libgcc_cmp_return_mode ();
3984 result = emit_library_call_value (libfunc, NULL_RTX, LCT_CONST,
3985 ret_mode, x, mode, y, mode);
3986
3987 /* There are two kinds of comparison routines. Biased routines
3988 return 0/1/2, and unbiased routines return -1/0/1. Other parts
3989 of gcc expect that the comparison operation is equivalent
3990 to the modified comparison. For signed comparisons compare the
3991 result against 1 in the biased case, and zero in the unbiased
3992 case. For unsigned comparisons always compare against 1 after
3993 biasing the unbiased result by adding 1. This gives us a way to
3994 represent LTU.
3995 The comparisons in the fixed-point helper library are always
3996 biased. */
3997 x = result;
3998 y = const1_rtx;
3999
4000 if (!TARGET_LIB_INT_CMP_BIASED && !ALL_FIXED_POINT_MODE_P (mode))
4001 {
4002 if (unsignedp)
4003 x = plus_constant (ret_mode, result, 1);
4004 else
4005 y = const0_rtx;
4006 }
4007
4008 *pmode = ret_mode;
4009 prepare_cmp_insn (x, y, comparison, NULL_RTX, unsignedp, methods,
4010 ptest, pmode);
4011 }
4012
4013 return;
4014
4015 fail:
4016 *ptest = NULL_RTX;
4017 }
4018
4019 /* Before emitting an insn with code ICODE, make sure that X, which is going
4020 to be used for operand OPNUM of the insn, is converted from mode MODE to
4021 WIDER_MODE (UNSIGNEDP determines whether it is an unsigned conversion), and
4022 that it is accepted by the operand predicate. Return the new value. */
4023
4024 rtx
prepare_operand(enum insn_code icode,rtx x,int opnum,machine_mode mode,machine_mode wider_mode,int unsignedp)4025 prepare_operand (enum insn_code icode, rtx x, int opnum, machine_mode mode,
4026 machine_mode wider_mode, int unsignedp)
4027 {
4028 if (mode != wider_mode)
4029 x = convert_modes (wider_mode, mode, x, unsignedp);
4030
4031 if (!insn_operand_matches (icode, opnum, x))
4032 {
4033 machine_mode op_mode = insn_data[(int) icode].operand[opnum].mode;
4034 if (reload_completed)
4035 return NULL_RTX;
4036 if (GET_MODE (x) != op_mode && GET_MODE (x) != VOIDmode)
4037 return NULL_RTX;
4038 x = copy_to_mode_reg (op_mode, x);
4039 }
4040
4041 return x;
4042 }
4043
4044 /* Subroutine of emit_cmp_and_jump_insns; this function is called when we know
4045 we can do the branch. */
4046
4047 static void
emit_cmp_and_jump_insn_1(rtx test,machine_mode mode,rtx label,profile_probability prob)4048 emit_cmp_and_jump_insn_1 (rtx test, machine_mode mode, rtx label,
4049 profile_probability prob)
4050 {
4051 machine_mode optab_mode;
4052 enum mode_class mclass;
4053 enum insn_code icode;
4054 rtx_insn *insn;
4055
4056 mclass = GET_MODE_CLASS (mode);
4057 optab_mode = (mclass == MODE_CC) ? CCmode : mode;
4058 icode = optab_handler (cbranch_optab, optab_mode);
4059
4060 gcc_assert (icode != CODE_FOR_nothing);
4061 gcc_assert (insn_operand_matches (icode, 0, test));
4062 insn = emit_jump_insn (GEN_FCN (icode) (test, XEXP (test, 0),
4063 XEXP (test, 1), label));
4064 if (prob.initialized_p ()
4065 && profile_status_for_fn (cfun) != PROFILE_ABSENT
4066 && insn
4067 && JUMP_P (insn)
4068 && any_condjump_p (insn)
4069 && !find_reg_note (insn, REG_BR_PROB, 0))
4070 add_reg_br_prob_note (insn, prob);
4071 }
4072
4073 /* Generate code to compare X with Y so that the condition codes are
4074 set and to jump to LABEL if the condition is true. If X is a
4075 constant and Y is not a constant, then the comparison is swapped to
4076 ensure that the comparison RTL has the canonical form.
4077
4078 UNSIGNEDP nonzero says that X and Y are unsigned; this matters if they
4079 need to be widened. UNSIGNEDP is also used to select the proper
4080 branch condition code.
4081
4082 If X and Y have mode BLKmode, then SIZE specifies the size of both X and Y.
4083
4084 MODE is the mode of the inputs (in case they are const_int).
4085
4086 COMPARISON is the rtl operator to compare with (EQ, NE, GT, etc.).
4087 It will be potentially converted into an unsigned variant based on
4088 UNSIGNEDP to select a proper jump instruction.
4089
4090 PROB is the probability of jumping to LABEL. */
4091
4092 void
emit_cmp_and_jump_insns(rtx x,rtx y,enum rtx_code comparison,rtx size,machine_mode mode,int unsignedp,rtx label,profile_probability prob)4093 emit_cmp_and_jump_insns (rtx x, rtx y, enum rtx_code comparison, rtx size,
4094 machine_mode mode, int unsignedp, rtx label,
4095 profile_probability prob)
4096 {
4097 rtx op0 = x, op1 = y;
4098 rtx test;
4099
4100 /* Swap operands and condition to ensure canonical RTL. */
4101 if (swap_commutative_operands_p (x, y)
4102 && can_compare_p (swap_condition (comparison), mode, ccp_jump))
4103 {
4104 op0 = y, op1 = x;
4105 comparison = swap_condition (comparison);
4106 }
4107
4108 /* If OP0 is still a constant, then both X and Y must be constants
4109 or the opposite comparison is not supported. Force X into a register
4110 to create canonical RTL. */
4111 if (CONSTANT_P (op0))
4112 op0 = force_reg (mode, op0);
4113
4114 if (unsignedp)
4115 comparison = unsigned_condition (comparison);
4116
4117 prepare_cmp_insn (op0, op1, comparison, size, unsignedp, OPTAB_LIB_WIDEN,
4118 &test, &mode);
4119 emit_cmp_and_jump_insn_1 (test, mode, label, prob);
4120 }
4121
4122
4123 /* Emit a library call comparison between floating point X and Y.
4124 COMPARISON is the rtl operator to compare with (EQ, NE, GT, etc.). */
4125
4126 static void
prepare_float_lib_cmp(rtx x,rtx y,enum rtx_code comparison,rtx * ptest,machine_mode * pmode)4127 prepare_float_lib_cmp (rtx x, rtx y, enum rtx_code comparison,
4128 rtx *ptest, machine_mode *pmode)
4129 {
4130 enum rtx_code swapped = swap_condition (comparison);
4131 enum rtx_code reversed = reverse_condition_maybe_unordered (comparison);
4132 machine_mode orig_mode = GET_MODE (x);
4133 machine_mode mode;
4134 rtx true_rtx, false_rtx;
4135 rtx value, target, equiv;
4136 rtx_insn *insns;
4137 rtx libfunc = 0;
4138 bool reversed_p = false;
4139 scalar_int_mode cmp_mode = targetm.libgcc_cmp_return_mode ();
4140
4141 FOR_EACH_MODE_FROM (mode, orig_mode)
4142 {
4143 if (code_to_optab (comparison)
4144 && (libfunc = optab_libfunc (code_to_optab (comparison), mode)))
4145 break;
4146
4147 if (code_to_optab (swapped)
4148 && (libfunc = optab_libfunc (code_to_optab (swapped), mode)))
4149 {
4150 std::swap (x, y);
4151 comparison = swapped;
4152 break;
4153 }
4154
4155 if (code_to_optab (reversed)
4156 && (libfunc = optab_libfunc (code_to_optab (reversed), mode)))
4157 {
4158 comparison = reversed;
4159 reversed_p = true;
4160 break;
4161 }
4162 }
4163
4164 gcc_assert (mode != VOIDmode);
4165
4166 if (mode != orig_mode)
4167 {
4168 x = convert_to_mode (mode, x, 0);
4169 y = convert_to_mode (mode, y, 0);
4170 }
4171
4172 /* Attach a REG_EQUAL note describing the semantics of the libcall to
4173 the RTL. The allows the RTL optimizers to delete the libcall if the
4174 condition can be determined at compile-time. */
4175 if (comparison == UNORDERED
4176 || FLOAT_LIB_COMPARE_RETURNS_BOOL (mode, comparison))
4177 {
4178 true_rtx = const_true_rtx;
4179 false_rtx = const0_rtx;
4180 }
4181 else
4182 {
4183 switch (comparison)
4184 {
4185 case EQ:
4186 true_rtx = const0_rtx;
4187 false_rtx = const_true_rtx;
4188 break;
4189
4190 case NE:
4191 true_rtx = const_true_rtx;
4192 false_rtx = const0_rtx;
4193 break;
4194
4195 case GT:
4196 true_rtx = const1_rtx;
4197 false_rtx = const0_rtx;
4198 break;
4199
4200 case GE:
4201 true_rtx = const0_rtx;
4202 false_rtx = constm1_rtx;
4203 break;
4204
4205 case LT:
4206 true_rtx = constm1_rtx;
4207 false_rtx = const0_rtx;
4208 break;
4209
4210 case LE:
4211 true_rtx = const0_rtx;
4212 false_rtx = const1_rtx;
4213 break;
4214
4215 default:
4216 gcc_unreachable ();
4217 }
4218 }
4219
4220 if (comparison == UNORDERED)
4221 {
4222 rtx temp = simplify_gen_relational (NE, cmp_mode, mode, x, x);
4223 equiv = simplify_gen_relational (NE, cmp_mode, mode, y, y);
4224 equiv = simplify_gen_ternary (IF_THEN_ELSE, cmp_mode, cmp_mode,
4225 temp, const_true_rtx, equiv);
4226 }
4227 else
4228 {
4229 equiv = simplify_gen_relational (comparison, cmp_mode, mode, x, y);
4230 if (! FLOAT_LIB_COMPARE_RETURNS_BOOL (mode, comparison))
4231 equiv = simplify_gen_ternary (IF_THEN_ELSE, cmp_mode, cmp_mode,
4232 equiv, true_rtx, false_rtx);
4233 }
4234
4235 start_sequence ();
4236 value = emit_library_call_value (libfunc, NULL_RTX, LCT_CONST,
4237 cmp_mode, x, mode, y, mode);
4238 insns = get_insns ();
4239 end_sequence ();
4240
4241 target = gen_reg_rtx (cmp_mode);
4242 emit_libcall_block (insns, target, value, equiv);
4243
4244 if (comparison == UNORDERED
4245 || FLOAT_LIB_COMPARE_RETURNS_BOOL (mode, comparison)
4246 || reversed_p)
4247 *ptest = gen_rtx_fmt_ee (reversed_p ? EQ : NE, VOIDmode, target, false_rtx);
4248 else
4249 *ptest = gen_rtx_fmt_ee (comparison, VOIDmode, target, const0_rtx);
4250
4251 *pmode = cmp_mode;
4252 }
4253
4254 /* Generate code to indirectly jump to a location given in the rtx LOC. */
4255
4256 void
emit_indirect_jump(rtx loc)4257 emit_indirect_jump (rtx loc)
4258 {
4259 if (!targetm.have_indirect_jump ())
4260 sorry ("indirect jumps are not available on this target");
4261 else
4262 {
4263 struct expand_operand ops[1];
4264 create_address_operand (&ops[0], loc);
4265 expand_jump_insn (targetm.code_for_indirect_jump, 1, ops);
4266 emit_barrier ();
4267 }
4268 }
4269
4270
4271 /* Emit a conditional move instruction if the machine supports one for that
4272 condition and machine mode.
4273
4274 OP0 and OP1 are the operands that should be compared using CODE. CMODE is
4275 the mode to use should they be constants. If it is VOIDmode, they cannot
4276 both be constants.
4277
4278 OP2 should be stored in TARGET if the comparison is true, otherwise OP3
4279 should be stored there. MODE is the mode to use should they be constants.
4280 If it is VOIDmode, they cannot both be constants.
4281
4282 The result is either TARGET (perhaps modified) or NULL_RTX if the operation
4283 is not supported. */
4284
4285 rtx
emit_conditional_move(rtx target,enum rtx_code code,rtx op0,rtx op1,machine_mode cmode,rtx op2,rtx op3,machine_mode mode,int unsignedp)4286 emit_conditional_move (rtx target, enum rtx_code code, rtx op0, rtx op1,
4287 machine_mode cmode, rtx op2, rtx op3,
4288 machine_mode mode, int unsignedp)
4289 {
4290 rtx comparison;
4291 rtx_insn *last;
4292 enum insn_code icode;
4293 enum rtx_code reversed;
4294
4295 /* If the two source operands are identical, that's just a move. */
4296
4297 if (rtx_equal_p (op2, op3))
4298 {
4299 if (!target)
4300 target = gen_reg_rtx (mode);
4301
4302 emit_move_insn (target, op3);
4303 return target;
4304 }
4305
4306 /* If one operand is constant, make it the second one. Only do this
4307 if the other operand is not constant as well. */
4308
4309 if (swap_commutative_operands_p (op0, op1))
4310 {
4311 std::swap (op0, op1);
4312 code = swap_condition (code);
4313 }
4314
4315 /* get_condition will prefer to generate LT and GT even if the old
4316 comparison was against zero, so undo that canonicalization here since
4317 comparisons against zero are cheaper. */
4318 if (code == LT && op1 == const1_rtx)
4319 code = LE, op1 = const0_rtx;
4320 else if (code == GT && op1 == constm1_rtx)
4321 code = GE, op1 = const0_rtx;
4322
4323 if (cmode == VOIDmode)
4324 cmode = GET_MODE (op0);
4325
4326 enum rtx_code orig_code = code;
4327 bool swapped = false;
4328 if (swap_commutative_operands_p (op2, op3)
4329 && ((reversed = reversed_comparison_code_parts (code, op0, op1, NULL))
4330 != UNKNOWN))
4331 {
4332 std::swap (op2, op3);
4333 code = reversed;
4334 swapped = true;
4335 }
4336
4337 if (mode == VOIDmode)
4338 mode = GET_MODE (op2);
4339
4340 icode = direct_optab_handler (movcc_optab, mode);
4341
4342 if (icode == CODE_FOR_nothing)
4343 return NULL_RTX;
4344
4345 if (!target)
4346 target = gen_reg_rtx (mode);
4347
4348 for (int pass = 0; ; pass++)
4349 {
4350 code = unsignedp ? unsigned_condition (code) : code;
4351 comparison = simplify_gen_relational (code, VOIDmode, cmode, op0, op1);
4352
4353 /* We can get const0_rtx or const_true_rtx in some circumstances. Just
4354 punt and let the caller figure out how best to deal with this
4355 situation. */
4356 if (COMPARISON_P (comparison))
4357 {
4358 saved_pending_stack_adjust save;
4359 save_pending_stack_adjust (&save);
4360 last = get_last_insn ();
4361 do_pending_stack_adjust ();
4362 machine_mode cmpmode = cmode;
4363 prepare_cmp_insn (XEXP (comparison, 0), XEXP (comparison, 1),
4364 GET_CODE (comparison), NULL_RTX, unsignedp,
4365 OPTAB_WIDEN, &comparison, &cmpmode);
4366 if (comparison)
4367 {
4368 struct expand_operand ops[4];
4369
4370 create_output_operand (&ops[0], target, mode);
4371 create_fixed_operand (&ops[1], comparison);
4372 create_input_operand (&ops[2], op2, mode);
4373 create_input_operand (&ops[3], op3, mode);
4374 if (maybe_expand_insn (icode, 4, ops))
4375 {
4376 if (ops[0].value != target)
4377 convert_move (target, ops[0].value, false);
4378 return target;
4379 }
4380 }
4381 delete_insns_since (last);
4382 restore_pending_stack_adjust (&save);
4383 }
4384
4385 if (pass == 1)
4386 return NULL_RTX;
4387
4388 /* If the preferred op2/op3 order is not usable, retry with other
4389 operand order, perhaps it will expand successfully. */
4390 if (swapped)
4391 code = orig_code;
4392 else if ((reversed = reversed_comparison_code_parts (orig_code, op0, op1,
4393 NULL))
4394 != UNKNOWN)
4395 code = reversed;
4396 else
4397 return NULL_RTX;
4398 std::swap (op2, op3);
4399 }
4400 }
4401
4402
4403 /* Emit a conditional negate or bitwise complement using the
4404 negcc or notcc optabs if available. Return NULL_RTX if such operations
4405 are not available. Otherwise return the RTX holding the result.
4406 TARGET is the desired destination of the result. COMP is the comparison
4407 on which to negate. If COND is true move into TARGET the negation
4408 or bitwise complement of OP1. Otherwise move OP2 into TARGET.
4409 CODE is either NEG or NOT. MODE is the machine mode in which the
4410 operation is performed. */
4411
4412 rtx
emit_conditional_neg_or_complement(rtx target,rtx_code code,machine_mode mode,rtx cond,rtx op1,rtx op2)4413 emit_conditional_neg_or_complement (rtx target, rtx_code code,
4414 machine_mode mode, rtx cond, rtx op1,
4415 rtx op2)
4416 {
4417 optab op = unknown_optab;
4418 if (code == NEG)
4419 op = negcc_optab;
4420 else if (code == NOT)
4421 op = notcc_optab;
4422 else
4423 gcc_unreachable ();
4424
4425 insn_code icode = direct_optab_handler (op, mode);
4426
4427 if (icode == CODE_FOR_nothing)
4428 return NULL_RTX;
4429
4430 if (!target)
4431 target = gen_reg_rtx (mode);
4432
4433 rtx_insn *last = get_last_insn ();
4434 struct expand_operand ops[4];
4435
4436 create_output_operand (&ops[0], target, mode);
4437 create_fixed_operand (&ops[1], cond);
4438 create_input_operand (&ops[2], op1, mode);
4439 create_input_operand (&ops[3], op2, mode);
4440
4441 if (maybe_expand_insn (icode, 4, ops))
4442 {
4443 if (ops[0].value != target)
4444 convert_move (target, ops[0].value, false);
4445
4446 return target;
4447 }
4448 delete_insns_since (last);
4449 return NULL_RTX;
4450 }
4451
4452 /* Emit a conditional addition instruction if the machine supports one for that
4453 condition and machine mode.
4454
4455 OP0 and OP1 are the operands that should be compared using CODE. CMODE is
4456 the mode to use should they be constants. If it is VOIDmode, they cannot
4457 both be constants.
4458
4459 OP2 should be stored in TARGET if the comparison is false, otherwise OP2+OP3
4460 should be stored there. MODE is the mode to use should they be constants.
4461 If it is VOIDmode, they cannot both be constants.
4462
4463 The result is either TARGET (perhaps modified) or NULL_RTX if the operation
4464 is not supported. */
4465
4466 rtx
emit_conditional_add(rtx target,enum rtx_code code,rtx op0,rtx op1,machine_mode cmode,rtx op2,rtx op3,machine_mode mode,int unsignedp)4467 emit_conditional_add (rtx target, enum rtx_code code, rtx op0, rtx op1,
4468 machine_mode cmode, rtx op2, rtx op3,
4469 machine_mode mode, int unsignedp)
4470 {
4471 rtx comparison;
4472 rtx_insn *last;
4473 enum insn_code icode;
4474
4475 /* If one operand is constant, make it the second one. Only do this
4476 if the other operand is not constant as well. */
4477
4478 if (swap_commutative_operands_p (op0, op1))
4479 {
4480 std::swap (op0, op1);
4481 code = swap_condition (code);
4482 }
4483
4484 /* get_condition will prefer to generate LT and GT even if the old
4485 comparison was against zero, so undo that canonicalization here since
4486 comparisons against zero are cheaper. */
4487 if (code == LT && op1 == const1_rtx)
4488 code = LE, op1 = const0_rtx;
4489 else if (code == GT && op1 == constm1_rtx)
4490 code = GE, op1 = const0_rtx;
4491
4492 if (cmode == VOIDmode)
4493 cmode = GET_MODE (op0);
4494
4495 if (mode == VOIDmode)
4496 mode = GET_MODE (op2);
4497
4498 icode = optab_handler (addcc_optab, mode);
4499
4500 if (icode == CODE_FOR_nothing)
4501 return 0;
4502
4503 if (!target)
4504 target = gen_reg_rtx (mode);
4505
4506 code = unsignedp ? unsigned_condition (code) : code;
4507 comparison = simplify_gen_relational (code, VOIDmode, cmode, op0, op1);
4508
4509 /* We can get const0_rtx or const_true_rtx in some circumstances. Just
4510 return NULL and let the caller figure out how best to deal with this
4511 situation. */
4512 if (!COMPARISON_P (comparison))
4513 return NULL_RTX;
4514
4515 do_pending_stack_adjust ();
4516 last = get_last_insn ();
4517 prepare_cmp_insn (XEXP (comparison, 0), XEXP (comparison, 1),
4518 GET_CODE (comparison), NULL_RTX, unsignedp, OPTAB_WIDEN,
4519 &comparison, &cmode);
4520 if (comparison)
4521 {
4522 struct expand_operand ops[4];
4523
4524 create_output_operand (&ops[0], target, mode);
4525 create_fixed_operand (&ops[1], comparison);
4526 create_input_operand (&ops[2], op2, mode);
4527 create_input_operand (&ops[3], op3, mode);
4528 if (maybe_expand_insn (icode, 4, ops))
4529 {
4530 if (ops[0].value != target)
4531 convert_move (target, ops[0].value, false);
4532 return target;
4533 }
4534 }
4535 delete_insns_since (last);
4536 return NULL_RTX;
4537 }
4538
4539 /* These functions attempt to generate an insn body, rather than
4540 emitting the insn, but if the gen function already emits them, we
4541 make no attempt to turn them back into naked patterns. */
4542
4543 /* Generate and return an insn body to add Y to X. */
4544
4545 rtx_insn *
gen_add2_insn(rtx x,rtx y)4546 gen_add2_insn (rtx x, rtx y)
4547 {
4548 enum insn_code icode = optab_handler (add_optab, GET_MODE (x));
4549
4550 gcc_assert (insn_operand_matches (icode, 0, x));
4551 gcc_assert (insn_operand_matches (icode, 1, x));
4552 gcc_assert (insn_operand_matches (icode, 2, y));
4553
4554 return GEN_FCN (icode) (x, x, y);
4555 }
4556
4557 /* Generate and return an insn body to add r1 and c,
4558 storing the result in r0. */
4559
4560 rtx_insn *
gen_add3_insn(rtx r0,rtx r1,rtx c)4561 gen_add3_insn (rtx r0, rtx r1, rtx c)
4562 {
4563 enum insn_code icode = optab_handler (add_optab, GET_MODE (r0));
4564
4565 if (icode == CODE_FOR_nothing
4566 || !insn_operand_matches (icode, 0, r0)
4567 || !insn_operand_matches (icode, 1, r1)
4568 || !insn_operand_matches (icode, 2, c))
4569 return NULL;
4570
4571 return GEN_FCN (icode) (r0, r1, c);
4572 }
4573
4574 int
have_add2_insn(rtx x,rtx y)4575 have_add2_insn (rtx x, rtx y)
4576 {
4577 enum insn_code icode;
4578
4579 gcc_assert (GET_MODE (x) != VOIDmode);
4580
4581 icode = optab_handler (add_optab, GET_MODE (x));
4582
4583 if (icode == CODE_FOR_nothing)
4584 return 0;
4585
4586 if (!insn_operand_matches (icode, 0, x)
4587 || !insn_operand_matches (icode, 1, x)
4588 || !insn_operand_matches (icode, 2, y))
4589 return 0;
4590
4591 return 1;
4592 }
4593
4594 /* Generate and return an insn body to add Y to X. */
4595
4596 rtx_insn *
gen_addptr3_insn(rtx x,rtx y,rtx z)4597 gen_addptr3_insn (rtx x, rtx y, rtx z)
4598 {
4599 enum insn_code icode = optab_handler (addptr3_optab, GET_MODE (x));
4600
4601 gcc_assert (insn_operand_matches (icode, 0, x));
4602 gcc_assert (insn_operand_matches (icode, 1, y));
4603 gcc_assert (insn_operand_matches (icode, 2, z));
4604
4605 return GEN_FCN (icode) (x, y, z);
4606 }
4607
4608 /* Return true if the target implements an addptr pattern and X, Y,
4609 and Z are valid for the pattern predicates. */
4610
4611 int
have_addptr3_insn(rtx x,rtx y,rtx z)4612 have_addptr3_insn (rtx x, rtx y, rtx z)
4613 {
4614 enum insn_code icode;
4615
4616 gcc_assert (GET_MODE (x) != VOIDmode);
4617
4618 icode = optab_handler (addptr3_optab, GET_MODE (x));
4619
4620 if (icode == CODE_FOR_nothing)
4621 return 0;
4622
4623 if (!insn_operand_matches (icode, 0, x)
4624 || !insn_operand_matches (icode, 1, y)
4625 || !insn_operand_matches (icode, 2, z))
4626 return 0;
4627
4628 return 1;
4629 }
4630
4631 /* Generate and return an insn body to subtract Y from X. */
4632
4633 rtx_insn *
gen_sub2_insn(rtx x,rtx y)4634 gen_sub2_insn (rtx x, rtx y)
4635 {
4636 enum insn_code icode = optab_handler (sub_optab, GET_MODE (x));
4637
4638 gcc_assert (insn_operand_matches (icode, 0, x));
4639 gcc_assert (insn_operand_matches (icode, 1, x));
4640 gcc_assert (insn_operand_matches (icode, 2, y));
4641
4642 return GEN_FCN (icode) (x, x, y);
4643 }
4644
4645 /* Generate and return an insn body to subtract r1 and c,
4646 storing the result in r0. */
4647
4648 rtx_insn *
gen_sub3_insn(rtx r0,rtx r1,rtx c)4649 gen_sub3_insn (rtx r0, rtx r1, rtx c)
4650 {
4651 enum insn_code icode = optab_handler (sub_optab, GET_MODE (r0));
4652
4653 if (icode == CODE_FOR_nothing
4654 || !insn_operand_matches (icode, 0, r0)
4655 || !insn_operand_matches (icode, 1, r1)
4656 || !insn_operand_matches (icode, 2, c))
4657 return NULL;
4658
4659 return GEN_FCN (icode) (r0, r1, c);
4660 }
4661
4662 int
have_sub2_insn(rtx x,rtx y)4663 have_sub2_insn (rtx x, rtx y)
4664 {
4665 enum insn_code icode;
4666
4667 gcc_assert (GET_MODE (x) != VOIDmode);
4668
4669 icode = optab_handler (sub_optab, GET_MODE (x));
4670
4671 if (icode == CODE_FOR_nothing)
4672 return 0;
4673
4674 if (!insn_operand_matches (icode, 0, x)
4675 || !insn_operand_matches (icode, 1, x)
4676 || !insn_operand_matches (icode, 2, y))
4677 return 0;
4678
4679 return 1;
4680 }
4681
4682 /* Generate the body of an insn to extend Y (with mode MFROM)
4683 into X (with mode MTO). Do zero-extension if UNSIGNEDP is nonzero. */
4684
4685 rtx_insn *
gen_extend_insn(rtx x,rtx y,machine_mode mto,machine_mode mfrom,int unsignedp)4686 gen_extend_insn (rtx x, rtx y, machine_mode mto,
4687 machine_mode mfrom, int unsignedp)
4688 {
4689 enum insn_code icode = can_extend_p (mto, mfrom, unsignedp);
4690 return GEN_FCN (icode) (x, y);
4691 }
4692
4693 /* Generate code to convert FROM to floating point
4694 and store in TO. FROM must be fixed point and not VOIDmode.
4695 UNSIGNEDP nonzero means regard FROM as unsigned.
4696 Normally this is done by correcting the final value
4697 if it is negative. */
4698
4699 void
expand_float(rtx to,rtx from,int unsignedp)4700 expand_float (rtx to, rtx from, int unsignedp)
4701 {
4702 enum insn_code icode;
4703 rtx target = to;
4704 scalar_mode from_mode, to_mode;
4705 machine_mode fmode, imode;
4706 bool can_do_signed = false;
4707
4708 /* Crash now, because we won't be able to decide which mode to use. */
4709 gcc_assert (GET_MODE (from) != VOIDmode);
4710
4711 /* Look for an insn to do the conversion. Do it in the specified
4712 modes if possible; otherwise convert either input, output or both to
4713 wider mode. If the integer mode is wider than the mode of FROM,
4714 we can do the conversion signed even if the input is unsigned. */
4715
4716 FOR_EACH_MODE_FROM (fmode, GET_MODE (to))
4717 FOR_EACH_MODE_FROM (imode, GET_MODE (from))
4718 {
4719 int doing_unsigned = unsignedp;
4720
4721 if (fmode != GET_MODE (to)
4722 && (significand_size (fmode)
4723 < GET_MODE_UNIT_PRECISION (GET_MODE (from))))
4724 continue;
4725
4726 icode = can_float_p (fmode, imode, unsignedp);
4727 if (icode == CODE_FOR_nothing && unsignedp)
4728 {
4729 enum insn_code scode = can_float_p (fmode, imode, 0);
4730 if (scode != CODE_FOR_nothing)
4731 can_do_signed = true;
4732 if (imode != GET_MODE (from))
4733 icode = scode, doing_unsigned = 0;
4734 }
4735
4736 if (icode != CODE_FOR_nothing)
4737 {
4738 if (imode != GET_MODE (from))
4739 from = convert_to_mode (imode, from, unsignedp);
4740
4741 if (fmode != GET_MODE (to))
4742 target = gen_reg_rtx (fmode);
4743
4744 emit_unop_insn (icode, target, from,
4745 doing_unsigned ? UNSIGNED_FLOAT : FLOAT);
4746
4747 if (target != to)
4748 convert_move (to, target, 0);
4749 return;
4750 }
4751 }
4752
4753 /* Unsigned integer, and no way to convert directly. Convert as signed,
4754 then unconditionally adjust the result. */
4755 if (unsignedp
4756 && can_do_signed
4757 && is_a <scalar_mode> (GET_MODE (to), &to_mode)
4758 && is_a <scalar_mode> (GET_MODE (from), &from_mode))
4759 {
4760 opt_scalar_mode fmode_iter;
4761 rtx_code_label *label = gen_label_rtx ();
4762 rtx temp;
4763 REAL_VALUE_TYPE offset;
4764
4765 /* Look for a usable floating mode FMODE wider than the source and at
4766 least as wide as the target. Using FMODE will avoid rounding woes
4767 with unsigned values greater than the signed maximum value. */
4768
4769 FOR_EACH_MODE_FROM (fmode_iter, to_mode)
4770 {
4771 scalar_mode fmode = fmode_iter.require ();
4772 if (GET_MODE_PRECISION (from_mode) < GET_MODE_BITSIZE (fmode)
4773 && can_float_p (fmode, from_mode, 0) != CODE_FOR_nothing)
4774 break;
4775 }
4776
4777 if (!fmode_iter.exists (&fmode))
4778 {
4779 /* There is no such mode. Pretend the target is wide enough. */
4780 fmode = to_mode;
4781
4782 /* Avoid double-rounding when TO is narrower than FROM. */
4783 if ((significand_size (fmode) + 1)
4784 < GET_MODE_PRECISION (from_mode))
4785 {
4786 rtx temp1;
4787 rtx_code_label *neglabel = gen_label_rtx ();
4788
4789 /* Don't use TARGET if it isn't a register, is a hard register,
4790 or is the wrong mode. */
4791 if (!REG_P (target)
4792 || REGNO (target) < FIRST_PSEUDO_REGISTER
4793 || GET_MODE (target) != fmode)
4794 target = gen_reg_rtx (fmode);
4795
4796 imode = from_mode;
4797 do_pending_stack_adjust ();
4798
4799 /* Test whether the sign bit is set. */
4800 emit_cmp_and_jump_insns (from, const0_rtx, LT, NULL_RTX, imode,
4801 0, neglabel);
4802
4803 /* The sign bit is not set. Convert as signed. */
4804 expand_float (target, from, 0);
4805 emit_jump_insn (targetm.gen_jump (label));
4806 emit_barrier ();
4807
4808 /* The sign bit is set.
4809 Convert to a usable (positive signed) value by shifting right
4810 one bit, while remembering if a nonzero bit was shifted
4811 out; i.e., compute (from & 1) | (from >> 1). */
4812
4813 emit_label (neglabel);
4814 temp = expand_binop (imode, and_optab, from, const1_rtx,
4815 NULL_RTX, 1, OPTAB_LIB_WIDEN);
4816 temp1 = expand_shift (RSHIFT_EXPR, imode, from, 1, NULL_RTX, 1);
4817 temp = expand_binop (imode, ior_optab, temp, temp1, temp, 1,
4818 OPTAB_LIB_WIDEN);
4819 expand_float (target, temp, 0);
4820
4821 /* Multiply by 2 to undo the shift above. */
4822 temp = expand_binop (fmode, add_optab, target, target,
4823 target, 0, OPTAB_LIB_WIDEN);
4824 if (temp != target)
4825 emit_move_insn (target, temp);
4826
4827 do_pending_stack_adjust ();
4828 emit_label (label);
4829 goto done;
4830 }
4831 }
4832
4833 /* If we are about to do some arithmetic to correct for an
4834 unsigned operand, do it in a pseudo-register. */
4835
4836 if (to_mode != fmode
4837 || !REG_P (to) || REGNO (to) < FIRST_PSEUDO_REGISTER)
4838 target = gen_reg_rtx (fmode);
4839
4840 /* Convert as signed integer to floating. */
4841 expand_float (target, from, 0);
4842
4843 /* If FROM is negative (and therefore TO is negative),
4844 correct its value by 2**bitwidth. */
4845
4846 do_pending_stack_adjust ();
4847 emit_cmp_and_jump_insns (from, const0_rtx, GE, NULL_RTX, from_mode,
4848 0, label);
4849
4850
4851 real_2expN (&offset, GET_MODE_PRECISION (from_mode), fmode);
4852 temp = expand_binop (fmode, add_optab, target,
4853 const_double_from_real_value (offset, fmode),
4854 target, 0, OPTAB_LIB_WIDEN);
4855 if (temp != target)
4856 emit_move_insn (target, temp);
4857
4858 do_pending_stack_adjust ();
4859 emit_label (label);
4860 goto done;
4861 }
4862
4863 /* No hardware instruction available; call a library routine. */
4864 {
4865 rtx libfunc;
4866 rtx_insn *insns;
4867 rtx value;
4868 convert_optab tab = unsignedp ? ufloat_optab : sfloat_optab;
4869
4870 if (is_narrower_int_mode (GET_MODE (from), SImode))
4871 from = convert_to_mode (SImode, from, unsignedp);
4872
4873 libfunc = convert_optab_libfunc (tab, GET_MODE (to), GET_MODE (from));
4874 gcc_assert (libfunc);
4875
4876 start_sequence ();
4877
4878 value = emit_library_call_value (libfunc, NULL_RTX, LCT_CONST,
4879 GET_MODE (to), from, GET_MODE (from));
4880 insns = get_insns ();
4881 end_sequence ();
4882
4883 emit_libcall_block (insns, target, value,
4884 gen_rtx_fmt_e (unsignedp ? UNSIGNED_FLOAT : FLOAT,
4885 GET_MODE (to), from));
4886 }
4887
4888 done:
4889
4890 /* Copy result to requested destination
4891 if we have been computing in a temp location. */
4892
4893 if (target != to)
4894 {
4895 if (GET_MODE (target) == GET_MODE (to))
4896 emit_move_insn (to, target);
4897 else
4898 convert_move (to, target, 0);
4899 }
4900 }
4901
4902 /* Generate code to convert FROM to fixed point and store in TO. FROM
4903 must be floating point. */
4904
4905 void
expand_fix(rtx to,rtx from,int unsignedp)4906 expand_fix (rtx to, rtx from, int unsignedp)
4907 {
4908 enum insn_code icode;
4909 rtx target = to;
4910 machine_mode fmode, imode;
4911 opt_scalar_mode fmode_iter;
4912 bool must_trunc = false;
4913
4914 /* We first try to find a pair of modes, one real and one integer, at
4915 least as wide as FROM and TO, respectively, in which we can open-code
4916 this conversion. If the integer mode is wider than the mode of TO,
4917 we can do the conversion either signed or unsigned. */
4918
4919 FOR_EACH_MODE_FROM (fmode, GET_MODE (from))
4920 FOR_EACH_MODE_FROM (imode, GET_MODE (to))
4921 {
4922 int doing_unsigned = unsignedp;
4923
4924 icode = can_fix_p (imode, fmode, unsignedp, &must_trunc);
4925 if (icode == CODE_FOR_nothing && imode != GET_MODE (to) && unsignedp)
4926 icode = can_fix_p (imode, fmode, 0, &must_trunc), doing_unsigned = 0;
4927
4928 if (icode != CODE_FOR_nothing)
4929 {
4930 rtx_insn *last = get_last_insn ();
4931 if (fmode != GET_MODE (from))
4932 from = convert_to_mode (fmode, from, 0);
4933
4934 if (must_trunc)
4935 {
4936 rtx temp = gen_reg_rtx (GET_MODE (from));
4937 from = expand_unop (GET_MODE (from), ftrunc_optab, from,
4938 temp, 0);
4939 }
4940
4941 if (imode != GET_MODE (to))
4942 target = gen_reg_rtx (imode);
4943
4944 if (maybe_emit_unop_insn (icode, target, from,
4945 doing_unsigned ? UNSIGNED_FIX : FIX))
4946 {
4947 if (target != to)
4948 convert_move (to, target, unsignedp);
4949 return;
4950 }
4951 delete_insns_since (last);
4952 }
4953 }
4954
4955 /* For an unsigned conversion, there is one more way to do it.
4956 If we have a signed conversion, we generate code that compares
4957 the real value to the largest representable positive number. If if
4958 is smaller, the conversion is done normally. Otherwise, subtract
4959 one plus the highest signed number, convert, and add it back.
4960
4961 We only need to check all real modes, since we know we didn't find
4962 anything with a wider integer mode.
4963
4964 This code used to extend FP value into mode wider than the destination.
4965 This is needed for decimal float modes which cannot accurately
4966 represent one plus the highest signed number of the same size, but
4967 not for binary modes. Consider, for instance conversion from SFmode
4968 into DImode.
4969
4970 The hot path through the code is dealing with inputs smaller than 2^63
4971 and doing just the conversion, so there is no bits to lose.
4972
4973 In the other path we know the value is positive in the range 2^63..2^64-1
4974 inclusive. (as for other input overflow happens and result is undefined)
4975 So we know that the most important bit set in mantissa corresponds to
4976 2^63. The subtraction of 2^63 should not generate any rounding as it
4977 simply clears out that bit. The rest is trivial. */
4978
4979 scalar_int_mode to_mode;
4980 if (unsignedp
4981 && is_a <scalar_int_mode> (GET_MODE (to), &to_mode)
4982 && HWI_COMPUTABLE_MODE_P (to_mode))
4983 FOR_EACH_MODE_FROM (fmode_iter, as_a <scalar_mode> (GET_MODE (from)))
4984 {
4985 scalar_mode fmode = fmode_iter.require ();
4986 if (CODE_FOR_nothing != can_fix_p (to_mode, fmode,
4987 0, &must_trunc)
4988 && (!DECIMAL_FLOAT_MODE_P (fmode)
4989 || (GET_MODE_BITSIZE (fmode) > GET_MODE_PRECISION (to_mode))))
4990 {
4991 int bitsize;
4992 REAL_VALUE_TYPE offset;
4993 rtx limit;
4994 rtx_code_label *lab1, *lab2;
4995 rtx_insn *insn;
4996
4997 bitsize = GET_MODE_PRECISION (to_mode);
4998 real_2expN (&offset, bitsize - 1, fmode);
4999 limit = const_double_from_real_value (offset, fmode);
5000 lab1 = gen_label_rtx ();
5001 lab2 = gen_label_rtx ();
5002
5003 if (fmode != GET_MODE (from))
5004 from = convert_to_mode (fmode, from, 0);
5005
5006 /* See if we need to do the subtraction. */
5007 do_pending_stack_adjust ();
5008 emit_cmp_and_jump_insns (from, limit, GE, NULL_RTX,
5009 GET_MODE (from), 0, lab1);
5010
5011 /* If not, do the signed "fix" and branch around fixup code. */
5012 expand_fix (to, from, 0);
5013 emit_jump_insn (targetm.gen_jump (lab2));
5014 emit_barrier ();
5015
5016 /* Otherwise, subtract 2**(N-1), convert to signed number,
5017 then add 2**(N-1). Do the addition using XOR since this
5018 will often generate better code. */
5019 emit_label (lab1);
5020 target = expand_binop (GET_MODE (from), sub_optab, from, limit,
5021 NULL_RTX, 0, OPTAB_LIB_WIDEN);
5022 expand_fix (to, target, 0);
5023 target = expand_binop (to_mode, xor_optab, to,
5024 gen_int_mode
5025 (HOST_WIDE_INT_1 << (bitsize - 1),
5026 to_mode),
5027 to, 1, OPTAB_LIB_WIDEN);
5028
5029 if (target != to)
5030 emit_move_insn (to, target);
5031
5032 emit_label (lab2);
5033
5034 if (optab_handler (mov_optab, to_mode) != CODE_FOR_nothing)
5035 {
5036 /* Make a place for a REG_NOTE and add it. */
5037 insn = emit_move_insn (to, to);
5038 set_dst_reg_note (insn, REG_EQUAL,
5039 gen_rtx_fmt_e (UNSIGNED_FIX, to_mode,
5040 copy_rtx (from)),
5041 to);
5042 }
5043
5044 return;
5045 }
5046 }
5047
5048 /* We can't do it with an insn, so use a library call. But first ensure
5049 that the mode of TO is at least as wide as SImode, since those are the
5050 only library calls we know about. */
5051
5052 if (is_narrower_int_mode (GET_MODE (to), SImode))
5053 {
5054 target = gen_reg_rtx (SImode);
5055
5056 expand_fix (target, from, unsignedp);
5057 }
5058 else
5059 {
5060 rtx_insn *insns;
5061 rtx value;
5062 rtx libfunc;
5063
5064 convert_optab tab = unsignedp ? ufix_optab : sfix_optab;
5065 libfunc = convert_optab_libfunc (tab, GET_MODE (to), GET_MODE (from));
5066 gcc_assert (libfunc);
5067
5068 start_sequence ();
5069
5070 value = emit_library_call_value (libfunc, NULL_RTX, LCT_CONST,
5071 GET_MODE (to), from, GET_MODE (from));
5072 insns = get_insns ();
5073 end_sequence ();
5074
5075 emit_libcall_block (insns, target, value,
5076 gen_rtx_fmt_e (unsignedp ? UNSIGNED_FIX : FIX,
5077 GET_MODE (to), from));
5078 }
5079
5080 if (target != to)
5081 {
5082 if (GET_MODE (to) == GET_MODE (target))
5083 emit_move_insn (to, target);
5084 else
5085 convert_move (to, target, 0);
5086 }
5087 }
5088
5089
5090 /* Promote integer arguments for a libcall if necessary.
5091 emit_library_call_value cannot do the promotion because it does not
5092 know if it should do a signed or unsigned promotion. This is because
5093 there are no tree types defined for libcalls. */
5094
5095 static rtx
prepare_libcall_arg(rtx arg,int uintp)5096 prepare_libcall_arg (rtx arg, int uintp)
5097 {
5098 scalar_int_mode mode;
5099 machine_mode arg_mode;
5100 if (is_a <scalar_int_mode> (GET_MODE (arg), &mode))
5101 {
5102 /* If we need to promote the integer function argument we need to do
5103 it here instead of inside emit_library_call_value because in
5104 emit_library_call_value we don't know if we should do a signed or
5105 unsigned promotion. */
5106
5107 int unsigned_p = 0;
5108 arg_mode = promote_function_mode (NULL_TREE, mode,
5109 &unsigned_p, NULL_TREE, 0);
5110 if (arg_mode != mode)
5111 return convert_to_mode (arg_mode, arg, uintp);
5112 }
5113 return arg;
5114 }
5115
5116 /* Generate code to convert FROM or TO a fixed-point.
5117 If UINTP is true, either TO or FROM is an unsigned integer.
5118 If SATP is true, we need to saturate the result. */
5119
5120 void
expand_fixed_convert(rtx to,rtx from,int uintp,int satp)5121 expand_fixed_convert (rtx to, rtx from, int uintp, int satp)
5122 {
5123 machine_mode to_mode = GET_MODE (to);
5124 machine_mode from_mode = GET_MODE (from);
5125 convert_optab tab;
5126 enum rtx_code this_code;
5127 enum insn_code code;
5128 rtx_insn *insns;
5129 rtx value;
5130 rtx libfunc;
5131
5132 if (to_mode == from_mode)
5133 {
5134 emit_move_insn (to, from);
5135 return;
5136 }
5137
5138 if (uintp)
5139 {
5140 tab = satp ? satfractuns_optab : fractuns_optab;
5141 this_code = satp ? UNSIGNED_SAT_FRACT : UNSIGNED_FRACT_CONVERT;
5142 }
5143 else
5144 {
5145 tab = satp ? satfract_optab : fract_optab;
5146 this_code = satp ? SAT_FRACT : FRACT_CONVERT;
5147 }
5148 code = convert_optab_handler (tab, to_mode, from_mode);
5149 if (code != CODE_FOR_nothing)
5150 {
5151 emit_unop_insn (code, to, from, this_code);
5152 return;
5153 }
5154
5155 libfunc = convert_optab_libfunc (tab, to_mode, from_mode);
5156 gcc_assert (libfunc);
5157
5158 from = prepare_libcall_arg (from, uintp);
5159 from_mode = GET_MODE (from);
5160
5161 start_sequence ();
5162 value = emit_library_call_value (libfunc, NULL_RTX, LCT_CONST, to_mode,
5163 from, from_mode);
5164 insns = get_insns ();
5165 end_sequence ();
5166
5167 emit_libcall_block (insns, to, value,
5168 gen_rtx_fmt_e (optab_to_code (tab), to_mode, from));
5169 }
5170
5171 /* Generate code to convert FROM to fixed point and store in TO. FROM
5172 must be floating point, TO must be signed. Use the conversion optab
5173 TAB to do the conversion. */
5174
5175 bool
expand_sfix_optab(rtx to,rtx from,convert_optab tab)5176 expand_sfix_optab (rtx to, rtx from, convert_optab tab)
5177 {
5178 enum insn_code icode;
5179 rtx target = to;
5180 machine_mode fmode, imode;
5181
5182 /* We first try to find a pair of modes, one real and one integer, at
5183 least as wide as FROM and TO, respectively, in which we can open-code
5184 this conversion. If the integer mode is wider than the mode of TO,
5185 we can do the conversion either signed or unsigned. */
5186
5187 FOR_EACH_MODE_FROM (fmode, GET_MODE (from))
5188 FOR_EACH_MODE_FROM (imode, GET_MODE (to))
5189 {
5190 icode = convert_optab_handler (tab, imode, fmode);
5191 if (icode != CODE_FOR_nothing)
5192 {
5193 rtx_insn *last = get_last_insn ();
5194 if (fmode != GET_MODE (from))
5195 from = convert_to_mode (fmode, from, 0);
5196
5197 if (imode != GET_MODE (to))
5198 target = gen_reg_rtx (imode);
5199
5200 if (!maybe_emit_unop_insn (icode, target, from, UNKNOWN))
5201 {
5202 delete_insns_since (last);
5203 continue;
5204 }
5205 if (target != to)
5206 convert_move (to, target, 0);
5207 return true;
5208 }
5209 }
5210
5211 return false;
5212 }
5213
5214 /* Report whether we have an instruction to perform the operation
5215 specified by CODE on operands of mode MODE. */
5216 int
have_insn_for(enum rtx_code code,machine_mode mode)5217 have_insn_for (enum rtx_code code, machine_mode mode)
5218 {
5219 return (code_to_optab (code)
5220 && (optab_handler (code_to_optab (code), mode)
5221 != CODE_FOR_nothing));
5222 }
5223
5224 /* Print information about the current contents of the optabs on
5225 STDERR. */
5226
5227 DEBUG_FUNCTION void
debug_optab_libfuncs(void)5228 debug_optab_libfuncs (void)
5229 {
5230 int i, j, k;
5231
5232 /* Dump the arithmetic optabs. */
5233 for (i = FIRST_NORM_OPTAB; i <= LAST_NORMLIB_OPTAB; ++i)
5234 for (j = 0; j < NUM_MACHINE_MODES; ++j)
5235 {
5236 rtx l = optab_libfunc ((optab) i, (machine_mode) j);
5237 if (l)
5238 {
5239 gcc_assert (GET_CODE (l) == SYMBOL_REF);
5240 fprintf (stderr, "%s\t%s:\t%s\n",
5241 GET_RTX_NAME (optab_to_code ((optab) i)),
5242 GET_MODE_NAME (j),
5243 XSTR (l, 0));
5244 }
5245 }
5246
5247 /* Dump the conversion optabs. */
5248 for (i = FIRST_CONV_OPTAB; i <= LAST_CONVLIB_OPTAB; ++i)
5249 for (j = 0; j < NUM_MACHINE_MODES; ++j)
5250 for (k = 0; k < NUM_MACHINE_MODES; ++k)
5251 {
5252 rtx l = convert_optab_libfunc ((optab) i, (machine_mode) j,
5253 (machine_mode) k);
5254 if (l)
5255 {
5256 gcc_assert (GET_CODE (l) == SYMBOL_REF);
5257 fprintf (stderr, "%s\t%s\t%s:\t%s\n",
5258 GET_RTX_NAME (optab_to_code ((optab) i)),
5259 GET_MODE_NAME (j),
5260 GET_MODE_NAME (k),
5261 XSTR (l, 0));
5262 }
5263 }
5264 }
5265
5266 /* Generate insns to trap with code TCODE if OP1 and OP2 satisfy condition
5267 CODE. Return 0 on failure. */
5268
5269 rtx_insn *
gen_cond_trap(enum rtx_code code,rtx op1,rtx op2,rtx tcode)5270 gen_cond_trap (enum rtx_code code, rtx op1, rtx op2, rtx tcode)
5271 {
5272 machine_mode mode = GET_MODE (op1);
5273 enum insn_code icode;
5274 rtx_insn *insn;
5275 rtx trap_rtx;
5276
5277 if (mode == VOIDmode)
5278 return 0;
5279
5280 icode = optab_handler (ctrap_optab, mode);
5281 if (icode == CODE_FOR_nothing)
5282 return 0;
5283
5284 /* Some targets only accept a zero trap code. */
5285 if (!insn_operand_matches (icode, 3, tcode))
5286 return 0;
5287
5288 do_pending_stack_adjust ();
5289 start_sequence ();
5290 prepare_cmp_insn (op1, op2, code, NULL_RTX, false, OPTAB_DIRECT,
5291 &trap_rtx, &mode);
5292 if (!trap_rtx)
5293 insn = NULL;
5294 else
5295 insn = GEN_FCN (icode) (trap_rtx, XEXP (trap_rtx, 0), XEXP (trap_rtx, 1),
5296 tcode);
5297
5298 /* If that failed, then give up. */
5299 if (insn == 0)
5300 {
5301 end_sequence ();
5302 return 0;
5303 }
5304
5305 emit_insn (insn);
5306 insn = get_insns ();
5307 end_sequence ();
5308 return insn;
5309 }
5310
5311 /* Return rtx code for TCODE. Use UNSIGNEDP to select signed
5312 or unsigned operation code. */
5313
5314 enum rtx_code
get_rtx_code(enum tree_code tcode,bool unsignedp)5315 get_rtx_code (enum tree_code tcode, bool unsignedp)
5316 {
5317 enum rtx_code code;
5318 switch (tcode)
5319 {
5320 case EQ_EXPR:
5321 code = EQ;
5322 break;
5323 case NE_EXPR:
5324 code = NE;
5325 break;
5326 case LT_EXPR:
5327 code = unsignedp ? LTU : LT;
5328 break;
5329 case LE_EXPR:
5330 code = unsignedp ? LEU : LE;
5331 break;
5332 case GT_EXPR:
5333 code = unsignedp ? GTU : GT;
5334 break;
5335 case GE_EXPR:
5336 code = unsignedp ? GEU : GE;
5337 break;
5338
5339 case UNORDERED_EXPR:
5340 code = UNORDERED;
5341 break;
5342 case ORDERED_EXPR:
5343 code = ORDERED;
5344 break;
5345 case UNLT_EXPR:
5346 code = UNLT;
5347 break;
5348 case UNLE_EXPR:
5349 code = UNLE;
5350 break;
5351 case UNGT_EXPR:
5352 code = UNGT;
5353 break;
5354 case UNGE_EXPR:
5355 code = UNGE;
5356 break;
5357 case UNEQ_EXPR:
5358 code = UNEQ;
5359 break;
5360 case LTGT_EXPR:
5361 code = LTGT;
5362 break;
5363
5364 case BIT_AND_EXPR:
5365 code = AND;
5366 break;
5367
5368 case BIT_IOR_EXPR:
5369 code = IOR;
5370 break;
5371
5372 default:
5373 gcc_unreachable ();
5374 }
5375 return code;
5376 }
5377
5378 /* Return a comparison rtx of mode CMP_MODE for COND. Use UNSIGNEDP to
5379 select signed or unsigned operators. OPNO holds the index of the
5380 first comparison operand for insn ICODE. Do not generate the
5381 compare instruction itself. */
5382
5383 static rtx
vector_compare_rtx(machine_mode cmp_mode,enum tree_code tcode,tree t_op0,tree t_op1,bool unsignedp,enum insn_code icode,unsigned int opno)5384 vector_compare_rtx (machine_mode cmp_mode, enum tree_code tcode,
5385 tree t_op0, tree t_op1, bool unsignedp,
5386 enum insn_code icode, unsigned int opno)
5387 {
5388 struct expand_operand ops[2];
5389 rtx rtx_op0, rtx_op1;
5390 machine_mode m0, m1;
5391 enum rtx_code rcode = get_rtx_code (tcode, unsignedp);
5392
5393 gcc_assert (TREE_CODE_CLASS (tcode) == tcc_comparison);
5394
5395 /* Expand operands. For vector types with scalar modes, e.g. where int64x1_t
5396 has mode DImode, this can produce a constant RTX of mode VOIDmode; in such
5397 cases, use the original mode. */
5398 rtx_op0 = expand_expr (t_op0, NULL_RTX, TYPE_MODE (TREE_TYPE (t_op0)),
5399 EXPAND_STACK_PARM);
5400 m0 = GET_MODE (rtx_op0);
5401 if (m0 == VOIDmode)
5402 m0 = TYPE_MODE (TREE_TYPE (t_op0));
5403
5404 rtx_op1 = expand_expr (t_op1, NULL_RTX, TYPE_MODE (TREE_TYPE (t_op1)),
5405 EXPAND_STACK_PARM);
5406 m1 = GET_MODE (rtx_op1);
5407 if (m1 == VOIDmode)
5408 m1 = TYPE_MODE (TREE_TYPE (t_op1));
5409
5410 create_input_operand (&ops[0], rtx_op0, m0);
5411 create_input_operand (&ops[1], rtx_op1, m1);
5412 if (!maybe_legitimize_operands (icode, opno, 2, ops))
5413 gcc_unreachable ();
5414 return gen_rtx_fmt_ee (rcode, cmp_mode, ops[0].value, ops[1].value);
5415 }
5416
5417 /* Check if vec_perm mask SEL is a constant equivalent to a shift of
5418 the first vec_perm operand, assuming the second operand is a constant
5419 vector of zeros. Return the shift distance in bits if so, or NULL_RTX
5420 if the vec_perm is not a shift. MODE is the mode of the value being
5421 shifted. */
5422 static rtx
shift_amt_for_vec_perm_mask(machine_mode mode,const vec_perm_indices & sel)5423 shift_amt_for_vec_perm_mask (machine_mode mode, const vec_perm_indices &sel)
5424 {
5425 unsigned int bitsize = GET_MODE_UNIT_BITSIZE (mode);
5426 poly_int64 first = sel[0];
5427 if (maybe_ge (sel[0], GET_MODE_NUNITS (mode)))
5428 return NULL_RTX;
5429
5430 if (!sel.series_p (0, 1, first, 1))
5431 {
5432 unsigned int nelt;
5433 if (!GET_MODE_NUNITS (mode).is_constant (&nelt))
5434 return NULL_RTX;
5435 for (unsigned int i = 1; i < nelt; i++)
5436 {
5437 poly_int64 expected = i + first;
5438 /* Indices into the second vector are all equivalent. */
5439 if (maybe_lt (sel[i], nelt)
5440 ? maybe_ne (sel[i], expected)
5441 : maybe_lt (expected, nelt))
5442 return NULL_RTX;
5443 }
5444 }
5445
5446 return gen_int_shift_amount (mode, first * bitsize);
5447 }
5448
5449 /* A subroutine of expand_vec_perm_var for expanding one vec_perm insn. */
5450
5451 static rtx
expand_vec_perm_1(enum insn_code icode,rtx target,rtx v0,rtx v1,rtx sel)5452 expand_vec_perm_1 (enum insn_code icode, rtx target,
5453 rtx v0, rtx v1, rtx sel)
5454 {
5455 machine_mode tmode = GET_MODE (target);
5456 machine_mode smode = GET_MODE (sel);
5457 struct expand_operand ops[4];
5458
5459 gcc_assert (GET_MODE_CLASS (smode) == MODE_VECTOR_INT
5460 || mode_for_int_vector (tmode).require () == smode);
5461 create_output_operand (&ops[0], target, tmode);
5462 create_input_operand (&ops[3], sel, smode);
5463
5464 /* Make an effort to preserve v0 == v1. The target expander is able to
5465 rely on this to determine if we're permuting a single input operand. */
5466 if (rtx_equal_p (v0, v1))
5467 {
5468 if (!insn_operand_matches (icode, 1, v0))
5469 v0 = force_reg (tmode, v0);
5470 gcc_checking_assert (insn_operand_matches (icode, 1, v0));
5471 gcc_checking_assert (insn_operand_matches (icode, 2, v0));
5472
5473 create_fixed_operand (&ops[1], v0);
5474 create_fixed_operand (&ops[2], v0);
5475 }
5476 else
5477 {
5478 create_input_operand (&ops[1], v0, tmode);
5479 create_input_operand (&ops[2], v1, tmode);
5480 }
5481
5482 if (maybe_expand_insn (icode, 4, ops))
5483 return ops[0].value;
5484 return NULL_RTX;
5485 }
5486
5487 /* Implement a permutation of vectors v0 and v1 using the permutation
5488 vector in SEL and return the result. Use TARGET to hold the result
5489 if nonnull and convenient.
5490
5491 MODE is the mode of the vectors being permuted (V0 and V1). SEL_MODE
5492 is the TYPE_MODE associated with SEL, or BLKmode if SEL isn't known
5493 to have a particular mode. */
5494
5495 rtx
expand_vec_perm_const(machine_mode mode,rtx v0,rtx v1,const vec_perm_builder & sel,machine_mode sel_mode,rtx target)5496 expand_vec_perm_const (machine_mode mode, rtx v0, rtx v1,
5497 const vec_perm_builder &sel, machine_mode sel_mode,
5498 rtx target)
5499 {
5500 if (!target || !register_operand (target, mode))
5501 target = gen_reg_rtx (mode);
5502
5503 /* Set QIMODE to a different vector mode with byte elements.
5504 If no such mode, or if MODE already has byte elements, use VOIDmode. */
5505 machine_mode qimode;
5506 if (!qimode_for_vec_perm (mode).exists (&qimode))
5507 qimode = VOIDmode;
5508
5509 rtx_insn *last = get_last_insn ();
5510
5511 bool single_arg_p = rtx_equal_p (v0, v1);
5512 /* Always specify two input vectors here and leave the target to handle
5513 cases in which the inputs are equal. Not all backends can cope with
5514 the single-input representation when testing for a double-input
5515 target instruction. */
5516 vec_perm_indices indices (sel, 2, GET_MODE_NUNITS (mode));
5517
5518 /* See if this can be handled with a vec_shr. We only do this if the
5519 second vector is all zeroes. */
5520 insn_code shift_code = optab_handler (vec_shr_optab, mode);
5521 insn_code shift_code_qi = ((qimode != VOIDmode && qimode != mode)
5522 ? optab_handler (vec_shr_optab, qimode)
5523 : CODE_FOR_nothing);
5524
5525 if (v1 == CONST0_RTX (GET_MODE (v1))
5526 && (shift_code != CODE_FOR_nothing
5527 || shift_code_qi != CODE_FOR_nothing))
5528 {
5529 rtx shift_amt = shift_amt_for_vec_perm_mask (mode, indices);
5530 if (shift_amt)
5531 {
5532 struct expand_operand ops[3];
5533 if (shift_amt == const0_rtx)
5534 return v0;
5535 if (shift_code != CODE_FOR_nothing)
5536 {
5537 create_output_operand (&ops[0], target, mode);
5538 create_input_operand (&ops[1], v0, mode);
5539 create_convert_operand_from_type (&ops[2], shift_amt, sizetype);
5540 if (maybe_expand_insn (shift_code, 3, ops))
5541 return ops[0].value;
5542 }
5543 if (shift_code_qi != CODE_FOR_nothing)
5544 {
5545 rtx tmp = gen_reg_rtx (qimode);
5546 create_output_operand (&ops[0], tmp, qimode);
5547 create_input_operand (&ops[1], gen_lowpart (qimode, v0), qimode);
5548 create_convert_operand_from_type (&ops[2], shift_amt, sizetype);
5549 if (maybe_expand_insn (shift_code_qi, 3, ops))
5550 return gen_lowpart (mode, ops[0].value);
5551 }
5552 }
5553 }
5554
5555 if (targetm.vectorize.vec_perm_const != NULL)
5556 {
5557 v0 = force_reg (mode, v0);
5558 if (single_arg_p)
5559 v1 = v0;
5560 else
5561 v1 = force_reg (mode, v1);
5562
5563 if (targetm.vectorize.vec_perm_const (mode, target, v0, v1, indices))
5564 return target;
5565 }
5566
5567 /* Fall back to a constant byte-based permutation. */
5568 vec_perm_indices qimode_indices;
5569 rtx target_qi = NULL_RTX, v0_qi = NULL_RTX, v1_qi = NULL_RTX;
5570 if (qimode != VOIDmode)
5571 {
5572 qimode_indices.new_expanded_vector (indices, GET_MODE_UNIT_SIZE (mode));
5573 target_qi = gen_reg_rtx (qimode);
5574 v0_qi = gen_lowpart (qimode, v0);
5575 v1_qi = gen_lowpart (qimode, v1);
5576 if (targetm.vectorize.vec_perm_const != NULL
5577 && targetm.vectorize.vec_perm_const (qimode, target_qi, v0_qi,
5578 v1_qi, qimode_indices))
5579 return gen_lowpart (mode, target_qi);
5580 }
5581
5582 /* Otherwise expand as a fully variable permuation. */
5583
5584 /* The optabs are only defined for selectors with the same width
5585 as the values being permuted. */
5586 machine_mode required_sel_mode;
5587 if (!mode_for_int_vector (mode).exists (&required_sel_mode)
5588 || !VECTOR_MODE_P (required_sel_mode))
5589 {
5590 delete_insns_since (last);
5591 return NULL_RTX;
5592 }
5593
5594 /* We know that it is semantically valid to treat SEL as having SEL_MODE.
5595 If that isn't the mode we want then we need to prove that using
5596 REQUIRED_SEL_MODE is OK. */
5597 if (sel_mode != required_sel_mode)
5598 {
5599 if (!selector_fits_mode_p (required_sel_mode, indices))
5600 {
5601 delete_insns_since (last);
5602 return NULL_RTX;
5603 }
5604 sel_mode = required_sel_mode;
5605 }
5606
5607 insn_code icode = direct_optab_handler (vec_perm_optab, mode);
5608 if (icode != CODE_FOR_nothing)
5609 {
5610 rtx sel_rtx = vec_perm_indices_to_rtx (sel_mode, indices);
5611 rtx tmp = expand_vec_perm_1 (icode, target, v0, v1, sel_rtx);
5612 if (tmp)
5613 return tmp;
5614 }
5615
5616 if (qimode != VOIDmode
5617 && selector_fits_mode_p (qimode, qimode_indices))
5618 {
5619 icode = direct_optab_handler (vec_perm_optab, qimode);
5620 if (icode != CODE_FOR_nothing)
5621 {
5622 rtx sel_qi = vec_perm_indices_to_rtx (qimode, qimode_indices);
5623 rtx tmp = expand_vec_perm_1 (icode, target_qi, v0_qi, v1_qi, sel_qi);
5624 if (tmp)
5625 return gen_lowpart (mode, tmp);
5626 }
5627 }
5628
5629 delete_insns_since (last);
5630 return NULL_RTX;
5631 }
5632
5633 /* Implement a permutation of vectors v0 and v1 using the permutation
5634 vector in SEL and return the result. Use TARGET to hold the result
5635 if nonnull and convenient.
5636
5637 MODE is the mode of the vectors being permuted (V0 and V1).
5638 SEL must have the integer equivalent of MODE and is known to be
5639 unsuitable for permutes with a constant permutation vector. */
5640
5641 rtx
expand_vec_perm_var(machine_mode mode,rtx v0,rtx v1,rtx sel,rtx target)5642 expand_vec_perm_var (machine_mode mode, rtx v0, rtx v1, rtx sel, rtx target)
5643 {
5644 enum insn_code icode;
5645 unsigned int i, u;
5646 rtx tmp, sel_qi;
5647
5648 u = GET_MODE_UNIT_SIZE (mode);
5649
5650 if (!target || GET_MODE (target) != mode)
5651 target = gen_reg_rtx (mode);
5652
5653 icode = direct_optab_handler (vec_perm_optab, mode);
5654 if (icode != CODE_FOR_nothing)
5655 {
5656 tmp = expand_vec_perm_1 (icode, target, v0, v1, sel);
5657 if (tmp)
5658 return tmp;
5659 }
5660
5661 /* As a special case to aid several targets, lower the element-based
5662 permutation to a byte-based permutation and try again. */
5663 machine_mode qimode;
5664 if (!qimode_for_vec_perm (mode).exists (&qimode)
5665 || maybe_gt (GET_MODE_NUNITS (qimode), GET_MODE_MASK (QImode) + 1))
5666 return NULL_RTX;
5667 icode = direct_optab_handler (vec_perm_optab, qimode);
5668 if (icode == CODE_FOR_nothing)
5669 return NULL_RTX;
5670
5671 /* Multiply each element by its byte size. */
5672 machine_mode selmode = GET_MODE (sel);
5673 if (u == 2)
5674 sel = expand_simple_binop (selmode, PLUS, sel, sel,
5675 NULL, 0, OPTAB_DIRECT);
5676 else
5677 sel = expand_simple_binop (selmode, ASHIFT, sel,
5678 gen_int_shift_amount (selmode, exact_log2 (u)),
5679 NULL, 0, OPTAB_DIRECT);
5680 gcc_assert (sel != NULL);
5681
5682 /* Broadcast the low byte each element into each of its bytes.
5683 The encoding has U interleaved stepped patterns, one for each
5684 byte of an element. */
5685 vec_perm_builder const_sel (GET_MODE_SIZE (mode), u, 3);
5686 unsigned int low_byte_in_u = BYTES_BIG_ENDIAN ? u - 1 : 0;
5687 for (i = 0; i < 3; ++i)
5688 for (unsigned int j = 0; j < u; ++j)
5689 const_sel.quick_push (i * u + low_byte_in_u);
5690 sel = gen_lowpart (qimode, sel);
5691 sel = expand_vec_perm_const (qimode, sel, sel, const_sel, qimode, NULL);
5692 gcc_assert (sel != NULL);
5693
5694 /* Add the byte offset to each byte element. */
5695 /* Note that the definition of the indicies here is memory ordering,
5696 so there should be no difference between big and little endian. */
5697 rtx_vector_builder byte_indices (qimode, u, 1);
5698 for (i = 0; i < u; ++i)
5699 byte_indices.quick_push (GEN_INT (i));
5700 tmp = byte_indices.build ();
5701 sel_qi = expand_simple_binop (qimode, PLUS, sel, tmp,
5702 sel, 0, OPTAB_DIRECT);
5703 gcc_assert (sel_qi != NULL);
5704
5705 tmp = mode != qimode ? gen_reg_rtx (qimode) : target;
5706 tmp = expand_vec_perm_1 (icode, tmp, gen_lowpart (qimode, v0),
5707 gen_lowpart (qimode, v1), sel_qi);
5708 if (tmp)
5709 tmp = gen_lowpart (mode, tmp);
5710 return tmp;
5711 }
5712
5713 /* Generate insns for a VEC_COND_EXPR with mask, given its TYPE and its
5714 three operands. */
5715
5716 rtx
expand_vec_cond_mask_expr(tree vec_cond_type,tree op0,tree op1,tree op2,rtx target)5717 expand_vec_cond_mask_expr (tree vec_cond_type, tree op0, tree op1, tree op2,
5718 rtx target)
5719 {
5720 struct expand_operand ops[4];
5721 machine_mode mode = TYPE_MODE (vec_cond_type);
5722 machine_mode mask_mode = TYPE_MODE (TREE_TYPE (op0));
5723 enum insn_code icode = get_vcond_mask_icode (mode, mask_mode);
5724 rtx mask, rtx_op1, rtx_op2;
5725
5726 if (icode == CODE_FOR_nothing)
5727 return 0;
5728
5729 mask = expand_normal (op0);
5730 rtx_op1 = expand_normal (op1);
5731 rtx_op2 = expand_normal (op2);
5732
5733 mask = force_reg (mask_mode, mask);
5734 rtx_op1 = force_reg (GET_MODE (rtx_op1), rtx_op1);
5735
5736 create_output_operand (&ops[0], target, mode);
5737 create_input_operand (&ops[1], rtx_op1, mode);
5738 create_input_operand (&ops[2], rtx_op2, mode);
5739 create_input_operand (&ops[3], mask, mask_mode);
5740 expand_insn (icode, 4, ops);
5741
5742 return ops[0].value;
5743 }
5744
5745 /* Generate insns for a VEC_COND_EXPR, given its TYPE and its
5746 three operands. */
5747
5748 rtx
expand_vec_cond_expr(tree vec_cond_type,tree op0,tree op1,tree op2,rtx target)5749 expand_vec_cond_expr (tree vec_cond_type, tree op0, tree op1, tree op2,
5750 rtx target)
5751 {
5752 struct expand_operand ops[6];
5753 enum insn_code icode;
5754 rtx comparison, rtx_op1, rtx_op2;
5755 machine_mode mode = TYPE_MODE (vec_cond_type);
5756 machine_mode cmp_op_mode;
5757 bool unsignedp;
5758 tree op0a, op0b;
5759 enum tree_code tcode;
5760
5761 if (COMPARISON_CLASS_P (op0))
5762 {
5763 op0a = TREE_OPERAND (op0, 0);
5764 op0b = TREE_OPERAND (op0, 1);
5765 tcode = TREE_CODE (op0);
5766 }
5767 else
5768 {
5769 gcc_assert (VECTOR_BOOLEAN_TYPE_P (TREE_TYPE (op0)));
5770 if (get_vcond_mask_icode (mode, TYPE_MODE (TREE_TYPE (op0)))
5771 != CODE_FOR_nothing)
5772 return expand_vec_cond_mask_expr (vec_cond_type, op0, op1,
5773 op2, target);
5774 /* Fake op0 < 0. */
5775 else
5776 {
5777 gcc_assert (GET_MODE_CLASS (TYPE_MODE (TREE_TYPE (op0)))
5778 == MODE_VECTOR_INT);
5779 op0a = op0;
5780 op0b = build_zero_cst (TREE_TYPE (op0));
5781 tcode = LT_EXPR;
5782 }
5783 }
5784 cmp_op_mode = TYPE_MODE (TREE_TYPE (op0a));
5785 unsignedp = TYPE_UNSIGNED (TREE_TYPE (op0a));
5786
5787
5788 gcc_assert (known_eq (GET_MODE_SIZE (mode), GET_MODE_SIZE (cmp_op_mode))
5789 && known_eq (GET_MODE_NUNITS (mode),
5790 GET_MODE_NUNITS (cmp_op_mode)));
5791
5792 icode = get_vcond_icode (mode, cmp_op_mode, unsignedp);
5793 if (icode == CODE_FOR_nothing)
5794 {
5795 if (tcode == LT_EXPR
5796 && op0a == op0
5797 && TREE_CODE (op0) == VECTOR_CST)
5798 {
5799 /* A VEC_COND_EXPR condition could be folded from EQ_EXPR/NE_EXPR
5800 into a constant when only get_vcond_eq_icode is supported.
5801 Verify < 0 and != 0 behave the same and change it to NE_EXPR. */
5802 unsigned HOST_WIDE_INT nelts;
5803 if (!VECTOR_CST_NELTS (op0).is_constant (&nelts))
5804 {
5805 if (VECTOR_CST_STEPPED_P (op0))
5806 return 0;
5807 nelts = vector_cst_encoded_nelts (op0);
5808 }
5809 for (unsigned int i = 0; i < nelts; ++i)
5810 if (tree_int_cst_sgn (vector_cst_elt (op0, i)) == 1)
5811 return 0;
5812 tcode = NE_EXPR;
5813 }
5814 if (tcode == EQ_EXPR || tcode == NE_EXPR)
5815 icode = get_vcond_eq_icode (mode, cmp_op_mode);
5816 if (icode == CODE_FOR_nothing)
5817 return 0;
5818 }
5819
5820 comparison = vector_compare_rtx (VOIDmode, tcode, op0a, op0b, unsignedp,
5821 icode, 4);
5822 rtx_op1 = expand_normal (op1);
5823 rtx_op2 = expand_normal (op2);
5824
5825 create_output_operand (&ops[0], target, mode);
5826 create_input_operand (&ops[1], rtx_op1, mode);
5827 create_input_operand (&ops[2], rtx_op2, mode);
5828 create_fixed_operand (&ops[3], comparison);
5829 create_fixed_operand (&ops[4], XEXP (comparison, 0));
5830 create_fixed_operand (&ops[5], XEXP (comparison, 1));
5831 expand_insn (icode, 6, ops);
5832 return ops[0].value;
5833 }
5834
5835 /* Generate VEC_SERIES_EXPR <OP0, OP1>, returning a value of mode VMODE.
5836 Use TARGET for the result if nonnull and convenient. */
5837
5838 rtx
expand_vec_series_expr(machine_mode vmode,rtx op0,rtx op1,rtx target)5839 expand_vec_series_expr (machine_mode vmode, rtx op0, rtx op1, rtx target)
5840 {
5841 struct expand_operand ops[3];
5842 enum insn_code icode;
5843 machine_mode emode = GET_MODE_INNER (vmode);
5844
5845 icode = direct_optab_handler (vec_series_optab, vmode);
5846 gcc_assert (icode != CODE_FOR_nothing);
5847
5848 create_output_operand (&ops[0], target, vmode);
5849 create_input_operand (&ops[1], op0, emode);
5850 create_input_operand (&ops[2], op1, emode);
5851
5852 expand_insn (icode, 3, ops);
5853 return ops[0].value;
5854 }
5855
5856 /* Generate insns for a vector comparison into a mask. */
5857
5858 rtx
expand_vec_cmp_expr(tree type,tree exp,rtx target)5859 expand_vec_cmp_expr (tree type, tree exp, rtx target)
5860 {
5861 struct expand_operand ops[4];
5862 enum insn_code icode;
5863 rtx comparison;
5864 machine_mode mask_mode = TYPE_MODE (type);
5865 machine_mode vmode;
5866 bool unsignedp;
5867 tree op0a, op0b;
5868 enum tree_code tcode;
5869
5870 op0a = TREE_OPERAND (exp, 0);
5871 op0b = TREE_OPERAND (exp, 1);
5872 tcode = TREE_CODE (exp);
5873
5874 unsignedp = TYPE_UNSIGNED (TREE_TYPE (op0a));
5875 vmode = TYPE_MODE (TREE_TYPE (op0a));
5876
5877 icode = get_vec_cmp_icode (vmode, mask_mode, unsignedp);
5878 if (icode == CODE_FOR_nothing)
5879 {
5880 if (tcode == EQ_EXPR || tcode == NE_EXPR)
5881 icode = get_vec_cmp_eq_icode (vmode, mask_mode);
5882 if (icode == CODE_FOR_nothing)
5883 return 0;
5884 }
5885
5886 comparison = vector_compare_rtx (mask_mode, tcode, op0a, op0b,
5887 unsignedp, icode, 2);
5888 create_output_operand (&ops[0], target, mask_mode);
5889 create_fixed_operand (&ops[1], comparison);
5890 create_fixed_operand (&ops[2], XEXP (comparison, 0));
5891 create_fixed_operand (&ops[3], XEXP (comparison, 1));
5892 expand_insn (icode, 4, ops);
5893 return ops[0].value;
5894 }
5895
5896 /* Expand a highpart multiply. */
5897
5898 rtx
expand_mult_highpart(machine_mode mode,rtx op0,rtx op1,rtx target,bool uns_p)5899 expand_mult_highpart (machine_mode mode, rtx op0, rtx op1,
5900 rtx target, bool uns_p)
5901 {
5902 struct expand_operand eops[3];
5903 enum insn_code icode;
5904 int method, i;
5905 machine_mode wmode;
5906 rtx m1, m2;
5907 optab tab1, tab2;
5908
5909 method = can_mult_highpart_p (mode, uns_p);
5910 switch (method)
5911 {
5912 case 0:
5913 return NULL_RTX;
5914 case 1:
5915 tab1 = uns_p ? umul_highpart_optab : smul_highpart_optab;
5916 return expand_binop (mode, tab1, op0, op1, target, uns_p,
5917 OPTAB_LIB_WIDEN);
5918 case 2:
5919 tab1 = uns_p ? vec_widen_umult_even_optab : vec_widen_smult_even_optab;
5920 tab2 = uns_p ? vec_widen_umult_odd_optab : vec_widen_smult_odd_optab;
5921 break;
5922 case 3:
5923 tab1 = uns_p ? vec_widen_umult_lo_optab : vec_widen_smult_lo_optab;
5924 tab2 = uns_p ? vec_widen_umult_hi_optab : vec_widen_smult_hi_optab;
5925 if (BYTES_BIG_ENDIAN)
5926 std::swap (tab1, tab2);
5927 break;
5928 default:
5929 gcc_unreachable ();
5930 }
5931
5932 icode = optab_handler (tab1, mode);
5933 wmode = insn_data[icode].operand[0].mode;
5934 gcc_checking_assert (known_eq (2 * GET_MODE_NUNITS (wmode),
5935 GET_MODE_NUNITS (mode)));
5936 gcc_checking_assert (known_eq (GET_MODE_SIZE (wmode), GET_MODE_SIZE (mode)));
5937
5938 create_output_operand (&eops[0], gen_reg_rtx (wmode), wmode);
5939 create_input_operand (&eops[1], op0, mode);
5940 create_input_operand (&eops[2], op1, mode);
5941 expand_insn (icode, 3, eops);
5942 m1 = gen_lowpart (mode, eops[0].value);
5943
5944 create_output_operand (&eops[0], gen_reg_rtx (wmode), wmode);
5945 create_input_operand (&eops[1], op0, mode);
5946 create_input_operand (&eops[2], op1, mode);
5947 expand_insn (optab_handler (tab2, mode), 3, eops);
5948 m2 = gen_lowpart (mode, eops[0].value);
5949
5950 vec_perm_builder sel;
5951 if (method == 2)
5952 {
5953 /* The encoding has 2 interleaved stepped patterns. */
5954 sel.new_vector (GET_MODE_NUNITS (mode), 2, 3);
5955 for (i = 0; i < 6; ++i)
5956 sel.quick_push (!BYTES_BIG_ENDIAN + (i & ~1)
5957 + ((i & 1) ? GET_MODE_NUNITS (mode) : 0));
5958 }
5959 else
5960 {
5961 /* The encoding has a single interleaved stepped pattern. */
5962 sel.new_vector (GET_MODE_NUNITS (mode), 1, 3);
5963 for (i = 0; i < 3; ++i)
5964 sel.quick_push (2 * i + (BYTES_BIG_ENDIAN ? 0 : 1));
5965 }
5966
5967 return expand_vec_perm_const (mode, m1, m2, sel, BLKmode, target);
5968 }
5969
5970 /* Helper function to find the MODE_CC set in a sync_compare_and_swap
5971 pattern. */
5972
5973 static void
find_cc_set(rtx x,const_rtx pat,void * data)5974 find_cc_set (rtx x, const_rtx pat, void *data)
5975 {
5976 if (REG_P (x) && GET_MODE_CLASS (GET_MODE (x)) == MODE_CC
5977 && GET_CODE (pat) == SET)
5978 {
5979 rtx *p_cc_reg = (rtx *) data;
5980 gcc_assert (!*p_cc_reg);
5981 *p_cc_reg = x;
5982 }
5983 }
5984
5985 /* This is a helper function for the other atomic operations. This function
5986 emits a loop that contains SEQ that iterates until a compare-and-swap
5987 operation at the end succeeds. MEM is the memory to be modified. SEQ is
5988 a set of instructions that takes a value from OLD_REG as an input and
5989 produces a value in NEW_REG as an output. Before SEQ, OLD_REG will be
5990 set to the current contents of MEM. After SEQ, a compare-and-swap will
5991 attempt to update MEM with NEW_REG. The function returns true when the
5992 loop was generated successfully. */
5993
5994 static bool
expand_compare_and_swap_loop(rtx mem,rtx old_reg,rtx new_reg,rtx seq)5995 expand_compare_and_swap_loop (rtx mem, rtx old_reg, rtx new_reg, rtx seq)
5996 {
5997 machine_mode mode = GET_MODE (mem);
5998 rtx_code_label *label;
5999 rtx cmp_reg, success, oldval;
6000
6001 /* The loop we want to generate looks like
6002
6003 cmp_reg = mem;
6004 label:
6005 old_reg = cmp_reg;
6006 seq;
6007 (success, cmp_reg) = compare-and-swap(mem, old_reg, new_reg)
6008 if (success)
6009 goto label;
6010
6011 Note that we only do the plain load from memory once. Subsequent
6012 iterations use the value loaded by the compare-and-swap pattern. */
6013
6014 label = gen_label_rtx ();
6015 cmp_reg = gen_reg_rtx (mode);
6016
6017 emit_move_insn (cmp_reg, mem);
6018 emit_label (label);
6019 emit_move_insn (old_reg, cmp_reg);
6020 if (seq)
6021 emit_insn (seq);
6022
6023 success = NULL_RTX;
6024 oldval = cmp_reg;
6025 if (!expand_atomic_compare_and_swap (&success, &oldval, mem, old_reg,
6026 new_reg, false, MEMMODEL_SYNC_SEQ_CST,
6027 MEMMODEL_RELAXED))
6028 return false;
6029
6030 if (oldval != cmp_reg)
6031 emit_move_insn (cmp_reg, oldval);
6032
6033 /* Mark this jump predicted not taken. */
6034 emit_cmp_and_jump_insns (success, const0_rtx, EQ, const0_rtx,
6035 GET_MODE (success), 1, label,
6036 profile_probability::guessed_never ());
6037 return true;
6038 }
6039
6040
6041 /* This function tries to emit an atomic_exchange intruction. VAL is written
6042 to *MEM using memory model MODEL. The previous contents of *MEM are returned,
6043 using TARGET if possible. */
6044
6045 static rtx
maybe_emit_atomic_exchange(rtx target,rtx mem,rtx val,enum memmodel model)6046 maybe_emit_atomic_exchange (rtx target, rtx mem, rtx val, enum memmodel model)
6047 {
6048 machine_mode mode = GET_MODE (mem);
6049 enum insn_code icode;
6050
6051 /* If the target supports the exchange directly, great. */
6052 icode = direct_optab_handler (atomic_exchange_optab, mode);
6053 if (icode != CODE_FOR_nothing)
6054 {
6055 struct expand_operand ops[4];
6056
6057 create_output_operand (&ops[0], target, mode);
6058 create_fixed_operand (&ops[1], mem);
6059 create_input_operand (&ops[2], val, mode);
6060 create_integer_operand (&ops[3], model);
6061 if (maybe_expand_insn (icode, 4, ops))
6062 return ops[0].value;
6063 }
6064
6065 return NULL_RTX;
6066 }
6067
6068 /* This function tries to implement an atomic exchange operation using
6069 __sync_lock_test_and_set. VAL is written to *MEM using memory model MODEL.
6070 The previous contents of *MEM are returned, using TARGET if possible.
6071 Since this instructionn is an acquire barrier only, stronger memory
6072 models may require additional barriers to be emitted. */
6073
6074 static rtx
maybe_emit_sync_lock_test_and_set(rtx target,rtx mem,rtx val,enum memmodel model)6075 maybe_emit_sync_lock_test_and_set (rtx target, rtx mem, rtx val,
6076 enum memmodel model)
6077 {
6078 machine_mode mode = GET_MODE (mem);
6079 enum insn_code icode;
6080 rtx_insn *last_insn = get_last_insn ();
6081
6082 icode = optab_handler (sync_lock_test_and_set_optab, mode);
6083
6084 /* Legacy sync_lock_test_and_set is an acquire barrier. If the pattern
6085 exists, and the memory model is stronger than acquire, add a release
6086 barrier before the instruction. */
6087
6088 if (is_mm_seq_cst (model) || is_mm_release (model) || is_mm_acq_rel (model))
6089 expand_mem_thread_fence (model);
6090
6091 if (icode != CODE_FOR_nothing)
6092 {
6093 struct expand_operand ops[3];
6094 create_output_operand (&ops[0], target, mode);
6095 create_fixed_operand (&ops[1], mem);
6096 create_input_operand (&ops[2], val, mode);
6097 if (maybe_expand_insn (icode, 3, ops))
6098 return ops[0].value;
6099 }
6100
6101 /* If an external test-and-set libcall is provided, use that instead of
6102 any external compare-and-swap that we might get from the compare-and-
6103 swap-loop expansion later. */
6104 if (!can_compare_and_swap_p (mode, false))
6105 {
6106 rtx libfunc = optab_libfunc (sync_lock_test_and_set_optab, mode);
6107 if (libfunc != NULL)
6108 {
6109 rtx addr;
6110
6111 addr = convert_memory_address (ptr_mode, XEXP (mem, 0));
6112 return emit_library_call_value (libfunc, NULL_RTX, LCT_NORMAL,
6113 mode, addr, ptr_mode,
6114 val, mode);
6115 }
6116 }
6117
6118 /* If the test_and_set can't be emitted, eliminate any barrier that might
6119 have been emitted. */
6120 delete_insns_since (last_insn);
6121 return NULL_RTX;
6122 }
6123
6124 /* This function tries to implement an atomic exchange operation using a
6125 compare_and_swap loop. VAL is written to *MEM. The previous contents of
6126 *MEM are returned, using TARGET if possible. No memory model is required
6127 since a compare_and_swap loop is seq-cst. */
6128
6129 static rtx
maybe_emit_compare_and_swap_exchange_loop(rtx target,rtx mem,rtx val)6130 maybe_emit_compare_and_swap_exchange_loop (rtx target, rtx mem, rtx val)
6131 {
6132 machine_mode mode = GET_MODE (mem);
6133
6134 if (can_compare_and_swap_p (mode, true))
6135 {
6136 if (!target || !register_operand (target, mode))
6137 target = gen_reg_rtx (mode);
6138 if (expand_compare_and_swap_loop (mem, target, val, NULL_RTX))
6139 return target;
6140 }
6141
6142 return NULL_RTX;
6143 }
6144
6145 /* This function tries to implement an atomic test-and-set operation
6146 using the atomic_test_and_set instruction pattern. A boolean value
6147 is returned from the operation, using TARGET if possible. */
6148
6149 static rtx
maybe_emit_atomic_test_and_set(rtx target,rtx mem,enum memmodel model)6150 maybe_emit_atomic_test_and_set (rtx target, rtx mem, enum memmodel model)
6151 {
6152 machine_mode pat_bool_mode;
6153 struct expand_operand ops[3];
6154
6155 if (!targetm.have_atomic_test_and_set ())
6156 return NULL_RTX;
6157
6158 /* While we always get QImode from __atomic_test_and_set, we get
6159 other memory modes from __sync_lock_test_and_set. Note that we
6160 use no endian adjustment here. This matches the 4.6 behavior
6161 in the Sparc backend. */
6162 enum insn_code icode = targetm.code_for_atomic_test_and_set;
6163 gcc_checking_assert (insn_data[icode].operand[1].mode == QImode);
6164 if (GET_MODE (mem) != QImode)
6165 mem = adjust_address_nv (mem, QImode, 0);
6166
6167 pat_bool_mode = insn_data[icode].operand[0].mode;
6168 create_output_operand (&ops[0], target, pat_bool_mode);
6169 create_fixed_operand (&ops[1], mem);
6170 create_integer_operand (&ops[2], model);
6171
6172 if (maybe_expand_insn (icode, 3, ops))
6173 return ops[0].value;
6174 return NULL_RTX;
6175 }
6176
6177 /* This function expands the legacy _sync_lock test_and_set operation which is
6178 generally an atomic exchange. Some limited targets only allow the
6179 constant 1 to be stored. This is an ACQUIRE operation.
6180
6181 TARGET is an optional place to stick the return value.
6182 MEM is where VAL is stored. */
6183
6184 rtx
expand_sync_lock_test_and_set(rtx target,rtx mem,rtx val)6185 expand_sync_lock_test_and_set (rtx target, rtx mem, rtx val)
6186 {
6187 rtx ret;
6188
6189 /* Try an atomic_exchange first. */
6190 ret = maybe_emit_atomic_exchange (target, mem, val, MEMMODEL_SYNC_ACQUIRE);
6191 if (ret)
6192 return ret;
6193
6194 ret = maybe_emit_sync_lock_test_and_set (target, mem, val,
6195 MEMMODEL_SYNC_ACQUIRE);
6196 if (ret)
6197 return ret;
6198
6199 ret = maybe_emit_compare_and_swap_exchange_loop (target, mem, val);
6200 if (ret)
6201 return ret;
6202
6203 /* If there are no other options, try atomic_test_and_set if the value
6204 being stored is 1. */
6205 if (val == const1_rtx)
6206 ret = maybe_emit_atomic_test_and_set (target, mem, MEMMODEL_SYNC_ACQUIRE);
6207
6208 return ret;
6209 }
6210
6211 /* This function expands the atomic test_and_set operation:
6212 atomically store a boolean TRUE into MEM and return the previous value.
6213
6214 MEMMODEL is the memory model variant to use.
6215 TARGET is an optional place to stick the return value. */
6216
6217 rtx
expand_atomic_test_and_set(rtx target,rtx mem,enum memmodel model)6218 expand_atomic_test_and_set (rtx target, rtx mem, enum memmodel model)
6219 {
6220 machine_mode mode = GET_MODE (mem);
6221 rtx ret, trueval, subtarget;
6222
6223 ret = maybe_emit_atomic_test_and_set (target, mem, model);
6224 if (ret)
6225 return ret;
6226
6227 /* Be binary compatible with non-default settings of trueval, and different
6228 cpu revisions. E.g. one revision may have atomic-test-and-set, but
6229 another only has atomic-exchange. */
6230 if (targetm.atomic_test_and_set_trueval == 1)
6231 {
6232 trueval = const1_rtx;
6233 subtarget = target ? target : gen_reg_rtx (mode);
6234 }
6235 else
6236 {
6237 trueval = gen_int_mode (targetm.atomic_test_and_set_trueval, mode);
6238 subtarget = gen_reg_rtx (mode);
6239 }
6240
6241 /* Try the atomic-exchange optab... */
6242 ret = maybe_emit_atomic_exchange (subtarget, mem, trueval, model);
6243
6244 /* ... then an atomic-compare-and-swap loop ... */
6245 if (!ret)
6246 ret = maybe_emit_compare_and_swap_exchange_loop (subtarget, mem, trueval);
6247
6248 /* ... before trying the vaguely defined legacy lock_test_and_set. */
6249 if (!ret)
6250 ret = maybe_emit_sync_lock_test_and_set (subtarget, mem, trueval, model);
6251
6252 /* Recall that the legacy lock_test_and_set optab was allowed to do magic
6253 things with the value 1. Thus we try again without trueval. */
6254 if (!ret && targetm.atomic_test_and_set_trueval != 1)
6255 ret = maybe_emit_sync_lock_test_and_set (subtarget, mem, const1_rtx, model);
6256
6257 /* Failing all else, assume a single threaded environment and simply
6258 perform the operation. */
6259 if (!ret)
6260 {
6261 /* If the result is ignored skip the move to target. */
6262 if (subtarget != const0_rtx)
6263 emit_move_insn (subtarget, mem);
6264
6265 emit_move_insn (mem, trueval);
6266 ret = subtarget;
6267 }
6268
6269 /* Recall that have to return a boolean value; rectify if trueval
6270 is not exactly one. */
6271 if (targetm.atomic_test_and_set_trueval != 1)
6272 ret = emit_store_flag_force (target, NE, ret, const0_rtx, mode, 0, 1);
6273
6274 return ret;
6275 }
6276
6277 /* This function expands the atomic exchange operation:
6278 atomically store VAL in MEM and return the previous value in MEM.
6279
6280 MEMMODEL is the memory model variant to use.
6281 TARGET is an optional place to stick the return value. */
6282
6283 rtx
expand_atomic_exchange(rtx target,rtx mem,rtx val,enum memmodel model)6284 expand_atomic_exchange (rtx target, rtx mem, rtx val, enum memmodel model)
6285 {
6286 machine_mode mode = GET_MODE (mem);
6287 rtx ret;
6288
6289 /* If loads are not atomic for the required size and we are not called to
6290 provide a __sync builtin, do not do anything so that we stay consistent
6291 with atomic loads of the same size. */
6292 if (!can_atomic_load_p (mode) && !is_mm_sync (model))
6293 return NULL_RTX;
6294
6295 ret = maybe_emit_atomic_exchange (target, mem, val, model);
6296
6297 /* Next try a compare-and-swap loop for the exchange. */
6298 if (!ret)
6299 ret = maybe_emit_compare_and_swap_exchange_loop (target, mem, val);
6300
6301 return ret;
6302 }
6303
6304 /* This function expands the atomic compare exchange operation:
6305
6306 *PTARGET_BOOL is an optional place to store the boolean success/failure.
6307 *PTARGET_OVAL is an optional place to store the old value from memory.
6308 Both target parameters may be NULL or const0_rtx to indicate that we do
6309 not care about that return value. Both target parameters are updated on
6310 success to the actual location of the corresponding result.
6311
6312 MEMMODEL is the memory model variant to use.
6313
6314 The return value of the function is true for success. */
6315
6316 bool
expand_atomic_compare_and_swap(rtx * ptarget_bool,rtx * ptarget_oval,rtx mem,rtx expected,rtx desired,bool is_weak,enum memmodel succ_model,enum memmodel fail_model)6317 expand_atomic_compare_and_swap (rtx *ptarget_bool, rtx *ptarget_oval,
6318 rtx mem, rtx expected, rtx desired,
6319 bool is_weak, enum memmodel succ_model,
6320 enum memmodel fail_model)
6321 {
6322 machine_mode mode = GET_MODE (mem);
6323 struct expand_operand ops[8];
6324 enum insn_code icode;
6325 rtx target_oval, target_bool = NULL_RTX;
6326 rtx libfunc;
6327
6328 /* If loads are not atomic for the required size and we are not called to
6329 provide a __sync builtin, do not do anything so that we stay consistent
6330 with atomic loads of the same size. */
6331 if (!can_atomic_load_p (mode) && !is_mm_sync (succ_model))
6332 return false;
6333
6334 /* Load expected into a register for the compare and swap. */
6335 if (MEM_P (expected))
6336 expected = copy_to_reg (expected);
6337
6338 /* Make sure we always have some place to put the return oldval.
6339 Further, make sure that place is distinct from the input expected,
6340 just in case we need that path down below. */
6341 if (ptarget_oval && *ptarget_oval == const0_rtx)
6342 ptarget_oval = NULL;
6343
6344 if (ptarget_oval == NULL
6345 || (target_oval = *ptarget_oval) == NULL
6346 || reg_overlap_mentioned_p (expected, target_oval))
6347 target_oval = gen_reg_rtx (mode);
6348
6349 icode = direct_optab_handler (atomic_compare_and_swap_optab, mode);
6350 if (icode != CODE_FOR_nothing)
6351 {
6352 machine_mode bool_mode = insn_data[icode].operand[0].mode;
6353
6354 if (ptarget_bool && *ptarget_bool == const0_rtx)
6355 ptarget_bool = NULL;
6356
6357 /* Make sure we always have a place for the bool operand. */
6358 if (ptarget_bool == NULL
6359 || (target_bool = *ptarget_bool) == NULL
6360 || GET_MODE (target_bool) != bool_mode)
6361 target_bool = gen_reg_rtx (bool_mode);
6362
6363 /* Emit the compare_and_swap. */
6364 create_output_operand (&ops[0], target_bool, bool_mode);
6365 create_output_operand (&ops[1], target_oval, mode);
6366 create_fixed_operand (&ops[2], mem);
6367 create_input_operand (&ops[3], expected, mode);
6368 create_input_operand (&ops[4], desired, mode);
6369 create_integer_operand (&ops[5], is_weak);
6370 create_integer_operand (&ops[6], succ_model);
6371 create_integer_operand (&ops[7], fail_model);
6372 if (maybe_expand_insn (icode, 8, ops))
6373 {
6374 /* Return success/failure. */
6375 target_bool = ops[0].value;
6376 target_oval = ops[1].value;
6377 goto success;
6378 }
6379 }
6380
6381 /* Otherwise fall back to the original __sync_val_compare_and_swap
6382 which is always seq-cst. */
6383 icode = optab_handler (sync_compare_and_swap_optab, mode);
6384 if (icode != CODE_FOR_nothing)
6385 {
6386 rtx cc_reg;
6387
6388 create_output_operand (&ops[0], target_oval, mode);
6389 create_fixed_operand (&ops[1], mem);
6390 create_input_operand (&ops[2], expected, mode);
6391 create_input_operand (&ops[3], desired, mode);
6392 if (!maybe_expand_insn (icode, 4, ops))
6393 return false;
6394
6395 target_oval = ops[0].value;
6396
6397 /* If the caller isn't interested in the boolean return value,
6398 skip the computation of it. */
6399 if (ptarget_bool == NULL)
6400 goto success;
6401
6402 /* Otherwise, work out if the compare-and-swap succeeded. */
6403 cc_reg = NULL_RTX;
6404 if (have_insn_for (COMPARE, CCmode))
6405 note_stores (PATTERN (get_last_insn ()), find_cc_set, &cc_reg);
6406 if (cc_reg)
6407 {
6408 target_bool = emit_store_flag_force (target_bool, EQ, cc_reg,
6409 const0_rtx, VOIDmode, 0, 1);
6410 goto success;
6411 }
6412 goto success_bool_from_val;
6413 }
6414
6415 /* Also check for library support for __sync_val_compare_and_swap. */
6416 libfunc = optab_libfunc (sync_compare_and_swap_optab, mode);
6417 if (libfunc != NULL)
6418 {
6419 rtx addr = convert_memory_address (ptr_mode, XEXP (mem, 0));
6420 rtx target = emit_library_call_value (libfunc, NULL_RTX, LCT_NORMAL,
6421 mode, addr, ptr_mode,
6422 expected, mode, desired, mode);
6423 emit_move_insn (target_oval, target);
6424
6425 /* Compute the boolean return value only if requested. */
6426 if (ptarget_bool)
6427 goto success_bool_from_val;
6428 else
6429 goto success;
6430 }
6431
6432 /* Failure. */
6433 return false;
6434
6435 success_bool_from_val:
6436 target_bool = emit_store_flag_force (target_bool, EQ, target_oval,
6437 expected, VOIDmode, 1, 1);
6438 success:
6439 /* Make sure that the oval output winds up where the caller asked. */
6440 if (ptarget_oval)
6441 *ptarget_oval = target_oval;
6442 if (ptarget_bool)
6443 *ptarget_bool = target_bool;
6444 return true;
6445 }
6446
6447 /* Generate asm volatile("" : : : "memory") as the memory blockage. */
6448
6449 static void
expand_asm_memory_blockage(void)6450 expand_asm_memory_blockage (void)
6451 {
6452 rtx asm_op, clob;
6453
6454 asm_op = gen_rtx_ASM_OPERANDS (VOIDmode, "", "", 0,
6455 rtvec_alloc (0), rtvec_alloc (0),
6456 rtvec_alloc (0), UNKNOWN_LOCATION);
6457 MEM_VOLATILE_P (asm_op) = 1;
6458
6459 clob = gen_rtx_SCRATCH (VOIDmode);
6460 clob = gen_rtx_MEM (BLKmode, clob);
6461 clob = gen_rtx_CLOBBER (VOIDmode, clob);
6462
6463 emit_insn (gen_rtx_PARALLEL (VOIDmode, gen_rtvec (2, asm_op, clob)));
6464 }
6465
6466 /* Do not propagate memory accesses across this point. */
6467
6468 static void
expand_memory_blockage(void)6469 expand_memory_blockage (void)
6470 {
6471 if (targetm.have_memory_blockage ())
6472 emit_insn (targetm.gen_memory_blockage ());
6473 else
6474 expand_asm_memory_blockage ();
6475 }
6476
6477 /* This routine will either emit the mem_thread_fence pattern or issue a
6478 sync_synchronize to generate a fence for memory model MEMMODEL. */
6479
6480 void
expand_mem_thread_fence(enum memmodel model)6481 expand_mem_thread_fence (enum memmodel model)
6482 {
6483 if (is_mm_relaxed (model))
6484 return;
6485 if (targetm.have_mem_thread_fence ())
6486 {
6487 emit_insn (targetm.gen_mem_thread_fence (GEN_INT (model)));
6488 expand_memory_blockage ();
6489 }
6490 else if (targetm.have_memory_barrier ())
6491 emit_insn (targetm.gen_memory_barrier ());
6492 else if (synchronize_libfunc != NULL_RTX)
6493 emit_library_call (synchronize_libfunc, LCT_NORMAL, VOIDmode);
6494 else
6495 expand_memory_blockage ();
6496 }
6497
6498 /* Emit a signal fence with given memory model. */
6499
6500 void
expand_mem_signal_fence(enum memmodel model)6501 expand_mem_signal_fence (enum memmodel model)
6502 {
6503 /* No machine barrier is required to implement a signal fence, but
6504 a compiler memory barrier must be issued, except for relaxed MM. */
6505 if (!is_mm_relaxed (model))
6506 expand_memory_blockage ();
6507 }
6508
6509 /* This function expands the atomic load operation:
6510 return the atomically loaded value in MEM.
6511
6512 MEMMODEL is the memory model variant to use.
6513 TARGET is an option place to stick the return value. */
6514
6515 rtx
expand_atomic_load(rtx target,rtx mem,enum memmodel model)6516 expand_atomic_load (rtx target, rtx mem, enum memmodel model)
6517 {
6518 machine_mode mode = GET_MODE (mem);
6519 enum insn_code icode;
6520
6521 /* If the target supports the load directly, great. */
6522 icode = direct_optab_handler (atomic_load_optab, mode);
6523 if (icode != CODE_FOR_nothing)
6524 {
6525 struct expand_operand ops[3];
6526 rtx_insn *last = get_last_insn ();
6527 if (is_mm_seq_cst (model))
6528 expand_memory_blockage ();
6529
6530 create_output_operand (&ops[0], target, mode);
6531 create_fixed_operand (&ops[1], mem);
6532 create_integer_operand (&ops[2], model);
6533 if (maybe_expand_insn (icode, 3, ops))
6534 {
6535 if (!is_mm_relaxed (model))
6536 expand_memory_blockage ();
6537 return ops[0].value;
6538 }
6539 delete_insns_since (last);
6540 }
6541
6542 /* If the size of the object is greater than word size on this target,
6543 then we assume that a load will not be atomic. We could try to
6544 emulate a load with a compare-and-swap operation, but the store that
6545 doing this could result in would be incorrect if this is a volatile
6546 atomic load or targetting read-only-mapped memory. */
6547 if (maybe_gt (GET_MODE_PRECISION (mode), BITS_PER_WORD))
6548 /* If there is no atomic load, leave the library call. */
6549 return NULL_RTX;
6550
6551 /* Otherwise assume loads are atomic, and emit the proper barriers. */
6552 if (!target || target == const0_rtx)
6553 target = gen_reg_rtx (mode);
6554
6555 /* For SEQ_CST, emit a barrier before the load. */
6556 if (is_mm_seq_cst (model))
6557 expand_mem_thread_fence (model);
6558
6559 emit_move_insn (target, mem);
6560
6561 /* Emit the appropriate barrier after the load. */
6562 expand_mem_thread_fence (model);
6563
6564 return target;
6565 }
6566
6567 /* This function expands the atomic store operation:
6568 Atomically store VAL in MEM.
6569 MEMMODEL is the memory model variant to use.
6570 USE_RELEASE is true if __sync_lock_release can be used as a fall back.
6571 function returns const0_rtx if a pattern was emitted. */
6572
6573 rtx
expand_atomic_store(rtx mem,rtx val,enum memmodel model,bool use_release)6574 expand_atomic_store (rtx mem, rtx val, enum memmodel model, bool use_release)
6575 {
6576 machine_mode mode = GET_MODE (mem);
6577 enum insn_code icode;
6578 struct expand_operand ops[3];
6579
6580 /* If the target supports the store directly, great. */
6581 icode = direct_optab_handler (atomic_store_optab, mode);
6582 if (icode != CODE_FOR_nothing)
6583 {
6584 rtx_insn *last = get_last_insn ();
6585 if (!is_mm_relaxed (model))
6586 expand_memory_blockage ();
6587 create_fixed_operand (&ops[0], mem);
6588 create_input_operand (&ops[1], val, mode);
6589 create_integer_operand (&ops[2], model);
6590 if (maybe_expand_insn (icode, 3, ops))
6591 {
6592 if (is_mm_seq_cst (model))
6593 expand_memory_blockage ();
6594 return const0_rtx;
6595 }
6596 delete_insns_since (last);
6597 }
6598
6599 /* If using __sync_lock_release is a viable alternative, try it.
6600 Note that this will not be set to true if we are expanding a generic
6601 __atomic_store_n. */
6602 if (use_release)
6603 {
6604 icode = direct_optab_handler (sync_lock_release_optab, mode);
6605 if (icode != CODE_FOR_nothing)
6606 {
6607 create_fixed_operand (&ops[0], mem);
6608 create_input_operand (&ops[1], const0_rtx, mode);
6609 if (maybe_expand_insn (icode, 2, ops))
6610 {
6611 /* lock_release is only a release barrier. */
6612 if (is_mm_seq_cst (model))
6613 expand_mem_thread_fence (model);
6614 return const0_rtx;
6615 }
6616 }
6617 }
6618
6619 /* If the size of the object is greater than word size on this target,
6620 a default store will not be atomic. */
6621 if (maybe_gt (GET_MODE_PRECISION (mode), BITS_PER_WORD))
6622 {
6623 /* If loads are atomic or we are called to provide a __sync builtin,
6624 we can try a atomic_exchange and throw away the result. Otherwise,
6625 don't do anything so that we do not create an inconsistency between
6626 loads and stores. */
6627 if (can_atomic_load_p (mode) || is_mm_sync (model))
6628 {
6629 rtx target = maybe_emit_atomic_exchange (NULL_RTX, mem, val, model);
6630 if (!target)
6631 target = maybe_emit_compare_and_swap_exchange_loop (NULL_RTX, mem,
6632 val);
6633 if (target)
6634 return const0_rtx;
6635 }
6636 return NULL_RTX;
6637 }
6638
6639 /* Otherwise assume stores are atomic, and emit the proper barriers. */
6640 expand_mem_thread_fence (model);
6641
6642 emit_move_insn (mem, val);
6643
6644 /* For SEQ_CST, also emit a barrier after the store. */
6645 if (is_mm_seq_cst (model))
6646 expand_mem_thread_fence (model);
6647
6648 return const0_rtx;
6649 }
6650
6651
6652 /* Structure containing the pointers and values required to process the
6653 various forms of the atomic_fetch_op and atomic_op_fetch builtins. */
6654
6655 struct atomic_op_functions
6656 {
6657 direct_optab mem_fetch_before;
6658 direct_optab mem_fetch_after;
6659 direct_optab mem_no_result;
6660 optab fetch_before;
6661 optab fetch_after;
6662 direct_optab no_result;
6663 enum rtx_code reverse_code;
6664 };
6665
6666
6667 /* Fill in structure pointed to by OP with the various optab entries for an
6668 operation of type CODE. */
6669
6670 static void
get_atomic_op_for_code(struct atomic_op_functions * op,enum rtx_code code)6671 get_atomic_op_for_code (struct atomic_op_functions *op, enum rtx_code code)
6672 {
6673 gcc_assert (op!= NULL);
6674
6675 /* If SWITCHABLE_TARGET is defined, then subtargets can be switched
6676 in the source code during compilation, and the optab entries are not
6677 computable until runtime. Fill in the values at runtime. */
6678 switch (code)
6679 {
6680 case PLUS:
6681 op->mem_fetch_before = atomic_fetch_add_optab;
6682 op->mem_fetch_after = atomic_add_fetch_optab;
6683 op->mem_no_result = atomic_add_optab;
6684 op->fetch_before = sync_old_add_optab;
6685 op->fetch_after = sync_new_add_optab;
6686 op->no_result = sync_add_optab;
6687 op->reverse_code = MINUS;
6688 break;
6689 case MINUS:
6690 op->mem_fetch_before = atomic_fetch_sub_optab;
6691 op->mem_fetch_after = atomic_sub_fetch_optab;
6692 op->mem_no_result = atomic_sub_optab;
6693 op->fetch_before = sync_old_sub_optab;
6694 op->fetch_after = sync_new_sub_optab;
6695 op->no_result = sync_sub_optab;
6696 op->reverse_code = PLUS;
6697 break;
6698 case XOR:
6699 op->mem_fetch_before = atomic_fetch_xor_optab;
6700 op->mem_fetch_after = atomic_xor_fetch_optab;
6701 op->mem_no_result = atomic_xor_optab;
6702 op->fetch_before = sync_old_xor_optab;
6703 op->fetch_after = sync_new_xor_optab;
6704 op->no_result = sync_xor_optab;
6705 op->reverse_code = XOR;
6706 break;
6707 case AND:
6708 op->mem_fetch_before = atomic_fetch_and_optab;
6709 op->mem_fetch_after = atomic_and_fetch_optab;
6710 op->mem_no_result = atomic_and_optab;
6711 op->fetch_before = sync_old_and_optab;
6712 op->fetch_after = sync_new_and_optab;
6713 op->no_result = sync_and_optab;
6714 op->reverse_code = UNKNOWN;
6715 break;
6716 case IOR:
6717 op->mem_fetch_before = atomic_fetch_or_optab;
6718 op->mem_fetch_after = atomic_or_fetch_optab;
6719 op->mem_no_result = atomic_or_optab;
6720 op->fetch_before = sync_old_ior_optab;
6721 op->fetch_after = sync_new_ior_optab;
6722 op->no_result = sync_ior_optab;
6723 op->reverse_code = UNKNOWN;
6724 break;
6725 case NOT:
6726 op->mem_fetch_before = atomic_fetch_nand_optab;
6727 op->mem_fetch_after = atomic_nand_fetch_optab;
6728 op->mem_no_result = atomic_nand_optab;
6729 op->fetch_before = sync_old_nand_optab;
6730 op->fetch_after = sync_new_nand_optab;
6731 op->no_result = sync_nand_optab;
6732 op->reverse_code = UNKNOWN;
6733 break;
6734 default:
6735 gcc_unreachable ();
6736 }
6737 }
6738
6739 /* See if there is a more optimal way to implement the operation "*MEM CODE VAL"
6740 using memory order MODEL. If AFTER is true the operation needs to return
6741 the value of *MEM after the operation, otherwise the previous value.
6742 TARGET is an optional place to place the result. The result is unused if
6743 it is const0_rtx.
6744 Return the result if there is a better sequence, otherwise NULL_RTX. */
6745
6746 static rtx
maybe_optimize_fetch_op(rtx target,rtx mem,rtx val,enum rtx_code code,enum memmodel model,bool after)6747 maybe_optimize_fetch_op (rtx target, rtx mem, rtx val, enum rtx_code code,
6748 enum memmodel model, bool after)
6749 {
6750 /* If the value is prefetched, or not used, it may be possible to replace
6751 the sequence with a native exchange operation. */
6752 if (!after || target == const0_rtx)
6753 {
6754 /* fetch_and (&x, 0, m) can be replaced with exchange (&x, 0, m). */
6755 if (code == AND && val == const0_rtx)
6756 {
6757 if (target == const0_rtx)
6758 target = gen_reg_rtx (GET_MODE (mem));
6759 return maybe_emit_atomic_exchange (target, mem, val, model);
6760 }
6761
6762 /* fetch_or (&x, -1, m) can be replaced with exchange (&x, -1, m). */
6763 if (code == IOR && val == constm1_rtx)
6764 {
6765 if (target == const0_rtx)
6766 target = gen_reg_rtx (GET_MODE (mem));
6767 return maybe_emit_atomic_exchange (target, mem, val, model);
6768 }
6769 }
6770
6771 return NULL_RTX;
6772 }
6773
6774 /* Try to emit an instruction for a specific operation varaition.
6775 OPTAB contains the OP functions.
6776 TARGET is an optional place to return the result. const0_rtx means unused.
6777 MEM is the memory location to operate on.
6778 VAL is the value to use in the operation.
6779 USE_MEMMODEL is TRUE if the variation with a memory model should be tried.
6780 MODEL is the memory model, if used.
6781 AFTER is true if the returned result is the value after the operation. */
6782
6783 static rtx
maybe_emit_op(const struct atomic_op_functions * optab,rtx target,rtx mem,rtx val,bool use_memmodel,enum memmodel model,bool after)6784 maybe_emit_op (const struct atomic_op_functions *optab, rtx target, rtx mem,
6785 rtx val, bool use_memmodel, enum memmodel model, bool after)
6786 {
6787 machine_mode mode = GET_MODE (mem);
6788 struct expand_operand ops[4];
6789 enum insn_code icode;
6790 int op_counter = 0;
6791 int num_ops;
6792
6793 /* Check to see if there is a result returned. */
6794 if (target == const0_rtx)
6795 {
6796 if (use_memmodel)
6797 {
6798 icode = direct_optab_handler (optab->mem_no_result, mode);
6799 create_integer_operand (&ops[2], model);
6800 num_ops = 3;
6801 }
6802 else
6803 {
6804 icode = direct_optab_handler (optab->no_result, mode);
6805 num_ops = 2;
6806 }
6807 }
6808 /* Otherwise, we need to generate a result. */
6809 else
6810 {
6811 if (use_memmodel)
6812 {
6813 icode = direct_optab_handler (after ? optab->mem_fetch_after
6814 : optab->mem_fetch_before, mode);
6815 create_integer_operand (&ops[3], model);
6816 num_ops = 4;
6817 }
6818 else
6819 {
6820 icode = optab_handler (after ? optab->fetch_after
6821 : optab->fetch_before, mode);
6822 num_ops = 3;
6823 }
6824 create_output_operand (&ops[op_counter++], target, mode);
6825 }
6826 if (icode == CODE_FOR_nothing)
6827 return NULL_RTX;
6828
6829 create_fixed_operand (&ops[op_counter++], mem);
6830 /* VAL may have been promoted to a wider mode. Shrink it if so. */
6831 create_convert_operand_to (&ops[op_counter++], val, mode, true);
6832
6833 if (maybe_expand_insn (icode, num_ops, ops))
6834 return (target == const0_rtx ? const0_rtx : ops[0].value);
6835
6836 return NULL_RTX;
6837 }
6838
6839
6840 /* This function expands an atomic fetch_OP or OP_fetch operation:
6841 TARGET is an option place to stick the return value. const0_rtx indicates
6842 the result is unused.
6843 atomically fetch MEM, perform the operation with VAL and return it to MEM.
6844 CODE is the operation being performed (OP)
6845 MEMMODEL is the memory model variant to use.
6846 AFTER is true to return the result of the operation (OP_fetch).
6847 AFTER is false to return the value before the operation (fetch_OP).
6848
6849 This function will *only* generate instructions if there is a direct
6850 optab. No compare and swap loops or libcalls will be generated. */
6851
6852 static rtx
expand_atomic_fetch_op_no_fallback(rtx target,rtx mem,rtx val,enum rtx_code code,enum memmodel model,bool after)6853 expand_atomic_fetch_op_no_fallback (rtx target, rtx mem, rtx val,
6854 enum rtx_code code, enum memmodel model,
6855 bool after)
6856 {
6857 machine_mode mode = GET_MODE (mem);
6858 struct atomic_op_functions optab;
6859 rtx result;
6860 bool unused_result = (target == const0_rtx);
6861
6862 get_atomic_op_for_code (&optab, code);
6863
6864 /* Check to see if there are any better instructions. */
6865 result = maybe_optimize_fetch_op (target, mem, val, code, model, after);
6866 if (result)
6867 return result;
6868
6869 /* Check for the case where the result isn't used and try those patterns. */
6870 if (unused_result)
6871 {
6872 /* Try the memory model variant first. */
6873 result = maybe_emit_op (&optab, target, mem, val, true, model, true);
6874 if (result)
6875 return result;
6876
6877 /* Next try the old style withuot a memory model. */
6878 result = maybe_emit_op (&optab, target, mem, val, false, model, true);
6879 if (result)
6880 return result;
6881
6882 /* There is no no-result pattern, so try patterns with a result. */
6883 target = NULL_RTX;
6884 }
6885
6886 /* Try the __atomic version. */
6887 result = maybe_emit_op (&optab, target, mem, val, true, model, after);
6888 if (result)
6889 return result;
6890
6891 /* Try the older __sync version. */
6892 result = maybe_emit_op (&optab, target, mem, val, false, model, after);
6893 if (result)
6894 return result;
6895
6896 /* If the fetch value can be calculated from the other variation of fetch,
6897 try that operation. */
6898 if (after || unused_result || optab.reverse_code != UNKNOWN)
6899 {
6900 /* Try the __atomic version, then the older __sync version. */
6901 result = maybe_emit_op (&optab, target, mem, val, true, model, !after);
6902 if (!result)
6903 result = maybe_emit_op (&optab, target, mem, val, false, model, !after);
6904
6905 if (result)
6906 {
6907 /* If the result isn't used, no need to do compensation code. */
6908 if (unused_result)
6909 return result;
6910
6911 /* Issue compensation code. Fetch_after == fetch_before OP val.
6912 Fetch_before == after REVERSE_OP val. */
6913 if (!after)
6914 code = optab.reverse_code;
6915 if (code == NOT)
6916 {
6917 result = expand_simple_binop (mode, AND, result, val, NULL_RTX,
6918 true, OPTAB_LIB_WIDEN);
6919 result = expand_simple_unop (mode, NOT, result, target, true);
6920 }
6921 else
6922 result = expand_simple_binop (mode, code, result, val, target,
6923 true, OPTAB_LIB_WIDEN);
6924 return result;
6925 }
6926 }
6927
6928 /* No direct opcode can be generated. */
6929 return NULL_RTX;
6930 }
6931
6932
6933
6934 /* This function expands an atomic fetch_OP or OP_fetch operation:
6935 TARGET is an option place to stick the return value. const0_rtx indicates
6936 the result is unused.
6937 atomically fetch MEM, perform the operation with VAL and return it to MEM.
6938 CODE is the operation being performed (OP)
6939 MEMMODEL is the memory model variant to use.
6940 AFTER is true to return the result of the operation (OP_fetch).
6941 AFTER is false to return the value before the operation (fetch_OP). */
6942 rtx
expand_atomic_fetch_op(rtx target,rtx mem,rtx val,enum rtx_code code,enum memmodel model,bool after)6943 expand_atomic_fetch_op (rtx target, rtx mem, rtx val, enum rtx_code code,
6944 enum memmodel model, bool after)
6945 {
6946 machine_mode mode = GET_MODE (mem);
6947 rtx result;
6948 bool unused_result = (target == const0_rtx);
6949
6950 /* If loads are not atomic for the required size and we are not called to
6951 provide a __sync builtin, do not do anything so that we stay consistent
6952 with atomic loads of the same size. */
6953 if (!can_atomic_load_p (mode) && !is_mm_sync (model))
6954 return NULL_RTX;
6955
6956 result = expand_atomic_fetch_op_no_fallback (target, mem, val, code, model,
6957 after);
6958
6959 if (result)
6960 return result;
6961
6962 /* Add/sub can be implemented by doing the reverse operation with -(val). */
6963 if (code == PLUS || code == MINUS)
6964 {
6965 rtx tmp;
6966 enum rtx_code reverse = (code == PLUS ? MINUS : PLUS);
6967
6968 start_sequence ();
6969 tmp = expand_simple_unop (mode, NEG, val, NULL_RTX, true);
6970 result = expand_atomic_fetch_op_no_fallback (target, mem, tmp, reverse,
6971 model, after);
6972 if (result)
6973 {
6974 /* PLUS worked so emit the insns and return. */
6975 tmp = get_insns ();
6976 end_sequence ();
6977 emit_insn (tmp);
6978 return result;
6979 }
6980
6981 /* PLUS did not work, so throw away the negation code and continue. */
6982 end_sequence ();
6983 }
6984
6985 /* Try the __sync libcalls only if we can't do compare-and-swap inline. */
6986 if (!can_compare_and_swap_p (mode, false))
6987 {
6988 rtx libfunc;
6989 bool fixup = false;
6990 enum rtx_code orig_code = code;
6991 struct atomic_op_functions optab;
6992
6993 get_atomic_op_for_code (&optab, code);
6994 libfunc = optab_libfunc (after ? optab.fetch_after
6995 : optab.fetch_before, mode);
6996 if (libfunc == NULL
6997 && (after || unused_result || optab.reverse_code != UNKNOWN))
6998 {
6999 fixup = true;
7000 if (!after)
7001 code = optab.reverse_code;
7002 libfunc = optab_libfunc (after ? optab.fetch_before
7003 : optab.fetch_after, mode);
7004 }
7005 if (libfunc != NULL)
7006 {
7007 rtx addr = convert_memory_address (ptr_mode, XEXP (mem, 0));
7008 result = emit_library_call_value (libfunc, NULL, LCT_NORMAL, mode,
7009 addr, ptr_mode, val, mode);
7010
7011 if (!unused_result && fixup)
7012 result = expand_simple_binop (mode, code, result, val, target,
7013 true, OPTAB_LIB_WIDEN);
7014 return result;
7015 }
7016
7017 /* We need the original code for any further attempts. */
7018 code = orig_code;
7019 }
7020
7021 /* If nothing else has succeeded, default to a compare and swap loop. */
7022 if (can_compare_and_swap_p (mode, true))
7023 {
7024 rtx_insn *insn;
7025 rtx t0 = gen_reg_rtx (mode), t1;
7026
7027 start_sequence ();
7028
7029 /* If the result is used, get a register for it. */
7030 if (!unused_result)
7031 {
7032 if (!target || !register_operand (target, mode))
7033 target = gen_reg_rtx (mode);
7034 /* If fetch_before, copy the value now. */
7035 if (!after)
7036 emit_move_insn (target, t0);
7037 }
7038 else
7039 target = const0_rtx;
7040
7041 t1 = t0;
7042 if (code == NOT)
7043 {
7044 t1 = expand_simple_binop (mode, AND, t1, val, NULL_RTX,
7045 true, OPTAB_LIB_WIDEN);
7046 t1 = expand_simple_unop (mode, code, t1, NULL_RTX, true);
7047 }
7048 else
7049 t1 = expand_simple_binop (mode, code, t1, val, NULL_RTX, true,
7050 OPTAB_LIB_WIDEN);
7051
7052 /* For after, copy the value now. */
7053 if (!unused_result && after)
7054 emit_move_insn (target, t1);
7055 insn = get_insns ();
7056 end_sequence ();
7057
7058 if (t1 != NULL && expand_compare_and_swap_loop (mem, t0, t1, insn))
7059 return target;
7060 }
7061
7062 return NULL_RTX;
7063 }
7064
7065 /* Return true if OPERAND is suitable for operand number OPNO of
7066 instruction ICODE. */
7067
7068 bool
insn_operand_matches(enum insn_code icode,unsigned int opno,rtx operand)7069 insn_operand_matches (enum insn_code icode, unsigned int opno, rtx operand)
7070 {
7071 return (!insn_data[(int) icode].operand[opno].predicate
7072 || (insn_data[(int) icode].operand[opno].predicate
7073 (operand, insn_data[(int) icode].operand[opno].mode)));
7074 }
7075
7076 /* TARGET is a target of a multiword operation that we are going to
7077 implement as a series of word-mode operations. Return true if
7078 TARGET is suitable for this purpose. */
7079
7080 bool
valid_multiword_target_p(rtx target)7081 valid_multiword_target_p (rtx target)
7082 {
7083 machine_mode mode;
7084 int i, size;
7085
7086 mode = GET_MODE (target);
7087 if (!GET_MODE_SIZE (mode).is_constant (&size))
7088 return false;
7089 for (i = 0; i < size; i += UNITS_PER_WORD)
7090 if (!validate_subreg (word_mode, mode, target, i))
7091 return false;
7092 return true;
7093 }
7094
7095 /* Make OP describe an input operand that has value INTVAL and that has
7096 no inherent mode. This function should only be used for operands that
7097 are always expand-time constants. The backend may request that INTVAL
7098 be copied into a different kind of rtx, but it must specify the mode
7099 of that rtx if so. */
7100
7101 void
create_integer_operand(struct expand_operand * op,poly_int64 intval)7102 create_integer_operand (struct expand_operand *op, poly_int64 intval)
7103 {
7104 create_expand_operand (op, EXPAND_INTEGER,
7105 gen_int_mode (intval, MAX_MODE_INT),
7106 VOIDmode, false, intval);
7107 }
7108
7109 /* Like maybe_legitimize_operand, but do not change the code of the
7110 current rtx value. */
7111
7112 static bool
maybe_legitimize_operand_same_code(enum insn_code icode,unsigned int opno,struct expand_operand * op)7113 maybe_legitimize_operand_same_code (enum insn_code icode, unsigned int opno,
7114 struct expand_operand *op)
7115 {
7116 /* See if the operand matches in its current form. */
7117 if (insn_operand_matches (icode, opno, op->value))
7118 return true;
7119
7120 /* If the operand is a memory whose address has no side effects,
7121 try forcing the address into a non-virtual pseudo register.
7122 The check for side effects is important because copy_to_mode_reg
7123 cannot handle things like auto-modified addresses. */
7124 if (insn_data[(int) icode].operand[opno].allows_mem && MEM_P (op->value))
7125 {
7126 rtx addr, mem;
7127
7128 mem = op->value;
7129 addr = XEXP (mem, 0);
7130 if (!(REG_P (addr) && REGNO (addr) > LAST_VIRTUAL_REGISTER)
7131 && !side_effects_p (addr))
7132 {
7133 rtx_insn *last;
7134 machine_mode mode;
7135
7136 last = get_last_insn ();
7137 mode = get_address_mode (mem);
7138 mem = replace_equiv_address (mem, copy_to_mode_reg (mode, addr));
7139 if (insn_operand_matches (icode, opno, mem))
7140 {
7141 op->value = mem;
7142 return true;
7143 }
7144 delete_insns_since (last);
7145 }
7146 }
7147
7148 return false;
7149 }
7150
7151 /* Try to make OP match operand OPNO of instruction ICODE. Return true
7152 on success, storing the new operand value back in OP. */
7153
7154 static bool
maybe_legitimize_operand(enum insn_code icode,unsigned int opno,struct expand_operand * op)7155 maybe_legitimize_operand (enum insn_code icode, unsigned int opno,
7156 struct expand_operand *op)
7157 {
7158 machine_mode mode, imode;
7159 bool old_volatile_ok, result;
7160
7161 mode = op->mode;
7162 switch (op->type)
7163 {
7164 case EXPAND_FIXED:
7165 old_volatile_ok = volatile_ok;
7166 volatile_ok = true;
7167 result = maybe_legitimize_operand_same_code (icode, opno, op);
7168 volatile_ok = old_volatile_ok;
7169 return result;
7170
7171 case EXPAND_OUTPUT:
7172 gcc_assert (mode != VOIDmode);
7173 if (op->value
7174 && op->value != const0_rtx
7175 && GET_MODE (op->value) == mode
7176 && maybe_legitimize_operand_same_code (icode, opno, op))
7177 return true;
7178
7179 op->value = gen_reg_rtx (mode);
7180 op->target = 0;
7181 break;
7182
7183 case EXPAND_INPUT:
7184 input:
7185 gcc_assert (mode != VOIDmode);
7186 gcc_assert (GET_MODE (op->value) == VOIDmode
7187 || GET_MODE (op->value) == mode);
7188 if (maybe_legitimize_operand_same_code (icode, opno, op))
7189 return true;
7190
7191 op->value = copy_to_mode_reg (mode, op->value);
7192 break;
7193
7194 case EXPAND_CONVERT_TO:
7195 gcc_assert (mode != VOIDmode);
7196 op->value = convert_to_mode (mode, op->value, op->unsigned_p);
7197 goto input;
7198
7199 case EXPAND_CONVERT_FROM:
7200 if (GET_MODE (op->value) != VOIDmode)
7201 mode = GET_MODE (op->value);
7202 else
7203 /* The caller must tell us what mode this value has. */
7204 gcc_assert (mode != VOIDmode);
7205
7206 imode = insn_data[(int) icode].operand[opno].mode;
7207 if (imode != VOIDmode && imode != mode)
7208 {
7209 op->value = convert_modes (imode, mode, op->value, op->unsigned_p);
7210 mode = imode;
7211 }
7212 goto input;
7213
7214 case EXPAND_ADDRESS:
7215 op->value = convert_memory_address (as_a <scalar_int_mode> (mode),
7216 op->value);
7217 goto input;
7218
7219 case EXPAND_INTEGER:
7220 mode = insn_data[(int) icode].operand[opno].mode;
7221 if (mode != VOIDmode
7222 && known_eq (trunc_int_for_mode (op->int_value, mode),
7223 op->int_value))
7224 {
7225 op->value = gen_int_mode (op->int_value, mode);
7226 goto input;
7227 }
7228 break;
7229 }
7230 return insn_operand_matches (icode, opno, op->value);
7231 }
7232
7233 /* Make OP describe an input operand that should have the same value
7234 as VALUE, after any mode conversion that the target might request.
7235 TYPE is the type of VALUE. */
7236
7237 void
create_convert_operand_from_type(struct expand_operand * op,rtx value,tree type)7238 create_convert_operand_from_type (struct expand_operand *op,
7239 rtx value, tree type)
7240 {
7241 create_convert_operand_from (op, value, TYPE_MODE (type),
7242 TYPE_UNSIGNED (type));
7243 }
7244
7245 /* Try to make operands [OPS, OPS + NOPS) match operands [OPNO, OPNO + NOPS)
7246 of instruction ICODE. Return true on success, leaving the new operand
7247 values in the OPS themselves. Emit no code on failure. */
7248
7249 bool
maybe_legitimize_operands(enum insn_code icode,unsigned int opno,unsigned int nops,struct expand_operand * ops)7250 maybe_legitimize_operands (enum insn_code icode, unsigned int opno,
7251 unsigned int nops, struct expand_operand *ops)
7252 {
7253 rtx_insn *last;
7254 unsigned int i;
7255
7256 last = get_last_insn ();
7257 for (i = 0; i < nops; i++)
7258 if (!maybe_legitimize_operand (icode, opno + i, &ops[i]))
7259 {
7260 delete_insns_since (last);
7261 return false;
7262 }
7263 return true;
7264 }
7265
7266 /* Try to generate instruction ICODE, using operands [OPS, OPS + NOPS)
7267 as its operands. Return the instruction pattern on success,
7268 and emit any necessary set-up code. Return null and emit no
7269 code on failure. */
7270
7271 rtx_insn *
maybe_gen_insn(enum insn_code icode,unsigned int nops,struct expand_operand * ops)7272 maybe_gen_insn (enum insn_code icode, unsigned int nops,
7273 struct expand_operand *ops)
7274 {
7275 gcc_assert (nops == (unsigned int) insn_data[(int) icode].n_generator_args);
7276 if (!maybe_legitimize_operands (icode, 0, nops, ops))
7277 return NULL;
7278
7279 switch (nops)
7280 {
7281 case 1:
7282 return GEN_FCN (icode) (ops[0].value);
7283 case 2:
7284 return GEN_FCN (icode) (ops[0].value, ops[1].value);
7285 case 3:
7286 return GEN_FCN (icode) (ops[0].value, ops[1].value, ops[2].value);
7287 case 4:
7288 return GEN_FCN (icode) (ops[0].value, ops[1].value, ops[2].value,
7289 ops[3].value);
7290 case 5:
7291 return GEN_FCN (icode) (ops[0].value, ops[1].value, ops[2].value,
7292 ops[3].value, ops[4].value);
7293 case 6:
7294 return GEN_FCN (icode) (ops[0].value, ops[1].value, ops[2].value,
7295 ops[3].value, ops[4].value, ops[5].value);
7296 case 7:
7297 return GEN_FCN (icode) (ops[0].value, ops[1].value, ops[2].value,
7298 ops[3].value, ops[4].value, ops[5].value,
7299 ops[6].value);
7300 case 8:
7301 return GEN_FCN (icode) (ops[0].value, ops[1].value, ops[2].value,
7302 ops[3].value, ops[4].value, ops[5].value,
7303 ops[6].value, ops[7].value);
7304 case 9:
7305 return GEN_FCN (icode) (ops[0].value, ops[1].value, ops[2].value,
7306 ops[3].value, ops[4].value, ops[5].value,
7307 ops[6].value, ops[7].value, ops[8].value);
7308 }
7309 gcc_unreachable ();
7310 }
7311
7312 /* Try to emit instruction ICODE, using operands [OPS, OPS + NOPS)
7313 as its operands. Return true on success and emit no code on failure. */
7314
7315 bool
maybe_expand_insn(enum insn_code icode,unsigned int nops,struct expand_operand * ops)7316 maybe_expand_insn (enum insn_code icode, unsigned int nops,
7317 struct expand_operand *ops)
7318 {
7319 rtx_insn *pat = maybe_gen_insn (icode, nops, ops);
7320 if (pat)
7321 {
7322 emit_insn (pat);
7323 return true;
7324 }
7325 return false;
7326 }
7327
7328 /* Like maybe_expand_insn, but for jumps. */
7329
7330 bool
maybe_expand_jump_insn(enum insn_code icode,unsigned int nops,struct expand_operand * ops)7331 maybe_expand_jump_insn (enum insn_code icode, unsigned int nops,
7332 struct expand_operand *ops)
7333 {
7334 rtx_insn *pat = maybe_gen_insn (icode, nops, ops);
7335 if (pat)
7336 {
7337 emit_jump_insn (pat);
7338 return true;
7339 }
7340 return false;
7341 }
7342
7343 /* Emit instruction ICODE, using operands [OPS, OPS + NOPS)
7344 as its operands. */
7345
7346 void
expand_insn(enum insn_code icode,unsigned int nops,struct expand_operand * ops)7347 expand_insn (enum insn_code icode, unsigned int nops,
7348 struct expand_operand *ops)
7349 {
7350 if (!maybe_expand_insn (icode, nops, ops))
7351 gcc_unreachable ();
7352 }
7353
7354 /* Like expand_insn, but for jumps. */
7355
7356 void
expand_jump_insn(enum insn_code icode,unsigned int nops,struct expand_operand * ops)7357 expand_jump_insn (enum insn_code icode, unsigned int nops,
7358 struct expand_operand *ops)
7359 {
7360 if (!maybe_expand_jump_insn (icode, nops, ops))
7361 gcc_unreachable ();
7362 }
7363