1 /* Expand the basic unary and binary arithmetic operations, for GNU compiler.
2 Copyright (C) 1987-2019 Free Software Foundation, Inc.
3
4 This file is part of GCC.
5
6 GCC is free software; you can redistribute it and/or modify it under
7 the terms of the GNU General Public License as published by the Free
8 Software Foundation; either version 3, or (at your option) any later
9 version.
10
11 GCC is distributed in the hope that it will be useful, but WITHOUT ANY
12 WARRANTY; without even the implied warranty of MERCHANTABILITY or
13 FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
14 for more details.
15
16 You should have received a copy of the GNU General Public License
17 along with GCC; see the file COPYING3. If not see
18 <http://www.gnu.org/licenses/>. */
19
20
21 #include "config.h"
22 #include "system.h"
23 #include "coretypes.h"
24 #include "backend.h"
25 #include "target.h"
26 #include "rtl.h"
27 #include "tree.h"
28 #include "memmodel.h"
29 #include "predict.h"
30 #include "tm_p.h"
31 #include "expmed.h"
32 #include "optabs.h"
33 #include "emit-rtl.h"
34 #include "recog.h"
35 #include "diagnostic-core.h"
36 #include "rtx-vector-builder.h"
37
38 /* Include insn-config.h before expr.h so that HAVE_conditional_move
39 is properly defined. */
40 #include "stor-layout.h"
41 #include "except.h"
42 #include "dojump.h"
43 #include "explow.h"
44 #include "expr.h"
45 #include "optabs-tree.h"
46 #include "libfuncs.h"
47
48 static void prepare_float_lib_cmp (rtx, rtx, enum rtx_code, rtx *,
49 machine_mode *);
50 static rtx expand_unop_direct (machine_mode, optab, rtx, rtx, int);
51 static void emit_libcall_block_1 (rtx_insn *, rtx, rtx, rtx, bool);
52
53 /* Debug facility for use in GDB. */
54 void debug_optab_libfuncs (void);
55
56 /* Add a REG_EQUAL note to the last insn in INSNS. TARGET is being set to
57 the result of operation CODE applied to OP0 (and OP1 if it is a binary
58 operation). OP0_MODE is OP0's mode.
59
60 If the last insn does not set TARGET, don't do anything, but return 1.
61
62 If the last insn or a previous insn sets TARGET and TARGET is one of OP0
63 or OP1, don't add the REG_EQUAL note but return 0. Our caller can then
64 try again, ensuring that TARGET is not one of the operands. */
65
66 static int
add_equal_note(rtx_insn * insns,rtx target,enum rtx_code code,rtx op0,rtx op1,machine_mode op0_mode)67 add_equal_note (rtx_insn *insns, rtx target, enum rtx_code code, rtx op0,
68 rtx op1, machine_mode op0_mode)
69 {
70 rtx_insn *last_insn;
71 rtx set;
72 rtx note;
73
74 gcc_assert (insns && INSN_P (insns) && NEXT_INSN (insns));
75
76 if (GET_RTX_CLASS (code) != RTX_COMM_ARITH
77 && GET_RTX_CLASS (code) != RTX_BIN_ARITH
78 && GET_RTX_CLASS (code) != RTX_COMM_COMPARE
79 && GET_RTX_CLASS (code) != RTX_COMPARE
80 && GET_RTX_CLASS (code) != RTX_UNARY)
81 return 1;
82
83 if (GET_CODE (target) == ZERO_EXTRACT)
84 return 1;
85
86 for (last_insn = insns;
87 NEXT_INSN (last_insn) != NULL_RTX;
88 last_insn = NEXT_INSN (last_insn))
89 ;
90
91 /* If TARGET is in OP0 or OP1, punt. We'd end up with a note referencing
92 a value changing in the insn, so the note would be invalid for CSE. */
93 if (reg_overlap_mentioned_p (target, op0)
94 || (op1 && reg_overlap_mentioned_p (target, op1)))
95 {
96 if (MEM_P (target)
97 && (rtx_equal_p (target, op0)
98 || (op1 && rtx_equal_p (target, op1))))
99 {
100 /* For MEM target, with MEM = MEM op X, prefer no REG_EQUAL note
101 over expanding it as temp = MEM op X, MEM = temp. If the target
102 supports MEM = MEM op X instructions, it is sometimes too hard
103 to reconstruct that form later, especially if X is also a memory,
104 and due to multiple occurrences of addresses the address might
105 be forced into register unnecessarily.
106 Note that not emitting the REG_EQUIV note might inhibit
107 CSE in some cases. */
108 set = single_set (last_insn);
109 if (set
110 && GET_CODE (SET_SRC (set)) == code
111 && MEM_P (SET_DEST (set))
112 && (rtx_equal_p (SET_DEST (set), XEXP (SET_SRC (set), 0))
113 || (op1 && rtx_equal_p (SET_DEST (set),
114 XEXP (SET_SRC (set), 1)))))
115 return 1;
116 }
117 return 0;
118 }
119
120 set = set_for_reg_notes (last_insn);
121 if (set == NULL_RTX)
122 return 1;
123
124 if (! rtx_equal_p (SET_DEST (set), target)
125 /* For a STRICT_LOW_PART, the REG_NOTE applies to what is inside it. */
126 && (GET_CODE (SET_DEST (set)) != STRICT_LOW_PART
127 || ! rtx_equal_p (XEXP (SET_DEST (set), 0), target)))
128 return 1;
129
130 if (GET_RTX_CLASS (code) == RTX_UNARY)
131 switch (code)
132 {
133 case FFS:
134 case CLZ:
135 case CTZ:
136 case CLRSB:
137 case POPCOUNT:
138 case PARITY:
139 case BSWAP:
140 if (op0_mode != VOIDmode && GET_MODE (target) != op0_mode)
141 {
142 note = gen_rtx_fmt_e (code, op0_mode, copy_rtx (op0));
143 if (GET_MODE_UNIT_SIZE (op0_mode)
144 > GET_MODE_UNIT_SIZE (GET_MODE (target)))
145 note = simplify_gen_unary (TRUNCATE, GET_MODE (target),
146 note, op0_mode);
147 else
148 note = simplify_gen_unary (ZERO_EXTEND, GET_MODE (target),
149 note, op0_mode);
150 break;
151 }
152 /* FALLTHRU */
153 default:
154 note = gen_rtx_fmt_e (code, GET_MODE (target), copy_rtx (op0));
155 break;
156 }
157 else
158 note = gen_rtx_fmt_ee (code, GET_MODE (target), copy_rtx (op0), copy_rtx (op1));
159
160 set_unique_reg_note (last_insn, REG_EQUAL, note);
161
162 return 1;
163 }
164
165 /* Given two input operands, OP0 and OP1, determine what the correct from_mode
166 for a widening operation would be. In most cases this would be OP0, but if
167 that's a constant it'll be VOIDmode, which isn't useful. */
168
169 static machine_mode
widened_mode(machine_mode to_mode,rtx op0,rtx op1)170 widened_mode (machine_mode to_mode, rtx op0, rtx op1)
171 {
172 machine_mode m0 = GET_MODE (op0);
173 machine_mode m1 = GET_MODE (op1);
174 machine_mode result;
175
176 if (m0 == VOIDmode && m1 == VOIDmode)
177 return to_mode;
178 else if (m0 == VOIDmode || GET_MODE_UNIT_SIZE (m0) < GET_MODE_UNIT_SIZE (m1))
179 result = m1;
180 else
181 result = m0;
182
183 if (GET_MODE_UNIT_SIZE (result) > GET_MODE_UNIT_SIZE (to_mode))
184 return to_mode;
185
186 return result;
187 }
188
189 /* Widen OP to MODE and return the rtx for the widened operand. UNSIGNEDP
190 says whether OP is signed or unsigned. NO_EXTEND is nonzero if we need
191 not actually do a sign-extend or zero-extend, but can leave the
192 higher-order bits of the result rtx undefined, for example, in the case
193 of logical operations, but not right shifts. */
194
195 static rtx
widen_operand(rtx op,machine_mode mode,machine_mode oldmode,int unsignedp,int no_extend)196 widen_operand (rtx op, machine_mode mode, machine_mode oldmode,
197 int unsignedp, int no_extend)
198 {
199 rtx result;
200 scalar_int_mode int_mode;
201
202 /* If we don't have to extend and this is a constant, return it. */
203 if (no_extend && GET_MODE (op) == VOIDmode)
204 return op;
205
206 /* If we must extend do so. If OP is a SUBREG for a promoted object, also
207 extend since it will be more efficient to do so unless the signedness of
208 a promoted object differs from our extension. */
209 if (! no_extend
210 || !is_a <scalar_int_mode> (mode, &int_mode)
211 || (GET_CODE (op) == SUBREG && SUBREG_PROMOTED_VAR_P (op)
212 && SUBREG_CHECK_PROMOTED_SIGN (op, unsignedp)))
213 return convert_modes (mode, oldmode, op, unsignedp);
214
215 /* If MODE is no wider than a single word, we return a lowpart or paradoxical
216 SUBREG. */
217 if (GET_MODE_SIZE (int_mode) <= UNITS_PER_WORD)
218 return gen_lowpart (int_mode, force_reg (GET_MODE (op), op));
219
220 /* Otherwise, get an object of MODE, clobber it, and set the low-order
221 part to OP. */
222
223 result = gen_reg_rtx (int_mode);
224 emit_clobber (result);
225 emit_move_insn (gen_lowpart (GET_MODE (op), result), op);
226 return result;
227 }
228
229 /* Expand vector widening operations.
230
231 There are two different classes of operations handled here:
232 1) Operations whose result is wider than all the arguments to the operation.
233 Examples: VEC_UNPACK_HI/LO_EXPR, VEC_WIDEN_MULT_HI/LO_EXPR
234 In this case OP0 and optionally OP1 would be initialized,
235 but WIDE_OP wouldn't (not relevant for this case).
236 2) Operations whose result is of the same size as the last argument to the
237 operation, but wider than all the other arguments to the operation.
238 Examples: WIDEN_SUM_EXPR, VEC_DOT_PROD_EXPR.
239 In the case WIDE_OP, OP0 and optionally OP1 would be initialized.
240
241 E.g, when called to expand the following operations, this is how
242 the arguments will be initialized:
243 nops OP0 OP1 WIDE_OP
244 widening-sum 2 oprnd0 - oprnd1
245 widening-dot-product 3 oprnd0 oprnd1 oprnd2
246 widening-mult 2 oprnd0 oprnd1 -
247 type-promotion (vec-unpack) 1 oprnd0 - - */
248
249 rtx
expand_widen_pattern_expr(sepops ops,rtx op0,rtx op1,rtx wide_op,rtx target,int unsignedp)250 expand_widen_pattern_expr (sepops ops, rtx op0, rtx op1, rtx wide_op,
251 rtx target, int unsignedp)
252 {
253 struct expand_operand eops[4];
254 tree oprnd0, oprnd1, oprnd2;
255 machine_mode wmode = VOIDmode, tmode0, tmode1 = VOIDmode;
256 optab widen_pattern_optab;
257 enum insn_code icode;
258 int nops = TREE_CODE_LENGTH (ops->code);
259 int op;
260 bool sbool = false;
261
262 oprnd0 = ops->op0;
263 tmode0 = TYPE_MODE (TREE_TYPE (oprnd0));
264 if (ops->code == VEC_UNPACK_FIX_TRUNC_HI_EXPR
265 || ops->code == VEC_UNPACK_FIX_TRUNC_LO_EXPR)
266 /* The sign is from the result type rather than operand's type
267 for these ops. */
268 widen_pattern_optab
269 = optab_for_tree_code (ops->code, ops->type, optab_default);
270 else if ((ops->code == VEC_UNPACK_HI_EXPR
271 || ops->code == VEC_UNPACK_LO_EXPR)
272 && VECTOR_BOOLEAN_TYPE_P (ops->type)
273 && VECTOR_BOOLEAN_TYPE_P (TREE_TYPE (oprnd0))
274 && TYPE_MODE (ops->type) == TYPE_MODE (TREE_TYPE (oprnd0))
275 && SCALAR_INT_MODE_P (TYPE_MODE (ops->type)))
276 {
277 /* For VEC_UNPACK_{LO,HI}_EXPR if the mode of op0 and result is
278 the same scalar mode for VECTOR_BOOLEAN_TYPE_P vectors, use
279 vec_unpacks_sbool_{lo,hi}_optab, so that we can pass in
280 the pattern number of elements in the wider vector. */
281 widen_pattern_optab
282 = (ops->code == VEC_UNPACK_HI_EXPR
283 ? vec_unpacks_sbool_hi_optab : vec_unpacks_sbool_lo_optab);
284 sbool = true;
285 }
286 else
287 widen_pattern_optab
288 = optab_for_tree_code (ops->code, TREE_TYPE (oprnd0), optab_default);
289 if (ops->code == WIDEN_MULT_PLUS_EXPR
290 || ops->code == WIDEN_MULT_MINUS_EXPR)
291 icode = find_widening_optab_handler (widen_pattern_optab,
292 TYPE_MODE (TREE_TYPE (ops->op2)),
293 tmode0);
294 else
295 icode = optab_handler (widen_pattern_optab, tmode0);
296 gcc_assert (icode != CODE_FOR_nothing);
297
298 if (nops >= 2)
299 {
300 oprnd1 = ops->op1;
301 tmode1 = TYPE_MODE (TREE_TYPE (oprnd1));
302 }
303 else if (sbool)
304 {
305 nops = 2;
306 op1 = GEN_INT (TYPE_VECTOR_SUBPARTS (TREE_TYPE (oprnd0)).to_constant ());
307 tmode1 = tmode0;
308 }
309
310 /* The last operand is of a wider mode than the rest of the operands. */
311 if (nops == 2)
312 wmode = tmode1;
313 else if (nops == 3)
314 {
315 gcc_assert (tmode1 == tmode0);
316 gcc_assert (op1);
317 oprnd2 = ops->op2;
318 wmode = TYPE_MODE (TREE_TYPE (oprnd2));
319 }
320
321 op = 0;
322 create_output_operand (&eops[op++], target, TYPE_MODE (ops->type));
323 create_convert_operand_from (&eops[op++], op0, tmode0, unsignedp);
324 if (op1)
325 create_convert_operand_from (&eops[op++], op1, tmode1, unsignedp);
326 if (wide_op)
327 create_convert_operand_from (&eops[op++], wide_op, wmode, unsignedp);
328 expand_insn (icode, op, eops);
329 return eops[0].value;
330 }
331
332 /* Generate code to perform an operation specified by TERNARY_OPTAB
333 on operands OP0, OP1 and OP2, with result having machine-mode MODE.
334
335 UNSIGNEDP is for the case where we have to widen the operands
336 to perform the operation. It says to use zero-extension.
337
338 If TARGET is nonzero, the value
339 is generated there, if it is convenient to do so.
340 In all cases an rtx is returned for the locus of the value;
341 this may or may not be TARGET. */
342
343 rtx
expand_ternary_op(machine_mode mode,optab ternary_optab,rtx op0,rtx op1,rtx op2,rtx target,int unsignedp)344 expand_ternary_op (machine_mode mode, optab ternary_optab, rtx op0,
345 rtx op1, rtx op2, rtx target, int unsignedp)
346 {
347 struct expand_operand ops[4];
348 enum insn_code icode = optab_handler (ternary_optab, mode);
349
350 gcc_assert (optab_handler (ternary_optab, mode) != CODE_FOR_nothing);
351
352 create_output_operand (&ops[0], target, mode);
353 create_convert_operand_from (&ops[1], op0, mode, unsignedp);
354 create_convert_operand_from (&ops[2], op1, mode, unsignedp);
355 create_convert_operand_from (&ops[3], op2, mode, unsignedp);
356 expand_insn (icode, 4, ops);
357 return ops[0].value;
358 }
359
360
361 /* Like expand_binop, but return a constant rtx if the result can be
362 calculated at compile time. The arguments and return value are
363 otherwise the same as for expand_binop. */
364
365 rtx
simplify_expand_binop(machine_mode mode,optab binoptab,rtx op0,rtx op1,rtx target,int unsignedp,enum optab_methods methods)366 simplify_expand_binop (machine_mode mode, optab binoptab,
367 rtx op0, rtx op1, rtx target, int unsignedp,
368 enum optab_methods methods)
369 {
370 if (CONSTANT_P (op0) && CONSTANT_P (op1))
371 {
372 rtx x = simplify_binary_operation (optab_to_code (binoptab),
373 mode, op0, op1);
374 if (x)
375 return x;
376 }
377
378 return expand_binop (mode, binoptab, op0, op1, target, unsignedp, methods);
379 }
380
381 /* Like simplify_expand_binop, but always put the result in TARGET.
382 Return true if the expansion succeeded. */
383
384 bool
force_expand_binop(machine_mode mode,optab binoptab,rtx op0,rtx op1,rtx target,int unsignedp,enum optab_methods methods)385 force_expand_binop (machine_mode mode, optab binoptab,
386 rtx op0, rtx op1, rtx target, int unsignedp,
387 enum optab_methods methods)
388 {
389 rtx x = simplify_expand_binop (mode, binoptab, op0, op1,
390 target, unsignedp, methods);
391 if (x == 0)
392 return false;
393 if (x != target)
394 emit_move_insn (target, x);
395 return true;
396 }
397
398 /* Create a new vector value in VMODE with all elements set to OP. The
399 mode of OP must be the element mode of VMODE. If OP is a constant,
400 then the return value will be a constant. */
401
402 rtx
expand_vector_broadcast(machine_mode vmode,rtx op)403 expand_vector_broadcast (machine_mode vmode, rtx op)
404 {
405 int n;
406 rtvec vec;
407
408 gcc_checking_assert (VECTOR_MODE_P (vmode));
409
410 if (valid_for_const_vector_p (vmode, op))
411 return gen_const_vec_duplicate (vmode, op);
412
413 insn_code icode = optab_handler (vec_duplicate_optab, vmode);
414 if (icode != CODE_FOR_nothing)
415 {
416 struct expand_operand ops[2];
417 create_output_operand (&ops[0], NULL_RTX, vmode);
418 create_input_operand (&ops[1], op, GET_MODE (op));
419 expand_insn (icode, 2, ops);
420 return ops[0].value;
421 }
422
423 if (!GET_MODE_NUNITS (vmode).is_constant (&n))
424 return NULL;
425
426 /* ??? If the target doesn't have a vec_init, then we have no easy way
427 of performing this operation. Most of this sort of generic support
428 is hidden away in the vector lowering support in gimple. */
429 icode = convert_optab_handler (vec_init_optab, vmode,
430 GET_MODE_INNER (vmode));
431 if (icode == CODE_FOR_nothing)
432 return NULL;
433
434 vec = rtvec_alloc (n);
435 for (int i = 0; i < n; ++i)
436 RTVEC_ELT (vec, i) = op;
437 rtx ret = gen_reg_rtx (vmode);
438 emit_insn (GEN_FCN (icode) (ret, gen_rtx_PARALLEL (vmode, vec)));
439
440 return ret;
441 }
442
443 /* This subroutine of expand_doubleword_shift handles the cases in which
444 the effective shift value is >= BITS_PER_WORD. The arguments and return
445 value are the same as for the parent routine, except that SUPERWORD_OP1
446 is the shift count to use when shifting OUTOF_INPUT into INTO_TARGET.
447 INTO_TARGET may be null if the caller has decided to calculate it. */
448
449 static bool
expand_superword_shift(optab binoptab,rtx outof_input,rtx superword_op1,rtx outof_target,rtx into_target,int unsignedp,enum optab_methods methods)450 expand_superword_shift (optab binoptab, rtx outof_input, rtx superword_op1,
451 rtx outof_target, rtx into_target,
452 int unsignedp, enum optab_methods methods)
453 {
454 if (into_target != 0)
455 if (!force_expand_binop (word_mode, binoptab, outof_input, superword_op1,
456 into_target, unsignedp, methods))
457 return false;
458
459 if (outof_target != 0)
460 {
461 /* For a signed right shift, we must fill OUTOF_TARGET with copies
462 of the sign bit, otherwise we must fill it with zeros. */
463 if (binoptab != ashr_optab)
464 emit_move_insn (outof_target, CONST0_RTX (word_mode));
465 else
466 if (!force_expand_binop (word_mode, binoptab, outof_input,
467 gen_int_shift_amount (word_mode,
468 BITS_PER_WORD - 1),
469 outof_target, unsignedp, methods))
470 return false;
471 }
472 return true;
473 }
474
475 /* This subroutine of expand_doubleword_shift handles the cases in which
476 the effective shift value is < BITS_PER_WORD. The arguments and return
477 value are the same as for the parent routine. */
478
479 static bool
expand_subword_shift(scalar_int_mode op1_mode,optab binoptab,rtx outof_input,rtx into_input,rtx op1,rtx outof_target,rtx into_target,int unsignedp,enum optab_methods methods,unsigned HOST_WIDE_INT shift_mask)480 expand_subword_shift (scalar_int_mode op1_mode, optab binoptab,
481 rtx outof_input, rtx into_input, rtx op1,
482 rtx outof_target, rtx into_target,
483 int unsignedp, enum optab_methods methods,
484 unsigned HOST_WIDE_INT shift_mask)
485 {
486 optab reverse_unsigned_shift, unsigned_shift;
487 rtx tmp, carries;
488
489 reverse_unsigned_shift = (binoptab == ashl_optab ? lshr_optab : ashl_optab);
490 unsigned_shift = (binoptab == ashl_optab ? ashl_optab : lshr_optab);
491
492 /* The low OP1 bits of INTO_TARGET come from the high bits of OUTOF_INPUT.
493 We therefore need to shift OUTOF_INPUT by (BITS_PER_WORD - OP1) bits in
494 the opposite direction to BINOPTAB. */
495 if (CONSTANT_P (op1) || shift_mask >= BITS_PER_WORD)
496 {
497 carries = outof_input;
498 tmp = immed_wide_int_const (wi::shwi (BITS_PER_WORD,
499 op1_mode), op1_mode);
500 tmp = simplify_expand_binop (op1_mode, sub_optab, tmp, op1,
501 0, true, methods);
502 }
503 else
504 {
505 /* We must avoid shifting by BITS_PER_WORD bits since that is either
506 the same as a zero shift (if shift_mask == BITS_PER_WORD - 1) or
507 has unknown behavior. Do a single shift first, then shift by the
508 remainder. It's OK to use ~OP1 as the remainder if shift counts
509 are truncated to the mode size. */
510 carries = expand_binop (word_mode, reverse_unsigned_shift,
511 outof_input, const1_rtx, 0, unsignedp, methods);
512 if (shift_mask == BITS_PER_WORD - 1)
513 {
514 tmp = immed_wide_int_const
515 (wi::minus_one (GET_MODE_PRECISION (op1_mode)), op1_mode);
516 tmp = simplify_expand_binop (op1_mode, xor_optab, op1, tmp,
517 0, true, methods);
518 }
519 else
520 {
521 tmp = immed_wide_int_const (wi::shwi (BITS_PER_WORD - 1,
522 op1_mode), op1_mode);
523 tmp = simplify_expand_binop (op1_mode, sub_optab, tmp, op1,
524 0, true, methods);
525 }
526 }
527 if (tmp == 0 || carries == 0)
528 return false;
529 carries = expand_binop (word_mode, reverse_unsigned_shift,
530 carries, tmp, 0, unsignedp, methods);
531 if (carries == 0)
532 return false;
533
534 /* Shift INTO_INPUT logically by OP1. This is the last use of INTO_INPUT
535 so the result can go directly into INTO_TARGET if convenient. */
536 tmp = expand_binop (word_mode, unsigned_shift, into_input, op1,
537 into_target, unsignedp, methods);
538 if (tmp == 0)
539 return false;
540
541 /* Now OR in the bits carried over from OUTOF_INPUT. */
542 if (!force_expand_binop (word_mode, ior_optab, tmp, carries,
543 into_target, unsignedp, methods))
544 return false;
545
546 /* Use a standard word_mode shift for the out-of half. */
547 if (outof_target != 0)
548 if (!force_expand_binop (word_mode, binoptab, outof_input, op1,
549 outof_target, unsignedp, methods))
550 return false;
551
552 return true;
553 }
554
555
556 /* Try implementing expand_doubleword_shift using conditional moves.
557 The shift is by < BITS_PER_WORD if (CMP_CODE CMP1 CMP2) is true,
558 otherwise it is by >= BITS_PER_WORD. SUBWORD_OP1 and SUPERWORD_OP1
559 are the shift counts to use in the former and latter case. All other
560 arguments are the same as the parent routine. */
561
562 static bool
expand_doubleword_shift_condmove(scalar_int_mode op1_mode,optab binoptab,enum rtx_code cmp_code,rtx cmp1,rtx cmp2,rtx outof_input,rtx into_input,rtx subword_op1,rtx superword_op1,rtx outof_target,rtx into_target,int unsignedp,enum optab_methods methods,unsigned HOST_WIDE_INT shift_mask)563 expand_doubleword_shift_condmove (scalar_int_mode op1_mode, optab binoptab,
564 enum rtx_code cmp_code, rtx cmp1, rtx cmp2,
565 rtx outof_input, rtx into_input,
566 rtx subword_op1, rtx superword_op1,
567 rtx outof_target, rtx into_target,
568 int unsignedp, enum optab_methods methods,
569 unsigned HOST_WIDE_INT shift_mask)
570 {
571 rtx outof_superword, into_superword;
572
573 /* Put the superword version of the output into OUTOF_SUPERWORD and
574 INTO_SUPERWORD. */
575 outof_superword = outof_target != 0 ? gen_reg_rtx (word_mode) : 0;
576 if (outof_target != 0 && subword_op1 == superword_op1)
577 {
578 /* The value INTO_TARGET >> SUBWORD_OP1, which we later store in
579 OUTOF_TARGET, is the same as the value of INTO_SUPERWORD. */
580 into_superword = outof_target;
581 if (!expand_superword_shift (binoptab, outof_input, superword_op1,
582 outof_superword, 0, unsignedp, methods))
583 return false;
584 }
585 else
586 {
587 into_superword = gen_reg_rtx (word_mode);
588 if (!expand_superword_shift (binoptab, outof_input, superword_op1,
589 outof_superword, into_superword,
590 unsignedp, methods))
591 return false;
592 }
593
594 /* Put the subword version directly in OUTOF_TARGET and INTO_TARGET. */
595 if (!expand_subword_shift (op1_mode, binoptab,
596 outof_input, into_input, subword_op1,
597 outof_target, into_target,
598 unsignedp, methods, shift_mask))
599 return false;
600
601 /* Select between them. Do the INTO half first because INTO_SUPERWORD
602 might be the current value of OUTOF_TARGET. */
603 if (!emit_conditional_move (into_target, cmp_code, cmp1, cmp2, op1_mode,
604 into_target, into_superword, word_mode, false))
605 return false;
606
607 if (outof_target != 0)
608 if (!emit_conditional_move (outof_target, cmp_code, cmp1, cmp2, op1_mode,
609 outof_target, outof_superword,
610 word_mode, false))
611 return false;
612
613 return true;
614 }
615
616 /* Expand a doubleword shift (ashl, ashr or lshr) using word-mode shifts.
617 OUTOF_INPUT and INTO_INPUT are the two word-sized halves of the first
618 input operand; the shift moves bits in the direction OUTOF_INPUT->
619 INTO_TARGET. OUTOF_TARGET and INTO_TARGET are the equivalent words
620 of the target. OP1 is the shift count and OP1_MODE is its mode.
621 If OP1 is constant, it will have been truncated as appropriate
622 and is known to be nonzero.
623
624 If SHIFT_MASK is zero, the result of word shifts is undefined when the
625 shift count is outside the range [0, BITS_PER_WORD). This routine must
626 avoid generating such shifts for OP1s in the range [0, BITS_PER_WORD * 2).
627
628 If SHIFT_MASK is nonzero, all word-mode shift counts are effectively
629 masked by it and shifts in the range [BITS_PER_WORD, SHIFT_MASK) will
630 fill with zeros or sign bits as appropriate.
631
632 If SHIFT_MASK is BITS_PER_WORD - 1, this routine will synthesize
633 a doubleword shift whose equivalent mask is BITS_PER_WORD * 2 - 1.
634 Doing this preserves semantics required by SHIFT_COUNT_TRUNCATED.
635 In all other cases, shifts by values outside [0, BITS_PER_UNIT * 2)
636 are undefined.
637
638 BINOPTAB, UNSIGNEDP and METHODS are as for expand_binop. This function
639 may not use INTO_INPUT after modifying INTO_TARGET, and similarly for
640 OUTOF_INPUT and OUTOF_TARGET. OUTOF_TARGET can be null if the parent
641 function wants to calculate it itself.
642
643 Return true if the shift could be successfully synthesized. */
644
645 static bool
expand_doubleword_shift(scalar_int_mode op1_mode,optab binoptab,rtx outof_input,rtx into_input,rtx op1,rtx outof_target,rtx into_target,int unsignedp,enum optab_methods methods,unsigned HOST_WIDE_INT shift_mask)646 expand_doubleword_shift (scalar_int_mode op1_mode, optab binoptab,
647 rtx outof_input, rtx into_input, rtx op1,
648 rtx outof_target, rtx into_target,
649 int unsignedp, enum optab_methods methods,
650 unsigned HOST_WIDE_INT shift_mask)
651 {
652 rtx superword_op1, tmp, cmp1, cmp2;
653 enum rtx_code cmp_code;
654
655 /* See if word-mode shifts by BITS_PER_WORD...BITS_PER_WORD * 2 - 1 will
656 fill the result with sign or zero bits as appropriate. If so, the value
657 of OUTOF_TARGET will always be (SHIFT OUTOF_INPUT OP1). Recursively call
658 this routine to calculate INTO_TARGET (which depends on both OUTOF_INPUT
659 and INTO_INPUT), then emit code to set up OUTOF_TARGET.
660
661 This isn't worthwhile for constant shifts since the optimizers will
662 cope better with in-range shift counts. */
663 if (shift_mask >= BITS_PER_WORD
664 && outof_target != 0
665 && !CONSTANT_P (op1))
666 {
667 if (!expand_doubleword_shift (op1_mode, binoptab,
668 outof_input, into_input, op1,
669 0, into_target,
670 unsignedp, methods, shift_mask))
671 return false;
672 if (!force_expand_binop (word_mode, binoptab, outof_input, op1,
673 outof_target, unsignedp, methods))
674 return false;
675 return true;
676 }
677
678 /* Set CMP_CODE, CMP1 and CMP2 so that the rtx (CMP_CODE CMP1 CMP2)
679 is true when the effective shift value is less than BITS_PER_WORD.
680 Set SUPERWORD_OP1 to the shift count that should be used to shift
681 OUTOF_INPUT into INTO_TARGET when the condition is false. */
682 tmp = immed_wide_int_const (wi::shwi (BITS_PER_WORD, op1_mode), op1_mode);
683 if (!CONSTANT_P (op1) && shift_mask == BITS_PER_WORD - 1)
684 {
685 /* Set CMP1 to OP1 & BITS_PER_WORD. The result is zero iff OP1
686 is a subword shift count. */
687 cmp1 = simplify_expand_binop (op1_mode, and_optab, op1, tmp,
688 0, true, methods);
689 cmp2 = CONST0_RTX (op1_mode);
690 cmp_code = EQ;
691 superword_op1 = op1;
692 }
693 else
694 {
695 /* Set CMP1 to OP1 - BITS_PER_WORD. */
696 cmp1 = simplify_expand_binop (op1_mode, sub_optab, op1, tmp,
697 0, true, methods);
698 cmp2 = CONST0_RTX (op1_mode);
699 cmp_code = LT;
700 superword_op1 = cmp1;
701 }
702 if (cmp1 == 0)
703 return false;
704
705 /* If we can compute the condition at compile time, pick the
706 appropriate subroutine. */
707 tmp = simplify_relational_operation (cmp_code, SImode, op1_mode, cmp1, cmp2);
708 if (tmp != 0 && CONST_INT_P (tmp))
709 {
710 if (tmp == const0_rtx)
711 return expand_superword_shift (binoptab, outof_input, superword_op1,
712 outof_target, into_target,
713 unsignedp, methods);
714 else
715 return expand_subword_shift (op1_mode, binoptab,
716 outof_input, into_input, op1,
717 outof_target, into_target,
718 unsignedp, methods, shift_mask);
719 }
720
721 /* Try using conditional moves to generate straight-line code. */
722 if (HAVE_conditional_move)
723 {
724 rtx_insn *start = get_last_insn ();
725 if (expand_doubleword_shift_condmove (op1_mode, binoptab,
726 cmp_code, cmp1, cmp2,
727 outof_input, into_input,
728 op1, superword_op1,
729 outof_target, into_target,
730 unsignedp, methods, shift_mask))
731 return true;
732 delete_insns_since (start);
733 }
734
735 /* As a last resort, use branches to select the correct alternative. */
736 rtx_code_label *subword_label = gen_label_rtx ();
737 rtx_code_label *done_label = gen_label_rtx ();
738
739 NO_DEFER_POP;
740 do_compare_rtx_and_jump (cmp1, cmp2, cmp_code, false, op1_mode,
741 0, 0, subword_label,
742 profile_probability::uninitialized ());
743 OK_DEFER_POP;
744
745 if (!expand_superword_shift (binoptab, outof_input, superword_op1,
746 outof_target, into_target,
747 unsignedp, methods))
748 return false;
749
750 emit_jump_insn (targetm.gen_jump (done_label));
751 emit_barrier ();
752 emit_label (subword_label);
753
754 if (!expand_subword_shift (op1_mode, binoptab,
755 outof_input, into_input, op1,
756 outof_target, into_target,
757 unsignedp, methods, shift_mask))
758 return false;
759
760 emit_label (done_label);
761 return true;
762 }
763
764 /* Subroutine of expand_binop. Perform a double word multiplication of
765 operands OP0 and OP1 both of mode MODE, which is exactly twice as wide
766 as the target's word_mode. This function return NULL_RTX if anything
767 goes wrong, in which case it may have already emitted instructions
768 which need to be deleted.
769
770 If we want to multiply two two-word values and have normal and widening
771 multiplies of single-word values, we can do this with three smaller
772 multiplications.
773
774 The multiplication proceeds as follows:
775 _______________________
776 [__op0_high_|__op0_low__]
777 _______________________
778 * [__op1_high_|__op1_low__]
779 _______________________________________________
780 _______________________
781 (1) [__op0_low__*__op1_low__]
782 _______________________
783 (2a) [__op0_low__*__op1_high_]
784 _______________________
785 (2b) [__op0_high_*__op1_low__]
786 _______________________
787 (3) [__op0_high_*__op1_high_]
788
789
790 This gives a 4-word result. Since we are only interested in the
791 lower 2 words, partial result (3) and the upper words of (2a) and
792 (2b) don't need to be calculated. Hence (2a) and (2b) can be
793 calculated using non-widening multiplication.
794
795 (1), however, needs to be calculated with an unsigned widening
796 multiplication. If this operation is not directly supported we
797 try using a signed widening multiplication and adjust the result.
798 This adjustment works as follows:
799
800 If both operands are positive then no adjustment is needed.
801
802 If the operands have different signs, for example op0_low < 0 and
803 op1_low >= 0, the instruction treats the most significant bit of
804 op0_low as a sign bit instead of a bit with significance
805 2**(BITS_PER_WORD-1), i.e. the instruction multiplies op1_low
806 with 2**BITS_PER_WORD - op0_low, and two's complements the
807 result. Conclusion: We need to add op1_low * 2**BITS_PER_WORD to
808 the result.
809
810 Similarly, if both operands are negative, we need to add
811 (op0_low + op1_low) * 2**BITS_PER_WORD.
812
813 We use a trick to adjust quickly. We logically shift op0_low right
814 (op1_low) BITS_PER_WORD-1 steps to get 0 or 1, and add this to
815 op0_high (op1_high) before it is used to calculate 2b (2a). If no
816 logical shift exists, we do an arithmetic right shift and subtract
817 the 0 or -1. */
818
819 static rtx
expand_doubleword_mult(machine_mode mode,rtx op0,rtx op1,rtx target,bool umulp,enum optab_methods methods)820 expand_doubleword_mult (machine_mode mode, rtx op0, rtx op1, rtx target,
821 bool umulp, enum optab_methods methods)
822 {
823 int low = (WORDS_BIG_ENDIAN ? 1 : 0);
824 int high = (WORDS_BIG_ENDIAN ? 0 : 1);
825 rtx wordm1 = (umulp ? NULL_RTX
826 : gen_int_shift_amount (word_mode, BITS_PER_WORD - 1));
827 rtx product, adjust, product_high, temp;
828
829 rtx op0_high = operand_subword_force (op0, high, mode);
830 rtx op0_low = operand_subword_force (op0, low, mode);
831 rtx op1_high = operand_subword_force (op1, high, mode);
832 rtx op1_low = operand_subword_force (op1, low, mode);
833
834 /* If we're using an unsigned multiply to directly compute the product
835 of the low-order words of the operands and perform any required
836 adjustments of the operands, we begin by trying two more multiplications
837 and then computing the appropriate sum.
838
839 We have checked above that the required addition is provided.
840 Full-word addition will normally always succeed, especially if
841 it is provided at all, so we don't worry about its failure. The
842 multiplication may well fail, however, so we do handle that. */
843
844 if (!umulp)
845 {
846 /* ??? This could be done with emit_store_flag where available. */
847 temp = expand_binop (word_mode, lshr_optab, op0_low, wordm1,
848 NULL_RTX, 1, methods);
849 if (temp)
850 op0_high = expand_binop (word_mode, add_optab, op0_high, temp,
851 NULL_RTX, 0, OPTAB_DIRECT);
852 else
853 {
854 temp = expand_binop (word_mode, ashr_optab, op0_low, wordm1,
855 NULL_RTX, 0, methods);
856 if (!temp)
857 return NULL_RTX;
858 op0_high = expand_binop (word_mode, sub_optab, op0_high, temp,
859 NULL_RTX, 0, OPTAB_DIRECT);
860 }
861
862 if (!op0_high)
863 return NULL_RTX;
864 }
865
866 adjust = expand_binop (word_mode, smul_optab, op0_high, op1_low,
867 NULL_RTX, 0, OPTAB_DIRECT);
868 if (!adjust)
869 return NULL_RTX;
870
871 /* OP0_HIGH should now be dead. */
872
873 if (!umulp)
874 {
875 /* ??? This could be done with emit_store_flag where available. */
876 temp = expand_binop (word_mode, lshr_optab, op1_low, wordm1,
877 NULL_RTX, 1, methods);
878 if (temp)
879 op1_high = expand_binop (word_mode, add_optab, op1_high, temp,
880 NULL_RTX, 0, OPTAB_DIRECT);
881 else
882 {
883 temp = expand_binop (word_mode, ashr_optab, op1_low, wordm1,
884 NULL_RTX, 0, methods);
885 if (!temp)
886 return NULL_RTX;
887 op1_high = expand_binop (word_mode, sub_optab, op1_high, temp,
888 NULL_RTX, 0, OPTAB_DIRECT);
889 }
890
891 if (!op1_high)
892 return NULL_RTX;
893 }
894
895 temp = expand_binop (word_mode, smul_optab, op1_high, op0_low,
896 NULL_RTX, 0, OPTAB_DIRECT);
897 if (!temp)
898 return NULL_RTX;
899
900 /* OP1_HIGH should now be dead. */
901
902 adjust = expand_binop (word_mode, add_optab, adjust, temp,
903 NULL_RTX, 0, OPTAB_DIRECT);
904
905 if (target && !REG_P (target))
906 target = NULL_RTX;
907
908 /* *_widen_optab needs to determine operand mode, make sure at least
909 one operand has non-VOID mode. */
910 if (GET_MODE (op0_low) == VOIDmode && GET_MODE (op1_low) == VOIDmode)
911 op0_low = force_reg (word_mode, op0_low);
912
913 if (umulp)
914 product = expand_binop (mode, umul_widen_optab, op0_low, op1_low,
915 target, 1, OPTAB_DIRECT);
916 else
917 product = expand_binop (mode, smul_widen_optab, op0_low, op1_low,
918 target, 1, OPTAB_DIRECT);
919
920 if (!product)
921 return NULL_RTX;
922
923 product_high = operand_subword (product, high, 1, mode);
924 adjust = expand_binop (word_mode, add_optab, product_high, adjust,
925 NULL_RTX, 0, OPTAB_DIRECT);
926 emit_move_insn (product_high, adjust);
927 return product;
928 }
929
930 /* Wrapper around expand_binop which takes an rtx code to specify
931 the operation to perform, not an optab pointer. All other
932 arguments are the same. */
933 rtx
expand_simple_binop(machine_mode mode,enum rtx_code code,rtx op0,rtx op1,rtx target,int unsignedp,enum optab_methods methods)934 expand_simple_binop (machine_mode mode, enum rtx_code code, rtx op0,
935 rtx op1, rtx target, int unsignedp,
936 enum optab_methods methods)
937 {
938 optab binop = code_to_optab (code);
939 gcc_assert (binop);
940
941 return expand_binop (mode, binop, op0, op1, target, unsignedp, methods);
942 }
943
944 /* Return whether OP0 and OP1 should be swapped when expanding a commutative
945 binop. Order them according to commutative_operand_precedence and, if
946 possible, try to put TARGET or a pseudo first. */
947 static bool
swap_commutative_operands_with_target(rtx target,rtx op0,rtx op1)948 swap_commutative_operands_with_target (rtx target, rtx op0, rtx op1)
949 {
950 int op0_prec = commutative_operand_precedence (op0);
951 int op1_prec = commutative_operand_precedence (op1);
952
953 if (op0_prec < op1_prec)
954 return true;
955
956 if (op0_prec > op1_prec)
957 return false;
958
959 /* With equal precedence, both orders are ok, but it is better if the
960 first operand is TARGET, or if both TARGET and OP0 are pseudos. */
961 if (target == 0 || REG_P (target))
962 return (REG_P (op1) && !REG_P (op0)) || target == op1;
963 else
964 return rtx_equal_p (op1, target);
965 }
966
967 /* Return true if BINOPTAB implements a shift operation. */
968
969 static bool
shift_optab_p(optab binoptab)970 shift_optab_p (optab binoptab)
971 {
972 switch (optab_to_code (binoptab))
973 {
974 case ASHIFT:
975 case SS_ASHIFT:
976 case US_ASHIFT:
977 case ASHIFTRT:
978 case LSHIFTRT:
979 case ROTATE:
980 case ROTATERT:
981 return true;
982
983 default:
984 return false;
985 }
986 }
987
988 /* Return true if BINOPTAB implements a commutative binary operation. */
989
990 static bool
commutative_optab_p(optab binoptab)991 commutative_optab_p (optab binoptab)
992 {
993 return (GET_RTX_CLASS (optab_to_code (binoptab)) == RTX_COMM_ARITH
994 || binoptab == smul_widen_optab
995 || binoptab == umul_widen_optab
996 || binoptab == smul_highpart_optab
997 || binoptab == umul_highpart_optab);
998 }
999
1000 /* X is to be used in mode MODE as operand OPN to BINOPTAB. If we're
1001 optimizing, and if the operand is a constant that costs more than
1002 1 instruction, force the constant into a register and return that
1003 register. Return X otherwise. UNSIGNEDP says whether X is unsigned. */
1004
1005 static rtx
avoid_expensive_constant(machine_mode mode,optab binoptab,int opn,rtx x,bool unsignedp)1006 avoid_expensive_constant (machine_mode mode, optab binoptab,
1007 int opn, rtx x, bool unsignedp)
1008 {
1009 bool speed = optimize_insn_for_speed_p ();
1010
1011 if (mode != VOIDmode
1012 && optimize
1013 && CONSTANT_P (x)
1014 && (rtx_cost (x, mode, optab_to_code (binoptab), opn, speed)
1015 > set_src_cost (x, mode, speed)))
1016 {
1017 if (CONST_INT_P (x))
1018 {
1019 HOST_WIDE_INT intval = trunc_int_for_mode (INTVAL (x), mode);
1020 if (intval != INTVAL (x))
1021 x = GEN_INT (intval);
1022 }
1023 else
1024 x = convert_modes (mode, VOIDmode, x, unsignedp);
1025 x = force_reg (mode, x);
1026 }
1027 return x;
1028 }
1029
1030 /* Helper function for expand_binop: handle the case where there
1031 is an insn ICODE that directly implements the indicated operation.
1032 Returns null if this is not possible. */
1033 static rtx
expand_binop_directly(enum insn_code icode,machine_mode mode,optab binoptab,rtx op0,rtx op1,rtx target,int unsignedp,enum optab_methods methods,rtx_insn * last)1034 expand_binop_directly (enum insn_code icode, machine_mode mode, optab binoptab,
1035 rtx op0, rtx op1,
1036 rtx target, int unsignedp, enum optab_methods methods,
1037 rtx_insn *last)
1038 {
1039 machine_mode xmode0 = insn_data[(int) icode].operand[1].mode;
1040 machine_mode xmode1 = insn_data[(int) icode].operand[2].mode;
1041 machine_mode mode0, mode1, tmp_mode;
1042 struct expand_operand ops[3];
1043 bool commutative_p;
1044 rtx_insn *pat;
1045 rtx xop0 = op0, xop1 = op1;
1046 bool canonicalize_op1 = false;
1047
1048 /* If it is a commutative operator and the modes would match
1049 if we would swap the operands, we can save the conversions. */
1050 commutative_p = commutative_optab_p (binoptab);
1051 if (commutative_p
1052 && GET_MODE (xop0) != xmode0 && GET_MODE (xop1) != xmode1
1053 && GET_MODE (xop0) == xmode1 && GET_MODE (xop1) == xmode1)
1054 std::swap (xop0, xop1);
1055
1056 /* If we are optimizing, force expensive constants into a register. */
1057 xop0 = avoid_expensive_constant (xmode0, binoptab, 0, xop0, unsignedp);
1058 if (!shift_optab_p (binoptab))
1059 xop1 = avoid_expensive_constant (xmode1, binoptab, 1, xop1, unsignedp);
1060 else
1061 /* Shifts and rotates often use a different mode for op1 from op0;
1062 for VOIDmode constants we don't know the mode, so force it
1063 to be canonicalized using convert_modes. */
1064 canonicalize_op1 = true;
1065
1066 /* In case the insn wants input operands in modes different from
1067 those of the actual operands, convert the operands. It would
1068 seem that we don't need to convert CONST_INTs, but we do, so
1069 that they're properly zero-extended, sign-extended or truncated
1070 for their mode. */
1071
1072 mode0 = GET_MODE (xop0) != VOIDmode ? GET_MODE (xop0) : mode;
1073 if (xmode0 != VOIDmode && xmode0 != mode0)
1074 {
1075 xop0 = convert_modes (xmode0, mode0, xop0, unsignedp);
1076 mode0 = xmode0;
1077 }
1078
1079 mode1 = ((GET_MODE (xop1) != VOIDmode || canonicalize_op1)
1080 ? GET_MODE (xop1) : mode);
1081 if (xmode1 != VOIDmode && xmode1 != mode1)
1082 {
1083 xop1 = convert_modes (xmode1, mode1, xop1, unsignedp);
1084 mode1 = xmode1;
1085 }
1086
1087 /* If operation is commutative,
1088 try to make the first operand a register.
1089 Even better, try to make it the same as the target.
1090 Also try to make the last operand a constant. */
1091 if (commutative_p
1092 && swap_commutative_operands_with_target (target, xop0, xop1))
1093 std::swap (xop0, xop1);
1094
1095 /* Now, if insn's predicates don't allow our operands, put them into
1096 pseudo regs. */
1097
1098 if (binoptab == vec_pack_trunc_optab
1099 || binoptab == vec_pack_usat_optab
1100 || binoptab == vec_pack_ssat_optab
1101 || binoptab == vec_pack_ufix_trunc_optab
1102 || binoptab == vec_pack_sfix_trunc_optab
1103 || binoptab == vec_packu_float_optab
1104 || binoptab == vec_packs_float_optab)
1105 {
1106 /* The mode of the result is different then the mode of the
1107 arguments. */
1108 tmp_mode = insn_data[(int) icode].operand[0].mode;
1109 if (VECTOR_MODE_P (mode)
1110 && maybe_ne (GET_MODE_NUNITS (tmp_mode), 2 * GET_MODE_NUNITS (mode)))
1111 {
1112 delete_insns_since (last);
1113 return NULL_RTX;
1114 }
1115 }
1116 else
1117 tmp_mode = mode;
1118
1119 create_output_operand (&ops[0], target, tmp_mode);
1120 create_input_operand (&ops[1], xop0, mode0);
1121 create_input_operand (&ops[2], xop1, mode1);
1122 pat = maybe_gen_insn (icode, 3, ops);
1123 if (pat)
1124 {
1125 /* If PAT is composed of more than one insn, try to add an appropriate
1126 REG_EQUAL note to it. If we can't because TEMP conflicts with an
1127 operand, call expand_binop again, this time without a target. */
1128 if (INSN_P (pat) && NEXT_INSN (pat) != NULL_RTX
1129 && ! add_equal_note (pat, ops[0].value,
1130 optab_to_code (binoptab),
1131 ops[1].value, ops[2].value, mode0))
1132 {
1133 delete_insns_since (last);
1134 return expand_binop (mode, binoptab, op0, op1, NULL_RTX,
1135 unsignedp, methods);
1136 }
1137
1138 emit_insn (pat);
1139 return ops[0].value;
1140 }
1141 delete_insns_since (last);
1142 return NULL_RTX;
1143 }
1144
1145 /* Generate code to perform an operation specified by BINOPTAB
1146 on operands OP0 and OP1, with result having machine-mode MODE.
1147
1148 UNSIGNEDP is for the case where we have to widen the operands
1149 to perform the operation. It says to use zero-extension.
1150
1151 If TARGET is nonzero, the value
1152 is generated there, if it is convenient to do so.
1153 In all cases an rtx is returned for the locus of the value;
1154 this may or may not be TARGET. */
1155
1156 rtx
expand_binop(machine_mode mode,optab binoptab,rtx op0,rtx op1,rtx target,int unsignedp,enum optab_methods methods)1157 expand_binop (machine_mode mode, optab binoptab, rtx op0, rtx op1,
1158 rtx target, int unsignedp, enum optab_methods methods)
1159 {
1160 enum optab_methods next_methods
1161 = (methods == OPTAB_LIB || methods == OPTAB_LIB_WIDEN
1162 ? OPTAB_WIDEN : methods);
1163 enum mode_class mclass;
1164 enum insn_code icode;
1165 machine_mode wider_mode;
1166 scalar_int_mode int_mode;
1167 rtx libfunc;
1168 rtx temp;
1169 rtx_insn *entry_last = get_last_insn ();
1170 rtx_insn *last;
1171
1172 mclass = GET_MODE_CLASS (mode);
1173
1174 /* If subtracting an integer constant, convert this into an addition of
1175 the negated constant. */
1176
1177 if (binoptab == sub_optab && CONST_INT_P (op1))
1178 {
1179 op1 = negate_rtx (mode, op1);
1180 binoptab = add_optab;
1181 }
1182 /* For shifts, constant invalid op1 might be expanded from different
1183 mode than MODE. As those are invalid, force them to a register
1184 to avoid further problems during expansion. */
1185 else if (CONST_INT_P (op1)
1186 && shift_optab_p (binoptab)
1187 && UINTVAL (op1) >= GET_MODE_BITSIZE (GET_MODE_INNER (mode)))
1188 {
1189 op1 = gen_int_mode (INTVAL (op1), GET_MODE_INNER (mode));
1190 op1 = force_reg (GET_MODE_INNER (mode), op1);
1191 }
1192
1193 /* Record where to delete back to if we backtrack. */
1194 last = get_last_insn ();
1195
1196 /* If we can do it with a three-operand insn, do so. */
1197
1198 if (methods != OPTAB_MUST_WIDEN)
1199 {
1200 if (convert_optab_p (binoptab))
1201 {
1202 machine_mode from_mode = widened_mode (mode, op0, op1);
1203 icode = find_widening_optab_handler (binoptab, mode, from_mode);
1204 }
1205 else
1206 icode = optab_handler (binoptab, mode);
1207 if (icode != CODE_FOR_nothing)
1208 {
1209 temp = expand_binop_directly (icode, mode, binoptab, op0, op1,
1210 target, unsignedp, methods, last);
1211 if (temp)
1212 return temp;
1213 }
1214 }
1215
1216 /* If we were trying to rotate, and that didn't work, try rotating
1217 the other direction before falling back to shifts and bitwise-or. */
1218 if (((binoptab == rotl_optab
1219 && (icode = optab_handler (rotr_optab, mode)) != CODE_FOR_nothing)
1220 || (binoptab == rotr_optab
1221 && (icode = optab_handler (rotl_optab, mode)) != CODE_FOR_nothing))
1222 && is_int_mode (mode, &int_mode))
1223 {
1224 optab otheroptab = (binoptab == rotl_optab ? rotr_optab : rotl_optab);
1225 rtx newop1;
1226 unsigned int bits = GET_MODE_PRECISION (int_mode);
1227
1228 if (CONST_INT_P (op1))
1229 newop1 = gen_int_shift_amount (int_mode, bits - INTVAL (op1));
1230 else if (targetm.shift_truncation_mask (int_mode) == bits - 1)
1231 newop1 = negate_rtx (GET_MODE (op1), op1);
1232 else
1233 newop1 = expand_binop (GET_MODE (op1), sub_optab,
1234 gen_int_mode (bits, GET_MODE (op1)), op1,
1235 NULL_RTX, unsignedp, OPTAB_DIRECT);
1236
1237 temp = expand_binop_directly (icode, int_mode, otheroptab, op0, newop1,
1238 target, unsignedp, methods, last);
1239 if (temp)
1240 return temp;
1241 }
1242
1243 /* If this is a multiply, see if we can do a widening operation that
1244 takes operands of this mode and makes a wider mode. */
1245
1246 if (binoptab == smul_optab
1247 && GET_MODE_2XWIDER_MODE (mode).exists (&wider_mode)
1248 && (convert_optab_handler ((unsignedp
1249 ? umul_widen_optab
1250 : smul_widen_optab),
1251 wider_mode, mode) != CODE_FOR_nothing))
1252 {
1253 /* *_widen_optab needs to determine operand mode, make sure at least
1254 one operand has non-VOID mode. */
1255 if (GET_MODE (op0) == VOIDmode && GET_MODE (op1) == VOIDmode)
1256 op0 = force_reg (mode, op0);
1257 temp = expand_binop (wider_mode,
1258 unsignedp ? umul_widen_optab : smul_widen_optab,
1259 op0, op1, NULL_RTX, unsignedp, OPTAB_DIRECT);
1260
1261 if (temp != 0)
1262 {
1263 if (GET_MODE_CLASS (mode) == MODE_INT
1264 && TRULY_NOOP_TRUNCATION_MODES_P (mode, GET_MODE (temp)))
1265 return gen_lowpart (mode, temp);
1266 else
1267 return convert_to_mode (mode, temp, unsignedp);
1268 }
1269 }
1270
1271 /* If this is a vector shift by a scalar, see if we can do a vector
1272 shift by a vector. If so, broadcast the scalar into a vector. */
1273 if (mclass == MODE_VECTOR_INT)
1274 {
1275 optab otheroptab = unknown_optab;
1276
1277 if (binoptab == ashl_optab)
1278 otheroptab = vashl_optab;
1279 else if (binoptab == ashr_optab)
1280 otheroptab = vashr_optab;
1281 else if (binoptab == lshr_optab)
1282 otheroptab = vlshr_optab;
1283 else if (binoptab == rotl_optab)
1284 otheroptab = vrotl_optab;
1285 else if (binoptab == rotr_optab)
1286 otheroptab = vrotr_optab;
1287
1288 if (otheroptab
1289 && (icode = optab_handler (otheroptab, mode)) != CODE_FOR_nothing)
1290 {
1291 /* The scalar may have been extended to be too wide. Truncate
1292 it back to the proper size to fit in the broadcast vector. */
1293 scalar_mode inner_mode = GET_MODE_INNER (mode);
1294 if (!CONST_INT_P (op1)
1295 && (GET_MODE_BITSIZE (as_a <scalar_int_mode> (GET_MODE (op1)))
1296 > GET_MODE_BITSIZE (inner_mode)))
1297 op1 = force_reg (inner_mode,
1298 simplify_gen_unary (TRUNCATE, inner_mode, op1,
1299 GET_MODE (op1)));
1300 rtx vop1 = expand_vector_broadcast (mode, op1);
1301 if (vop1)
1302 {
1303 temp = expand_binop_directly (icode, mode, otheroptab, op0, vop1,
1304 target, unsignedp, methods, last);
1305 if (temp)
1306 return temp;
1307 }
1308 }
1309 }
1310
1311 /* Look for a wider mode of the same class for which we think we
1312 can open-code the operation. Check for a widening multiply at the
1313 wider mode as well. */
1314
1315 if (CLASS_HAS_WIDER_MODES_P (mclass)
1316 && methods != OPTAB_DIRECT && methods != OPTAB_LIB)
1317 FOR_EACH_WIDER_MODE (wider_mode, mode)
1318 {
1319 machine_mode next_mode;
1320 if (optab_handler (binoptab, wider_mode) != CODE_FOR_nothing
1321 || (binoptab == smul_optab
1322 && GET_MODE_WIDER_MODE (wider_mode).exists (&next_mode)
1323 && (find_widening_optab_handler ((unsignedp
1324 ? umul_widen_optab
1325 : smul_widen_optab),
1326 next_mode, mode)
1327 != CODE_FOR_nothing)))
1328 {
1329 rtx xop0 = op0, xop1 = op1;
1330 int no_extend = 0;
1331
1332 /* For certain integer operations, we need not actually extend
1333 the narrow operands, as long as we will truncate
1334 the results to the same narrowness. */
1335
1336 if ((binoptab == ior_optab || binoptab == and_optab
1337 || binoptab == xor_optab
1338 || binoptab == add_optab || binoptab == sub_optab
1339 || binoptab == smul_optab || binoptab == ashl_optab)
1340 && mclass == MODE_INT)
1341 {
1342 no_extend = 1;
1343 xop0 = avoid_expensive_constant (mode, binoptab, 0,
1344 xop0, unsignedp);
1345 if (binoptab != ashl_optab)
1346 xop1 = avoid_expensive_constant (mode, binoptab, 1,
1347 xop1, unsignedp);
1348 }
1349
1350 xop0 = widen_operand (xop0, wider_mode, mode, unsignedp, no_extend);
1351
1352 /* The second operand of a shift must always be extended. */
1353 xop1 = widen_operand (xop1, wider_mode, mode, unsignedp,
1354 no_extend && binoptab != ashl_optab);
1355
1356 temp = expand_binop (wider_mode, binoptab, xop0, xop1, NULL_RTX,
1357 unsignedp, OPTAB_DIRECT);
1358 if (temp)
1359 {
1360 if (mclass != MODE_INT
1361 || !TRULY_NOOP_TRUNCATION_MODES_P (mode, wider_mode))
1362 {
1363 if (target == 0)
1364 target = gen_reg_rtx (mode);
1365 convert_move (target, temp, 0);
1366 return target;
1367 }
1368 else
1369 return gen_lowpart (mode, temp);
1370 }
1371 else
1372 delete_insns_since (last);
1373 }
1374 }
1375
1376 /* If operation is commutative,
1377 try to make the first operand a register.
1378 Even better, try to make it the same as the target.
1379 Also try to make the last operand a constant. */
1380 if (commutative_optab_p (binoptab)
1381 && swap_commutative_operands_with_target (target, op0, op1))
1382 std::swap (op0, op1);
1383
1384 /* These can be done a word at a time. */
1385 if ((binoptab == and_optab || binoptab == ior_optab || binoptab == xor_optab)
1386 && is_int_mode (mode, &int_mode)
1387 && GET_MODE_SIZE (int_mode) > UNITS_PER_WORD
1388 && optab_handler (binoptab, word_mode) != CODE_FOR_nothing)
1389 {
1390 int i;
1391 rtx_insn *insns;
1392
1393 /* If TARGET is the same as one of the operands, the REG_EQUAL note
1394 won't be accurate, so use a new target. */
1395 if (target == 0
1396 || target == op0
1397 || target == op1
1398 || reg_overlap_mentioned_p (target, op0)
1399 || reg_overlap_mentioned_p (target, op1)
1400 || !valid_multiword_target_p (target))
1401 target = gen_reg_rtx (int_mode);
1402
1403 start_sequence ();
1404
1405 /* Do the actual arithmetic. */
1406 machine_mode op0_mode = GET_MODE (op0);
1407 machine_mode op1_mode = GET_MODE (op1);
1408 if (op0_mode == VOIDmode)
1409 op0_mode = int_mode;
1410 if (op1_mode == VOIDmode)
1411 op1_mode = int_mode;
1412 for (i = 0; i < GET_MODE_BITSIZE (int_mode) / BITS_PER_WORD; i++)
1413 {
1414 rtx target_piece = operand_subword (target, i, 1, int_mode);
1415 rtx x = expand_binop (word_mode, binoptab,
1416 operand_subword_force (op0, i, op0_mode),
1417 operand_subword_force (op1, i, op1_mode),
1418 target_piece, unsignedp, next_methods);
1419
1420 if (x == 0)
1421 break;
1422
1423 if (target_piece != x)
1424 emit_move_insn (target_piece, x);
1425 }
1426
1427 insns = get_insns ();
1428 end_sequence ();
1429
1430 if (i == GET_MODE_BITSIZE (int_mode) / BITS_PER_WORD)
1431 {
1432 emit_insn (insns);
1433 return target;
1434 }
1435 }
1436
1437 /* Synthesize double word shifts from single word shifts. */
1438 if ((binoptab == lshr_optab || binoptab == ashl_optab
1439 || binoptab == ashr_optab)
1440 && is_int_mode (mode, &int_mode)
1441 && (CONST_INT_P (op1) || optimize_insn_for_speed_p ())
1442 && GET_MODE_SIZE (int_mode) == 2 * UNITS_PER_WORD
1443 && GET_MODE_PRECISION (int_mode) == GET_MODE_BITSIZE (int_mode)
1444 && optab_handler (binoptab, word_mode) != CODE_FOR_nothing
1445 && optab_handler (ashl_optab, word_mode) != CODE_FOR_nothing
1446 && optab_handler (lshr_optab, word_mode) != CODE_FOR_nothing)
1447 {
1448 unsigned HOST_WIDE_INT shift_mask, double_shift_mask;
1449 scalar_int_mode op1_mode;
1450
1451 double_shift_mask = targetm.shift_truncation_mask (int_mode);
1452 shift_mask = targetm.shift_truncation_mask (word_mode);
1453 op1_mode = (GET_MODE (op1) != VOIDmode
1454 ? as_a <scalar_int_mode> (GET_MODE (op1))
1455 : word_mode);
1456
1457 /* Apply the truncation to constant shifts. */
1458 if (double_shift_mask > 0 && CONST_INT_P (op1))
1459 op1 = gen_int_mode (INTVAL (op1) & double_shift_mask, op1_mode);
1460
1461 if (op1 == CONST0_RTX (op1_mode))
1462 return op0;
1463
1464 /* Make sure that this is a combination that expand_doubleword_shift
1465 can handle. See the comments there for details. */
1466 if (double_shift_mask == 0
1467 || (shift_mask == BITS_PER_WORD - 1
1468 && double_shift_mask == BITS_PER_WORD * 2 - 1))
1469 {
1470 rtx_insn *insns;
1471 rtx into_target, outof_target;
1472 rtx into_input, outof_input;
1473 int left_shift, outof_word;
1474
1475 /* If TARGET is the same as one of the operands, the REG_EQUAL note
1476 won't be accurate, so use a new target. */
1477 if (target == 0
1478 || target == op0
1479 || target == op1
1480 || reg_overlap_mentioned_p (target, op0)
1481 || reg_overlap_mentioned_p (target, op1)
1482 || !valid_multiword_target_p (target))
1483 target = gen_reg_rtx (int_mode);
1484
1485 start_sequence ();
1486
1487 /* OUTOF_* is the word we are shifting bits away from, and
1488 INTO_* is the word that we are shifting bits towards, thus
1489 they differ depending on the direction of the shift and
1490 WORDS_BIG_ENDIAN. */
1491
1492 left_shift = binoptab == ashl_optab;
1493 outof_word = left_shift ^ ! WORDS_BIG_ENDIAN;
1494
1495 outof_target = operand_subword (target, outof_word, 1, int_mode);
1496 into_target = operand_subword (target, 1 - outof_word, 1, int_mode);
1497
1498 outof_input = operand_subword_force (op0, outof_word, int_mode);
1499 into_input = operand_subword_force (op0, 1 - outof_word, int_mode);
1500
1501 if (expand_doubleword_shift (op1_mode, binoptab,
1502 outof_input, into_input, op1,
1503 outof_target, into_target,
1504 unsignedp, next_methods, shift_mask))
1505 {
1506 insns = get_insns ();
1507 end_sequence ();
1508
1509 emit_insn (insns);
1510 return target;
1511 }
1512 end_sequence ();
1513 }
1514 }
1515
1516 /* Synthesize double word rotates from single word shifts. */
1517 if ((binoptab == rotl_optab || binoptab == rotr_optab)
1518 && is_int_mode (mode, &int_mode)
1519 && CONST_INT_P (op1)
1520 && GET_MODE_PRECISION (int_mode) == 2 * BITS_PER_WORD
1521 && optab_handler (ashl_optab, word_mode) != CODE_FOR_nothing
1522 && optab_handler (lshr_optab, word_mode) != CODE_FOR_nothing)
1523 {
1524 rtx_insn *insns;
1525 rtx into_target, outof_target;
1526 rtx into_input, outof_input;
1527 rtx inter;
1528 int shift_count, left_shift, outof_word;
1529
1530 /* If TARGET is the same as one of the operands, the REG_EQUAL note
1531 won't be accurate, so use a new target. Do this also if target is not
1532 a REG, first because having a register instead may open optimization
1533 opportunities, and second because if target and op0 happen to be MEMs
1534 designating the same location, we would risk clobbering it too early
1535 in the code sequence we generate below. */
1536 if (target == 0
1537 || target == op0
1538 || target == op1
1539 || !REG_P (target)
1540 || reg_overlap_mentioned_p (target, op0)
1541 || reg_overlap_mentioned_p (target, op1)
1542 || !valid_multiword_target_p (target))
1543 target = gen_reg_rtx (int_mode);
1544
1545 start_sequence ();
1546
1547 shift_count = INTVAL (op1);
1548
1549 /* OUTOF_* is the word we are shifting bits away from, and
1550 INTO_* is the word that we are shifting bits towards, thus
1551 they differ depending on the direction of the shift and
1552 WORDS_BIG_ENDIAN. */
1553
1554 left_shift = (binoptab == rotl_optab);
1555 outof_word = left_shift ^ ! WORDS_BIG_ENDIAN;
1556
1557 outof_target = operand_subword (target, outof_word, 1, int_mode);
1558 into_target = operand_subword (target, 1 - outof_word, 1, int_mode);
1559
1560 outof_input = operand_subword_force (op0, outof_word, int_mode);
1561 into_input = operand_subword_force (op0, 1 - outof_word, int_mode);
1562
1563 if (shift_count == BITS_PER_WORD)
1564 {
1565 /* This is just a word swap. */
1566 emit_move_insn (outof_target, into_input);
1567 emit_move_insn (into_target, outof_input);
1568 inter = const0_rtx;
1569 }
1570 else
1571 {
1572 rtx into_temp1, into_temp2, outof_temp1, outof_temp2;
1573 HOST_WIDE_INT first_shift_count, second_shift_count;
1574 optab reverse_unsigned_shift, unsigned_shift;
1575
1576 reverse_unsigned_shift = (left_shift ^ (shift_count < BITS_PER_WORD)
1577 ? lshr_optab : ashl_optab);
1578
1579 unsigned_shift = (left_shift ^ (shift_count < BITS_PER_WORD)
1580 ? ashl_optab : lshr_optab);
1581
1582 if (shift_count > BITS_PER_WORD)
1583 {
1584 first_shift_count = shift_count - BITS_PER_WORD;
1585 second_shift_count = 2 * BITS_PER_WORD - shift_count;
1586 }
1587 else
1588 {
1589 first_shift_count = BITS_PER_WORD - shift_count;
1590 second_shift_count = shift_count;
1591 }
1592 rtx first_shift_count_rtx
1593 = gen_int_shift_amount (word_mode, first_shift_count);
1594 rtx second_shift_count_rtx
1595 = gen_int_shift_amount (word_mode, second_shift_count);
1596
1597 into_temp1 = expand_binop (word_mode, unsigned_shift,
1598 outof_input, first_shift_count_rtx,
1599 NULL_RTX, unsignedp, next_methods);
1600 into_temp2 = expand_binop (word_mode, reverse_unsigned_shift,
1601 into_input, second_shift_count_rtx,
1602 NULL_RTX, unsignedp, next_methods);
1603
1604 if (into_temp1 != 0 && into_temp2 != 0)
1605 inter = expand_binop (word_mode, ior_optab, into_temp1, into_temp2,
1606 into_target, unsignedp, next_methods);
1607 else
1608 inter = 0;
1609
1610 if (inter != 0 && inter != into_target)
1611 emit_move_insn (into_target, inter);
1612
1613 outof_temp1 = expand_binop (word_mode, unsigned_shift,
1614 into_input, first_shift_count_rtx,
1615 NULL_RTX, unsignedp, next_methods);
1616 outof_temp2 = expand_binop (word_mode, reverse_unsigned_shift,
1617 outof_input, second_shift_count_rtx,
1618 NULL_RTX, unsignedp, next_methods);
1619
1620 if (inter != 0 && outof_temp1 != 0 && outof_temp2 != 0)
1621 inter = expand_binop (word_mode, ior_optab,
1622 outof_temp1, outof_temp2,
1623 outof_target, unsignedp, next_methods);
1624
1625 if (inter != 0 && inter != outof_target)
1626 emit_move_insn (outof_target, inter);
1627 }
1628
1629 insns = get_insns ();
1630 end_sequence ();
1631
1632 if (inter != 0)
1633 {
1634 emit_insn (insns);
1635 return target;
1636 }
1637 }
1638
1639 /* These can be done a word at a time by propagating carries. */
1640 if ((binoptab == add_optab || binoptab == sub_optab)
1641 && is_int_mode (mode, &int_mode)
1642 && GET_MODE_SIZE (int_mode) >= 2 * UNITS_PER_WORD
1643 && optab_handler (binoptab, word_mode) != CODE_FOR_nothing)
1644 {
1645 unsigned int i;
1646 optab otheroptab = binoptab == add_optab ? sub_optab : add_optab;
1647 const unsigned int nwords = GET_MODE_BITSIZE (int_mode) / BITS_PER_WORD;
1648 rtx carry_in = NULL_RTX, carry_out = NULL_RTX;
1649 rtx xop0, xop1, xtarget;
1650
1651 /* We can handle either a 1 or -1 value for the carry. If STORE_FLAG
1652 value is one of those, use it. Otherwise, use 1 since it is the
1653 one easiest to get. */
1654 #if STORE_FLAG_VALUE == 1 || STORE_FLAG_VALUE == -1
1655 int normalizep = STORE_FLAG_VALUE;
1656 #else
1657 int normalizep = 1;
1658 #endif
1659
1660 /* Prepare the operands. */
1661 xop0 = force_reg (int_mode, op0);
1662 xop1 = force_reg (int_mode, op1);
1663
1664 xtarget = gen_reg_rtx (int_mode);
1665
1666 if (target == 0 || !REG_P (target) || !valid_multiword_target_p (target))
1667 target = xtarget;
1668
1669 /* Indicate for flow that the entire target reg is being set. */
1670 if (REG_P (target))
1671 emit_clobber (xtarget);
1672
1673 /* Do the actual arithmetic. */
1674 for (i = 0; i < nwords; i++)
1675 {
1676 int index = (WORDS_BIG_ENDIAN ? nwords - i - 1 : i);
1677 rtx target_piece = operand_subword (xtarget, index, 1, int_mode);
1678 rtx op0_piece = operand_subword_force (xop0, index, int_mode);
1679 rtx op1_piece = operand_subword_force (xop1, index, int_mode);
1680 rtx x;
1681
1682 /* Main add/subtract of the input operands. */
1683 x = expand_binop (word_mode, binoptab,
1684 op0_piece, op1_piece,
1685 target_piece, unsignedp, next_methods);
1686 if (x == 0)
1687 break;
1688
1689 if (i + 1 < nwords)
1690 {
1691 /* Store carry from main add/subtract. */
1692 carry_out = gen_reg_rtx (word_mode);
1693 carry_out = emit_store_flag_force (carry_out,
1694 (binoptab == add_optab
1695 ? LT : GT),
1696 x, op0_piece,
1697 word_mode, 1, normalizep);
1698 }
1699
1700 if (i > 0)
1701 {
1702 rtx newx;
1703
1704 /* Add/subtract previous carry to main result. */
1705 newx = expand_binop (word_mode,
1706 normalizep == 1 ? binoptab : otheroptab,
1707 x, carry_in,
1708 NULL_RTX, 1, next_methods);
1709
1710 if (i + 1 < nwords)
1711 {
1712 /* Get out carry from adding/subtracting carry in. */
1713 rtx carry_tmp = gen_reg_rtx (word_mode);
1714 carry_tmp = emit_store_flag_force (carry_tmp,
1715 (binoptab == add_optab
1716 ? LT : GT),
1717 newx, x,
1718 word_mode, 1, normalizep);
1719
1720 /* Logical-ior the two poss. carry together. */
1721 carry_out = expand_binop (word_mode, ior_optab,
1722 carry_out, carry_tmp,
1723 carry_out, 0, next_methods);
1724 if (carry_out == 0)
1725 break;
1726 }
1727 emit_move_insn (target_piece, newx);
1728 }
1729 else
1730 {
1731 if (x != target_piece)
1732 emit_move_insn (target_piece, x);
1733 }
1734
1735 carry_in = carry_out;
1736 }
1737
1738 if (i == GET_MODE_BITSIZE (int_mode) / (unsigned) BITS_PER_WORD)
1739 {
1740 if (optab_handler (mov_optab, int_mode) != CODE_FOR_nothing
1741 || ! rtx_equal_p (target, xtarget))
1742 {
1743 rtx_insn *temp = emit_move_insn (target, xtarget);
1744
1745 set_dst_reg_note (temp, REG_EQUAL,
1746 gen_rtx_fmt_ee (optab_to_code (binoptab),
1747 int_mode, copy_rtx (xop0),
1748 copy_rtx (xop1)),
1749 target);
1750 }
1751 else
1752 target = xtarget;
1753
1754 return target;
1755 }
1756
1757 else
1758 delete_insns_since (last);
1759 }
1760
1761 /* Attempt to synthesize double word multiplies using a sequence of word
1762 mode multiplications. We first attempt to generate a sequence using a
1763 more efficient unsigned widening multiply, and if that fails we then
1764 try using a signed widening multiply. */
1765
1766 if (binoptab == smul_optab
1767 && is_int_mode (mode, &int_mode)
1768 && GET_MODE_SIZE (int_mode) == 2 * UNITS_PER_WORD
1769 && optab_handler (smul_optab, word_mode) != CODE_FOR_nothing
1770 && optab_handler (add_optab, word_mode) != CODE_FOR_nothing)
1771 {
1772 rtx product = NULL_RTX;
1773 if (convert_optab_handler (umul_widen_optab, int_mode, word_mode)
1774 != CODE_FOR_nothing)
1775 {
1776 product = expand_doubleword_mult (int_mode, op0, op1, target,
1777 true, methods);
1778 if (!product)
1779 delete_insns_since (last);
1780 }
1781
1782 if (product == NULL_RTX
1783 && (convert_optab_handler (smul_widen_optab, int_mode, word_mode)
1784 != CODE_FOR_nothing))
1785 {
1786 product = expand_doubleword_mult (int_mode, op0, op1, target,
1787 false, methods);
1788 if (!product)
1789 delete_insns_since (last);
1790 }
1791
1792 if (product != NULL_RTX)
1793 {
1794 if (optab_handler (mov_optab, int_mode) != CODE_FOR_nothing)
1795 {
1796 rtx_insn *move = emit_move_insn (target ? target : product,
1797 product);
1798 set_dst_reg_note (move,
1799 REG_EQUAL,
1800 gen_rtx_fmt_ee (MULT, int_mode,
1801 copy_rtx (op0),
1802 copy_rtx (op1)),
1803 target ? target : product);
1804 }
1805 return product;
1806 }
1807 }
1808
1809 /* It can't be open-coded in this mode.
1810 Use a library call if one is available and caller says that's ok. */
1811
1812 libfunc = optab_libfunc (binoptab, mode);
1813 if (libfunc
1814 && (methods == OPTAB_LIB || methods == OPTAB_LIB_WIDEN))
1815 {
1816 rtx_insn *insns;
1817 rtx op1x = op1;
1818 machine_mode op1_mode = mode;
1819 rtx value;
1820
1821 start_sequence ();
1822
1823 if (shift_optab_p (binoptab))
1824 {
1825 op1_mode = targetm.libgcc_shift_count_mode ();
1826 /* Specify unsigned here,
1827 since negative shift counts are meaningless. */
1828 op1x = convert_to_mode (op1_mode, op1, 1);
1829 }
1830
1831 if (GET_MODE (op0) != VOIDmode
1832 && GET_MODE (op0) != mode)
1833 op0 = convert_to_mode (mode, op0, unsignedp);
1834
1835 /* Pass 1 for NO_QUEUE so we don't lose any increments
1836 if the libcall is cse'd or moved. */
1837 value = emit_library_call_value (libfunc,
1838 NULL_RTX, LCT_CONST, mode,
1839 op0, mode, op1x, op1_mode);
1840
1841 insns = get_insns ();
1842 end_sequence ();
1843
1844 bool trapv = trapv_binoptab_p (binoptab);
1845 target = gen_reg_rtx (mode);
1846 emit_libcall_block_1 (insns, target, value,
1847 trapv ? NULL_RTX
1848 : gen_rtx_fmt_ee (optab_to_code (binoptab),
1849 mode, op0, op1), trapv);
1850
1851 return target;
1852 }
1853
1854 delete_insns_since (last);
1855
1856 /* It can't be done in this mode. Can we do it in a wider mode? */
1857
1858 if (! (methods == OPTAB_WIDEN || methods == OPTAB_LIB_WIDEN
1859 || methods == OPTAB_MUST_WIDEN))
1860 {
1861 /* Caller says, don't even try. */
1862 delete_insns_since (entry_last);
1863 return 0;
1864 }
1865
1866 /* Compute the value of METHODS to pass to recursive calls.
1867 Don't allow widening to be tried recursively. */
1868
1869 methods = (methods == OPTAB_LIB_WIDEN ? OPTAB_LIB : OPTAB_DIRECT);
1870
1871 /* Look for a wider mode of the same class for which it appears we can do
1872 the operation. */
1873
1874 if (CLASS_HAS_WIDER_MODES_P (mclass))
1875 {
1876 /* This code doesn't make sense for conversion optabs, since we
1877 wouldn't then want to extend the operands to be the same size
1878 as the result. */
1879 gcc_assert (!convert_optab_p (binoptab));
1880 FOR_EACH_WIDER_MODE (wider_mode, mode)
1881 {
1882 if (optab_handler (binoptab, wider_mode)
1883 || (methods == OPTAB_LIB
1884 && optab_libfunc (binoptab, wider_mode)))
1885 {
1886 rtx xop0 = op0, xop1 = op1;
1887 int no_extend = 0;
1888
1889 /* For certain integer operations, we need not actually extend
1890 the narrow operands, as long as we will truncate
1891 the results to the same narrowness. */
1892
1893 if ((binoptab == ior_optab || binoptab == and_optab
1894 || binoptab == xor_optab
1895 || binoptab == add_optab || binoptab == sub_optab
1896 || binoptab == smul_optab || binoptab == ashl_optab)
1897 && mclass == MODE_INT)
1898 no_extend = 1;
1899
1900 xop0 = widen_operand (xop0, wider_mode, mode,
1901 unsignedp, no_extend);
1902
1903 /* The second operand of a shift must always be extended. */
1904 xop1 = widen_operand (xop1, wider_mode, mode, unsignedp,
1905 no_extend && binoptab != ashl_optab);
1906
1907 temp = expand_binop (wider_mode, binoptab, xop0, xop1, NULL_RTX,
1908 unsignedp, methods);
1909 if (temp)
1910 {
1911 if (mclass != MODE_INT
1912 || !TRULY_NOOP_TRUNCATION_MODES_P (mode, wider_mode))
1913 {
1914 if (target == 0)
1915 target = gen_reg_rtx (mode);
1916 convert_move (target, temp, 0);
1917 return target;
1918 }
1919 else
1920 return gen_lowpart (mode, temp);
1921 }
1922 else
1923 delete_insns_since (last);
1924 }
1925 }
1926 }
1927
1928 delete_insns_since (entry_last);
1929 return 0;
1930 }
1931
1932 /* Expand a binary operator which has both signed and unsigned forms.
1933 UOPTAB is the optab for unsigned operations, and SOPTAB is for
1934 signed operations.
1935
1936 If we widen unsigned operands, we may use a signed wider operation instead
1937 of an unsigned wider operation, since the result would be the same. */
1938
1939 rtx
sign_expand_binop(machine_mode mode,optab uoptab,optab soptab,rtx op0,rtx op1,rtx target,int unsignedp,enum optab_methods methods)1940 sign_expand_binop (machine_mode mode, optab uoptab, optab soptab,
1941 rtx op0, rtx op1, rtx target, int unsignedp,
1942 enum optab_methods methods)
1943 {
1944 rtx temp;
1945 optab direct_optab = unsignedp ? uoptab : soptab;
1946 bool save_enable;
1947
1948 /* Do it without widening, if possible. */
1949 temp = expand_binop (mode, direct_optab, op0, op1, target,
1950 unsignedp, OPTAB_DIRECT);
1951 if (temp || methods == OPTAB_DIRECT)
1952 return temp;
1953
1954 /* Try widening to a signed int. Disable any direct use of any
1955 signed insn in the current mode. */
1956 save_enable = swap_optab_enable (soptab, mode, false);
1957
1958 temp = expand_binop (mode, soptab, op0, op1, target,
1959 unsignedp, OPTAB_WIDEN);
1960
1961 /* For unsigned operands, try widening to an unsigned int. */
1962 if (!temp && unsignedp)
1963 temp = expand_binop (mode, uoptab, op0, op1, target,
1964 unsignedp, OPTAB_WIDEN);
1965 if (temp || methods == OPTAB_WIDEN)
1966 goto egress;
1967
1968 /* Use the right width libcall if that exists. */
1969 temp = expand_binop (mode, direct_optab, op0, op1, target,
1970 unsignedp, OPTAB_LIB);
1971 if (temp || methods == OPTAB_LIB)
1972 goto egress;
1973
1974 /* Must widen and use a libcall, use either signed or unsigned. */
1975 temp = expand_binop (mode, soptab, op0, op1, target,
1976 unsignedp, methods);
1977 if (!temp && unsignedp)
1978 temp = expand_binop (mode, uoptab, op0, op1, target,
1979 unsignedp, methods);
1980
1981 egress:
1982 /* Undo the fiddling above. */
1983 if (save_enable)
1984 swap_optab_enable (soptab, mode, true);
1985 return temp;
1986 }
1987
1988 /* Generate code to perform an operation specified by UNOPPTAB
1989 on operand OP0, with two results to TARG0 and TARG1.
1990 We assume that the order of the operands for the instruction
1991 is TARG0, TARG1, OP0.
1992
1993 Either TARG0 or TARG1 may be zero, but what that means is that
1994 the result is not actually wanted. We will generate it into
1995 a dummy pseudo-reg and discard it. They may not both be zero.
1996
1997 Returns 1 if this operation can be performed; 0 if not. */
1998
1999 int
expand_twoval_unop(optab unoptab,rtx op0,rtx targ0,rtx targ1,int unsignedp)2000 expand_twoval_unop (optab unoptab, rtx op0, rtx targ0, rtx targ1,
2001 int unsignedp)
2002 {
2003 machine_mode mode = GET_MODE (targ0 ? targ0 : targ1);
2004 enum mode_class mclass;
2005 machine_mode wider_mode;
2006 rtx_insn *entry_last = get_last_insn ();
2007 rtx_insn *last;
2008
2009 mclass = GET_MODE_CLASS (mode);
2010
2011 if (!targ0)
2012 targ0 = gen_reg_rtx (mode);
2013 if (!targ1)
2014 targ1 = gen_reg_rtx (mode);
2015
2016 /* Record where to go back to if we fail. */
2017 last = get_last_insn ();
2018
2019 if (optab_handler (unoptab, mode) != CODE_FOR_nothing)
2020 {
2021 struct expand_operand ops[3];
2022 enum insn_code icode = optab_handler (unoptab, mode);
2023
2024 create_fixed_operand (&ops[0], targ0);
2025 create_fixed_operand (&ops[1], targ1);
2026 create_convert_operand_from (&ops[2], op0, mode, unsignedp);
2027 if (maybe_expand_insn (icode, 3, ops))
2028 return 1;
2029 }
2030
2031 /* It can't be done in this mode. Can we do it in a wider mode? */
2032
2033 if (CLASS_HAS_WIDER_MODES_P (mclass))
2034 {
2035 FOR_EACH_WIDER_MODE (wider_mode, mode)
2036 {
2037 if (optab_handler (unoptab, wider_mode) != CODE_FOR_nothing)
2038 {
2039 rtx t0 = gen_reg_rtx (wider_mode);
2040 rtx t1 = gen_reg_rtx (wider_mode);
2041 rtx cop0 = convert_modes (wider_mode, mode, op0, unsignedp);
2042
2043 if (expand_twoval_unop (unoptab, cop0, t0, t1, unsignedp))
2044 {
2045 convert_move (targ0, t0, unsignedp);
2046 convert_move (targ1, t1, unsignedp);
2047 return 1;
2048 }
2049 else
2050 delete_insns_since (last);
2051 }
2052 }
2053 }
2054
2055 delete_insns_since (entry_last);
2056 return 0;
2057 }
2058
2059 /* Generate code to perform an operation specified by BINOPTAB
2060 on operands OP0 and OP1, with two results to TARG1 and TARG2.
2061 We assume that the order of the operands for the instruction
2062 is TARG0, OP0, OP1, TARG1, which would fit a pattern like
2063 [(set TARG0 (operate OP0 OP1)) (set TARG1 (operate ...))].
2064
2065 Either TARG0 or TARG1 may be zero, but what that means is that
2066 the result is not actually wanted. We will generate it into
2067 a dummy pseudo-reg and discard it. They may not both be zero.
2068
2069 Returns 1 if this operation can be performed; 0 if not. */
2070
2071 int
expand_twoval_binop(optab binoptab,rtx op0,rtx op1,rtx targ0,rtx targ1,int unsignedp)2072 expand_twoval_binop (optab binoptab, rtx op0, rtx op1, rtx targ0, rtx targ1,
2073 int unsignedp)
2074 {
2075 machine_mode mode = GET_MODE (targ0 ? targ0 : targ1);
2076 enum mode_class mclass;
2077 machine_mode wider_mode;
2078 rtx_insn *entry_last = get_last_insn ();
2079 rtx_insn *last;
2080
2081 mclass = GET_MODE_CLASS (mode);
2082
2083 if (!targ0)
2084 targ0 = gen_reg_rtx (mode);
2085 if (!targ1)
2086 targ1 = gen_reg_rtx (mode);
2087
2088 /* Record where to go back to if we fail. */
2089 last = get_last_insn ();
2090
2091 if (optab_handler (binoptab, mode) != CODE_FOR_nothing)
2092 {
2093 struct expand_operand ops[4];
2094 enum insn_code icode = optab_handler (binoptab, mode);
2095 machine_mode mode0 = insn_data[icode].operand[1].mode;
2096 machine_mode mode1 = insn_data[icode].operand[2].mode;
2097 rtx xop0 = op0, xop1 = op1;
2098
2099 /* If we are optimizing, force expensive constants into a register. */
2100 xop0 = avoid_expensive_constant (mode0, binoptab, 0, xop0, unsignedp);
2101 xop1 = avoid_expensive_constant (mode1, binoptab, 1, xop1, unsignedp);
2102
2103 create_fixed_operand (&ops[0], targ0);
2104 create_convert_operand_from (&ops[1], op0, mode, unsignedp);
2105 create_convert_operand_from (&ops[2], op1, mode, unsignedp);
2106 create_fixed_operand (&ops[3], targ1);
2107 if (maybe_expand_insn (icode, 4, ops))
2108 return 1;
2109 delete_insns_since (last);
2110 }
2111
2112 /* It can't be done in this mode. Can we do it in a wider mode? */
2113
2114 if (CLASS_HAS_WIDER_MODES_P (mclass))
2115 {
2116 FOR_EACH_WIDER_MODE (wider_mode, mode)
2117 {
2118 if (optab_handler (binoptab, wider_mode) != CODE_FOR_nothing)
2119 {
2120 rtx t0 = gen_reg_rtx (wider_mode);
2121 rtx t1 = gen_reg_rtx (wider_mode);
2122 rtx cop0 = convert_modes (wider_mode, mode, op0, unsignedp);
2123 rtx cop1 = convert_modes (wider_mode, mode, op1, unsignedp);
2124
2125 if (expand_twoval_binop (binoptab, cop0, cop1,
2126 t0, t1, unsignedp))
2127 {
2128 convert_move (targ0, t0, unsignedp);
2129 convert_move (targ1, t1, unsignedp);
2130 return 1;
2131 }
2132 else
2133 delete_insns_since (last);
2134 }
2135 }
2136 }
2137
2138 delete_insns_since (entry_last);
2139 return 0;
2140 }
2141
2142 /* Expand the two-valued library call indicated by BINOPTAB, but
2143 preserve only one of the values. If TARG0 is non-NULL, the first
2144 value is placed into TARG0; otherwise the second value is placed
2145 into TARG1. Exactly one of TARG0 and TARG1 must be non-NULL. The
2146 value stored into TARG0 or TARG1 is equivalent to (CODE OP0 OP1).
2147 This routine assumes that the value returned by the library call is
2148 as if the return value was of an integral mode twice as wide as the
2149 mode of OP0. Returns 1 if the call was successful. */
2150
2151 bool
expand_twoval_binop_libfunc(optab binoptab,rtx op0,rtx op1,rtx targ0,rtx targ1,enum rtx_code code)2152 expand_twoval_binop_libfunc (optab binoptab, rtx op0, rtx op1,
2153 rtx targ0, rtx targ1, enum rtx_code code)
2154 {
2155 machine_mode mode;
2156 machine_mode libval_mode;
2157 rtx libval;
2158 rtx_insn *insns;
2159 rtx libfunc;
2160
2161 /* Exactly one of TARG0 or TARG1 should be non-NULL. */
2162 gcc_assert (!targ0 != !targ1);
2163
2164 mode = GET_MODE (op0);
2165 libfunc = optab_libfunc (binoptab, mode);
2166 if (!libfunc)
2167 return false;
2168
2169 /* The value returned by the library function will have twice as
2170 many bits as the nominal MODE. */
2171 libval_mode = smallest_int_mode_for_size (2 * GET_MODE_BITSIZE (mode));
2172 start_sequence ();
2173 libval = emit_library_call_value (libfunc, NULL_RTX, LCT_CONST,
2174 libval_mode,
2175 op0, mode,
2176 op1, mode);
2177 /* Get the part of VAL containing the value that we want. */
2178 libval = simplify_gen_subreg (mode, libval, libval_mode,
2179 targ0 ? 0 : GET_MODE_SIZE (mode));
2180 insns = get_insns ();
2181 end_sequence ();
2182 /* Move the into the desired location. */
2183 emit_libcall_block (insns, targ0 ? targ0 : targ1, libval,
2184 gen_rtx_fmt_ee (code, mode, op0, op1));
2185
2186 return true;
2187 }
2188
2189
2190 /* Wrapper around expand_unop which takes an rtx code to specify
2191 the operation to perform, not an optab pointer. All other
2192 arguments are the same. */
2193 rtx
expand_simple_unop(machine_mode mode,enum rtx_code code,rtx op0,rtx target,int unsignedp)2194 expand_simple_unop (machine_mode mode, enum rtx_code code, rtx op0,
2195 rtx target, int unsignedp)
2196 {
2197 optab unop = code_to_optab (code);
2198 gcc_assert (unop);
2199
2200 return expand_unop (mode, unop, op0, target, unsignedp);
2201 }
2202
2203 /* Try calculating
2204 (clz:narrow x)
2205 as
2206 (clz:wide (zero_extend:wide x)) - ((width wide) - (width narrow)).
2207
2208 A similar operation can be used for clrsb. UNOPTAB says which operation
2209 we are trying to expand. */
2210 static rtx
widen_leading(scalar_int_mode mode,rtx op0,rtx target,optab unoptab)2211 widen_leading (scalar_int_mode mode, rtx op0, rtx target, optab unoptab)
2212 {
2213 opt_scalar_int_mode wider_mode_iter;
2214 FOR_EACH_WIDER_MODE (wider_mode_iter, mode)
2215 {
2216 scalar_int_mode wider_mode = wider_mode_iter.require ();
2217 if (optab_handler (unoptab, wider_mode) != CODE_FOR_nothing)
2218 {
2219 rtx xop0, temp;
2220 rtx_insn *last;
2221
2222 last = get_last_insn ();
2223
2224 if (target == 0)
2225 target = gen_reg_rtx (mode);
2226 xop0 = widen_operand (op0, wider_mode, mode,
2227 unoptab != clrsb_optab, false);
2228 temp = expand_unop (wider_mode, unoptab, xop0, NULL_RTX,
2229 unoptab != clrsb_optab);
2230 if (temp != 0)
2231 temp = expand_binop
2232 (wider_mode, sub_optab, temp,
2233 gen_int_mode (GET_MODE_PRECISION (wider_mode)
2234 - GET_MODE_PRECISION (mode),
2235 wider_mode),
2236 target, true, OPTAB_DIRECT);
2237 if (temp == 0)
2238 delete_insns_since (last);
2239
2240 return temp;
2241 }
2242 }
2243 return 0;
2244 }
2245
2246 /* Try calculating clz of a double-word quantity as two clz's of word-sized
2247 quantities, choosing which based on whether the high word is nonzero. */
2248 static rtx
expand_doubleword_clz(scalar_int_mode mode,rtx op0,rtx target)2249 expand_doubleword_clz (scalar_int_mode mode, rtx op0, rtx target)
2250 {
2251 rtx xop0 = force_reg (mode, op0);
2252 rtx subhi = gen_highpart (word_mode, xop0);
2253 rtx sublo = gen_lowpart (word_mode, xop0);
2254 rtx_code_label *hi0_label = gen_label_rtx ();
2255 rtx_code_label *after_label = gen_label_rtx ();
2256 rtx_insn *seq;
2257 rtx temp, result;
2258
2259 /* If we were not given a target, use a word_mode register, not a
2260 'mode' register. The result will fit, and nobody is expecting
2261 anything bigger (the return type of __builtin_clz* is int). */
2262 if (!target)
2263 target = gen_reg_rtx (word_mode);
2264
2265 /* In any case, write to a word_mode scratch in both branches of the
2266 conditional, so we can ensure there is a single move insn setting
2267 'target' to tag a REG_EQUAL note on. */
2268 result = gen_reg_rtx (word_mode);
2269
2270 start_sequence ();
2271
2272 /* If the high word is not equal to zero,
2273 then clz of the full value is clz of the high word. */
2274 emit_cmp_and_jump_insns (subhi, CONST0_RTX (word_mode), EQ, 0,
2275 word_mode, true, hi0_label);
2276
2277 temp = expand_unop_direct (word_mode, clz_optab, subhi, result, true);
2278 if (!temp)
2279 goto fail;
2280
2281 if (temp != result)
2282 convert_move (result, temp, true);
2283
2284 emit_jump_insn (targetm.gen_jump (after_label));
2285 emit_barrier ();
2286
2287 /* Else clz of the full value is clz of the low word plus the number
2288 of bits in the high word. */
2289 emit_label (hi0_label);
2290
2291 temp = expand_unop_direct (word_mode, clz_optab, sublo, 0, true);
2292 if (!temp)
2293 goto fail;
2294 temp = expand_binop (word_mode, add_optab, temp,
2295 gen_int_mode (GET_MODE_BITSIZE (word_mode), word_mode),
2296 result, true, OPTAB_DIRECT);
2297 if (!temp)
2298 goto fail;
2299 if (temp != result)
2300 convert_move (result, temp, true);
2301
2302 emit_label (after_label);
2303 convert_move (target, result, true);
2304
2305 seq = get_insns ();
2306 end_sequence ();
2307
2308 add_equal_note (seq, target, CLZ, xop0, NULL_RTX, mode);
2309 emit_insn (seq);
2310 return target;
2311
2312 fail:
2313 end_sequence ();
2314 return 0;
2315 }
2316
2317 /* Try calculating popcount of a double-word quantity as two popcount's of
2318 word-sized quantities and summing up the results. */
2319 static rtx
expand_doubleword_popcount(scalar_int_mode mode,rtx op0,rtx target)2320 expand_doubleword_popcount (scalar_int_mode mode, rtx op0, rtx target)
2321 {
2322 rtx t0, t1, t;
2323 rtx_insn *seq;
2324
2325 start_sequence ();
2326
2327 t0 = expand_unop_direct (word_mode, popcount_optab,
2328 operand_subword_force (op0, 0, mode), NULL_RTX,
2329 true);
2330 t1 = expand_unop_direct (word_mode, popcount_optab,
2331 operand_subword_force (op0, 1, mode), NULL_RTX,
2332 true);
2333 if (!t0 || !t1)
2334 {
2335 end_sequence ();
2336 return NULL_RTX;
2337 }
2338
2339 /* If we were not given a target, use a word_mode register, not a
2340 'mode' register. The result will fit, and nobody is expecting
2341 anything bigger (the return type of __builtin_popcount* is int). */
2342 if (!target)
2343 target = gen_reg_rtx (word_mode);
2344
2345 t = expand_binop (word_mode, add_optab, t0, t1, target, 0, OPTAB_DIRECT);
2346
2347 seq = get_insns ();
2348 end_sequence ();
2349
2350 add_equal_note (seq, t, POPCOUNT, op0, NULL_RTX, mode);
2351 emit_insn (seq);
2352 return t;
2353 }
2354
2355 /* Try calculating
2356 (parity:wide x)
2357 as
2358 (parity:narrow (low (x) ^ high (x))) */
2359 static rtx
expand_doubleword_parity(scalar_int_mode mode,rtx op0,rtx target)2360 expand_doubleword_parity (scalar_int_mode mode, rtx op0, rtx target)
2361 {
2362 rtx t = expand_binop (word_mode, xor_optab,
2363 operand_subword_force (op0, 0, mode),
2364 operand_subword_force (op0, 1, mode),
2365 NULL_RTX, 0, OPTAB_DIRECT);
2366 return expand_unop (word_mode, parity_optab, t, target, true);
2367 }
2368
2369 /* Try calculating
2370 (bswap:narrow x)
2371 as
2372 (lshiftrt:wide (bswap:wide x) ((width wide) - (width narrow))). */
2373 static rtx
widen_bswap(scalar_int_mode mode,rtx op0,rtx target)2374 widen_bswap (scalar_int_mode mode, rtx op0, rtx target)
2375 {
2376 rtx x;
2377 rtx_insn *last;
2378 opt_scalar_int_mode wider_mode_iter;
2379
2380 FOR_EACH_WIDER_MODE (wider_mode_iter, mode)
2381 if (optab_handler (bswap_optab, wider_mode_iter.require ())
2382 != CODE_FOR_nothing)
2383 break;
2384
2385 if (!wider_mode_iter.exists ())
2386 return NULL_RTX;
2387
2388 scalar_int_mode wider_mode = wider_mode_iter.require ();
2389 last = get_last_insn ();
2390
2391 x = widen_operand (op0, wider_mode, mode, true, true);
2392 x = expand_unop (wider_mode, bswap_optab, x, NULL_RTX, true);
2393
2394 gcc_assert (GET_MODE_PRECISION (wider_mode) == GET_MODE_BITSIZE (wider_mode)
2395 && GET_MODE_PRECISION (mode) == GET_MODE_BITSIZE (mode));
2396 if (x != 0)
2397 x = expand_shift (RSHIFT_EXPR, wider_mode, x,
2398 GET_MODE_BITSIZE (wider_mode)
2399 - GET_MODE_BITSIZE (mode),
2400 NULL_RTX, true);
2401
2402 if (x != 0)
2403 {
2404 if (target == 0)
2405 target = gen_reg_rtx (mode);
2406 emit_move_insn (target, gen_lowpart (mode, x));
2407 }
2408 else
2409 delete_insns_since (last);
2410
2411 return target;
2412 }
2413
2414 /* Try calculating bswap as two bswaps of two word-sized operands. */
2415
2416 static rtx
expand_doubleword_bswap(machine_mode mode,rtx op,rtx target)2417 expand_doubleword_bswap (machine_mode mode, rtx op, rtx target)
2418 {
2419 rtx t0, t1;
2420
2421 t1 = expand_unop (word_mode, bswap_optab,
2422 operand_subword_force (op, 0, mode), NULL_RTX, true);
2423 t0 = expand_unop (word_mode, bswap_optab,
2424 operand_subword_force (op, 1, mode), NULL_RTX, true);
2425
2426 if (target == 0 || !valid_multiword_target_p (target))
2427 target = gen_reg_rtx (mode);
2428 if (REG_P (target))
2429 emit_clobber (target);
2430 emit_move_insn (operand_subword (target, 0, 1, mode), t0);
2431 emit_move_insn (operand_subword (target, 1, 1, mode), t1);
2432
2433 return target;
2434 }
2435
2436 /* Try calculating (parity x) as (and (popcount x) 1), where
2437 popcount can also be done in a wider mode. */
2438 static rtx
expand_parity(scalar_int_mode mode,rtx op0,rtx target)2439 expand_parity (scalar_int_mode mode, rtx op0, rtx target)
2440 {
2441 enum mode_class mclass = GET_MODE_CLASS (mode);
2442 opt_scalar_int_mode wider_mode_iter;
2443 FOR_EACH_MODE_FROM (wider_mode_iter, mode)
2444 {
2445 scalar_int_mode wider_mode = wider_mode_iter.require ();
2446 if (optab_handler (popcount_optab, wider_mode) != CODE_FOR_nothing)
2447 {
2448 rtx xop0, temp;
2449 rtx_insn *last;
2450
2451 last = get_last_insn ();
2452
2453 if (target == 0 || GET_MODE (target) != wider_mode)
2454 target = gen_reg_rtx (wider_mode);
2455
2456 xop0 = widen_operand (op0, wider_mode, mode, true, false);
2457 temp = expand_unop (wider_mode, popcount_optab, xop0, NULL_RTX,
2458 true);
2459 if (temp != 0)
2460 temp = expand_binop (wider_mode, and_optab, temp, const1_rtx,
2461 target, true, OPTAB_DIRECT);
2462
2463 if (temp)
2464 {
2465 if (mclass != MODE_INT
2466 || !TRULY_NOOP_TRUNCATION_MODES_P (mode, wider_mode))
2467 return convert_to_mode (mode, temp, 0);
2468 else
2469 return gen_lowpart (mode, temp);
2470 }
2471 else
2472 delete_insns_since (last);
2473 }
2474 }
2475 return 0;
2476 }
2477
2478 /* Try calculating ctz(x) as K - clz(x & -x) ,
2479 where K is GET_MODE_PRECISION(mode) - 1.
2480
2481 Both __builtin_ctz and __builtin_clz are undefined at zero, so we
2482 don't have to worry about what the hardware does in that case. (If
2483 the clz instruction produces the usual value at 0, which is K, the
2484 result of this code sequence will be -1; expand_ffs, below, relies
2485 on this. It might be nice to have it be K instead, for consistency
2486 with the (very few) processors that provide a ctz with a defined
2487 value, but that would take one more instruction, and it would be
2488 less convenient for expand_ffs anyway. */
2489
2490 static rtx
expand_ctz(scalar_int_mode mode,rtx op0,rtx target)2491 expand_ctz (scalar_int_mode mode, rtx op0, rtx target)
2492 {
2493 rtx_insn *seq;
2494 rtx temp;
2495
2496 if (optab_handler (clz_optab, mode) == CODE_FOR_nothing)
2497 return 0;
2498
2499 start_sequence ();
2500
2501 temp = expand_unop_direct (mode, neg_optab, op0, NULL_RTX, true);
2502 if (temp)
2503 temp = expand_binop (mode, and_optab, op0, temp, NULL_RTX,
2504 true, OPTAB_DIRECT);
2505 if (temp)
2506 temp = expand_unop_direct (mode, clz_optab, temp, NULL_RTX, true);
2507 if (temp)
2508 temp = expand_binop (mode, sub_optab,
2509 gen_int_mode (GET_MODE_PRECISION (mode) - 1, mode),
2510 temp, target,
2511 true, OPTAB_DIRECT);
2512 if (temp == 0)
2513 {
2514 end_sequence ();
2515 return 0;
2516 }
2517
2518 seq = get_insns ();
2519 end_sequence ();
2520
2521 add_equal_note (seq, temp, CTZ, op0, NULL_RTX, mode);
2522 emit_insn (seq);
2523 return temp;
2524 }
2525
2526
2527 /* Try calculating ffs(x) using ctz(x) if we have that instruction, or
2528 else with the sequence used by expand_clz.
2529
2530 The ffs builtin promises to return zero for a zero value and ctz/clz
2531 may have an undefined value in that case. If they do not give us a
2532 convenient value, we have to generate a test and branch. */
2533 static rtx
expand_ffs(scalar_int_mode mode,rtx op0,rtx target)2534 expand_ffs (scalar_int_mode mode, rtx op0, rtx target)
2535 {
2536 HOST_WIDE_INT val = 0;
2537 bool defined_at_zero = false;
2538 rtx temp;
2539 rtx_insn *seq;
2540
2541 if (optab_handler (ctz_optab, mode) != CODE_FOR_nothing)
2542 {
2543 start_sequence ();
2544
2545 temp = expand_unop_direct (mode, ctz_optab, op0, 0, true);
2546 if (!temp)
2547 goto fail;
2548
2549 defined_at_zero = (CTZ_DEFINED_VALUE_AT_ZERO (mode, val) == 2);
2550 }
2551 else if (optab_handler (clz_optab, mode) != CODE_FOR_nothing)
2552 {
2553 start_sequence ();
2554 temp = expand_ctz (mode, op0, 0);
2555 if (!temp)
2556 goto fail;
2557
2558 if (CLZ_DEFINED_VALUE_AT_ZERO (mode, val) == 2)
2559 {
2560 defined_at_zero = true;
2561 val = (GET_MODE_PRECISION (mode) - 1) - val;
2562 }
2563 }
2564 else
2565 return 0;
2566
2567 if (defined_at_zero && val == -1)
2568 /* No correction needed at zero. */;
2569 else
2570 {
2571 /* We don't try to do anything clever with the situation found
2572 on some processors (eg Alpha) where ctz(0:mode) ==
2573 bitsize(mode). If someone can think of a way to send N to -1
2574 and leave alone all values in the range 0..N-1 (where N is a
2575 power of two), cheaper than this test-and-branch, please add it.
2576
2577 The test-and-branch is done after the operation itself, in case
2578 the operation sets condition codes that can be recycled for this.
2579 (This is true on i386, for instance.) */
2580
2581 rtx_code_label *nonzero_label = gen_label_rtx ();
2582 emit_cmp_and_jump_insns (op0, CONST0_RTX (mode), NE, 0,
2583 mode, true, nonzero_label);
2584
2585 convert_move (temp, GEN_INT (-1), false);
2586 emit_label (nonzero_label);
2587 }
2588
2589 /* temp now has a value in the range -1..bitsize-1. ffs is supposed
2590 to produce a value in the range 0..bitsize. */
2591 temp = expand_binop (mode, add_optab, temp, gen_int_mode (1, mode),
2592 target, false, OPTAB_DIRECT);
2593 if (!temp)
2594 goto fail;
2595
2596 seq = get_insns ();
2597 end_sequence ();
2598
2599 add_equal_note (seq, temp, FFS, op0, NULL_RTX, mode);
2600 emit_insn (seq);
2601 return temp;
2602
2603 fail:
2604 end_sequence ();
2605 return 0;
2606 }
2607
2608 /* Extract the OMODE lowpart from VAL, which has IMODE. Under certain
2609 conditions, VAL may already be a SUBREG against which we cannot generate
2610 a further SUBREG. In this case, we expect forcing the value into a
2611 register will work around the situation. */
2612
2613 static rtx
lowpart_subreg_maybe_copy(machine_mode omode,rtx val,machine_mode imode)2614 lowpart_subreg_maybe_copy (machine_mode omode, rtx val,
2615 machine_mode imode)
2616 {
2617 rtx ret;
2618 ret = lowpart_subreg (omode, val, imode);
2619 if (ret == NULL)
2620 {
2621 val = force_reg (imode, val);
2622 ret = lowpart_subreg (omode, val, imode);
2623 gcc_assert (ret != NULL);
2624 }
2625 return ret;
2626 }
2627
2628 /* Expand a floating point absolute value or negation operation via a
2629 logical operation on the sign bit. */
2630
2631 static rtx
expand_absneg_bit(enum rtx_code code,scalar_float_mode mode,rtx op0,rtx target)2632 expand_absneg_bit (enum rtx_code code, scalar_float_mode mode,
2633 rtx op0, rtx target)
2634 {
2635 const struct real_format *fmt;
2636 int bitpos, word, nwords, i;
2637 scalar_int_mode imode;
2638 rtx temp;
2639 rtx_insn *insns;
2640
2641 /* The format has to have a simple sign bit. */
2642 fmt = REAL_MODE_FORMAT (mode);
2643 if (fmt == NULL)
2644 return NULL_RTX;
2645
2646 bitpos = fmt->signbit_rw;
2647 if (bitpos < 0)
2648 return NULL_RTX;
2649
2650 /* Don't create negative zeros if the format doesn't support them. */
2651 if (code == NEG && !fmt->has_signed_zero)
2652 return NULL_RTX;
2653
2654 if (GET_MODE_SIZE (mode) <= UNITS_PER_WORD)
2655 {
2656 if (!int_mode_for_mode (mode).exists (&imode))
2657 return NULL_RTX;
2658 word = 0;
2659 nwords = 1;
2660 }
2661 else
2662 {
2663 imode = word_mode;
2664
2665 if (FLOAT_WORDS_BIG_ENDIAN)
2666 word = (GET_MODE_BITSIZE (mode) - bitpos) / BITS_PER_WORD;
2667 else
2668 word = bitpos / BITS_PER_WORD;
2669 bitpos = bitpos % BITS_PER_WORD;
2670 nwords = (GET_MODE_BITSIZE (mode) + BITS_PER_WORD - 1) / BITS_PER_WORD;
2671 }
2672
2673 wide_int mask = wi::set_bit_in_zero (bitpos, GET_MODE_PRECISION (imode));
2674 if (code == ABS)
2675 mask = ~mask;
2676
2677 if (target == 0
2678 || target == op0
2679 || reg_overlap_mentioned_p (target, op0)
2680 || (nwords > 1 && !valid_multiword_target_p (target)))
2681 target = gen_reg_rtx (mode);
2682
2683 if (nwords > 1)
2684 {
2685 start_sequence ();
2686
2687 for (i = 0; i < nwords; ++i)
2688 {
2689 rtx targ_piece = operand_subword (target, i, 1, mode);
2690 rtx op0_piece = operand_subword_force (op0, i, mode);
2691
2692 if (i == word)
2693 {
2694 temp = expand_binop (imode, code == ABS ? and_optab : xor_optab,
2695 op0_piece,
2696 immed_wide_int_const (mask, imode),
2697 targ_piece, 1, OPTAB_LIB_WIDEN);
2698 if (temp != targ_piece)
2699 emit_move_insn (targ_piece, temp);
2700 }
2701 else
2702 emit_move_insn (targ_piece, op0_piece);
2703 }
2704
2705 insns = get_insns ();
2706 end_sequence ();
2707
2708 emit_insn (insns);
2709 }
2710 else
2711 {
2712 temp = expand_binop (imode, code == ABS ? and_optab : xor_optab,
2713 gen_lowpart (imode, op0),
2714 immed_wide_int_const (mask, imode),
2715 gen_lowpart (imode, target), 1, OPTAB_LIB_WIDEN);
2716 target = lowpart_subreg_maybe_copy (mode, temp, imode);
2717
2718 set_dst_reg_note (get_last_insn (), REG_EQUAL,
2719 gen_rtx_fmt_e (code, mode, copy_rtx (op0)),
2720 target);
2721 }
2722
2723 return target;
2724 }
2725
2726 /* As expand_unop, but will fail rather than attempt the operation in a
2727 different mode or with a libcall. */
2728 static rtx
expand_unop_direct(machine_mode mode,optab unoptab,rtx op0,rtx target,int unsignedp)2729 expand_unop_direct (machine_mode mode, optab unoptab, rtx op0, rtx target,
2730 int unsignedp)
2731 {
2732 if (optab_handler (unoptab, mode) != CODE_FOR_nothing)
2733 {
2734 struct expand_operand ops[2];
2735 enum insn_code icode = optab_handler (unoptab, mode);
2736 rtx_insn *last = get_last_insn ();
2737 rtx_insn *pat;
2738
2739 create_output_operand (&ops[0], target, mode);
2740 create_convert_operand_from (&ops[1], op0, mode, unsignedp);
2741 pat = maybe_gen_insn (icode, 2, ops);
2742 if (pat)
2743 {
2744 if (INSN_P (pat) && NEXT_INSN (pat) != NULL_RTX
2745 && ! add_equal_note (pat, ops[0].value,
2746 optab_to_code (unoptab),
2747 ops[1].value, NULL_RTX, mode))
2748 {
2749 delete_insns_since (last);
2750 return expand_unop (mode, unoptab, op0, NULL_RTX, unsignedp);
2751 }
2752
2753 emit_insn (pat);
2754
2755 return ops[0].value;
2756 }
2757 }
2758 return 0;
2759 }
2760
2761 /* Generate code to perform an operation specified by UNOPTAB
2762 on operand OP0, with result having machine-mode MODE.
2763
2764 UNSIGNEDP is for the case where we have to widen the operands
2765 to perform the operation. It says to use zero-extension.
2766
2767 If TARGET is nonzero, the value
2768 is generated there, if it is convenient to do so.
2769 In all cases an rtx is returned for the locus of the value;
2770 this may or may not be TARGET. */
2771
2772 rtx
expand_unop(machine_mode mode,optab unoptab,rtx op0,rtx target,int unsignedp)2773 expand_unop (machine_mode mode, optab unoptab, rtx op0, rtx target,
2774 int unsignedp)
2775 {
2776 enum mode_class mclass = GET_MODE_CLASS (mode);
2777 machine_mode wider_mode;
2778 scalar_int_mode int_mode;
2779 scalar_float_mode float_mode;
2780 rtx temp;
2781 rtx libfunc;
2782
2783 temp = expand_unop_direct (mode, unoptab, op0, target, unsignedp);
2784 if (temp)
2785 return temp;
2786
2787 /* It can't be done in this mode. Can we open-code it in a wider mode? */
2788
2789 /* Widening (or narrowing) clz needs special treatment. */
2790 if (unoptab == clz_optab)
2791 {
2792 if (is_a <scalar_int_mode> (mode, &int_mode))
2793 {
2794 temp = widen_leading (int_mode, op0, target, unoptab);
2795 if (temp)
2796 return temp;
2797
2798 if (GET_MODE_SIZE (int_mode) == 2 * UNITS_PER_WORD
2799 && optab_handler (unoptab, word_mode) != CODE_FOR_nothing)
2800 {
2801 temp = expand_doubleword_clz (int_mode, op0, target);
2802 if (temp)
2803 return temp;
2804 }
2805 }
2806
2807 goto try_libcall;
2808 }
2809
2810 if (unoptab == clrsb_optab)
2811 {
2812 if (is_a <scalar_int_mode> (mode, &int_mode))
2813 {
2814 temp = widen_leading (int_mode, op0, target, unoptab);
2815 if (temp)
2816 return temp;
2817 }
2818 goto try_libcall;
2819 }
2820
2821 if (unoptab == popcount_optab
2822 && is_a <scalar_int_mode> (mode, &int_mode)
2823 && GET_MODE_SIZE (int_mode) == 2 * UNITS_PER_WORD
2824 && optab_handler (unoptab, word_mode) != CODE_FOR_nothing
2825 && optimize_insn_for_speed_p ())
2826 {
2827 temp = expand_doubleword_popcount (int_mode, op0, target);
2828 if (temp)
2829 return temp;
2830 }
2831
2832 if (unoptab == parity_optab
2833 && is_a <scalar_int_mode> (mode, &int_mode)
2834 && GET_MODE_SIZE (int_mode) == 2 * UNITS_PER_WORD
2835 && (optab_handler (unoptab, word_mode) != CODE_FOR_nothing
2836 || optab_handler (popcount_optab, word_mode) != CODE_FOR_nothing)
2837 && optimize_insn_for_speed_p ())
2838 {
2839 temp = expand_doubleword_parity (int_mode, op0, target);
2840 if (temp)
2841 return temp;
2842 }
2843
2844 /* Widening (or narrowing) bswap needs special treatment. */
2845 if (unoptab == bswap_optab)
2846 {
2847 /* HImode is special because in this mode BSWAP is equivalent to ROTATE
2848 or ROTATERT. First try these directly; if this fails, then try the
2849 obvious pair of shifts with allowed widening, as this will probably
2850 be always more efficient than the other fallback methods. */
2851 if (mode == HImode)
2852 {
2853 rtx_insn *last;
2854 rtx temp1, temp2;
2855
2856 if (optab_handler (rotl_optab, mode) != CODE_FOR_nothing)
2857 {
2858 temp = expand_binop (mode, rotl_optab, op0,
2859 gen_int_shift_amount (mode, 8),
2860 target, unsignedp, OPTAB_DIRECT);
2861 if (temp)
2862 return temp;
2863 }
2864
2865 if (optab_handler (rotr_optab, mode) != CODE_FOR_nothing)
2866 {
2867 temp = expand_binop (mode, rotr_optab, op0,
2868 gen_int_shift_amount (mode, 8),
2869 target, unsignedp, OPTAB_DIRECT);
2870 if (temp)
2871 return temp;
2872 }
2873
2874 last = get_last_insn ();
2875
2876 temp1 = expand_binop (mode, ashl_optab, op0,
2877 gen_int_shift_amount (mode, 8), NULL_RTX,
2878 unsignedp, OPTAB_WIDEN);
2879 temp2 = expand_binop (mode, lshr_optab, op0,
2880 gen_int_shift_amount (mode, 8), NULL_RTX,
2881 unsignedp, OPTAB_WIDEN);
2882 if (temp1 && temp2)
2883 {
2884 temp = expand_binop (mode, ior_optab, temp1, temp2, target,
2885 unsignedp, OPTAB_WIDEN);
2886 if (temp)
2887 return temp;
2888 }
2889
2890 delete_insns_since (last);
2891 }
2892
2893 if (is_a <scalar_int_mode> (mode, &int_mode))
2894 {
2895 temp = widen_bswap (int_mode, op0, target);
2896 if (temp)
2897 return temp;
2898
2899 if (GET_MODE_SIZE (int_mode) == 2 * UNITS_PER_WORD
2900 && optab_handler (unoptab, word_mode) != CODE_FOR_nothing)
2901 {
2902 temp = expand_doubleword_bswap (mode, op0, target);
2903 if (temp)
2904 return temp;
2905 }
2906 }
2907
2908 goto try_libcall;
2909 }
2910
2911 if (CLASS_HAS_WIDER_MODES_P (mclass))
2912 FOR_EACH_WIDER_MODE (wider_mode, mode)
2913 {
2914 if (optab_handler (unoptab, wider_mode) != CODE_FOR_nothing)
2915 {
2916 rtx xop0 = op0;
2917 rtx_insn *last = get_last_insn ();
2918
2919 /* For certain operations, we need not actually extend
2920 the narrow operand, as long as we will truncate the
2921 results to the same narrowness. */
2922
2923 xop0 = widen_operand (xop0, wider_mode, mode, unsignedp,
2924 (unoptab == neg_optab
2925 || unoptab == one_cmpl_optab)
2926 && mclass == MODE_INT);
2927
2928 temp = expand_unop (wider_mode, unoptab, xop0, NULL_RTX,
2929 unsignedp);
2930
2931 if (temp)
2932 {
2933 if (mclass != MODE_INT
2934 || !TRULY_NOOP_TRUNCATION_MODES_P (mode, wider_mode))
2935 {
2936 if (target == 0)
2937 target = gen_reg_rtx (mode);
2938 convert_move (target, temp, 0);
2939 return target;
2940 }
2941 else
2942 return gen_lowpart (mode, temp);
2943 }
2944 else
2945 delete_insns_since (last);
2946 }
2947 }
2948
2949 /* These can be done a word at a time. */
2950 if (unoptab == one_cmpl_optab
2951 && is_int_mode (mode, &int_mode)
2952 && GET_MODE_SIZE (int_mode) > UNITS_PER_WORD
2953 && optab_handler (unoptab, word_mode) != CODE_FOR_nothing)
2954 {
2955 int i;
2956 rtx_insn *insns;
2957
2958 if (target == 0
2959 || target == op0
2960 || reg_overlap_mentioned_p (target, op0)
2961 || !valid_multiword_target_p (target))
2962 target = gen_reg_rtx (int_mode);
2963
2964 start_sequence ();
2965
2966 /* Do the actual arithmetic. */
2967 for (i = 0; i < GET_MODE_BITSIZE (int_mode) / BITS_PER_WORD; i++)
2968 {
2969 rtx target_piece = operand_subword (target, i, 1, int_mode);
2970 rtx x = expand_unop (word_mode, unoptab,
2971 operand_subword_force (op0, i, int_mode),
2972 target_piece, unsignedp);
2973
2974 if (target_piece != x)
2975 emit_move_insn (target_piece, x);
2976 }
2977
2978 insns = get_insns ();
2979 end_sequence ();
2980
2981 emit_insn (insns);
2982 return target;
2983 }
2984
2985 if (optab_to_code (unoptab) == NEG)
2986 {
2987 /* Try negating floating point values by flipping the sign bit. */
2988 if (is_a <scalar_float_mode> (mode, &float_mode))
2989 {
2990 temp = expand_absneg_bit (NEG, float_mode, op0, target);
2991 if (temp)
2992 return temp;
2993 }
2994
2995 /* If there is no negation pattern, and we have no negative zero,
2996 try subtracting from zero. */
2997 if (!HONOR_SIGNED_ZEROS (mode))
2998 {
2999 temp = expand_binop (mode, (unoptab == negv_optab
3000 ? subv_optab : sub_optab),
3001 CONST0_RTX (mode), op0, target,
3002 unsignedp, OPTAB_DIRECT);
3003 if (temp)
3004 return temp;
3005 }
3006 }
3007
3008 /* Try calculating parity (x) as popcount (x) % 2. */
3009 if (unoptab == parity_optab && is_a <scalar_int_mode> (mode, &int_mode))
3010 {
3011 temp = expand_parity (int_mode, op0, target);
3012 if (temp)
3013 return temp;
3014 }
3015
3016 /* Try implementing ffs (x) in terms of clz (x). */
3017 if (unoptab == ffs_optab && is_a <scalar_int_mode> (mode, &int_mode))
3018 {
3019 temp = expand_ffs (int_mode, op0, target);
3020 if (temp)
3021 return temp;
3022 }
3023
3024 /* Try implementing ctz (x) in terms of clz (x). */
3025 if (unoptab == ctz_optab && is_a <scalar_int_mode> (mode, &int_mode))
3026 {
3027 temp = expand_ctz (int_mode, op0, target);
3028 if (temp)
3029 return temp;
3030 }
3031
3032 try_libcall:
3033 /* Now try a library call in this mode. */
3034 libfunc = optab_libfunc (unoptab, mode);
3035 if (libfunc)
3036 {
3037 rtx_insn *insns;
3038 rtx value;
3039 rtx eq_value;
3040 machine_mode outmode = mode;
3041
3042 /* All of these functions return small values. Thus we choose to
3043 have them return something that isn't a double-word. */
3044 if (unoptab == ffs_optab || unoptab == clz_optab || unoptab == ctz_optab
3045 || unoptab == clrsb_optab || unoptab == popcount_optab
3046 || unoptab == parity_optab)
3047 outmode
3048 = GET_MODE (hard_libcall_value (TYPE_MODE (integer_type_node),
3049 optab_libfunc (unoptab, mode)));
3050
3051 start_sequence ();
3052
3053 /* Pass 1 for NO_QUEUE so we don't lose any increments
3054 if the libcall is cse'd or moved. */
3055 value = emit_library_call_value (libfunc, NULL_RTX, LCT_CONST, outmode,
3056 op0, mode);
3057 insns = get_insns ();
3058 end_sequence ();
3059
3060 target = gen_reg_rtx (outmode);
3061 bool trapv = trapv_unoptab_p (unoptab);
3062 if (trapv)
3063 eq_value = NULL_RTX;
3064 else
3065 {
3066 eq_value = gen_rtx_fmt_e (optab_to_code (unoptab), mode, op0);
3067 if (GET_MODE_UNIT_SIZE (outmode) < GET_MODE_UNIT_SIZE (mode))
3068 eq_value = simplify_gen_unary (TRUNCATE, outmode, eq_value, mode);
3069 else if (GET_MODE_UNIT_SIZE (outmode) > GET_MODE_UNIT_SIZE (mode))
3070 eq_value = simplify_gen_unary (ZERO_EXTEND,
3071 outmode, eq_value, mode);
3072 }
3073 emit_libcall_block_1 (insns, target, value, eq_value, trapv);
3074
3075 return target;
3076 }
3077
3078 /* It can't be done in this mode. Can we do it in a wider mode? */
3079
3080 if (CLASS_HAS_WIDER_MODES_P (mclass))
3081 {
3082 FOR_EACH_WIDER_MODE (wider_mode, mode)
3083 {
3084 if (optab_handler (unoptab, wider_mode) != CODE_FOR_nothing
3085 || optab_libfunc (unoptab, wider_mode))
3086 {
3087 rtx xop0 = op0;
3088 rtx_insn *last = get_last_insn ();
3089
3090 /* For certain operations, we need not actually extend
3091 the narrow operand, as long as we will truncate the
3092 results to the same narrowness. */
3093 xop0 = widen_operand (xop0, wider_mode, mode, unsignedp,
3094 (unoptab == neg_optab
3095 || unoptab == one_cmpl_optab
3096 || unoptab == bswap_optab)
3097 && mclass == MODE_INT);
3098
3099 temp = expand_unop (wider_mode, unoptab, xop0, NULL_RTX,
3100 unsignedp);
3101
3102 /* If we are generating clz using wider mode, adjust the
3103 result. Similarly for clrsb. */
3104 if ((unoptab == clz_optab || unoptab == clrsb_optab)
3105 && temp != 0)
3106 {
3107 scalar_int_mode wider_int_mode
3108 = as_a <scalar_int_mode> (wider_mode);
3109 int_mode = as_a <scalar_int_mode> (mode);
3110 temp = expand_binop
3111 (wider_mode, sub_optab, temp,
3112 gen_int_mode (GET_MODE_PRECISION (wider_int_mode)
3113 - GET_MODE_PRECISION (int_mode),
3114 wider_int_mode),
3115 target, true, OPTAB_DIRECT);
3116 }
3117
3118 /* Likewise for bswap. */
3119 if (unoptab == bswap_optab && temp != 0)
3120 {
3121 scalar_int_mode wider_int_mode
3122 = as_a <scalar_int_mode> (wider_mode);
3123 int_mode = as_a <scalar_int_mode> (mode);
3124 gcc_assert (GET_MODE_PRECISION (wider_int_mode)
3125 == GET_MODE_BITSIZE (wider_int_mode)
3126 && GET_MODE_PRECISION (int_mode)
3127 == GET_MODE_BITSIZE (int_mode));
3128
3129 temp = expand_shift (RSHIFT_EXPR, wider_int_mode, temp,
3130 GET_MODE_BITSIZE (wider_int_mode)
3131 - GET_MODE_BITSIZE (int_mode),
3132 NULL_RTX, true);
3133 }
3134
3135 if (temp)
3136 {
3137 if (mclass != MODE_INT)
3138 {
3139 if (target == 0)
3140 target = gen_reg_rtx (mode);
3141 convert_move (target, temp, 0);
3142 return target;
3143 }
3144 else
3145 return gen_lowpart (mode, temp);
3146 }
3147 else
3148 delete_insns_since (last);
3149 }
3150 }
3151 }
3152
3153 /* One final attempt at implementing negation via subtraction,
3154 this time allowing widening of the operand. */
3155 if (optab_to_code (unoptab) == NEG && !HONOR_SIGNED_ZEROS (mode))
3156 {
3157 rtx temp;
3158 temp = expand_binop (mode,
3159 unoptab == negv_optab ? subv_optab : sub_optab,
3160 CONST0_RTX (mode), op0,
3161 target, unsignedp, OPTAB_LIB_WIDEN);
3162 if (temp)
3163 return temp;
3164 }
3165
3166 return 0;
3167 }
3168
3169 /* Emit code to compute the absolute value of OP0, with result to
3170 TARGET if convenient. (TARGET may be 0.) The return value says
3171 where the result actually is to be found.
3172
3173 MODE is the mode of the operand; the mode of the result is
3174 different but can be deduced from MODE.
3175
3176 */
3177
3178 rtx
expand_abs_nojump(machine_mode mode,rtx op0,rtx target,int result_unsignedp)3179 expand_abs_nojump (machine_mode mode, rtx op0, rtx target,
3180 int result_unsignedp)
3181 {
3182 rtx temp;
3183
3184 if (GET_MODE_CLASS (mode) != MODE_INT
3185 || ! flag_trapv)
3186 result_unsignedp = 1;
3187
3188 /* First try to do it with a special abs instruction. */
3189 temp = expand_unop (mode, result_unsignedp ? abs_optab : absv_optab,
3190 op0, target, 0);
3191 if (temp != 0)
3192 return temp;
3193
3194 /* For floating point modes, try clearing the sign bit. */
3195 scalar_float_mode float_mode;
3196 if (is_a <scalar_float_mode> (mode, &float_mode))
3197 {
3198 temp = expand_absneg_bit (ABS, float_mode, op0, target);
3199 if (temp)
3200 return temp;
3201 }
3202
3203 /* If we have a MAX insn, we can do this as MAX (x, -x). */
3204 if (optab_handler (smax_optab, mode) != CODE_FOR_nothing
3205 && !HONOR_SIGNED_ZEROS (mode))
3206 {
3207 rtx_insn *last = get_last_insn ();
3208
3209 temp = expand_unop (mode, result_unsignedp ? neg_optab : negv_optab,
3210 op0, NULL_RTX, 0);
3211 if (temp != 0)
3212 temp = expand_binop (mode, smax_optab, op0, temp, target, 0,
3213 OPTAB_WIDEN);
3214
3215 if (temp != 0)
3216 return temp;
3217
3218 delete_insns_since (last);
3219 }
3220
3221 /* If this machine has expensive jumps, we can do integer absolute
3222 value of X as (((signed) x >> (W-1)) ^ x) - ((signed) x >> (W-1)),
3223 where W is the width of MODE. */
3224
3225 scalar_int_mode int_mode;
3226 if (is_int_mode (mode, &int_mode)
3227 && BRANCH_COST (optimize_insn_for_speed_p (),
3228 false) >= 2)
3229 {
3230 rtx extended = expand_shift (RSHIFT_EXPR, int_mode, op0,
3231 GET_MODE_PRECISION (int_mode) - 1,
3232 NULL_RTX, 0);
3233
3234 temp = expand_binop (int_mode, xor_optab, extended, op0, target, 0,
3235 OPTAB_LIB_WIDEN);
3236 if (temp != 0)
3237 temp = expand_binop (int_mode,
3238 result_unsignedp ? sub_optab : subv_optab,
3239 temp, extended, target, 0, OPTAB_LIB_WIDEN);
3240
3241 if (temp != 0)
3242 return temp;
3243 }
3244
3245 return NULL_RTX;
3246 }
3247
3248 rtx
expand_abs(machine_mode mode,rtx op0,rtx target,int result_unsignedp,int safe)3249 expand_abs (machine_mode mode, rtx op0, rtx target,
3250 int result_unsignedp, int safe)
3251 {
3252 rtx temp;
3253 rtx_code_label *op1;
3254
3255 if (GET_MODE_CLASS (mode) != MODE_INT
3256 || ! flag_trapv)
3257 result_unsignedp = 1;
3258
3259 temp = expand_abs_nojump (mode, op0, target, result_unsignedp);
3260 if (temp != 0)
3261 return temp;
3262
3263 /* If that does not win, use conditional jump and negate. */
3264
3265 /* It is safe to use the target if it is the same
3266 as the source if this is also a pseudo register */
3267 if (op0 == target && REG_P (op0)
3268 && REGNO (op0) >= FIRST_PSEUDO_REGISTER)
3269 safe = 1;
3270
3271 op1 = gen_label_rtx ();
3272 if (target == 0 || ! safe
3273 || GET_MODE (target) != mode
3274 || (MEM_P (target) && MEM_VOLATILE_P (target))
3275 || (REG_P (target)
3276 && REGNO (target) < FIRST_PSEUDO_REGISTER))
3277 target = gen_reg_rtx (mode);
3278
3279 emit_move_insn (target, op0);
3280 NO_DEFER_POP;
3281
3282 do_compare_rtx_and_jump (target, CONST0_RTX (mode), GE, 0, mode,
3283 NULL_RTX, NULL, op1,
3284 profile_probability::uninitialized ());
3285
3286 op0 = expand_unop (mode, result_unsignedp ? neg_optab : negv_optab,
3287 target, target, 0);
3288 if (op0 != target)
3289 emit_move_insn (target, op0);
3290 emit_label (op1);
3291 OK_DEFER_POP;
3292 return target;
3293 }
3294
3295 /* Emit code to compute the one's complement absolute value of OP0
3296 (if (OP0 < 0) OP0 = ~OP0), with result to TARGET if convenient.
3297 (TARGET may be NULL_RTX.) The return value says where the result
3298 actually is to be found.
3299
3300 MODE is the mode of the operand; the mode of the result is
3301 different but can be deduced from MODE. */
3302
3303 rtx
expand_one_cmpl_abs_nojump(machine_mode mode,rtx op0,rtx target)3304 expand_one_cmpl_abs_nojump (machine_mode mode, rtx op0, rtx target)
3305 {
3306 rtx temp;
3307
3308 /* Not applicable for floating point modes. */
3309 if (FLOAT_MODE_P (mode))
3310 return NULL_RTX;
3311
3312 /* If we have a MAX insn, we can do this as MAX (x, ~x). */
3313 if (optab_handler (smax_optab, mode) != CODE_FOR_nothing)
3314 {
3315 rtx_insn *last = get_last_insn ();
3316
3317 temp = expand_unop (mode, one_cmpl_optab, op0, NULL_RTX, 0);
3318 if (temp != 0)
3319 temp = expand_binop (mode, smax_optab, op0, temp, target, 0,
3320 OPTAB_WIDEN);
3321
3322 if (temp != 0)
3323 return temp;
3324
3325 delete_insns_since (last);
3326 }
3327
3328 /* If this machine has expensive jumps, we can do one's complement
3329 absolute value of X as (((signed) x >> (W-1)) ^ x). */
3330
3331 scalar_int_mode int_mode;
3332 if (is_int_mode (mode, &int_mode)
3333 && BRANCH_COST (optimize_insn_for_speed_p (),
3334 false) >= 2)
3335 {
3336 rtx extended = expand_shift (RSHIFT_EXPR, int_mode, op0,
3337 GET_MODE_PRECISION (int_mode) - 1,
3338 NULL_RTX, 0);
3339
3340 temp = expand_binop (int_mode, xor_optab, extended, op0, target, 0,
3341 OPTAB_LIB_WIDEN);
3342
3343 if (temp != 0)
3344 return temp;
3345 }
3346
3347 return NULL_RTX;
3348 }
3349
3350 /* A subroutine of expand_copysign, perform the copysign operation using the
3351 abs and neg primitives advertised to exist on the target. The assumption
3352 is that we have a split register file, and leaving op0 in fp registers,
3353 and not playing with subregs so much, will help the register allocator. */
3354
3355 static rtx
expand_copysign_absneg(scalar_float_mode mode,rtx op0,rtx op1,rtx target,int bitpos,bool op0_is_abs)3356 expand_copysign_absneg (scalar_float_mode mode, rtx op0, rtx op1, rtx target,
3357 int bitpos, bool op0_is_abs)
3358 {
3359 scalar_int_mode imode;
3360 enum insn_code icode;
3361 rtx sign;
3362 rtx_code_label *label;
3363
3364 if (target == op1)
3365 target = NULL_RTX;
3366
3367 /* Check if the back end provides an insn that handles signbit for the
3368 argument's mode. */
3369 icode = optab_handler (signbit_optab, mode);
3370 if (icode != CODE_FOR_nothing)
3371 {
3372 imode = as_a <scalar_int_mode> (insn_data[(int) icode].operand[0].mode);
3373 sign = gen_reg_rtx (imode);
3374 emit_unop_insn (icode, sign, op1, UNKNOWN);
3375 }
3376 else
3377 {
3378 if (GET_MODE_SIZE (mode) <= UNITS_PER_WORD)
3379 {
3380 if (!int_mode_for_mode (mode).exists (&imode))
3381 return NULL_RTX;
3382 op1 = gen_lowpart (imode, op1);
3383 }
3384 else
3385 {
3386 int word;
3387
3388 imode = word_mode;
3389 if (FLOAT_WORDS_BIG_ENDIAN)
3390 word = (GET_MODE_BITSIZE (mode) - bitpos) / BITS_PER_WORD;
3391 else
3392 word = bitpos / BITS_PER_WORD;
3393 bitpos = bitpos % BITS_PER_WORD;
3394 op1 = operand_subword_force (op1, word, mode);
3395 }
3396
3397 wide_int mask = wi::set_bit_in_zero (bitpos, GET_MODE_PRECISION (imode));
3398 sign = expand_binop (imode, and_optab, op1,
3399 immed_wide_int_const (mask, imode),
3400 NULL_RTX, 1, OPTAB_LIB_WIDEN);
3401 }
3402
3403 if (!op0_is_abs)
3404 {
3405 op0 = expand_unop (mode, abs_optab, op0, target, 0);
3406 if (op0 == NULL)
3407 return NULL_RTX;
3408 target = op0;
3409 }
3410 else
3411 {
3412 if (target == NULL_RTX)
3413 target = copy_to_reg (op0);
3414 else
3415 emit_move_insn (target, op0);
3416 }
3417
3418 label = gen_label_rtx ();
3419 emit_cmp_and_jump_insns (sign, const0_rtx, EQ, NULL_RTX, imode, 1, label);
3420
3421 if (CONST_DOUBLE_AS_FLOAT_P (op0))
3422 op0 = simplify_unary_operation (NEG, mode, op0, mode);
3423 else
3424 op0 = expand_unop (mode, neg_optab, op0, target, 0);
3425 if (op0 != target)
3426 emit_move_insn (target, op0);
3427
3428 emit_label (label);
3429
3430 return target;
3431 }
3432
3433
3434 /* A subroutine of expand_copysign, perform the entire copysign operation
3435 with integer bitmasks. BITPOS is the position of the sign bit; OP0_IS_ABS
3436 is true if op0 is known to have its sign bit clear. */
3437
3438 static rtx
expand_copysign_bit(scalar_float_mode mode,rtx op0,rtx op1,rtx target,int bitpos,bool op0_is_abs)3439 expand_copysign_bit (scalar_float_mode mode, rtx op0, rtx op1, rtx target,
3440 int bitpos, bool op0_is_abs)
3441 {
3442 scalar_int_mode imode;
3443 int word, nwords, i;
3444 rtx temp;
3445 rtx_insn *insns;
3446
3447 if (GET_MODE_SIZE (mode) <= UNITS_PER_WORD)
3448 {
3449 if (!int_mode_for_mode (mode).exists (&imode))
3450 return NULL_RTX;
3451 word = 0;
3452 nwords = 1;
3453 }
3454 else
3455 {
3456 imode = word_mode;
3457
3458 if (FLOAT_WORDS_BIG_ENDIAN)
3459 word = (GET_MODE_BITSIZE (mode) - bitpos) / BITS_PER_WORD;
3460 else
3461 word = bitpos / BITS_PER_WORD;
3462 bitpos = bitpos % BITS_PER_WORD;
3463 nwords = (GET_MODE_BITSIZE (mode) + BITS_PER_WORD - 1) / BITS_PER_WORD;
3464 }
3465
3466 wide_int mask = wi::set_bit_in_zero (bitpos, GET_MODE_PRECISION (imode));
3467
3468 if (target == 0
3469 || target == op0
3470 || target == op1
3471 || reg_overlap_mentioned_p (target, op0)
3472 || reg_overlap_mentioned_p (target, op1)
3473 || (nwords > 1 && !valid_multiword_target_p (target)))
3474 target = gen_reg_rtx (mode);
3475
3476 if (nwords > 1)
3477 {
3478 start_sequence ();
3479
3480 for (i = 0; i < nwords; ++i)
3481 {
3482 rtx targ_piece = operand_subword (target, i, 1, mode);
3483 rtx op0_piece = operand_subword_force (op0, i, mode);
3484
3485 if (i == word)
3486 {
3487 if (!op0_is_abs)
3488 op0_piece
3489 = expand_binop (imode, and_optab, op0_piece,
3490 immed_wide_int_const (~mask, imode),
3491 NULL_RTX, 1, OPTAB_LIB_WIDEN);
3492 op1 = expand_binop (imode, and_optab,
3493 operand_subword_force (op1, i, mode),
3494 immed_wide_int_const (mask, imode),
3495 NULL_RTX, 1, OPTAB_LIB_WIDEN);
3496
3497 temp = expand_binop (imode, ior_optab, op0_piece, op1,
3498 targ_piece, 1, OPTAB_LIB_WIDEN);
3499 if (temp != targ_piece)
3500 emit_move_insn (targ_piece, temp);
3501 }
3502 else
3503 emit_move_insn (targ_piece, op0_piece);
3504 }
3505
3506 insns = get_insns ();
3507 end_sequence ();
3508
3509 emit_insn (insns);
3510 }
3511 else
3512 {
3513 op1 = expand_binop (imode, and_optab, gen_lowpart (imode, op1),
3514 immed_wide_int_const (mask, imode),
3515 NULL_RTX, 1, OPTAB_LIB_WIDEN);
3516
3517 op0 = gen_lowpart (imode, op0);
3518 if (!op0_is_abs)
3519 op0 = expand_binop (imode, and_optab, op0,
3520 immed_wide_int_const (~mask, imode),
3521 NULL_RTX, 1, OPTAB_LIB_WIDEN);
3522
3523 temp = expand_binop (imode, ior_optab, op0, op1,
3524 gen_lowpart (imode, target), 1, OPTAB_LIB_WIDEN);
3525 target = lowpart_subreg_maybe_copy (mode, temp, imode);
3526 }
3527
3528 return target;
3529 }
3530
3531 /* Expand the C99 copysign operation. OP0 and OP1 must be the same
3532 scalar floating point mode. Return NULL if we do not know how to
3533 expand the operation inline. */
3534
3535 rtx
expand_copysign(rtx op0,rtx op1,rtx target)3536 expand_copysign (rtx op0, rtx op1, rtx target)
3537 {
3538 scalar_float_mode mode;
3539 const struct real_format *fmt;
3540 bool op0_is_abs;
3541 rtx temp;
3542
3543 mode = as_a <scalar_float_mode> (GET_MODE (op0));
3544 gcc_assert (GET_MODE (op1) == mode);
3545
3546 /* First try to do it with a special instruction. */
3547 temp = expand_binop (mode, copysign_optab, op0, op1,
3548 target, 0, OPTAB_DIRECT);
3549 if (temp)
3550 return temp;
3551
3552 fmt = REAL_MODE_FORMAT (mode);
3553 if (fmt == NULL || !fmt->has_signed_zero)
3554 return NULL_RTX;
3555
3556 op0_is_abs = false;
3557 if (CONST_DOUBLE_AS_FLOAT_P (op0))
3558 {
3559 if (real_isneg (CONST_DOUBLE_REAL_VALUE (op0)))
3560 op0 = simplify_unary_operation (ABS, mode, op0, mode);
3561 op0_is_abs = true;
3562 }
3563
3564 if (fmt->signbit_ro >= 0
3565 && (CONST_DOUBLE_AS_FLOAT_P (op0)
3566 || (optab_handler (neg_optab, mode) != CODE_FOR_nothing
3567 && optab_handler (abs_optab, mode) != CODE_FOR_nothing)))
3568 {
3569 temp = expand_copysign_absneg (mode, op0, op1, target,
3570 fmt->signbit_ro, op0_is_abs);
3571 if (temp)
3572 return temp;
3573 }
3574
3575 if (fmt->signbit_rw < 0)
3576 return NULL_RTX;
3577 return expand_copysign_bit (mode, op0, op1, target,
3578 fmt->signbit_rw, op0_is_abs);
3579 }
3580
3581 /* Generate an instruction whose insn-code is INSN_CODE,
3582 with two operands: an output TARGET and an input OP0.
3583 TARGET *must* be nonzero, and the output is always stored there.
3584 CODE is an rtx code such that (CODE OP0) is an rtx that describes
3585 the value that is stored into TARGET.
3586
3587 Return false if expansion failed. */
3588
3589 bool
maybe_emit_unop_insn(enum insn_code icode,rtx target,rtx op0,enum rtx_code code)3590 maybe_emit_unop_insn (enum insn_code icode, rtx target, rtx op0,
3591 enum rtx_code code)
3592 {
3593 struct expand_operand ops[2];
3594 rtx_insn *pat;
3595
3596 create_output_operand (&ops[0], target, GET_MODE (target));
3597 create_input_operand (&ops[1], op0, GET_MODE (op0));
3598 pat = maybe_gen_insn (icode, 2, ops);
3599 if (!pat)
3600 return false;
3601
3602 if (INSN_P (pat) && NEXT_INSN (pat) != NULL_RTX
3603 && code != UNKNOWN)
3604 add_equal_note (pat, ops[0].value, code, ops[1].value, NULL_RTX,
3605 GET_MODE (op0));
3606
3607 emit_insn (pat);
3608
3609 if (ops[0].value != target)
3610 emit_move_insn (target, ops[0].value);
3611 return true;
3612 }
3613 /* Generate an instruction whose insn-code is INSN_CODE,
3614 with two operands: an output TARGET and an input OP0.
3615 TARGET *must* be nonzero, and the output is always stored there.
3616 CODE is an rtx code such that (CODE OP0) is an rtx that describes
3617 the value that is stored into TARGET. */
3618
3619 void
emit_unop_insn(enum insn_code icode,rtx target,rtx op0,enum rtx_code code)3620 emit_unop_insn (enum insn_code icode, rtx target, rtx op0, enum rtx_code code)
3621 {
3622 bool ok = maybe_emit_unop_insn (icode, target, op0, code);
3623 gcc_assert (ok);
3624 }
3625
3626 struct no_conflict_data
3627 {
3628 rtx target;
3629 rtx_insn *first, *insn;
3630 bool must_stay;
3631 };
3632
3633 /* Called via note_stores by emit_libcall_block. Set P->must_stay if
3634 the currently examined clobber / store has to stay in the list of
3635 insns that constitute the actual libcall block. */
3636 static void
no_conflict_move_test(rtx dest,const_rtx set,void * p0)3637 no_conflict_move_test (rtx dest, const_rtx set, void *p0)
3638 {
3639 struct no_conflict_data *p= (struct no_conflict_data *) p0;
3640
3641 /* If this inns directly contributes to setting the target, it must stay. */
3642 if (reg_overlap_mentioned_p (p->target, dest))
3643 p->must_stay = true;
3644 /* If we haven't committed to keeping any other insns in the list yet,
3645 there is nothing more to check. */
3646 else if (p->insn == p->first)
3647 return;
3648 /* If this insn sets / clobbers a register that feeds one of the insns
3649 already in the list, this insn has to stay too. */
3650 else if (reg_overlap_mentioned_p (dest, PATTERN (p->first))
3651 || (CALL_P (p->first) && (find_reg_fusage (p->first, USE, dest)))
3652 || reg_used_between_p (dest, p->first, p->insn)
3653 /* Likewise if this insn depends on a register set by a previous
3654 insn in the list, or if it sets a result (presumably a hard
3655 register) that is set or clobbered by a previous insn.
3656 N.B. the modified_*_p (SET_DEST...) tests applied to a MEM
3657 SET_DEST perform the former check on the address, and the latter
3658 check on the MEM. */
3659 || (GET_CODE (set) == SET
3660 && (modified_in_p (SET_SRC (set), p->first)
3661 || modified_in_p (SET_DEST (set), p->first)
3662 || modified_between_p (SET_SRC (set), p->first, p->insn)
3663 || modified_between_p (SET_DEST (set), p->first, p->insn))))
3664 p->must_stay = true;
3665 }
3666
3667
3668 /* Emit code to make a call to a constant function or a library call.
3669
3670 INSNS is a list containing all insns emitted in the call.
3671 These insns leave the result in RESULT. Our block is to copy RESULT
3672 to TARGET, which is logically equivalent to EQUIV.
3673
3674 We first emit any insns that set a pseudo on the assumption that these are
3675 loading constants into registers; doing so allows them to be safely cse'ed
3676 between blocks. Then we emit all the other insns in the block, followed by
3677 an insn to move RESULT to TARGET. This last insn will have a REQ_EQUAL
3678 note with an operand of EQUIV. */
3679
3680 static void
emit_libcall_block_1(rtx_insn * insns,rtx target,rtx result,rtx equiv,bool equiv_may_trap)3681 emit_libcall_block_1 (rtx_insn *insns, rtx target, rtx result, rtx equiv,
3682 bool equiv_may_trap)
3683 {
3684 rtx final_dest = target;
3685 rtx_insn *next, *last, *insn;
3686
3687 /* If this is a reg with REG_USERVAR_P set, then it could possibly turn
3688 into a MEM later. Protect the libcall block from this change. */
3689 if (! REG_P (target) || REG_USERVAR_P (target))
3690 target = gen_reg_rtx (GET_MODE (target));
3691
3692 /* If we're using non-call exceptions, a libcall corresponding to an
3693 operation that may trap may also trap. */
3694 /* ??? See the comment in front of make_reg_eh_region_note. */
3695 if (cfun->can_throw_non_call_exceptions
3696 && (equiv_may_trap || may_trap_p (equiv)))
3697 {
3698 for (insn = insns; insn; insn = NEXT_INSN (insn))
3699 if (CALL_P (insn))
3700 {
3701 rtx note = find_reg_note (insn, REG_EH_REGION, NULL_RTX);
3702 if (note)
3703 {
3704 int lp_nr = INTVAL (XEXP (note, 0));
3705 if (lp_nr == 0 || lp_nr == INT_MIN)
3706 remove_note (insn, note);
3707 }
3708 }
3709 }
3710 else
3711 {
3712 /* Look for any CALL_INSNs in this sequence, and attach a REG_EH_REGION
3713 reg note to indicate that this call cannot throw or execute a nonlocal
3714 goto (unless there is already a REG_EH_REGION note, in which case
3715 we update it). */
3716 for (insn = insns; insn; insn = NEXT_INSN (insn))
3717 if (CALL_P (insn))
3718 make_reg_eh_region_note_nothrow_nononlocal (insn);
3719 }
3720
3721 /* First emit all insns that set pseudos. Remove them from the list as
3722 we go. Avoid insns that set pseudos which were referenced in previous
3723 insns. These can be generated by move_by_pieces, for example,
3724 to update an address. Similarly, avoid insns that reference things
3725 set in previous insns. */
3726
3727 for (insn = insns; insn; insn = next)
3728 {
3729 rtx set = single_set (insn);
3730
3731 next = NEXT_INSN (insn);
3732
3733 if (set != 0 && REG_P (SET_DEST (set))
3734 && REGNO (SET_DEST (set)) >= FIRST_PSEUDO_REGISTER)
3735 {
3736 struct no_conflict_data data;
3737
3738 data.target = const0_rtx;
3739 data.first = insns;
3740 data.insn = insn;
3741 data.must_stay = 0;
3742 note_stores (PATTERN (insn), no_conflict_move_test, &data);
3743 if (! data.must_stay)
3744 {
3745 if (PREV_INSN (insn))
3746 SET_NEXT_INSN (PREV_INSN (insn)) = next;
3747 else
3748 insns = next;
3749
3750 if (next)
3751 SET_PREV_INSN (next) = PREV_INSN (insn);
3752
3753 add_insn (insn);
3754 }
3755 }
3756
3757 /* Some ports use a loop to copy large arguments onto the stack.
3758 Don't move anything outside such a loop. */
3759 if (LABEL_P (insn))
3760 break;
3761 }
3762
3763 /* Write the remaining insns followed by the final copy. */
3764 for (insn = insns; insn; insn = next)
3765 {
3766 next = NEXT_INSN (insn);
3767
3768 add_insn (insn);
3769 }
3770
3771 last = emit_move_insn (target, result);
3772 if (equiv)
3773 set_dst_reg_note (last, REG_EQUAL, copy_rtx (equiv), target);
3774
3775 if (final_dest != target)
3776 emit_move_insn (final_dest, target);
3777 }
3778
3779 void
emit_libcall_block(rtx_insn * insns,rtx target,rtx result,rtx equiv)3780 emit_libcall_block (rtx_insn *insns, rtx target, rtx result, rtx equiv)
3781 {
3782 emit_libcall_block_1 (insns, target, result, equiv, false);
3783 }
3784
3785 /* Nonzero if we can perform a comparison of mode MODE straightforwardly.
3786 PURPOSE describes how this comparison will be used. CODE is the rtx
3787 comparison code we will be using.
3788
3789 ??? Actually, CODE is slightly weaker than that. A target is still
3790 required to implement all of the normal bcc operations, but not
3791 required to implement all (or any) of the unordered bcc operations. */
3792
3793 int
can_compare_p(enum rtx_code code,machine_mode mode,enum can_compare_purpose purpose)3794 can_compare_p (enum rtx_code code, machine_mode mode,
3795 enum can_compare_purpose purpose)
3796 {
3797 rtx test;
3798 test = gen_rtx_fmt_ee (code, mode, const0_rtx, const0_rtx);
3799 do
3800 {
3801 enum insn_code icode;
3802
3803 if (purpose == ccp_jump
3804 && (icode = optab_handler (cbranch_optab, mode)) != CODE_FOR_nothing
3805 && insn_operand_matches (icode, 0, test))
3806 return 1;
3807 if (purpose == ccp_store_flag
3808 && (icode = optab_handler (cstore_optab, mode)) != CODE_FOR_nothing
3809 && insn_operand_matches (icode, 1, test))
3810 return 1;
3811 if (purpose == ccp_cmov
3812 && optab_handler (cmov_optab, mode) != CODE_FOR_nothing)
3813 return 1;
3814
3815 mode = GET_MODE_WIDER_MODE (mode).else_void ();
3816 PUT_MODE (test, mode);
3817 }
3818 while (mode != VOIDmode);
3819
3820 return 0;
3821 }
3822
3823 /* This function is called when we are going to emit a compare instruction that
3824 compares the values found in X and Y, using the rtl operator COMPARISON.
3825
3826 If they have mode BLKmode, then SIZE specifies the size of both operands.
3827
3828 UNSIGNEDP nonzero says that the operands are unsigned;
3829 this matters if they need to be widened (as given by METHODS).
3830
3831 *PTEST is where the resulting comparison RTX is returned or NULL_RTX
3832 if we failed to produce one.
3833
3834 *PMODE is the mode of the inputs (in case they are const_int).
3835
3836 This function performs all the setup necessary so that the caller only has
3837 to emit a single comparison insn. This setup can involve doing a BLKmode
3838 comparison or emitting a library call to perform the comparison if no insn
3839 is available to handle it.
3840 The values which are passed in through pointers can be modified; the caller
3841 should perform the comparison on the modified values. Constant
3842 comparisons must have already been folded. */
3843
3844 static void
prepare_cmp_insn(rtx x,rtx y,enum rtx_code comparison,rtx size,int unsignedp,enum optab_methods methods,rtx * ptest,machine_mode * pmode)3845 prepare_cmp_insn (rtx x, rtx y, enum rtx_code comparison, rtx size,
3846 int unsignedp, enum optab_methods methods,
3847 rtx *ptest, machine_mode *pmode)
3848 {
3849 machine_mode mode = *pmode;
3850 rtx libfunc, test;
3851 machine_mode cmp_mode;
3852 enum mode_class mclass;
3853
3854 /* The other methods are not needed. */
3855 gcc_assert (methods == OPTAB_DIRECT || methods == OPTAB_WIDEN
3856 || methods == OPTAB_LIB_WIDEN);
3857
3858 if (CONST_SCALAR_INT_P (y))
3859 canonicalize_comparison (mode, &comparison, &y);
3860
3861 /* If we are optimizing, force expensive constants into a register. */
3862 if (CONSTANT_P (x) && optimize
3863 && (rtx_cost (x, mode, COMPARE, 0, optimize_insn_for_speed_p ())
3864 > COSTS_N_INSNS (1)))
3865 x = force_reg (mode, x);
3866
3867 if (CONSTANT_P (y) && optimize
3868 && (rtx_cost (y, mode, COMPARE, 1, optimize_insn_for_speed_p ())
3869 > COSTS_N_INSNS (1)))
3870 y = force_reg (mode, y);
3871
3872 #if HAVE_cc0
3873 /* Make sure if we have a canonical comparison. The RTL
3874 documentation states that canonical comparisons are required only
3875 for targets which have cc0. */
3876 gcc_assert (!CONSTANT_P (x) || CONSTANT_P (y));
3877 #endif
3878
3879 /* Don't let both operands fail to indicate the mode. */
3880 if (GET_MODE (x) == VOIDmode && GET_MODE (y) == VOIDmode)
3881 x = force_reg (mode, x);
3882 if (mode == VOIDmode)
3883 mode = GET_MODE (x) != VOIDmode ? GET_MODE (x) : GET_MODE (y);
3884
3885 /* Handle all BLKmode compares. */
3886
3887 if (mode == BLKmode)
3888 {
3889 machine_mode result_mode;
3890 enum insn_code cmp_code;
3891 rtx result;
3892 rtx opalign
3893 = GEN_INT (MIN (MEM_ALIGN (x), MEM_ALIGN (y)) / BITS_PER_UNIT);
3894
3895 gcc_assert (size);
3896
3897 /* Try to use a memory block compare insn - either cmpstr
3898 or cmpmem will do. */
3899 opt_scalar_int_mode cmp_mode_iter;
3900 FOR_EACH_MODE_IN_CLASS (cmp_mode_iter, MODE_INT)
3901 {
3902 scalar_int_mode cmp_mode = cmp_mode_iter.require ();
3903 cmp_code = direct_optab_handler (cmpmem_optab, cmp_mode);
3904 if (cmp_code == CODE_FOR_nothing)
3905 cmp_code = direct_optab_handler (cmpstr_optab, cmp_mode);
3906 if (cmp_code == CODE_FOR_nothing)
3907 cmp_code = direct_optab_handler (cmpstrn_optab, cmp_mode);
3908 if (cmp_code == CODE_FOR_nothing)
3909 continue;
3910
3911 /* Must make sure the size fits the insn's mode. */
3912 if (CONST_INT_P (size)
3913 ? UINTVAL (size) > GET_MODE_MASK (cmp_mode)
3914 : (GET_MODE_BITSIZE (as_a <scalar_int_mode> (GET_MODE (size)))
3915 > GET_MODE_BITSIZE (cmp_mode)))
3916 continue;
3917
3918 result_mode = insn_data[cmp_code].operand[0].mode;
3919 result = gen_reg_rtx (result_mode);
3920 size = convert_to_mode (cmp_mode, size, 1);
3921 emit_insn (GEN_FCN (cmp_code) (result, x, y, size, opalign));
3922
3923 *ptest = gen_rtx_fmt_ee (comparison, VOIDmode, result, const0_rtx);
3924 *pmode = result_mode;
3925 return;
3926 }
3927
3928 if (methods != OPTAB_LIB && methods != OPTAB_LIB_WIDEN)
3929 goto fail;
3930
3931 /* Otherwise call a library function. */
3932 result = emit_block_comp_via_libcall (x, y, size);
3933
3934 x = result;
3935 y = const0_rtx;
3936 mode = TYPE_MODE (integer_type_node);
3937 methods = OPTAB_LIB_WIDEN;
3938 unsignedp = false;
3939 }
3940
3941 /* Don't allow operands to the compare to trap, as that can put the
3942 compare and branch in different basic blocks. */
3943 if (cfun->can_throw_non_call_exceptions)
3944 {
3945 if (may_trap_p (x))
3946 x = copy_to_reg (x);
3947 if (may_trap_p (y))
3948 y = copy_to_reg (y);
3949 }
3950
3951 if (GET_MODE_CLASS (mode) == MODE_CC)
3952 {
3953 enum insn_code icode = optab_handler (cbranch_optab, CCmode);
3954 test = gen_rtx_fmt_ee (comparison, VOIDmode, x, y);
3955 gcc_assert (icode != CODE_FOR_nothing
3956 && insn_operand_matches (icode, 0, test));
3957 *ptest = test;
3958 return;
3959 }
3960
3961 mclass = GET_MODE_CLASS (mode);
3962 test = gen_rtx_fmt_ee (comparison, VOIDmode, x, y);
3963 FOR_EACH_MODE_FROM (cmp_mode, mode)
3964 {
3965 enum insn_code icode;
3966 icode = optab_handler (cbranch_optab, cmp_mode);
3967 if (icode != CODE_FOR_nothing
3968 && insn_operand_matches (icode, 0, test))
3969 {
3970 rtx_insn *last = get_last_insn ();
3971 rtx op0 = prepare_operand (icode, x, 1, mode, cmp_mode, unsignedp);
3972 rtx op1 = prepare_operand (icode, y, 2, mode, cmp_mode, unsignedp);
3973 if (op0 && op1
3974 && insn_operand_matches (icode, 1, op0)
3975 && insn_operand_matches (icode, 2, op1))
3976 {
3977 XEXP (test, 0) = op0;
3978 XEXP (test, 1) = op1;
3979 *ptest = test;
3980 *pmode = cmp_mode;
3981 return;
3982 }
3983 delete_insns_since (last);
3984 }
3985
3986 if (methods == OPTAB_DIRECT || !CLASS_HAS_WIDER_MODES_P (mclass))
3987 break;
3988 }
3989
3990 if (methods != OPTAB_LIB_WIDEN)
3991 goto fail;
3992
3993 if (SCALAR_FLOAT_MODE_P (mode))
3994 {
3995 /* Small trick if UNORDERED isn't implemented by the hardware. */
3996 if (comparison == UNORDERED && rtx_equal_p (x, y))
3997 {
3998 prepare_cmp_insn (x, y, UNLT, NULL_RTX, unsignedp, OPTAB_WIDEN,
3999 ptest, pmode);
4000 if (*ptest)
4001 return;
4002 }
4003
4004 prepare_float_lib_cmp (x, y, comparison, ptest, pmode);
4005 }
4006 else
4007 {
4008 rtx result;
4009 machine_mode ret_mode;
4010
4011 /* Handle a libcall just for the mode we are using. */
4012 libfunc = optab_libfunc (cmp_optab, mode);
4013 gcc_assert (libfunc);
4014
4015 /* If we want unsigned, and this mode has a distinct unsigned
4016 comparison routine, use that. */
4017 if (unsignedp)
4018 {
4019 rtx ulibfunc = optab_libfunc (ucmp_optab, mode);
4020 if (ulibfunc)
4021 libfunc = ulibfunc;
4022 }
4023
4024 ret_mode = targetm.libgcc_cmp_return_mode ();
4025 result = emit_library_call_value (libfunc, NULL_RTX, LCT_CONST,
4026 ret_mode, x, mode, y, mode);
4027
4028 /* There are two kinds of comparison routines. Biased routines
4029 return 0/1/2, and unbiased routines return -1/0/1. Other parts
4030 of gcc expect that the comparison operation is equivalent
4031 to the modified comparison. For signed comparisons compare the
4032 result against 1 in the biased case, and zero in the unbiased
4033 case. For unsigned comparisons always compare against 1 after
4034 biasing the unbiased result by adding 1. This gives us a way to
4035 represent LTU.
4036 The comparisons in the fixed-point helper library are always
4037 biased. */
4038 x = result;
4039 y = const1_rtx;
4040
4041 if (!TARGET_LIB_INT_CMP_BIASED && !ALL_FIXED_POINT_MODE_P (mode))
4042 {
4043 if (unsignedp)
4044 x = plus_constant (ret_mode, result, 1);
4045 else
4046 y = const0_rtx;
4047 }
4048
4049 *pmode = ret_mode;
4050 prepare_cmp_insn (x, y, comparison, NULL_RTX, unsignedp, methods,
4051 ptest, pmode);
4052 }
4053
4054 return;
4055
4056 fail:
4057 *ptest = NULL_RTX;
4058 }
4059
4060 /* Before emitting an insn with code ICODE, make sure that X, which is going
4061 to be used for operand OPNUM of the insn, is converted from mode MODE to
4062 WIDER_MODE (UNSIGNEDP determines whether it is an unsigned conversion), and
4063 that it is accepted by the operand predicate. Return the new value. */
4064
4065 rtx
prepare_operand(enum insn_code icode,rtx x,int opnum,machine_mode mode,machine_mode wider_mode,int unsignedp)4066 prepare_operand (enum insn_code icode, rtx x, int opnum, machine_mode mode,
4067 machine_mode wider_mode, int unsignedp)
4068 {
4069 if (mode != wider_mode)
4070 x = convert_modes (wider_mode, mode, x, unsignedp);
4071
4072 if (!insn_operand_matches (icode, opnum, x))
4073 {
4074 machine_mode op_mode = insn_data[(int) icode].operand[opnum].mode;
4075 if (reload_completed)
4076 return NULL_RTX;
4077 if (GET_MODE (x) != op_mode && GET_MODE (x) != VOIDmode)
4078 return NULL_RTX;
4079 x = copy_to_mode_reg (op_mode, x);
4080 }
4081
4082 return x;
4083 }
4084
4085 /* Subroutine of emit_cmp_and_jump_insns; this function is called when we know
4086 we can do the branch. */
4087
4088 static void
emit_cmp_and_jump_insn_1(rtx test,machine_mode mode,rtx label,profile_probability prob)4089 emit_cmp_and_jump_insn_1 (rtx test, machine_mode mode, rtx label,
4090 profile_probability prob)
4091 {
4092 machine_mode optab_mode;
4093 enum mode_class mclass;
4094 enum insn_code icode;
4095 rtx_insn *insn;
4096
4097 mclass = GET_MODE_CLASS (mode);
4098 optab_mode = (mclass == MODE_CC) ? CCmode : mode;
4099 icode = optab_handler (cbranch_optab, optab_mode);
4100
4101 gcc_assert (icode != CODE_FOR_nothing);
4102 gcc_assert (insn_operand_matches (icode, 0, test));
4103 insn = emit_jump_insn (GEN_FCN (icode) (test, XEXP (test, 0),
4104 XEXP (test, 1), label));
4105 if (prob.initialized_p ()
4106 && profile_status_for_fn (cfun) != PROFILE_ABSENT
4107 && insn
4108 && JUMP_P (insn)
4109 && any_condjump_p (insn)
4110 && !find_reg_note (insn, REG_BR_PROB, 0))
4111 add_reg_br_prob_note (insn, prob);
4112 }
4113
4114 /* Generate code to compare X with Y so that the condition codes are
4115 set and to jump to LABEL if the condition is true. If X is a
4116 constant and Y is not a constant, then the comparison is swapped to
4117 ensure that the comparison RTL has the canonical form.
4118
4119 UNSIGNEDP nonzero says that X and Y are unsigned; this matters if they
4120 need to be widened. UNSIGNEDP is also used to select the proper
4121 branch condition code.
4122
4123 If X and Y have mode BLKmode, then SIZE specifies the size of both X and Y.
4124
4125 MODE is the mode of the inputs (in case they are const_int).
4126
4127 COMPARISON is the rtl operator to compare with (EQ, NE, GT, etc.).
4128 It will be potentially converted into an unsigned variant based on
4129 UNSIGNEDP to select a proper jump instruction.
4130
4131 PROB is the probability of jumping to LABEL. */
4132
4133 void
emit_cmp_and_jump_insns(rtx x,rtx y,enum rtx_code comparison,rtx size,machine_mode mode,int unsignedp,rtx label,profile_probability prob)4134 emit_cmp_and_jump_insns (rtx x, rtx y, enum rtx_code comparison, rtx size,
4135 machine_mode mode, int unsignedp, rtx label,
4136 profile_probability prob)
4137 {
4138 rtx op0 = x, op1 = y;
4139 rtx test;
4140
4141 /* Swap operands and condition to ensure canonical RTL. */
4142 if (swap_commutative_operands_p (x, y)
4143 && can_compare_p (swap_condition (comparison), mode, ccp_jump))
4144 {
4145 op0 = y, op1 = x;
4146 comparison = swap_condition (comparison);
4147 }
4148
4149 /* If OP0 is still a constant, then both X and Y must be constants
4150 or the opposite comparison is not supported. Force X into a register
4151 to create canonical RTL. */
4152 if (CONSTANT_P (op0))
4153 op0 = force_reg (mode, op0);
4154
4155 if (unsignedp)
4156 comparison = unsigned_condition (comparison);
4157
4158 prepare_cmp_insn (op0, op1, comparison, size, unsignedp, OPTAB_LIB_WIDEN,
4159 &test, &mode);
4160 emit_cmp_and_jump_insn_1 (test, mode, label, prob);
4161 }
4162
4163
4164 /* Emit a library call comparison between floating point X and Y.
4165 COMPARISON is the rtl operator to compare with (EQ, NE, GT, etc.). */
4166
4167 static void
prepare_float_lib_cmp(rtx x,rtx y,enum rtx_code comparison,rtx * ptest,machine_mode * pmode)4168 prepare_float_lib_cmp (rtx x, rtx y, enum rtx_code comparison,
4169 rtx *ptest, machine_mode *pmode)
4170 {
4171 enum rtx_code swapped = swap_condition (comparison);
4172 enum rtx_code reversed = reverse_condition_maybe_unordered (comparison);
4173 machine_mode orig_mode = GET_MODE (x);
4174 machine_mode mode;
4175 rtx true_rtx, false_rtx;
4176 rtx value, target, equiv;
4177 rtx_insn *insns;
4178 rtx libfunc = 0;
4179 bool reversed_p = false;
4180 scalar_int_mode cmp_mode = targetm.libgcc_cmp_return_mode ();
4181
4182 FOR_EACH_MODE_FROM (mode, orig_mode)
4183 {
4184 if (code_to_optab (comparison)
4185 && (libfunc = optab_libfunc (code_to_optab (comparison), mode)))
4186 break;
4187
4188 if (code_to_optab (swapped)
4189 && (libfunc = optab_libfunc (code_to_optab (swapped), mode)))
4190 {
4191 std::swap (x, y);
4192 comparison = swapped;
4193 break;
4194 }
4195
4196 if (code_to_optab (reversed)
4197 && (libfunc = optab_libfunc (code_to_optab (reversed), mode)))
4198 {
4199 comparison = reversed;
4200 reversed_p = true;
4201 break;
4202 }
4203 }
4204
4205 gcc_assert (mode != VOIDmode);
4206
4207 if (mode != orig_mode)
4208 {
4209 x = convert_to_mode (mode, x, 0);
4210 y = convert_to_mode (mode, y, 0);
4211 }
4212
4213 /* Attach a REG_EQUAL note describing the semantics of the libcall to
4214 the RTL. The allows the RTL optimizers to delete the libcall if the
4215 condition can be determined at compile-time. */
4216 if (comparison == UNORDERED
4217 || FLOAT_LIB_COMPARE_RETURNS_BOOL (mode, comparison))
4218 {
4219 true_rtx = const_true_rtx;
4220 false_rtx = const0_rtx;
4221 }
4222 else
4223 {
4224 switch (comparison)
4225 {
4226 case EQ:
4227 true_rtx = const0_rtx;
4228 false_rtx = const_true_rtx;
4229 break;
4230
4231 case NE:
4232 true_rtx = const_true_rtx;
4233 false_rtx = const0_rtx;
4234 break;
4235
4236 case GT:
4237 true_rtx = const1_rtx;
4238 false_rtx = const0_rtx;
4239 break;
4240
4241 case GE:
4242 true_rtx = const0_rtx;
4243 false_rtx = constm1_rtx;
4244 break;
4245
4246 case LT:
4247 true_rtx = constm1_rtx;
4248 false_rtx = const0_rtx;
4249 break;
4250
4251 case LE:
4252 true_rtx = const0_rtx;
4253 false_rtx = const1_rtx;
4254 break;
4255
4256 default:
4257 gcc_unreachable ();
4258 }
4259 }
4260
4261 if (comparison == UNORDERED)
4262 {
4263 rtx temp = simplify_gen_relational (NE, cmp_mode, mode, x, x);
4264 equiv = simplify_gen_relational (NE, cmp_mode, mode, y, y);
4265 equiv = simplify_gen_ternary (IF_THEN_ELSE, cmp_mode, cmp_mode,
4266 temp, const_true_rtx, equiv);
4267 }
4268 else
4269 {
4270 equiv = simplify_gen_relational (comparison, cmp_mode, mode, x, y);
4271 if (! FLOAT_LIB_COMPARE_RETURNS_BOOL (mode, comparison))
4272 equiv = simplify_gen_ternary (IF_THEN_ELSE, cmp_mode, cmp_mode,
4273 equiv, true_rtx, false_rtx);
4274 }
4275
4276 start_sequence ();
4277 value = emit_library_call_value (libfunc, NULL_RTX, LCT_CONST,
4278 cmp_mode, x, mode, y, mode);
4279 insns = get_insns ();
4280 end_sequence ();
4281
4282 target = gen_reg_rtx (cmp_mode);
4283 emit_libcall_block (insns, target, value, equiv);
4284
4285 if (comparison == UNORDERED
4286 || FLOAT_LIB_COMPARE_RETURNS_BOOL (mode, comparison)
4287 || reversed_p)
4288 *ptest = gen_rtx_fmt_ee (reversed_p ? EQ : NE, VOIDmode, target, false_rtx);
4289 else
4290 *ptest = gen_rtx_fmt_ee (comparison, VOIDmode, target, const0_rtx);
4291
4292 *pmode = cmp_mode;
4293 }
4294
4295 /* Generate code to indirectly jump to a location given in the rtx LOC. */
4296
4297 void
emit_indirect_jump(rtx loc)4298 emit_indirect_jump (rtx loc)
4299 {
4300 if (!targetm.have_indirect_jump ())
4301 sorry ("indirect jumps are not available on this target");
4302 else
4303 {
4304 struct expand_operand ops[1];
4305 create_address_operand (&ops[0], loc);
4306 expand_jump_insn (targetm.code_for_indirect_jump, 1, ops);
4307 emit_barrier ();
4308 }
4309 }
4310
4311
4312 /* Emit a conditional move instruction if the machine supports one for that
4313 condition and machine mode.
4314
4315 OP0 and OP1 are the operands that should be compared using CODE. CMODE is
4316 the mode to use should they be constants. If it is VOIDmode, they cannot
4317 both be constants.
4318
4319 OP2 should be stored in TARGET if the comparison is true, otherwise OP3
4320 should be stored there. MODE is the mode to use should they be constants.
4321 If it is VOIDmode, they cannot both be constants.
4322
4323 The result is either TARGET (perhaps modified) or NULL_RTX if the operation
4324 is not supported. */
4325
4326 rtx
emit_conditional_move(rtx target,enum rtx_code code,rtx op0,rtx op1,machine_mode cmode,rtx op2,rtx op3,machine_mode mode,int unsignedp)4327 emit_conditional_move (rtx target, enum rtx_code code, rtx op0, rtx op1,
4328 machine_mode cmode, rtx op2, rtx op3,
4329 machine_mode mode, int unsignedp)
4330 {
4331 rtx comparison;
4332 rtx_insn *last;
4333 enum insn_code icode;
4334 enum rtx_code reversed;
4335
4336 /* If the two source operands are identical, that's just a move. */
4337
4338 if (rtx_equal_p (op2, op3))
4339 {
4340 if (!target)
4341 target = gen_reg_rtx (mode);
4342
4343 emit_move_insn (target, op3);
4344 return target;
4345 }
4346
4347 /* If one operand is constant, make it the second one. Only do this
4348 if the other operand is not constant as well. */
4349
4350 if (swap_commutative_operands_p (op0, op1))
4351 {
4352 std::swap (op0, op1);
4353 code = swap_condition (code);
4354 }
4355
4356 /* get_condition will prefer to generate LT and GT even if the old
4357 comparison was against zero, so undo that canonicalization here since
4358 comparisons against zero are cheaper. */
4359 if (code == LT && op1 == const1_rtx)
4360 code = LE, op1 = const0_rtx;
4361 else if (code == GT && op1 == constm1_rtx)
4362 code = GE, op1 = const0_rtx;
4363
4364 if (cmode == VOIDmode)
4365 cmode = GET_MODE (op0);
4366
4367 enum rtx_code orig_code = code;
4368 bool swapped = false;
4369 if (swap_commutative_operands_p (op2, op3)
4370 && ((reversed = reversed_comparison_code_parts (code, op0, op1, NULL))
4371 != UNKNOWN))
4372 {
4373 std::swap (op2, op3);
4374 code = reversed;
4375 swapped = true;
4376 }
4377
4378 if (mode == VOIDmode)
4379 mode = GET_MODE (op2);
4380
4381 icode = direct_optab_handler (movcc_optab, mode);
4382
4383 if (icode == CODE_FOR_nothing)
4384 return NULL_RTX;
4385
4386 if (!target)
4387 target = gen_reg_rtx (mode);
4388
4389 for (int pass = 0; ; pass++)
4390 {
4391 code = unsignedp ? unsigned_condition (code) : code;
4392 comparison = simplify_gen_relational (code, VOIDmode, cmode, op0, op1);
4393
4394 /* We can get const0_rtx or const_true_rtx in some circumstances. Just
4395 punt and let the caller figure out how best to deal with this
4396 situation. */
4397 if (COMPARISON_P (comparison))
4398 {
4399 saved_pending_stack_adjust save;
4400 save_pending_stack_adjust (&save);
4401 last = get_last_insn ();
4402 do_pending_stack_adjust ();
4403 machine_mode cmpmode = cmode;
4404 prepare_cmp_insn (XEXP (comparison, 0), XEXP (comparison, 1),
4405 GET_CODE (comparison), NULL_RTX, unsignedp,
4406 OPTAB_WIDEN, &comparison, &cmpmode);
4407 if (comparison)
4408 {
4409 struct expand_operand ops[4];
4410
4411 create_output_operand (&ops[0], target, mode);
4412 create_fixed_operand (&ops[1], comparison);
4413 create_input_operand (&ops[2], op2, mode);
4414 create_input_operand (&ops[3], op3, mode);
4415 if (maybe_expand_insn (icode, 4, ops))
4416 {
4417 if (ops[0].value != target)
4418 convert_move (target, ops[0].value, false);
4419 return target;
4420 }
4421 }
4422 delete_insns_since (last);
4423 restore_pending_stack_adjust (&save);
4424 }
4425
4426 if (pass == 1)
4427 return NULL_RTX;
4428
4429 /* If the preferred op2/op3 order is not usable, retry with other
4430 operand order, perhaps it will expand successfully. */
4431 if (swapped)
4432 code = orig_code;
4433 else if ((reversed = reversed_comparison_code_parts (orig_code, op0, op1,
4434 NULL))
4435 != UNKNOWN)
4436 code = reversed;
4437 else
4438 return NULL_RTX;
4439 std::swap (op2, op3);
4440 }
4441 }
4442
4443
4444 /* Emit a conditional negate or bitwise complement using the
4445 negcc or notcc optabs if available. Return NULL_RTX if such operations
4446 are not available. Otherwise return the RTX holding the result.
4447 TARGET is the desired destination of the result. COMP is the comparison
4448 on which to negate. If COND is true move into TARGET the negation
4449 or bitwise complement of OP1. Otherwise move OP2 into TARGET.
4450 CODE is either NEG or NOT. MODE is the machine mode in which the
4451 operation is performed. */
4452
4453 rtx
emit_conditional_neg_or_complement(rtx target,rtx_code code,machine_mode mode,rtx cond,rtx op1,rtx op2)4454 emit_conditional_neg_or_complement (rtx target, rtx_code code,
4455 machine_mode mode, rtx cond, rtx op1,
4456 rtx op2)
4457 {
4458 optab op = unknown_optab;
4459 if (code == NEG)
4460 op = negcc_optab;
4461 else if (code == NOT)
4462 op = notcc_optab;
4463 else
4464 gcc_unreachable ();
4465
4466 insn_code icode = direct_optab_handler (op, mode);
4467
4468 if (icode == CODE_FOR_nothing)
4469 return NULL_RTX;
4470
4471 if (!target)
4472 target = gen_reg_rtx (mode);
4473
4474 rtx_insn *last = get_last_insn ();
4475 struct expand_operand ops[4];
4476
4477 create_output_operand (&ops[0], target, mode);
4478 create_fixed_operand (&ops[1], cond);
4479 create_input_operand (&ops[2], op1, mode);
4480 create_input_operand (&ops[3], op2, mode);
4481
4482 if (maybe_expand_insn (icode, 4, ops))
4483 {
4484 if (ops[0].value != target)
4485 convert_move (target, ops[0].value, false);
4486
4487 return target;
4488 }
4489 delete_insns_since (last);
4490 return NULL_RTX;
4491 }
4492
4493 /* Emit a conditional addition instruction if the machine supports one for that
4494 condition and machine mode.
4495
4496 OP0 and OP1 are the operands that should be compared using CODE. CMODE is
4497 the mode to use should they be constants. If it is VOIDmode, they cannot
4498 both be constants.
4499
4500 OP2 should be stored in TARGET if the comparison is false, otherwise OP2+OP3
4501 should be stored there. MODE is the mode to use should they be constants.
4502 If it is VOIDmode, they cannot both be constants.
4503
4504 The result is either TARGET (perhaps modified) or NULL_RTX if the operation
4505 is not supported. */
4506
4507 rtx
emit_conditional_add(rtx target,enum rtx_code code,rtx op0,rtx op1,machine_mode cmode,rtx op2,rtx op3,machine_mode mode,int unsignedp)4508 emit_conditional_add (rtx target, enum rtx_code code, rtx op0, rtx op1,
4509 machine_mode cmode, rtx op2, rtx op3,
4510 machine_mode mode, int unsignedp)
4511 {
4512 rtx comparison;
4513 rtx_insn *last;
4514 enum insn_code icode;
4515
4516 /* If one operand is constant, make it the second one. Only do this
4517 if the other operand is not constant as well. */
4518
4519 if (swap_commutative_operands_p (op0, op1))
4520 {
4521 std::swap (op0, op1);
4522 code = swap_condition (code);
4523 }
4524
4525 /* get_condition will prefer to generate LT and GT even if the old
4526 comparison was against zero, so undo that canonicalization here since
4527 comparisons against zero are cheaper. */
4528 if (code == LT && op1 == const1_rtx)
4529 code = LE, op1 = const0_rtx;
4530 else if (code == GT && op1 == constm1_rtx)
4531 code = GE, op1 = const0_rtx;
4532
4533 if (cmode == VOIDmode)
4534 cmode = GET_MODE (op0);
4535
4536 if (mode == VOIDmode)
4537 mode = GET_MODE (op2);
4538
4539 icode = optab_handler (addcc_optab, mode);
4540
4541 if (icode == CODE_FOR_nothing)
4542 return 0;
4543
4544 if (!target)
4545 target = gen_reg_rtx (mode);
4546
4547 code = unsignedp ? unsigned_condition (code) : code;
4548 comparison = simplify_gen_relational (code, VOIDmode, cmode, op0, op1);
4549
4550 /* We can get const0_rtx or const_true_rtx in some circumstances. Just
4551 return NULL and let the caller figure out how best to deal with this
4552 situation. */
4553 if (!COMPARISON_P (comparison))
4554 return NULL_RTX;
4555
4556 do_pending_stack_adjust ();
4557 last = get_last_insn ();
4558 prepare_cmp_insn (XEXP (comparison, 0), XEXP (comparison, 1),
4559 GET_CODE (comparison), NULL_RTX, unsignedp, OPTAB_WIDEN,
4560 &comparison, &cmode);
4561 if (comparison)
4562 {
4563 struct expand_operand ops[4];
4564
4565 create_output_operand (&ops[0], target, mode);
4566 create_fixed_operand (&ops[1], comparison);
4567 create_input_operand (&ops[2], op2, mode);
4568 create_input_operand (&ops[3], op3, mode);
4569 if (maybe_expand_insn (icode, 4, ops))
4570 {
4571 if (ops[0].value != target)
4572 convert_move (target, ops[0].value, false);
4573 return target;
4574 }
4575 }
4576 delete_insns_since (last);
4577 return NULL_RTX;
4578 }
4579
4580 /* These functions attempt to generate an insn body, rather than
4581 emitting the insn, but if the gen function already emits them, we
4582 make no attempt to turn them back into naked patterns. */
4583
4584 /* Generate and return an insn body to add Y to X. */
4585
4586 rtx_insn *
gen_add2_insn(rtx x,rtx y)4587 gen_add2_insn (rtx x, rtx y)
4588 {
4589 enum insn_code icode = optab_handler (add_optab, GET_MODE (x));
4590
4591 gcc_assert (insn_operand_matches (icode, 0, x));
4592 gcc_assert (insn_operand_matches (icode, 1, x));
4593 gcc_assert (insn_operand_matches (icode, 2, y));
4594
4595 return GEN_FCN (icode) (x, x, y);
4596 }
4597
4598 /* Generate and return an insn body to add r1 and c,
4599 storing the result in r0. */
4600
4601 rtx_insn *
gen_add3_insn(rtx r0,rtx r1,rtx c)4602 gen_add3_insn (rtx r0, rtx r1, rtx c)
4603 {
4604 enum insn_code icode = optab_handler (add_optab, GET_MODE (r0));
4605
4606 if (icode == CODE_FOR_nothing
4607 || !insn_operand_matches (icode, 0, r0)
4608 || !insn_operand_matches (icode, 1, r1)
4609 || !insn_operand_matches (icode, 2, c))
4610 return NULL;
4611
4612 return GEN_FCN (icode) (r0, r1, c);
4613 }
4614
4615 int
have_add2_insn(rtx x,rtx y)4616 have_add2_insn (rtx x, rtx y)
4617 {
4618 enum insn_code icode;
4619
4620 gcc_assert (GET_MODE (x) != VOIDmode);
4621
4622 icode = optab_handler (add_optab, GET_MODE (x));
4623
4624 if (icode == CODE_FOR_nothing)
4625 return 0;
4626
4627 if (!insn_operand_matches (icode, 0, x)
4628 || !insn_operand_matches (icode, 1, x)
4629 || !insn_operand_matches (icode, 2, y))
4630 return 0;
4631
4632 return 1;
4633 }
4634
4635 /* Generate and return an insn body to add Y to X. */
4636
4637 rtx_insn *
gen_addptr3_insn(rtx x,rtx y,rtx z)4638 gen_addptr3_insn (rtx x, rtx y, rtx z)
4639 {
4640 enum insn_code icode = optab_handler (addptr3_optab, GET_MODE (x));
4641
4642 gcc_assert (insn_operand_matches (icode, 0, x));
4643 gcc_assert (insn_operand_matches (icode, 1, y));
4644 gcc_assert (insn_operand_matches (icode, 2, z));
4645
4646 return GEN_FCN (icode) (x, y, z);
4647 }
4648
4649 /* Return true if the target implements an addptr pattern and X, Y,
4650 and Z are valid for the pattern predicates. */
4651
4652 int
have_addptr3_insn(rtx x,rtx y,rtx z)4653 have_addptr3_insn (rtx x, rtx y, rtx z)
4654 {
4655 enum insn_code icode;
4656
4657 gcc_assert (GET_MODE (x) != VOIDmode);
4658
4659 icode = optab_handler (addptr3_optab, GET_MODE (x));
4660
4661 if (icode == CODE_FOR_nothing)
4662 return 0;
4663
4664 if (!insn_operand_matches (icode, 0, x)
4665 || !insn_operand_matches (icode, 1, y)
4666 || !insn_operand_matches (icode, 2, z))
4667 return 0;
4668
4669 return 1;
4670 }
4671
4672 /* Generate and return an insn body to subtract Y from X. */
4673
4674 rtx_insn *
gen_sub2_insn(rtx x,rtx y)4675 gen_sub2_insn (rtx x, rtx y)
4676 {
4677 enum insn_code icode = optab_handler (sub_optab, GET_MODE (x));
4678
4679 gcc_assert (insn_operand_matches (icode, 0, x));
4680 gcc_assert (insn_operand_matches (icode, 1, x));
4681 gcc_assert (insn_operand_matches (icode, 2, y));
4682
4683 return GEN_FCN (icode) (x, x, y);
4684 }
4685
4686 /* Generate and return an insn body to subtract r1 and c,
4687 storing the result in r0. */
4688
4689 rtx_insn *
gen_sub3_insn(rtx r0,rtx r1,rtx c)4690 gen_sub3_insn (rtx r0, rtx r1, rtx c)
4691 {
4692 enum insn_code icode = optab_handler (sub_optab, GET_MODE (r0));
4693
4694 if (icode == CODE_FOR_nothing
4695 || !insn_operand_matches (icode, 0, r0)
4696 || !insn_operand_matches (icode, 1, r1)
4697 || !insn_operand_matches (icode, 2, c))
4698 return NULL;
4699
4700 return GEN_FCN (icode) (r0, r1, c);
4701 }
4702
4703 int
have_sub2_insn(rtx x,rtx y)4704 have_sub2_insn (rtx x, rtx y)
4705 {
4706 enum insn_code icode;
4707
4708 gcc_assert (GET_MODE (x) != VOIDmode);
4709
4710 icode = optab_handler (sub_optab, GET_MODE (x));
4711
4712 if (icode == CODE_FOR_nothing)
4713 return 0;
4714
4715 if (!insn_operand_matches (icode, 0, x)
4716 || !insn_operand_matches (icode, 1, x)
4717 || !insn_operand_matches (icode, 2, y))
4718 return 0;
4719
4720 return 1;
4721 }
4722
4723 /* Generate the body of an insn to extend Y (with mode MFROM)
4724 into X (with mode MTO). Do zero-extension if UNSIGNEDP is nonzero. */
4725
4726 rtx_insn *
gen_extend_insn(rtx x,rtx y,machine_mode mto,machine_mode mfrom,int unsignedp)4727 gen_extend_insn (rtx x, rtx y, machine_mode mto,
4728 machine_mode mfrom, int unsignedp)
4729 {
4730 enum insn_code icode = can_extend_p (mto, mfrom, unsignedp);
4731 return GEN_FCN (icode) (x, y);
4732 }
4733
4734 /* Generate code to convert FROM to floating point
4735 and store in TO. FROM must be fixed point and not VOIDmode.
4736 UNSIGNEDP nonzero means regard FROM as unsigned.
4737 Normally this is done by correcting the final value
4738 if it is negative. */
4739
4740 void
expand_float(rtx to,rtx from,int unsignedp)4741 expand_float (rtx to, rtx from, int unsignedp)
4742 {
4743 enum insn_code icode;
4744 rtx target = to;
4745 scalar_mode from_mode, to_mode;
4746 machine_mode fmode, imode;
4747 bool can_do_signed = false;
4748
4749 /* Crash now, because we won't be able to decide which mode to use. */
4750 gcc_assert (GET_MODE (from) != VOIDmode);
4751
4752 /* Look for an insn to do the conversion. Do it in the specified
4753 modes if possible; otherwise convert either input, output or both to
4754 wider mode. If the integer mode is wider than the mode of FROM,
4755 we can do the conversion signed even if the input is unsigned. */
4756
4757 FOR_EACH_MODE_FROM (fmode, GET_MODE (to))
4758 FOR_EACH_MODE_FROM (imode, GET_MODE (from))
4759 {
4760 int doing_unsigned = unsignedp;
4761
4762 if (fmode != GET_MODE (to)
4763 && (significand_size (fmode)
4764 < GET_MODE_UNIT_PRECISION (GET_MODE (from))))
4765 continue;
4766
4767 icode = can_float_p (fmode, imode, unsignedp);
4768 if (icode == CODE_FOR_nothing && unsignedp)
4769 {
4770 enum insn_code scode = can_float_p (fmode, imode, 0);
4771 if (scode != CODE_FOR_nothing)
4772 can_do_signed = true;
4773 if (imode != GET_MODE (from))
4774 icode = scode, doing_unsigned = 0;
4775 }
4776
4777 if (icode != CODE_FOR_nothing)
4778 {
4779 if (imode != GET_MODE (from))
4780 from = convert_to_mode (imode, from, unsignedp);
4781
4782 if (fmode != GET_MODE (to))
4783 target = gen_reg_rtx (fmode);
4784
4785 emit_unop_insn (icode, target, from,
4786 doing_unsigned ? UNSIGNED_FLOAT : FLOAT);
4787
4788 if (target != to)
4789 convert_move (to, target, 0);
4790 return;
4791 }
4792 }
4793
4794 /* Unsigned integer, and no way to convert directly. Convert as signed,
4795 then unconditionally adjust the result. */
4796 if (unsignedp
4797 && can_do_signed
4798 && is_a <scalar_mode> (GET_MODE (to), &to_mode)
4799 && is_a <scalar_mode> (GET_MODE (from), &from_mode))
4800 {
4801 opt_scalar_mode fmode_iter;
4802 rtx_code_label *label = gen_label_rtx ();
4803 rtx temp;
4804 REAL_VALUE_TYPE offset;
4805
4806 /* Look for a usable floating mode FMODE wider than the source and at
4807 least as wide as the target. Using FMODE will avoid rounding woes
4808 with unsigned values greater than the signed maximum value. */
4809
4810 FOR_EACH_MODE_FROM (fmode_iter, to_mode)
4811 {
4812 scalar_mode fmode = fmode_iter.require ();
4813 if (GET_MODE_PRECISION (from_mode) < GET_MODE_BITSIZE (fmode)
4814 && can_float_p (fmode, from_mode, 0) != CODE_FOR_nothing)
4815 break;
4816 }
4817
4818 if (!fmode_iter.exists (&fmode))
4819 {
4820 /* There is no such mode. Pretend the target is wide enough. */
4821 fmode = to_mode;
4822
4823 /* Avoid double-rounding when TO is narrower than FROM. */
4824 if ((significand_size (fmode) + 1)
4825 < GET_MODE_PRECISION (from_mode))
4826 {
4827 rtx temp1;
4828 rtx_code_label *neglabel = gen_label_rtx ();
4829
4830 /* Don't use TARGET if it isn't a register, is a hard register,
4831 or is the wrong mode. */
4832 if (!REG_P (target)
4833 || REGNO (target) < FIRST_PSEUDO_REGISTER
4834 || GET_MODE (target) != fmode)
4835 target = gen_reg_rtx (fmode);
4836
4837 imode = from_mode;
4838 do_pending_stack_adjust ();
4839
4840 /* Test whether the sign bit is set. */
4841 emit_cmp_and_jump_insns (from, const0_rtx, LT, NULL_RTX, imode,
4842 0, neglabel);
4843
4844 /* The sign bit is not set. Convert as signed. */
4845 expand_float (target, from, 0);
4846 emit_jump_insn (targetm.gen_jump (label));
4847 emit_barrier ();
4848
4849 /* The sign bit is set.
4850 Convert to a usable (positive signed) value by shifting right
4851 one bit, while remembering if a nonzero bit was shifted
4852 out; i.e., compute (from & 1) | (from >> 1). */
4853
4854 emit_label (neglabel);
4855 temp = expand_binop (imode, and_optab, from, const1_rtx,
4856 NULL_RTX, 1, OPTAB_LIB_WIDEN);
4857 temp1 = expand_shift (RSHIFT_EXPR, imode, from, 1, NULL_RTX, 1);
4858 temp = expand_binop (imode, ior_optab, temp, temp1, temp, 1,
4859 OPTAB_LIB_WIDEN);
4860 expand_float (target, temp, 0);
4861
4862 /* Multiply by 2 to undo the shift above. */
4863 temp = expand_binop (fmode, add_optab, target, target,
4864 target, 0, OPTAB_LIB_WIDEN);
4865 if (temp != target)
4866 emit_move_insn (target, temp);
4867
4868 do_pending_stack_adjust ();
4869 emit_label (label);
4870 goto done;
4871 }
4872 }
4873
4874 /* If we are about to do some arithmetic to correct for an
4875 unsigned operand, do it in a pseudo-register. */
4876
4877 if (to_mode != fmode
4878 || !REG_P (to) || REGNO (to) < FIRST_PSEUDO_REGISTER)
4879 target = gen_reg_rtx (fmode);
4880
4881 /* Convert as signed integer to floating. */
4882 expand_float (target, from, 0);
4883
4884 /* If FROM is negative (and therefore TO is negative),
4885 correct its value by 2**bitwidth. */
4886
4887 do_pending_stack_adjust ();
4888 emit_cmp_and_jump_insns (from, const0_rtx, GE, NULL_RTX, from_mode,
4889 0, label);
4890
4891
4892 real_2expN (&offset, GET_MODE_PRECISION (from_mode), fmode);
4893 temp = expand_binop (fmode, add_optab, target,
4894 const_double_from_real_value (offset, fmode),
4895 target, 0, OPTAB_LIB_WIDEN);
4896 if (temp != target)
4897 emit_move_insn (target, temp);
4898
4899 do_pending_stack_adjust ();
4900 emit_label (label);
4901 goto done;
4902 }
4903
4904 /* No hardware instruction available; call a library routine. */
4905 {
4906 rtx libfunc;
4907 rtx_insn *insns;
4908 rtx value;
4909 convert_optab tab = unsignedp ? ufloat_optab : sfloat_optab;
4910
4911 if (is_narrower_int_mode (GET_MODE (from), SImode))
4912 from = convert_to_mode (SImode, from, unsignedp);
4913
4914 libfunc = convert_optab_libfunc (tab, GET_MODE (to), GET_MODE (from));
4915 gcc_assert (libfunc);
4916
4917 start_sequence ();
4918
4919 value = emit_library_call_value (libfunc, NULL_RTX, LCT_CONST,
4920 GET_MODE (to), from, GET_MODE (from));
4921 insns = get_insns ();
4922 end_sequence ();
4923
4924 emit_libcall_block (insns, target, value,
4925 gen_rtx_fmt_e (unsignedp ? UNSIGNED_FLOAT : FLOAT,
4926 GET_MODE (to), from));
4927 }
4928
4929 done:
4930
4931 /* Copy result to requested destination
4932 if we have been computing in a temp location. */
4933
4934 if (target != to)
4935 {
4936 if (GET_MODE (target) == GET_MODE (to))
4937 emit_move_insn (to, target);
4938 else
4939 convert_move (to, target, 0);
4940 }
4941 }
4942
4943 /* Generate code to convert FROM to fixed point and store in TO. FROM
4944 must be floating point. */
4945
4946 void
expand_fix(rtx to,rtx from,int unsignedp)4947 expand_fix (rtx to, rtx from, int unsignedp)
4948 {
4949 enum insn_code icode;
4950 rtx target = to;
4951 machine_mode fmode, imode;
4952 opt_scalar_mode fmode_iter;
4953 bool must_trunc = false;
4954
4955 /* We first try to find a pair of modes, one real and one integer, at
4956 least as wide as FROM and TO, respectively, in which we can open-code
4957 this conversion. If the integer mode is wider than the mode of TO,
4958 we can do the conversion either signed or unsigned. */
4959
4960 FOR_EACH_MODE_FROM (fmode, GET_MODE (from))
4961 FOR_EACH_MODE_FROM (imode, GET_MODE (to))
4962 {
4963 int doing_unsigned = unsignedp;
4964
4965 icode = can_fix_p (imode, fmode, unsignedp, &must_trunc);
4966 if (icode == CODE_FOR_nothing && imode != GET_MODE (to) && unsignedp)
4967 icode = can_fix_p (imode, fmode, 0, &must_trunc), doing_unsigned = 0;
4968
4969 if (icode != CODE_FOR_nothing)
4970 {
4971 rtx_insn *last = get_last_insn ();
4972 if (fmode != GET_MODE (from))
4973 from = convert_to_mode (fmode, from, 0);
4974
4975 if (must_trunc)
4976 {
4977 rtx temp = gen_reg_rtx (GET_MODE (from));
4978 from = expand_unop (GET_MODE (from), ftrunc_optab, from,
4979 temp, 0);
4980 }
4981
4982 if (imode != GET_MODE (to))
4983 target = gen_reg_rtx (imode);
4984
4985 if (maybe_emit_unop_insn (icode, target, from,
4986 doing_unsigned ? UNSIGNED_FIX : FIX))
4987 {
4988 if (target != to)
4989 convert_move (to, target, unsignedp);
4990 return;
4991 }
4992 delete_insns_since (last);
4993 }
4994 }
4995
4996 /* For an unsigned conversion, there is one more way to do it.
4997 If we have a signed conversion, we generate code that compares
4998 the real value to the largest representable positive number. If if
4999 is smaller, the conversion is done normally. Otherwise, subtract
5000 one plus the highest signed number, convert, and add it back.
5001
5002 We only need to check all real modes, since we know we didn't find
5003 anything with a wider integer mode.
5004
5005 This code used to extend FP value into mode wider than the destination.
5006 This is needed for decimal float modes which cannot accurately
5007 represent one plus the highest signed number of the same size, but
5008 not for binary modes. Consider, for instance conversion from SFmode
5009 into DImode.
5010
5011 The hot path through the code is dealing with inputs smaller than 2^63
5012 and doing just the conversion, so there is no bits to lose.
5013
5014 In the other path we know the value is positive in the range 2^63..2^64-1
5015 inclusive. (as for other input overflow happens and result is undefined)
5016 So we know that the most important bit set in mantissa corresponds to
5017 2^63. The subtraction of 2^63 should not generate any rounding as it
5018 simply clears out that bit. The rest is trivial. */
5019
5020 scalar_int_mode to_mode;
5021 if (unsignedp
5022 && is_a <scalar_int_mode> (GET_MODE (to), &to_mode)
5023 && HWI_COMPUTABLE_MODE_P (to_mode))
5024 FOR_EACH_MODE_FROM (fmode_iter, as_a <scalar_mode> (GET_MODE (from)))
5025 {
5026 scalar_mode fmode = fmode_iter.require ();
5027 if (CODE_FOR_nothing != can_fix_p (to_mode, fmode,
5028 0, &must_trunc)
5029 && (!DECIMAL_FLOAT_MODE_P (fmode)
5030 || (GET_MODE_BITSIZE (fmode) > GET_MODE_PRECISION (to_mode))))
5031 {
5032 int bitsize;
5033 REAL_VALUE_TYPE offset;
5034 rtx limit;
5035 rtx_code_label *lab1, *lab2;
5036 rtx_insn *insn;
5037
5038 bitsize = GET_MODE_PRECISION (to_mode);
5039 real_2expN (&offset, bitsize - 1, fmode);
5040 limit = const_double_from_real_value (offset, fmode);
5041 lab1 = gen_label_rtx ();
5042 lab2 = gen_label_rtx ();
5043
5044 if (fmode != GET_MODE (from))
5045 from = convert_to_mode (fmode, from, 0);
5046
5047 /* See if we need to do the subtraction. */
5048 do_pending_stack_adjust ();
5049 emit_cmp_and_jump_insns (from, limit, GE, NULL_RTX,
5050 GET_MODE (from), 0, lab1);
5051
5052 /* If not, do the signed "fix" and branch around fixup code. */
5053 expand_fix (to, from, 0);
5054 emit_jump_insn (targetm.gen_jump (lab2));
5055 emit_barrier ();
5056
5057 /* Otherwise, subtract 2**(N-1), convert to signed number,
5058 then add 2**(N-1). Do the addition using XOR since this
5059 will often generate better code. */
5060 emit_label (lab1);
5061 target = expand_binop (GET_MODE (from), sub_optab, from, limit,
5062 NULL_RTX, 0, OPTAB_LIB_WIDEN);
5063 expand_fix (to, target, 0);
5064 target = expand_binop (to_mode, xor_optab, to,
5065 gen_int_mode
5066 (HOST_WIDE_INT_1 << (bitsize - 1),
5067 to_mode),
5068 to, 1, OPTAB_LIB_WIDEN);
5069
5070 if (target != to)
5071 emit_move_insn (to, target);
5072
5073 emit_label (lab2);
5074
5075 if (optab_handler (mov_optab, to_mode) != CODE_FOR_nothing)
5076 {
5077 /* Make a place for a REG_NOTE and add it. */
5078 insn = emit_move_insn (to, to);
5079 set_dst_reg_note (insn, REG_EQUAL,
5080 gen_rtx_fmt_e (UNSIGNED_FIX, to_mode,
5081 copy_rtx (from)),
5082 to);
5083 }
5084
5085 return;
5086 }
5087 }
5088
5089 /* We can't do it with an insn, so use a library call. But first ensure
5090 that the mode of TO is at least as wide as SImode, since those are the
5091 only library calls we know about. */
5092
5093 if (is_narrower_int_mode (GET_MODE (to), SImode))
5094 {
5095 target = gen_reg_rtx (SImode);
5096
5097 expand_fix (target, from, unsignedp);
5098 }
5099 else
5100 {
5101 rtx_insn *insns;
5102 rtx value;
5103 rtx libfunc;
5104
5105 convert_optab tab = unsignedp ? ufix_optab : sfix_optab;
5106 libfunc = convert_optab_libfunc (tab, GET_MODE (to), GET_MODE (from));
5107 gcc_assert (libfunc);
5108
5109 start_sequence ();
5110
5111 value = emit_library_call_value (libfunc, NULL_RTX, LCT_CONST,
5112 GET_MODE (to), from, GET_MODE (from));
5113 insns = get_insns ();
5114 end_sequence ();
5115
5116 emit_libcall_block (insns, target, value,
5117 gen_rtx_fmt_e (unsignedp ? UNSIGNED_FIX : FIX,
5118 GET_MODE (to), from));
5119 }
5120
5121 if (target != to)
5122 {
5123 if (GET_MODE (to) == GET_MODE (target))
5124 emit_move_insn (to, target);
5125 else
5126 convert_move (to, target, 0);
5127 }
5128 }
5129
5130
5131 /* Promote integer arguments for a libcall if necessary.
5132 emit_library_call_value cannot do the promotion because it does not
5133 know if it should do a signed or unsigned promotion. This is because
5134 there are no tree types defined for libcalls. */
5135
5136 static rtx
prepare_libcall_arg(rtx arg,int uintp)5137 prepare_libcall_arg (rtx arg, int uintp)
5138 {
5139 scalar_int_mode mode;
5140 machine_mode arg_mode;
5141 if (is_a <scalar_int_mode> (GET_MODE (arg), &mode))
5142 {
5143 /* If we need to promote the integer function argument we need to do
5144 it here instead of inside emit_library_call_value because in
5145 emit_library_call_value we don't know if we should do a signed or
5146 unsigned promotion. */
5147
5148 int unsigned_p = 0;
5149 arg_mode = promote_function_mode (NULL_TREE, mode,
5150 &unsigned_p, NULL_TREE, 0);
5151 if (arg_mode != mode)
5152 return convert_to_mode (arg_mode, arg, uintp);
5153 }
5154 return arg;
5155 }
5156
5157 /* Generate code to convert FROM or TO a fixed-point.
5158 If UINTP is true, either TO or FROM is an unsigned integer.
5159 If SATP is true, we need to saturate the result. */
5160
5161 void
expand_fixed_convert(rtx to,rtx from,int uintp,int satp)5162 expand_fixed_convert (rtx to, rtx from, int uintp, int satp)
5163 {
5164 machine_mode to_mode = GET_MODE (to);
5165 machine_mode from_mode = GET_MODE (from);
5166 convert_optab tab;
5167 enum rtx_code this_code;
5168 enum insn_code code;
5169 rtx_insn *insns;
5170 rtx value;
5171 rtx libfunc;
5172
5173 if (to_mode == from_mode)
5174 {
5175 emit_move_insn (to, from);
5176 return;
5177 }
5178
5179 if (uintp)
5180 {
5181 tab = satp ? satfractuns_optab : fractuns_optab;
5182 this_code = satp ? UNSIGNED_SAT_FRACT : UNSIGNED_FRACT_CONVERT;
5183 }
5184 else
5185 {
5186 tab = satp ? satfract_optab : fract_optab;
5187 this_code = satp ? SAT_FRACT : FRACT_CONVERT;
5188 }
5189 code = convert_optab_handler (tab, to_mode, from_mode);
5190 if (code != CODE_FOR_nothing)
5191 {
5192 emit_unop_insn (code, to, from, this_code);
5193 return;
5194 }
5195
5196 libfunc = convert_optab_libfunc (tab, to_mode, from_mode);
5197 gcc_assert (libfunc);
5198
5199 from = prepare_libcall_arg (from, uintp);
5200 from_mode = GET_MODE (from);
5201
5202 start_sequence ();
5203 value = emit_library_call_value (libfunc, NULL_RTX, LCT_CONST, to_mode,
5204 from, from_mode);
5205 insns = get_insns ();
5206 end_sequence ();
5207
5208 emit_libcall_block (insns, to, value,
5209 gen_rtx_fmt_e (optab_to_code (tab), to_mode, from));
5210 }
5211
5212 /* Generate code to convert FROM to fixed point and store in TO. FROM
5213 must be floating point, TO must be signed. Use the conversion optab
5214 TAB to do the conversion. */
5215
5216 bool
expand_sfix_optab(rtx to,rtx from,convert_optab tab)5217 expand_sfix_optab (rtx to, rtx from, convert_optab tab)
5218 {
5219 enum insn_code icode;
5220 rtx target = to;
5221 machine_mode fmode, imode;
5222
5223 /* We first try to find a pair of modes, one real and one integer, at
5224 least as wide as FROM and TO, respectively, in which we can open-code
5225 this conversion. If the integer mode is wider than the mode of TO,
5226 we can do the conversion either signed or unsigned. */
5227
5228 FOR_EACH_MODE_FROM (fmode, GET_MODE (from))
5229 FOR_EACH_MODE_FROM (imode, GET_MODE (to))
5230 {
5231 icode = convert_optab_handler (tab, imode, fmode);
5232 if (icode != CODE_FOR_nothing)
5233 {
5234 rtx_insn *last = get_last_insn ();
5235 if (fmode != GET_MODE (from))
5236 from = convert_to_mode (fmode, from, 0);
5237
5238 if (imode != GET_MODE (to))
5239 target = gen_reg_rtx (imode);
5240
5241 if (!maybe_emit_unop_insn (icode, target, from, UNKNOWN))
5242 {
5243 delete_insns_since (last);
5244 continue;
5245 }
5246 if (target != to)
5247 convert_move (to, target, 0);
5248 return true;
5249 }
5250 }
5251
5252 return false;
5253 }
5254
5255 /* Report whether we have an instruction to perform the operation
5256 specified by CODE on operands of mode MODE. */
5257 int
have_insn_for(enum rtx_code code,machine_mode mode)5258 have_insn_for (enum rtx_code code, machine_mode mode)
5259 {
5260 return (code_to_optab (code)
5261 && (optab_handler (code_to_optab (code), mode)
5262 != CODE_FOR_nothing));
5263 }
5264
5265 /* Print information about the current contents of the optabs on
5266 STDERR. */
5267
5268 DEBUG_FUNCTION void
debug_optab_libfuncs(void)5269 debug_optab_libfuncs (void)
5270 {
5271 int i, j, k;
5272
5273 /* Dump the arithmetic optabs. */
5274 for (i = FIRST_NORM_OPTAB; i <= LAST_NORMLIB_OPTAB; ++i)
5275 for (j = 0; j < NUM_MACHINE_MODES; ++j)
5276 {
5277 rtx l = optab_libfunc ((optab) i, (machine_mode) j);
5278 if (l)
5279 {
5280 gcc_assert (GET_CODE (l) == SYMBOL_REF);
5281 fprintf (stderr, "%s\t%s:\t%s\n",
5282 GET_RTX_NAME (optab_to_code ((optab) i)),
5283 GET_MODE_NAME (j),
5284 XSTR (l, 0));
5285 }
5286 }
5287
5288 /* Dump the conversion optabs. */
5289 for (i = FIRST_CONV_OPTAB; i <= LAST_CONVLIB_OPTAB; ++i)
5290 for (j = 0; j < NUM_MACHINE_MODES; ++j)
5291 for (k = 0; k < NUM_MACHINE_MODES; ++k)
5292 {
5293 rtx l = convert_optab_libfunc ((optab) i, (machine_mode) j,
5294 (machine_mode) k);
5295 if (l)
5296 {
5297 gcc_assert (GET_CODE (l) == SYMBOL_REF);
5298 fprintf (stderr, "%s\t%s\t%s:\t%s\n",
5299 GET_RTX_NAME (optab_to_code ((optab) i)),
5300 GET_MODE_NAME (j),
5301 GET_MODE_NAME (k),
5302 XSTR (l, 0));
5303 }
5304 }
5305 }
5306
5307 /* Generate insns to trap with code TCODE if OP1 and OP2 satisfy condition
5308 CODE. Return 0 on failure. */
5309
5310 rtx_insn *
gen_cond_trap(enum rtx_code code,rtx op1,rtx op2,rtx tcode)5311 gen_cond_trap (enum rtx_code code, rtx op1, rtx op2, rtx tcode)
5312 {
5313 machine_mode mode = GET_MODE (op1);
5314 enum insn_code icode;
5315 rtx_insn *insn;
5316 rtx trap_rtx;
5317
5318 if (mode == VOIDmode)
5319 return 0;
5320
5321 icode = optab_handler (ctrap_optab, mode);
5322 if (icode == CODE_FOR_nothing)
5323 return 0;
5324
5325 /* Some targets only accept a zero trap code. */
5326 if (!insn_operand_matches (icode, 3, tcode))
5327 return 0;
5328
5329 do_pending_stack_adjust ();
5330 start_sequence ();
5331 prepare_cmp_insn (op1, op2, code, NULL_RTX, false, OPTAB_DIRECT,
5332 &trap_rtx, &mode);
5333 if (!trap_rtx)
5334 insn = NULL;
5335 else
5336 insn = GEN_FCN (icode) (trap_rtx, XEXP (trap_rtx, 0), XEXP (trap_rtx, 1),
5337 tcode);
5338
5339 /* If that failed, then give up. */
5340 if (insn == 0)
5341 {
5342 end_sequence ();
5343 return 0;
5344 }
5345
5346 emit_insn (insn);
5347 insn = get_insns ();
5348 end_sequence ();
5349 return insn;
5350 }
5351
5352 /* Return rtx code for TCODE. Use UNSIGNEDP to select signed
5353 or unsigned operation code. */
5354
5355 enum rtx_code
get_rtx_code(enum tree_code tcode,bool unsignedp)5356 get_rtx_code (enum tree_code tcode, bool unsignedp)
5357 {
5358 enum rtx_code code;
5359 switch (tcode)
5360 {
5361 case EQ_EXPR:
5362 code = EQ;
5363 break;
5364 case NE_EXPR:
5365 code = NE;
5366 break;
5367 case LT_EXPR:
5368 code = unsignedp ? LTU : LT;
5369 break;
5370 case LE_EXPR:
5371 code = unsignedp ? LEU : LE;
5372 break;
5373 case GT_EXPR:
5374 code = unsignedp ? GTU : GT;
5375 break;
5376 case GE_EXPR:
5377 code = unsignedp ? GEU : GE;
5378 break;
5379
5380 case UNORDERED_EXPR:
5381 code = UNORDERED;
5382 break;
5383 case ORDERED_EXPR:
5384 code = ORDERED;
5385 break;
5386 case UNLT_EXPR:
5387 code = UNLT;
5388 break;
5389 case UNLE_EXPR:
5390 code = UNLE;
5391 break;
5392 case UNGT_EXPR:
5393 code = UNGT;
5394 break;
5395 case UNGE_EXPR:
5396 code = UNGE;
5397 break;
5398 case UNEQ_EXPR:
5399 code = UNEQ;
5400 break;
5401 case LTGT_EXPR:
5402 code = LTGT;
5403 break;
5404
5405 case BIT_AND_EXPR:
5406 code = AND;
5407 break;
5408
5409 case BIT_IOR_EXPR:
5410 code = IOR;
5411 break;
5412
5413 default:
5414 gcc_unreachable ();
5415 }
5416 return code;
5417 }
5418
5419 /* Return a comparison rtx of mode CMP_MODE for COND. Use UNSIGNEDP to
5420 select signed or unsigned operators. OPNO holds the index of the
5421 first comparison operand for insn ICODE. Do not generate the
5422 compare instruction itself. */
5423
5424 static rtx
vector_compare_rtx(machine_mode cmp_mode,enum tree_code tcode,tree t_op0,tree t_op1,bool unsignedp,enum insn_code icode,unsigned int opno)5425 vector_compare_rtx (machine_mode cmp_mode, enum tree_code tcode,
5426 tree t_op0, tree t_op1, bool unsignedp,
5427 enum insn_code icode, unsigned int opno)
5428 {
5429 struct expand_operand ops[2];
5430 rtx rtx_op0, rtx_op1;
5431 machine_mode m0, m1;
5432 enum rtx_code rcode = get_rtx_code (tcode, unsignedp);
5433
5434 gcc_assert (TREE_CODE_CLASS (tcode) == tcc_comparison);
5435
5436 /* Expand operands. For vector types with scalar modes, e.g. where int64x1_t
5437 has mode DImode, this can produce a constant RTX of mode VOIDmode; in such
5438 cases, use the original mode. */
5439 rtx_op0 = expand_expr (t_op0, NULL_RTX, TYPE_MODE (TREE_TYPE (t_op0)),
5440 EXPAND_STACK_PARM);
5441 m0 = GET_MODE (rtx_op0);
5442 if (m0 == VOIDmode)
5443 m0 = TYPE_MODE (TREE_TYPE (t_op0));
5444
5445 rtx_op1 = expand_expr (t_op1, NULL_RTX, TYPE_MODE (TREE_TYPE (t_op1)),
5446 EXPAND_STACK_PARM);
5447 m1 = GET_MODE (rtx_op1);
5448 if (m1 == VOIDmode)
5449 m1 = TYPE_MODE (TREE_TYPE (t_op1));
5450
5451 create_input_operand (&ops[0], rtx_op0, m0);
5452 create_input_operand (&ops[1], rtx_op1, m1);
5453 if (!maybe_legitimize_operands (icode, opno, 2, ops))
5454 gcc_unreachable ();
5455 return gen_rtx_fmt_ee (rcode, cmp_mode, ops[0].value, ops[1].value);
5456 }
5457
5458 /* Check if vec_perm mask SEL is a constant equivalent to a shift of
5459 the first vec_perm operand, assuming the second operand is a constant
5460 vector of zeros. Return the shift distance in bits if so, or NULL_RTX
5461 if the vec_perm is not a shift. MODE is the mode of the value being
5462 shifted. */
5463 static rtx
shift_amt_for_vec_perm_mask(machine_mode mode,const vec_perm_indices & sel)5464 shift_amt_for_vec_perm_mask (machine_mode mode, const vec_perm_indices &sel)
5465 {
5466 unsigned int bitsize = GET_MODE_UNIT_BITSIZE (mode);
5467 poly_int64 first = sel[0];
5468 if (maybe_ge (sel[0], GET_MODE_NUNITS (mode)))
5469 return NULL_RTX;
5470
5471 if (!sel.series_p (0, 1, first, 1))
5472 {
5473 unsigned int nelt;
5474 if (!GET_MODE_NUNITS (mode).is_constant (&nelt))
5475 return NULL_RTX;
5476 for (unsigned int i = 1; i < nelt; i++)
5477 {
5478 poly_int64 expected = i + first;
5479 /* Indices into the second vector are all equivalent. */
5480 if (maybe_lt (sel[i], nelt)
5481 ? maybe_ne (sel[i], expected)
5482 : maybe_lt (expected, nelt))
5483 return NULL_RTX;
5484 }
5485 }
5486
5487 return gen_int_shift_amount (mode, first * bitsize);
5488 }
5489
5490 /* A subroutine of expand_vec_perm_var for expanding one vec_perm insn. */
5491
5492 static rtx
expand_vec_perm_1(enum insn_code icode,rtx target,rtx v0,rtx v1,rtx sel)5493 expand_vec_perm_1 (enum insn_code icode, rtx target,
5494 rtx v0, rtx v1, rtx sel)
5495 {
5496 machine_mode tmode = GET_MODE (target);
5497 machine_mode smode = GET_MODE (sel);
5498 struct expand_operand ops[4];
5499
5500 gcc_assert (GET_MODE_CLASS (smode) == MODE_VECTOR_INT
5501 || mode_for_int_vector (tmode).require () == smode);
5502 create_output_operand (&ops[0], target, tmode);
5503 create_input_operand (&ops[3], sel, smode);
5504
5505 /* Make an effort to preserve v0 == v1. The target expander is able to
5506 rely on this to determine if we're permuting a single input operand. */
5507 if (rtx_equal_p (v0, v1))
5508 {
5509 if (!insn_operand_matches (icode, 1, v0))
5510 v0 = force_reg (tmode, v0);
5511 gcc_checking_assert (insn_operand_matches (icode, 1, v0));
5512 gcc_checking_assert (insn_operand_matches (icode, 2, v0));
5513
5514 create_fixed_operand (&ops[1], v0);
5515 create_fixed_operand (&ops[2], v0);
5516 }
5517 else
5518 {
5519 create_input_operand (&ops[1], v0, tmode);
5520 create_input_operand (&ops[2], v1, tmode);
5521 }
5522
5523 if (maybe_expand_insn (icode, 4, ops))
5524 return ops[0].value;
5525 return NULL_RTX;
5526 }
5527
5528 /* Implement a permutation of vectors v0 and v1 using the permutation
5529 vector in SEL and return the result. Use TARGET to hold the result
5530 if nonnull and convenient.
5531
5532 MODE is the mode of the vectors being permuted (V0 and V1). SEL_MODE
5533 is the TYPE_MODE associated with SEL, or BLKmode if SEL isn't known
5534 to have a particular mode. */
5535
5536 rtx
expand_vec_perm_const(machine_mode mode,rtx v0,rtx v1,const vec_perm_builder & sel,machine_mode sel_mode,rtx target)5537 expand_vec_perm_const (machine_mode mode, rtx v0, rtx v1,
5538 const vec_perm_builder &sel, machine_mode sel_mode,
5539 rtx target)
5540 {
5541 if (!target || !register_operand (target, mode))
5542 target = gen_reg_rtx (mode);
5543
5544 /* Set QIMODE to a different vector mode with byte elements.
5545 If no such mode, or if MODE already has byte elements, use VOIDmode. */
5546 machine_mode qimode;
5547 if (!qimode_for_vec_perm (mode).exists (&qimode))
5548 qimode = VOIDmode;
5549
5550 rtx_insn *last = get_last_insn ();
5551
5552 bool single_arg_p = rtx_equal_p (v0, v1);
5553 /* Always specify two input vectors here and leave the target to handle
5554 cases in which the inputs are equal. Not all backends can cope with
5555 the single-input representation when testing for a double-input
5556 target instruction. */
5557 vec_perm_indices indices (sel, 2, GET_MODE_NUNITS (mode));
5558
5559 /* See if this can be handled with a vec_shr. We only do this if the
5560 second vector is all zeroes. */
5561 insn_code shift_code = optab_handler (vec_shr_optab, mode);
5562 insn_code shift_code_qi = ((qimode != VOIDmode && qimode != mode)
5563 ? optab_handler (vec_shr_optab, qimode)
5564 : CODE_FOR_nothing);
5565
5566 if (v1 == CONST0_RTX (GET_MODE (v1))
5567 && (shift_code != CODE_FOR_nothing
5568 || shift_code_qi != CODE_FOR_nothing))
5569 {
5570 rtx shift_amt = shift_amt_for_vec_perm_mask (mode, indices);
5571 if (shift_amt)
5572 {
5573 struct expand_operand ops[3];
5574 if (shift_amt == const0_rtx)
5575 return v0;
5576 if (shift_code != CODE_FOR_nothing)
5577 {
5578 create_output_operand (&ops[0], target, mode);
5579 create_input_operand (&ops[1], v0, mode);
5580 create_convert_operand_from_type (&ops[2], shift_amt, sizetype);
5581 if (maybe_expand_insn (shift_code, 3, ops))
5582 return ops[0].value;
5583 }
5584 if (shift_code_qi != CODE_FOR_nothing)
5585 {
5586 rtx tmp = gen_reg_rtx (qimode);
5587 create_output_operand (&ops[0], tmp, qimode);
5588 create_input_operand (&ops[1], gen_lowpart (qimode, v0), qimode);
5589 create_convert_operand_from_type (&ops[2], shift_amt, sizetype);
5590 if (maybe_expand_insn (shift_code_qi, 3, ops))
5591 return gen_lowpart (mode, ops[0].value);
5592 }
5593 }
5594 }
5595
5596 if (targetm.vectorize.vec_perm_const != NULL)
5597 {
5598 v0 = force_reg (mode, v0);
5599 if (single_arg_p)
5600 v1 = v0;
5601 else
5602 v1 = force_reg (mode, v1);
5603
5604 if (targetm.vectorize.vec_perm_const (mode, target, v0, v1, indices))
5605 return target;
5606 }
5607
5608 /* Fall back to a constant byte-based permutation. */
5609 vec_perm_indices qimode_indices;
5610 rtx target_qi = NULL_RTX, v0_qi = NULL_RTX, v1_qi = NULL_RTX;
5611 if (qimode != VOIDmode)
5612 {
5613 qimode_indices.new_expanded_vector (indices, GET_MODE_UNIT_SIZE (mode));
5614 target_qi = gen_reg_rtx (qimode);
5615 v0_qi = gen_lowpart (qimode, v0);
5616 v1_qi = gen_lowpart (qimode, v1);
5617 if (targetm.vectorize.vec_perm_const != NULL
5618 && targetm.vectorize.vec_perm_const (qimode, target_qi, v0_qi,
5619 v1_qi, qimode_indices))
5620 return gen_lowpart (mode, target_qi);
5621 }
5622
5623 /* Otherwise expand as a fully variable permuation. */
5624
5625 /* The optabs are only defined for selectors with the same width
5626 as the values being permuted. */
5627 machine_mode required_sel_mode;
5628 if (!mode_for_int_vector (mode).exists (&required_sel_mode)
5629 || !VECTOR_MODE_P (required_sel_mode))
5630 {
5631 delete_insns_since (last);
5632 return NULL_RTX;
5633 }
5634
5635 /* We know that it is semantically valid to treat SEL as having SEL_MODE.
5636 If that isn't the mode we want then we need to prove that using
5637 REQUIRED_SEL_MODE is OK. */
5638 if (sel_mode != required_sel_mode)
5639 {
5640 if (!selector_fits_mode_p (required_sel_mode, indices))
5641 {
5642 delete_insns_since (last);
5643 return NULL_RTX;
5644 }
5645 sel_mode = required_sel_mode;
5646 }
5647
5648 insn_code icode = direct_optab_handler (vec_perm_optab, mode);
5649 if (icode != CODE_FOR_nothing)
5650 {
5651 rtx sel_rtx = vec_perm_indices_to_rtx (sel_mode, indices);
5652 rtx tmp = expand_vec_perm_1 (icode, target, v0, v1, sel_rtx);
5653 if (tmp)
5654 return tmp;
5655 }
5656
5657 if (qimode != VOIDmode
5658 && selector_fits_mode_p (qimode, qimode_indices))
5659 {
5660 icode = direct_optab_handler (vec_perm_optab, qimode);
5661 if (icode != CODE_FOR_nothing)
5662 {
5663 rtx sel_qi = vec_perm_indices_to_rtx (qimode, qimode_indices);
5664 rtx tmp = expand_vec_perm_1 (icode, target_qi, v0_qi, v1_qi, sel_qi);
5665 if (tmp)
5666 return gen_lowpart (mode, tmp);
5667 }
5668 }
5669
5670 delete_insns_since (last);
5671 return NULL_RTX;
5672 }
5673
5674 /* Implement a permutation of vectors v0 and v1 using the permutation
5675 vector in SEL and return the result. Use TARGET to hold the result
5676 if nonnull and convenient.
5677
5678 MODE is the mode of the vectors being permuted (V0 and V1).
5679 SEL must have the integer equivalent of MODE and is known to be
5680 unsuitable for permutes with a constant permutation vector. */
5681
5682 rtx
expand_vec_perm_var(machine_mode mode,rtx v0,rtx v1,rtx sel,rtx target)5683 expand_vec_perm_var (machine_mode mode, rtx v0, rtx v1, rtx sel, rtx target)
5684 {
5685 enum insn_code icode;
5686 unsigned int i, u;
5687 rtx tmp, sel_qi;
5688
5689 u = GET_MODE_UNIT_SIZE (mode);
5690
5691 if (!target || GET_MODE (target) != mode)
5692 target = gen_reg_rtx (mode);
5693
5694 icode = direct_optab_handler (vec_perm_optab, mode);
5695 if (icode != CODE_FOR_nothing)
5696 {
5697 tmp = expand_vec_perm_1 (icode, target, v0, v1, sel);
5698 if (tmp)
5699 return tmp;
5700 }
5701
5702 /* As a special case to aid several targets, lower the element-based
5703 permutation to a byte-based permutation and try again. */
5704 machine_mode qimode;
5705 if (!qimode_for_vec_perm (mode).exists (&qimode)
5706 || maybe_gt (GET_MODE_NUNITS (qimode), GET_MODE_MASK (QImode) + 1))
5707 return NULL_RTX;
5708 icode = direct_optab_handler (vec_perm_optab, qimode);
5709 if (icode == CODE_FOR_nothing)
5710 return NULL_RTX;
5711
5712 /* Multiply each element by its byte size. */
5713 machine_mode selmode = GET_MODE (sel);
5714 if (u == 2)
5715 sel = expand_simple_binop (selmode, PLUS, sel, sel,
5716 NULL, 0, OPTAB_DIRECT);
5717 else
5718 sel = expand_simple_binop (selmode, ASHIFT, sel,
5719 gen_int_shift_amount (selmode, exact_log2 (u)),
5720 NULL, 0, OPTAB_DIRECT);
5721 gcc_assert (sel != NULL);
5722
5723 /* Broadcast the low byte each element into each of its bytes.
5724 The encoding has U interleaved stepped patterns, one for each
5725 byte of an element. */
5726 vec_perm_builder const_sel (GET_MODE_SIZE (mode), u, 3);
5727 unsigned int low_byte_in_u = BYTES_BIG_ENDIAN ? u - 1 : 0;
5728 for (i = 0; i < 3; ++i)
5729 for (unsigned int j = 0; j < u; ++j)
5730 const_sel.quick_push (i * u + low_byte_in_u);
5731 sel = gen_lowpart (qimode, sel);
5732 sel = expand_vec_perm_const (qimode, sel, sel, const_sel, qimode, NULL);
5733 gcc_assert (sel != NULL);
5734
5735 /* Add the byte offset to each byte element. */
5736 /* Note that the definition of the indicies here is memory ordering,
5737 so there should be no difference between big and little endian. */
5738 rtx_vector_builder byte_indices (qimode, u, 1);
5739 for (i = 0; i < u; ++i)
5740 byte_indices.quick_push (GEN_INT (i));
5741 tmp = byte_indices.build ();
5742 sel_qi = expand_simple_binop (qimode, PLUS, sel, tmp,
5743 sel, 0, OPTAB_DIRECT);
5744 gcc_assert (sel_qi != NULL);
5745
5746 tmp = mode != qimode ? gen_reg_rtx (qimode) : target;
5747 tmp = expand_vec_perm_1 (icode, tmp, gen_lowpart (qimode, v0),
5748 gen_lowpart (qimode, v1), sel_qi);
5749 if (tmp)
5750 tmp = gen_lowpart (mode, tmp);
5751 return tmp;
5752 }
5753
5754 /* Generate insns for a VEC_COND_EXPR with mask, given its TYPE and its
5755 three operands. */
5756
5757 rtx
expand_vec_cond_mask_expr(tree vec_cond_type,tree op0,tree op1,tree op2,rtx target)5758 expand_vec_cond_mask_expr (tree vec_cond_type, tree op0, tree op1, tree op2,
5759 rtx target)
5760 {
5761 struct expand_operand ops[4];
5762 machine_mode mode = TYPE_MODE (vec_cond_type);
5763 machine_mode mask_mode = TYPE_MODE (TREE_TYPE (op0));
5764 enum insn_code icode = get_vcond_mask_icode (mode, mask_mode);
5765 rtx mask, rtx_op1, rtx_op2;
5766
5767 if (icode == CODE_FOR_nothing)
5768 return 0;
5769
5770 mask = expand_normal (op0);
5771 rtx_op1 = expand_normal (op1);
5772 rtx_op2 = expand_normal (op2);
5773
5774 mask = force_reg (mask_mode, mask);
5775 rtx_op1 = force_reg (GET_MODE (rtx_op1), rtx_op1);
5776
5777 create_output_operand (&ops[0], target, mode);
5778 create_input_operand (&ops[1], rtx_op1, mode);
5779 create_input_operand (&ops[2], rtx_op2, mode);
5780 create_input_operand (&ops[3], mask, mask_mode);
5781 expand_insn (icode, 4, ops);
5782
5783 return ops[0].value;
5784 }
5785
5786 /* Generate insns for a VEC_COND_EXPR, given its TYPE and its
5787 three operands. */
5788
5789 rtx
expand_vec_cond_expr(tree vec_cond_type,tree op0,tree op1,tree op2,rtx target)5790 expand_vec_cond_expr (tree vec_cond_type, tree op0, tree op1, tree op2,
5791 rtx target)
5792 {
5793 struct expand_operand ops[6];
5794 enum insn_code icode;
5795 rtx comparison, rtx_op1, rtx_op2;
5796 machine_mode mode = TYPE_MODE (vec_cond_type);
5797 machine_mode cmp_op_mode;
5798 bool unsignedp;
5799 tree op0a, op0b;
5800 enum tree_code tcode;
5801
5802 if (COMPARISON_CLASS_P (op0))
5803 {
5804 op0a = TREE_OPERAND (op0, 0);
5805 op0b = TREE_OPERAND (op0, 1);
5806 tcode = TREE_CODE (op0);
5807 }
5808 else
5809 {
5810 gcc_assert (VECTOR_BOOLEAN_TYPE_P (TREE_TYPE (op0)));
5811 if (get_vcond_mask_icode (mode, TYPE_MODE (TREE_TYPE (op0)))
5812 != CODE_FOR_nothing)
5813 return expand_vec_cond_mask_expr (vec_cond_type, op0, op1,
5814 op2, target);
5815 /* Fake op0 < 0. */
5816 else
5817 {
5818 gcc_assert (GET_MODE_CLASS (TYPE_MODE (TREE_TYPE (op0)))
5819 == MODE_VECTOR_INT);
5820 op0a = op0;
5821 op0b = build_zero_cst (TREE_TYPE (op0));
5822 tcode = LT_EXPR;
5823 }
5824 }
5825 cmp_op_mode = TYPE_MODE (TREE_TYPE (op0a));
5826 unsignedp = TYPE_UNSIGNED (TREE_TYPE (op0a));
5827
5828
5829 gcc_assert (known_eq (GET_MODE_SIZE (mode), GET_MODE_SIZE (cmp_op_mode))
5830 && known_eq (GET_MODE_NUNITS (mode),
5831 GET_MODE_NUNITS (cmp_op_mode)));
5832
5833 icode = get_vcond_icode (mode, cmp_op_mode, unsignedp);
5834 if (icode == CODE_FOR_nothing)
5835 {
5836 if (tcode == LT_EXPR
5837 && op0a == op0
5838 && TREE_CODE (op0) == VECTOR_CST)
5839 {
5840 /* A VEC_COND_EXPR condition could be folded from EQ_EXPR/NE_EXPR
5841 into a constant when only get_vcond_eq_icode is supported.
5842 Verify < 0 and != 0 behave the same and change it to NE_EXPR. */
5843 unsigned HOST_WIDE_INT nelts;
5844 if (!VECTOR_CST_NELTS (op0).is_constant (&nelts))
5845 {
5846 if (VECTOR_CST_STEPPED_P (op0))
5847 return 0;
5848 nelts = vector_cst_encoded_nelts (op0);
5849 }
5850 for (unsigned int i = 0; i < nelts; ++i)
5851 if (tree_int_cst_sgn (vector_cst_elt (op0, i)) == 1)
5852 return 0;
5853 tcode = NE_EXPR;
5854 }
5855 if (tcode == EQ_EXPR || tcode == NE_EXPR)
5856 icode = get_vcond_eq_icode (mode, cmp_op_mode);
5857 if (icode == CODE_FOR_nothing)
5858 return 0;
5859 }
5860
5861 comparison = vector_compare_rtx (VOIDmode, tcode, op0a, op0b, unsignedp,
5862 icode, 4);
5863 rtx_op1 = expand_normal (op1);
5864 rtx_op2 = expand_normal (op2);
5865
5866 create_output_operand (&ops[0], target, mode);
5867 create_input_operand (&ops[1], rtx_op1, mode);
5868 create_input_operand (&ops[2], rtx_op2, mode);
5869 create_fixed_operand (&ops[3], comparison);
5870 create_fixed_operand (&ops[4], XEXP (comparison, 0));
5871 create_fixed_operand (&ops[5], XEXP (comparison, 1));
5872 expand_insn (icode, 6, ops);
5873 return ops[0].value;
5874 }
5875
5876 /* Generate VEC_SERIES_EXPR <OP0, OP1>, returning a value of mode VMODE.
5877 Use TARGET for the result if nonnull and convenient. */
5878
5879 rtx
expand_vec_series_expr(machine_mode vmode,rtx op0,rtx op1,rtx target)5880 expand_vec_series_expr (machine_mode vmode, rtx op0, rtx op1, rtx target)
5881 {
5882 struct expand_operand ops[3];
5883 enum insn_code icode;
5884 machine_mode emode = GET_MODE_INNER (vmode);
5885
5886 icode = direct_optab_handler (vec_series_optab, vmode);
5887 gcc_assert (icode != CODE_FOR_nothing);
5888
5889 create_output_operand (&ops[0], target, vmode);
5890 create_input_operand (&ops[1], op0, emode);
5891 create_input_operand (&ops[2], op1, emode);
5892
5893 expand_insn (icode, 3, ops);
5894 return ops[0].value;
5895 }
5896
5897 /* Generate insns for a vector comparison into a mask. */
5898
5899 rtx
expand_vec_cmp_expr(tree type,tree exp,rtx target)5900 expand_vec_cmp_expr (tree type, tree exp, rtx target)
5901 {
5902 struct expand_operand ops[4];
5903 enum insn_code icode;
5904 rtx comparison;
5905 machine_mode mask_mode = TYPE_MODE (type);
5906 machine_mode vmode;
5907 bool unsignedp;
5908 tree op0a, op0b;
5909 enum tree_code tcode;
5910
5911 op0a = TREE_OPERAND (exp, 0);
5912 op0b = TREE_OPERAND (exp, 1);
5913 tcode = TREE_CODE (exp);
5914
5915 unsignedp = TYPE_UNSIGNED (TREE_TYPE (op0a));
5916 vmode = TYPE_MODE (TREE_TYPE (op0a));
5917
5918 icode = get_vec_cmp_icode (vmode, mask_mode, unsignedp);
5919 if (icode == CODE_FOR_nothing)
5920 {
5921 if (tcode == EQ_EXPR || tcode == NE_EXPR)
5922 icode = get_vec_cmp_eq_icode (vmode, mask_mode);
5923 if (icode == CODE_FOR_nothing)
5924 return 0;
5925 }
5926
5927 comparison = vector_compare_rtx (mask_mode, tcode, op0a, op0b,
5928 unsignedp, icode, 2);
5929 create_output_operand (&ops[0], target, mask_mode);
5930 create_fixed_operand (&ops[1], comparison);
5931 create_fixed_operand (&ops[2], XEXP (comparison, 0));
5932 create_fixed_operand (&ops[3], XEXP (comparison, 1));
5933 expand_insn (icode, 4, ops);
5934 return ops[0].value;
5935 }
5936
5937 /* Expand a highpart multiply. */
5938
5939 rtx
expand_mult_highpart(machine_mode mode,rtx op0,rtx op1,rtx target,bool uns_p)5940 expand_mult_highpart (machine_mode mode, rtx op0, rtx op1,
5941 rtx target, bool uns_p)
5942 {
5943 struct expand_operand eops[3];
5944 enum insn_code icode;
5945 int method, i;
5946 machine_mode wmode;
5947 rtx m1, m2;
5948 optab tab1, tab2;
5949
5950 method = can_mult_highpart_p (mode, uns_p);
5951 switch (method)
5952 {
5953 case 0:
5954 return NULL_RTX;
5955 case 1:
5956 tab1 = uns_p ? umul_highpart_optab : smul_highpart_optab;
5957 return expand_binop (mode, tab1, op0, op1, target, uns_p,
5958 OPTAB_LIB_WIDEN);
5959 case 2:
5960 tab1 = uns_p ? vec_widen_umult_even_optab : vec_widen_smult_even_optab;
5961 tab2 = uns_p ? vec_widen_umult_odd_optab : vec_widen_smult_odd_optab;
5962 break;
5963 case 3:
5964 tab1 = uns_p ? vec_widen_umult_lo_optab : vec_widen_smult_lo_optab;
5965 tab2 = uns_p ? vec_widen_umult_hi_optab : vec_widen_smult_hi_optab;
5966 if (BYTES_BIG_ENDIAN)
5967 std::swap (tab1, tab2);
5968 break;
5969 default:
5970 gcc_unreachable ();
5971 }
5972
5973 icode = optab_handler (tab1, mode);
5974 wmode = insn_data[icode].operand[0].mode;
5975 gcc_checking_assert (known_eq (2 * GET_MODE_NUNITS (wmode),
5976 GET_MODE_NUNITS (mode)));
5977 gcc_checking_assert (known_eq (GET_MODE_SIZE (wmode), GET_MODE_SIZE (mode)));
5978
5979 create_output_operand (&eops[0], gen_reg_rtx (wmode), wmode);
5980 create_input_operand (&eops[1], op0, mode);
5981 create_input_operand (&eops[2], op1, mode);
5982 expand_insn (icode, 3, eops);
5983 m1 = gen_lowpart (mode, eops[0].value);
5984
5985 create_output_operand (&eops[0], gen_reg_rtx (wmode), wmode);
5986 create_input_operand (&eops[1], op0, mode);
5987 create_input_operand (&eops[2], op1, mode);
5988 expand_insn (optab_handler (tab2, mode), 3, eops);
5989 m2 = gen_lowpart (mode, eops[0].value);
5990
5991 vec_perm_builder sel;
5992 if (method == 2)
5993 {
5994 /* The encoding has 2 interleaved stepped patterns. */
5995 sel.new_vector (GET_MODE_NUNITS (mode), 2, 3);
5996 for (i = 0; i < 6; ++i)
5997 sel.quick_push (!BYTES_BIG_ENDIAN + (i & ~1)
5998 + ((i & 1) ? GET_MODE_NUNITS (mode) : 0));
5999 }
6000 else
6001 {
6002 /* The encoding has a single interleaved stepped pattern. */
6003 sel.new_vector (GET_MODE_NUNITS (mode), 1, 3);
6004 for (i = 0; i < 3; ++i)
6005 sel.quick_push (2 * i + (BYTES_BIG_ENDIAN ? 0 : 1));
6006 }
6007
6008 return expand_vec_perm_const (mode, m1, m2, sel, BLKmode, target);
6009 }
6010
6011 /* Helper function to find the MODE_CC set in a sync_compare_and_swap
6012 pattern. */
6013
6014 static void
find_cc_set(rtx x,const_rtx pat,void * data)6015 find_cc_set (rtx x, const_rtx pat, void *data)
6016 {
6017 if (REG_P (x) && GET_MODE_CLASS (GET_MODE (x)) == MODE_CC
6018 && GET_CODE (pat) == SET)
6019 {
6020 rtx *p_cc_reg = (rtx *) data;
6021 gcc_assert (!*p_cc_reg);
6022 *p_cc_reg = x;
6023 }
6024 }
6025
6026 /* This is a helper function for the other atomic operations. This function
6027 emits a loop that contains SEQ that iterates until a compare-and-swap
6028 operation at the end succeeds. MEM is the memory to be modified. SEQ is
6029 a set of instructions that takes a value from OLD_REG as an input and
6030 produces a value in NEW_REG as an output. Before SEQ, OLD_REG will be
6031 set to the current contents of MEM. After SEQ, a compare-and-swap will
6032 attempt to update MEM with NEW_REG. The function returns true when the
6033 loop was generated successfully. */
6034
6035 static bool
expand_compare_and_swap_loop(rtx mem,rtx old_reg,rtx new_reg,rtx seq)6036 expand_compare_and_swap_loop (rtx mem, rtx old_reg, rtx new_reg, rtx seq)
6037 {
6038 machine_mode mode = GET_MODE (mem);
6039 rtx_code_label *label;
6040 rtx cmp_reg, success, oldval;
6041
6042 /* The loop we want to generate looks like
6043
6044 cmp_reg = mem;
6045 label:
6046 old_reg = cmp_reg;
6047 seq;
6048 (success, cmp_reg) = compare-and-swap(mem, old_reg, new_reg)
6049 if (success)
6050 goto label;
6051
6052 Note that we only do the plain load from memory once. Subsequent
6053 iterations use the value loaded by the compare-and-swap pattern. */
6054
6055 label = gen_label_rtx ();
6056 cmp_reg = gen_reg_rtx (mode);
6057
6058 emit_move_insn (cmp_reg, mem);
6059 emit_label (label);
6060 emit_move_insn (old_reg, cmp_reg);
6061 if (seq)
6062 emit_insn (seq);
6063
6064 success = NULL_RTX;
6065 oldval = cmp_reg;
6066 if (!expand_atomic_compare_and_swap (&success, &oldval, mem, old_reg,
6067 new_reg, false, MEMMODEL_SYNC_SEQ_CST,
6068 MEMMODEL_RELAXED))
6069 return false;
6070
6071 if (oldval != cmp_reg)
6072 emit_move_insn (cmp_reg, oldval);
6073
6074 /* Mark this jump predicted not taken. */
6075 emit_cmp_and_jump_insns (success, const0_rtx, EQ, const0_rtx,
6076 GET_MODE (success), 1, label,
6077 profile_probability::guessed_never ());
6078 return true;
6079 }
6080
6081
6082 /* This function tries to emit an atomic_exchange intruction. VAL is written
6083 to *MEM using memory model MODEL. The previous contents of *MEM are returned,
6084 using TARGET if possible. */
6085
6086 static rtx
maybe_emit_atomic_exchange(rtx target,rtx mem,rtx val,enum memmodel model)6087 maybe_emit_atomic_exchange (rtx target, rtx mem, rtx val, enum memmodel model)
6088 {
6089 machine_mode mode = GET_MODE (mem);
6090 enum insn_code icode;
6091
6092 /* If the target supports the exchange directly, great. */
6093 icode = direct_optab_handler (atomic_exchange_optab, mode);
6094 if (icode != CODE_FOR_nothing)
6095 {
6096 struct expand_operand ops[4];
6097
6098 create_output_operand (&ops[0], target, mode);
6099 create_fixed_operand (&ops[1], mem);
6100 create_input_operand (&ops[2], val, mode);
6101 create_integer_operand (&ops[3], model);
6102 if (maybe_expand_insn (icode, 4, ops))
6103 return ops[0].value;
6104 }
6105
6106 return NULL_RTX;
6107 }
6108
6109 /* This function tries to implement an atomic exchange operation using
6110 __sync_lock_test_and_set. VAL is written to *MEM using memory model MODEL.
6111 The previous contents of *MEM are returned, using TARGET if possible.
6112 Since this instructionn is an acquire barrier only, stronger memory
6113 models may require additional barriers to be emitted. */
6114
6115 static rtx
maybe_emit_sync_lock_test_and_set(rtx target,rtx mem,rtx val,enum memmodel model)6116 maybe_emit_sync_lock_test_and_set (rtx target, rtx mem, rtx val,
6117 enum memmodel model)
6118 {
6119 machine_mode mode = GET_MODE (mem);
6120 enum insn_code icode;
6121 rtx_insn *last_insn = get_last_insn ();
6122
6123 icode = optab_handler (sync_lock_test_and_set_optab, mode);
6124
6125 /* Legacy sync_lock_test_and_set is an acquire barrier. If the pattern
6126 exists, and the memory model is stronger than acquire, add a release
6127 barrier before the instruction. */
6128
6129 if (is_mm_seq_cst (model) || is_mm_release (model) || is_mm_acq_rel (model))
6130 expand_mem_thread_fence (model);
6131
6132 if (icode != CODE_FOR_nothing)
6133 {
6134 struct expand_operand ops[3];
6135 create_output_operand (&ops[0], target, mode);
6136 create_fixed_operand (&ops[1], mem);
6137 create_input_operand (&ops[2], val, mode);
6138 if (maybe_expand_insn (icode, 3, ops))
6139 return ops[0].value;
6140 }
6141
6142 /* If an external test-and-set libcall is provided, use that instead of
6143 any external compare-and-swap that we might get from the compare-and-
6144 swap-loop expansion later. */
6145 if (!can_compare_and_swap_p (mode, false))
6146 {
6147 rtx libfunc = optab_libfunc (sync_lock_test_and_set_optab, mode);
6148 if (libfunc != NULL)
6149 {
6150 rtx addr;
6151
6152 addr = convert_memory_address (ptr_mode, XEXP (mem, 0));
6153 return emit_library_call_value (libfunc, NULL_RTX, LCT_NORMAL,
6154 mode, addr, ptr_mode,
6155 val, mode);
6156 }
6157 }
6158
6159 /* If the test_and_set can't be emitted, eliminate any barrier that might
6160 have been emitted. */
6161 delete_insns_since (last_insn);
6162 return NULL_RTX;
6163 }
6164
6165 /* This function tries to implement an atomic exchange operation using a
6166 compare_and_swap loop. VAL is written to *MEM. The previous contents of
6167 *MEM are returned, using TARGET if possible. No memory model is required
6168 since a compare_and_swap loop is seq-cst. */
6169
6170 static rtx
maybe_emit_compare_and_swap_exchange_loop(rtx target,rtx mem,rtx val)6171 maybe_emit_compare_and_swap_exchange_loop (rtx target, rtx mem, rtx val)
6172 {
6173 machine_mode mode = GET_MODE (mem);
6174
6175 if (can_compare_and_swap_p (mode, true))
6176 {
6177 if (!target || !register_operand (target, mode))
6178 target = gen_reg_rtx (mode);
6179 if (expand_compare_and_swap_loop (mem, target, val, NULL_RTX))
6180 return target;
6181 }
6182
6183 return NULL_RTX;
6184 }
6185
6186 /* This function tries to implement an atomic test-and-set operation
6187 using the atomic_test_and_set instruction pattern. A boolean value
6188 is returned from the operation, using TARGET if possible. */
6189
6190 static rtx
maybe_emit_atomic_test_and_set(rtx target,rtx mem,enum memmodel model)6191 maybe_emit_atomic_test_and_set (rtx target, rtx mem, enum memmodel model)
6192 {
6193 machine_mode pat_bool_mode;
6194 struct expand_operand ops[3];
6195
6196 if (!targetm.have_atomic_test_and_set ())
6197 return NULL_RTX;
6198
6199 /* While we always get QImode from __atomic_test_and_set, we get
6200 other memory modes from __sync_lock_test_and_set. Note that we
6201 use no endian adjustment here. This matches the 4.6 behavior
6202 in the Sparc backend. */
6203 enum insn_code icode = targetm.code_for_atomic_test_and_set;
6204 gcc_checking_assert (insn_data[icode].operand[1].mode == QImode);
6205 if (GET_MODE (mem) != QImode)
6206 mem = adjust_address_nv (mem, QImode, 0);
6207
6208 pat_bool_mode = insn_data[icode].operand[0].mode;
6209 create_output_operand (&ops[0], target, pat_bool_mode);
6210 create_fixed_operand (&ops[1], mem);
6211 create_integer_operand (&ops[2], model);
6212
6213 if (maybe_expand_insn (icode, 3, ops))
6214 return ops[0].value;
6215 return NULL_RTX;
6216 }
6217
6218 /* This function expands the legacy _sync_lock test_and_set operation which is
6219 generally an atomic exchange. Some limited targets only allow the
6220 constant 1 to be stored. This is an ACQUIRE operation.
6221
6222 TARGET is an optional place to stick the return value.
6223 MEM is where VAL is stored. */
6224
6225 rtx
expand_sync_lock_test_and_set(rtx target,rtx mem,rtx val)6226 expand_sync_lock_test_and_set (rtx target, rtx mem, rtx val)
6227 {
6228 rtx ret;
6229
6230 /* Try an atomic_exchange first. */
6231 ret = maybe_emit_atomic_exchange (target, mem, val, MEMMODEL_SYNC_ACQUIRE);
6232 if (ret)
6233 return ret;
6234
6235 ret = maybe_emit_sync_lock_test_and_set (target, mem, val,
6236 MEMMODEL_SYNC_ACQUIRE);
6237 if (ret)
6238 return ret;
6239
6240 ret = maybe_emit_compare_and_swap_exchange_loop (target, mem, val);
6241 if (ret)
6242 return ret;
6243
6244 /* If there are no other options, try atomic_test_and_set if the value
6245 being stored is 1. */
6246 if (val == const1_rtx)
6247 ret = maybe_emit_atomic_test_and_set (target, mem, MEMMODEL_SYNC_ACQUIRE);
6248
6249 return ret;
6250 }
6251
6252 /* This function expands the atomic test_and_set operation:
6253 atomically store a boolean TRUE into MEM and return the previous value.
6254
6255 MEMMODEL is the memory model variant to use.
6256 TARGET is an optional place to stick the return value. */
6257
6258 rtx
expand_atomic_test_and_set(rtx target,rtx mem,enum memmodel model)6259 expand_atomic_test_and_set (rtx target, rtx mem, enum memmodel model)
6260 {
6261 machine_mode mode = GET_MODE (mem);
6262 rtx ret, trueval, subtarget;
6263
6264 ret = maybe_emit_atomic_test_and_set (target, mem, model);
6265 if (ret)
6266 return ret;
6267
6268 /* Be binary compatible with non-default settings of trueval, and different
6269 cpu revisions. E.g. one revision may have atomic-test-and-set, but
6270 another only has atomic-exchange. */
6271 if (targetm.atomic_test_and_set_trueval == 1)
6272 {
6273 trueval = const1_rtx;
6274 subtarget = target ? target : gen_reg_rtx (mode);
6275 }
6276 else
6277 {
6278 trueval = gen_int_mode (targetm.atomic_test_and_set_trueval, mode);
6279 subtarget = gen_reg_rtx (mode);
6280 }
6281
6282 /* Try the atomic-exchange optab... */
6283 ret = maybe_emit_atomic_exchange (subtarget, mem, trueval, model);
6284
6285 /* ... then an atomic-compare-and-swap loop ... */
6286 if (!ret)
6287 ret = maybe_emit_compare_and_swap_exchange_loop (subtarget, mem, trueval);
6288
6289 /* ... before trying the vaguely defined legacy lock_test_and_set. */
6290 if (!ret)
6291 ret = maybe_emit_sync_lock_test_and_set (subtarget, mem, trueval, model);
6292
6293 /* Recall that the legacy lock_test_and_set optab was allowed to do magic
6294 things with the value 1. Thus we try again without trueval. */
6295 if (!ret && targetm.atomic_test_and_set_trueval != 1)
6296 ret = maybe_emit_sync_lock_test_and_set (subtarget, mem, const1_rtx, model);
6297
6298 /* Failing all else, assume a single threaded environment and simply
6299 perform the operation. */
6300 if (!ret)
6301 {
6302 /* If the result is ignored skip the move to target. */
6303 if (subtarget != const0_rtx)
6304 emit_move_insn (subtarget, mem);
6305
6306 emit_move_insn (mem, trueval);
6307 ret = subtarget;
6308 }
6309
6310 /* Recall that have to return a boolean value; rectify if trueval
6311 is not exactly one. */
6312 if (targetm.atomic_test_and_set_trueval != 1)
6313 ret = emit_store_flag_force (target, NE, ret, const0_rtx, mode, 0, 1);
6314
6315 return ret;
6316 }
6317
6318 /* This function expands the atomic exchange operation:
6319 atomically store VAL in MEM and return the previous value in MEM.
6320
6321 MEMMODEL is the memory model variant to use.
6322 TARGET is an optional place to stick the return value. */
6323
6324 rtx
expand_atomic_exchange(rtx target,rtx mem,rtx val,enum memmodel model)6325 expand_atomic_exchange (rtx target, rtx mem, rtx val, enum memmodel model)
6326 {
6327 machine_mode mode = GET_MODE (mem);
6328 rtx ret;
6329
6330 /* If loads are not atomic for the required size and we are not called to
6331 provide a __sync builtin, do not do anything so that we stay consistent
6332 with atomic loads of the same size. */
6333 if (!can_atomic_load_p (mode) && !is_mm_sync (model))
6334 return NULL_RTX;
6335
6336 ret = maybe_emit_atomic_exchange (target, mem, val, model);
6337
6338 /* Next try a compare-and-swap loop for the exchange. */
6339 if (!ret)
6340 ret = maybe_emit_compare_and_swap_exchange_loop (target, mem, val);
6341
6342 return ret;
6343 }
6344
6345 /* This function expands the atomic compare exchange operation:
6346
6347 *PTARGET_BOOL is an optional place to store the boolean success/failure.
6348 *PTARGET_OVAL is an optional place to store the old value from memory.
6349 Both target parameters may be NULL or const0_rtx to indicate that we do
6350 not care about that return value. Both target parameters are updated on
6351 success to the actual location of the corresponding result.
6352
6353 MEMMODEL is the memory model variant to use.
6354
6355 The return value of the function is true for success. */
6356
6357 bool
expand_atomic_compare_and_swap(rtx * ptarget_bool,rtx * ptarget_oval,rtx mem,rtx expected,rtx desired,bool is_weak,enum memmodel succ_model,enum memmodel fail_model)6358 expand_atomic_compare_and_swap (rtx *ptarget_bool, rtx *ptarget_oval,
6359 rtx mem, rtx expected, rtx desired,
6360 bool is_weak, enum memmodel succ_model,
6361 enum memmodel fail_model)
6362 {
6363 machine_mode mode = GET_MODE (mem);
6364 struct expand_operand ops[8];
6365 enum insn_code icode;
6366 rtx target_oval, target_bool = NULL_RTX;
6367 rtx libfunc;
6368
6369 /* If loads are not atomic for the required size and we are not called to
6370 provide a __sync builtin, do not do anything so that we stay consistent
6371 with atomic loads of the same size. */
6372 if (!can_atomic_load_p (mode) && !is_mm_sync (succ_model))
6373 return false;
6374
6375 /* Load expected into a register for the compare and swap. */
6376 if (MEM_P (expected))
6377 expected = copy_to_reg (expected);
6378
6379 /* Make sure we always have some place to put the return oldval.
6380 Further, make sure that place is distinct from the input expected,
6381 just in case we need that path down below. */
6382 if (ptarget_oval && *ptarget_oval == const0_rtx)
6383 ptarget_oval = NULL;
6384
6385 if (ptarget_oval == NULL
6386 || (target_oval = *ptarget_oval) == NULL
6387 || reg_overlap_mentioned_p (expected, target_oval))
6388 target_oval = gen_reg_rtx (mode);
6389
6390 icode = direct_optab_handler (atomic_compare_and_swap_optab, mode);
6391 if (icode != CODE_FOR_nothing)
6392 {
6393 machine_mode bool_mode = insn_data[icode].operand[0].mode;
6394
6395 if (ptarget_bool && *ptarget_bool == const0_rtx)
6396 ptarget_bool = NULL;
6397
6398 /* Make sure we always have a place for the bool operand. */
6399 if (ptarget_bool == NULL
6400 || (target_bool = *ptarget_bool) == NULL
6401 || GET_MODE (target_bool) != bool_mode)
6402 target_bool = gen_reg_rtx (bool_mode);
6403
6404 /* Emit the compare_and_swap. */
6405 create_output_operand (&ops[0], target_bool, bool_mode);
6406 create_output_operand (&ops[1], target_oval, mode);
6407 create_fixed_operand (&ops[2], mem);
6408 create_input_operand (&ops[3], expected, mode);
6409 create_input_operand (&ops[4], desired, mode);
6410 create_integer_operand (&ops[5], is_weak);
6411 create_integer_operand (&ops[6], succ_model);
6412 create_integer_operand (&ops[7], fail_model);
6413 if (maybe_expand_insn (icode, 8, ops))
6414 {
6415 /* Return success/failure. */
6416 target_bool = ops[0].value;
6417 target_oval = ops[1].value;
6418 goto success;
6419 }
6420 }
6421
6422 /* Otherwise fall back to the original __sync_val_compare_and_swap
6423 which is always seq-cst. */
6424 icode = optab_handler (sync_compare_and_swap_optab, mode);
6425 if (icode != CODE_FOR_nothing)
6426 {
6427 rtx cc_reg;
6428
6429 create_output_operand (&ops[0], target_oval, mode);
6430 create_fixed_operand (&ops[1], mem);
6431 create_input_operand (&ops[2], expected, mode);
6432 create_input_operand (&ops[3], desired, mode);
6433 if (!maybe_expand_insn (icode, 4, ops))
6434 return false;
6435
6436 target_oval = ops[0].value;
6437
6438 /* If the caller isn't interested in the boolean return value,
6439 skip the computation of it. */
6440 if (ptarget_bool == NULL)
6441 goto success;
6442
6443 /* Otherwise, work out if the compare-and-swap succeeded. */
6444 cc_reg = NULL_RTX;
6445 if (have_insn_for (COMPARE, CCmode))
6446 note_stores (PATTERN (get_last_insn ()), find_cc_set, &cc_reg);
6447 if (cc_reg)
6448 {
6449 target_bool = emit_store_flag_force (target_bool, EQ, cc_reg,
6450 const0_rtx, VOIDmode, 0, 1);
6451 goto success;
6452 }
6453 goto success_bool_from_val;
6454 }
6455
6456 /* Also check for library support for __sync_val_compare_and_swap. */
6457 libfunc = optab_libfunc (sync_compare_and_swap_optab, mode);
6458 if (libfunc != NULL)
6459 {
6460 rtx addr = convert_memory_address (ptr_mode, XEXP (mem, 0));
6461 rtx target = emit_library_call_value (libfunc, NULL_RTX, LCT_NORMAL,
6462 mode, addr, ptr_mode,
6463 expected, mode, desired, mode);
6464 emit_move_insn (target_oval, target);
6465
6466 /* Compute the boolean return value only if requested. */
6467 if (ptarget_bool)
6468 goto success_bool_from_val;
6469 else
6470 goto success;
6471 }
6472
6473 /* Failure. */
6474 return false;
6475
6476 success_bool_from_val:
6477 target_bool = emit_store_flag_force (target_bool, EQ, target_oval,
6478 expected, VOIDmode, 1, 1);
6479 success:
6480 /* Make sure that the oval output winds up where the caller asked. */
6481 if (ptarget_oval)
6482 *ptarget_oval = target_oval;
6483 if (ptarget_bool)
6484 *ptarget_bool = target_bool;
6485 return true;
6486 }
6487
6488 /* Generate asm volatile("" : : : "memory") as the memory blockage. */
6489
6490 static void
expand_asm_memory_blockage(void)6491 expand_asm_memory_blockage (void)
6492 {
6493 rtx asm_op, clob;
6494
6495 asm_op = gen_rtx_ASM_OPERANDS (VOIDmode, "", "", 0,
6496 rtvec_alloc (0), rtvec_alloc (0),
6497 rtvec_alloc (0), UNKNOWN_LOCATION);
6498 MEM_VOLATILE_P (asm_op) = 1;
6499
6500 clob = gen_rtx_SCRATCH (VOIDmode);
6501 clob = gen_rtx_MEM (BLKmode, clob);
6502 clob = gen_rtx_CLOBBER (VOIDmode, clob);
6503
6504 emit_insn (gen_rtx_PARALLEL (VOIDmode, gen_rtvec (2, asm_op, clob)));
6505 }
6506
6507 /* Do not propagate memory accesses across this point. */
6508
6509 static void
expand_memory_blockage(void)6510 expand_memory_blockage (void)
6511 {
6512 if (targetm.have_memory_blockage ())
6513 emit_insn (targetm.gen_memory_blockage ());
6514 else
6515 expand_asm_memory_blockage ();
6516 }
6517
6518 /* This routine will either emit the mem_thread_fence pattern or issue a
6519 sync_synchronize to generate a fence for memory model MEMMODEL. */
6520
6521 void
expand_mem_thread_fence(enum memmodel model)6522 expand_mem_thread_fence (enum memmodel model)
6523 {
6524 if (is_mm_relaxed (model))
6525 return;
6526 if (targetm.have_mem_thread_fence ())
6527 {
6528 emit_insn (targetm.gen_mem_thread_fence (GEN_INT (model)));
6529 expand_memory_blockage ();
6530 }
6531 else if (targetm.have_memory_barrier ())
6532 emit_insn (targetm.gen_memory_barrier ());
6533 else if (synchronize_libfunc != NULL_RTX)
6534 emit_library_call (synchronize_libfunc, LCT_NORMAL, VOIDmode);
6535 else
6536 expand_memory_blockage ();
6537 }
6538
6539 /* Emit a signal fence with given memory model. */
6540
6541 void
expand_mem_signal_fence(enum memmodel model)6542 expand_mem_signal_fence (enum memmodel model)
6543 {
6544 /* No machine barrier is required to implement a signal fence, but
6545 a compiler memory barrier must be issued, except for relaxed MM. */
6546 if (!is_mm_relaxed (model))
6547 expand_memory_blockage ();
6548 }
6549
6550 /* This function expands the atomic load operation:
6551 return the atomically loaded value in MEM.
6552
6553 MEMMODEL is the memory model variant to use.
6554 TARGET is an option place to stick the return value. */
6555
6556 rtx
expand_atomic_load(rtx target,rtx mem,enum memmodel model)6557 expand_atomic_load (rtx target, rtx mem, enum memmodel model)
6558 {
6559 machine_mode mode = GET_MODE (mem);
6560 enum insn_code icode;
6561
6562 /* If the target supports the load directly, great. */
6563 icode = direct_optab_handler (atomic_load_optab, mode);
6564 if (icode != CODE_FOR_nothing)
6565 {
6566 struct expand_operand ops[3];
6567 rtx_insn *last = get_last_insn ();
6568 if (is_mm_seq_cst (model))
6569 expand_memory_blockage ();
6570
6571 create_output_operand (&ops[0], target, mode);
6572 create_fixed_operand (&ops[1], mem);
6573 create_integer_operand (&ops[2], model);
6574 if (maybe_expand_insn (icode, 3, ops))
6575 {
6576 if (!is_mm_relaxed (model))
6577 expand_memory_blockage ();
6578 return ops[0].value;
6579 }
6580 delete_insns_since (last);
6581 }
6582
6583 /* If the size of the object is greater than word size on this target,
6584 then we assume that a load will not be atomic. We could try to
6585 emulate a load with a compare-and-swap operation, but the store that
6586 doing this could result in would be incorrect if this is a volatile
6587 atomic load or targetting read-only-mapped memory. */
6588 if (maybe_gt (GET_MODE_PRECISION (mode), BITS_PER_WORD))
6589 /* If there is no atomic load, leave the library call. */
6590 return NULL_RTX;
6591
6592 /* Otherwise assume loads are atomic, and emit the proper barriers. */
6593 if (!target || target == const0_rtx)
6594 target = gen_reg_rtx (mode);
6595
6596 /* For SEQ_CST, emit a barrier before the load. */
6597 if (is_mm_seq_cst (model))
6598 expand_mem_thread_fence (model);
6599
6600 emit_move_insn (target, mem);
6601
6602 /* Emit the appropriate barrier after the load. */
6603 expand_mem_thread_fence (model);
6604
6605 return target;
6606 }
6607
6608 /* This function expands the atomic store operation:
6609 Atomically store VAL in MEM.
6610 MEMMODEL is the memory model variant to use.
6611 USE_RELEASE is true if __sync_lock_release can be used as a fall back.
6612 function returns const0_rtx if a pattern was emitted. */
6613
6614 rtx
expand_atomic_store(rtx mem,rtx val,enum memmodel model,bool use_release)6615 expand_atomic_store (rtx mem, rtx val, enum memmodel model, bool use_release)
6616 {
6617 machine_mode mode = GET_MODE (mem);
6618 enum insn_code icode;
6619 struct expand_operand ops[3];
6620
6621 /* If the target supports the store directly, great. */
6622 icode = direct_optab_handler (atomic_store_optab, mode);
6623 if (icode != CODE_FOR_nothing)
6624 {
6625 rtx_insn *last = get_last_insn ();
6626 if (!is_mm_relaxed (model))
6627 expand_memory_blockage ();
6628 create_fixed_operand (&ops[0], mem);
6629 create_input_operand (&ops[1], val, mode);
6630 create_integer_operand (&ops[2], model);
6631 if (maybe_expand_insn (icode, 3, ops))
6632 {
6633 if (is_mm_seq_cst (model))
6634 expand_memory_blockage ();
6635 return const0_rtx;
6636 }
6637 delete_insns_since (last);
6638 }
6639
6640 /* If using __sync_lock_release is a viable alternative, try it.
6641 Note that this will not be set to true if we are expanding a generic
6642 __atomic_store_n. */
6643 if (use_release)
6644 {
6645 icode = direct_optab_handler (sync_lock_release_optab, mode);
6646 if (icode != CODE_FOR_nothing)
6647 {
6648 create_fixed_operand (&ops[0], mem);
6649 create_input_operand (&ops[1], const0_rtx, mode);
6650 if (maybe_expand_insn (icode, 2, ops))
6651 {
6652 /* lock_release is only a release barrier. */
6653 if (is_mm_seq_cst (model))
6654 expand_mem_thread_fence (model);
6655 return const0_rtx;
6656 }
6657 }
6658 }
6659
6660 /* If the size of the object is greater than word size on this target,
6661 a default store will not be atomic. */
6662 if (maybe_gt (GET_MODE_PRECISION (mode), BITS_PER_WORD))
6663 {
6664 /* If loads are atomic or we are called to provide a __sync builtin,
6665 we can try a atomic_exchange and throw away the result. Otherwise,
6666 don't do anything so that we do not create an inconsistency between
6667 loads and stores. */
6668 if (can_atomic_load_p (mode) || is_mm_sync (model))
6669 {
6670 rtx target = maybe_emit_atomic_exchange (NULL_RTX, mem, val, model);
6671 if (!target)
6672 target = maybe_emit_compare_and_swap_exchange_loop (NULL_RTX, mem,
6673 val);
6674 if (target)
6675 return const0_rtx;
6676 }
6677 return NULL_RTX;
6678 }
6679
6680 /* Otherwise assume stores are atomic, and emit the proper barriers. */
6681 expand_mem_thread_fence (model);
6682
6683 emit_move_insn (mem, val);
6684
6685 /* For SEQ_CST, also emit a barrier after the store. */
6686 if (is_mm_seq_cst (model))
6687 expand_mem_thread_fence (model);
6688
6689 return const0_rtx;
6690 }
6691
6692
6693 /* Structure containing the pointers and values required to process the
6694 various forms of the atomic_fetch_op and atomic_op_fetch builtins. */
6695
6696 struct atomic_op_functions
6697 {
6698 direct_optab mem_fetch_before;
6699 direct_optab mem_fetch_after;
6700 direct_optab mem_no_result;
6701 optab fetch_before;
6702 optab fetch_after;
6703 direct_optab no_result;
6704 enum rtx_code reverse_code;
6705 };
6706
6707
6708 /* Fill in structure pointed to by OP with the various optab entries for an
6709 operation of type CODE. */
6710
6711 static void
get_atomic_op_for_code(struct atomic_op_functions * op,enum rtx_code code)6712 get_atomic_op_for_code (struct atomic_op_functions *op, enum rtx_code code)
6713 {
6714 gcc_assert (op!= NULL);
6715
6716 /* If SWITCHABLE_TARGET is defined, then subtargets can be switched
6717 in the source code during compilation, and the optab entries are not
6718 computable until runtime. Fill in the values at runtime. */
6719 switch (code)
6720 {
6721 case PLUS:
6722 op->mem_fetch_before = atomic_fetch_add_optab;
6723 op->mem_fetch_after = atomic_add_fetch_optab;
6724 op->mem_no_result = atomic_add_optab;
6725 op->fetch_before = sync_old_add_optab;
6726 op->fetch_after = sync_new_add_optab;
6727 op->no_result = sync_add_optab;
6728 op->reverse_code = MINUS;
6729 break;
6730 case MINUS:
6731 op->mem_fetch_before = atomic_fetch_sub_optab;
6732 op->mem_fetch_after = atomic_sub_fetch_optab;
6733 op->mem_no_result = atomic_sub_optab;
6734 op->fetch_before = sync_old_sub_optab;
6735 op->fetch_after = sync_new_sub_optab;
6736 op->no_result = sync_sub_optab;
6737 op->reverse_code = PLUS;
6738 break;
6739 case XOR:
6740 op->mem_fetch_before = atomic_fetch_xor_optab;
6741 op->mem_fetch_after = atomic_xor_fetch_optab;
6742 op->mem_no_result = atomic_xor_optab;
6743 op->fetch_before = sync_old_xor_optab;
6744 op->fetch_after = sync_new_xor_optab;
6745 op->no_result = sync_xor_optab;
6746 op->reverse_code = XOR;
6747 break;
6748 case AND:
6749 op->mem_fetch_before = atomic_fetch_and_optab;
6750 op->mem_fetch_after = atomic_and_fetch_optab;
6751 op->mem_no_result = atomic_and_optab;
6752 op->fetch_before = sync_old_and_optab;
6753 op->fetch_after = sync_new_and_optab;
6754 op->no_result = sync_and_optab;
6755 op->reverse_code = UNKNOWN;
6756 break;
6757 case IOR:
6758 op->mem_fetch_before = atomic_fetch_or_optab;
6759 op->mem_fetch_after = atomic_or_fetch_optab;
6760 op->mem_no_result = atomic_or_optab;
6761 op->fetch_before = sync_old_ior_optab;
6762 op->fetch_after = sync_new_ior_optab;
6763 op->no_result = sync_ior_optab;
6764 op->reverse_code = UNKNOWN;
6765 break;
6766 case NOT:
6767 op->mem_fetch_before = atomic_fetch_nand_optab;
6768 op->mem_fetch_after = atomic_nand_fetch_optab;
6769 op->mem_no_result = atomic_nand_optab;
6770 op->fetch_before = sync_old_nand_optab;
6771 op->fetch_after = sync_new_nand_optab;
6772 op->no_result = sync_nand_optab;
6773 op->reverse_code = UNKNOWN;
6774 break;
6775 default:
6776 gcc_unreachable ();
6777 }
6778 }
6779
6780 /* See if there is a more optimal way to implement the operation "*MEM CODE VAL"
6781 using memory order MODEL. If AFTER is true the operation needs to return
6782 the value of *MEM after the operation, otherwise the previous value.
6783 TARGET is an optional place to place the result. The result is unused if
6784 it is const0_rtx.
6785 Return the result if there is a better sequence, otherwise NULL_RTX. */
6786
6787 static rtx
maybe_optimize_fetch_op(rtx target,rtx mem,rtx val,enum rtx_code code,enum memmodel model,bool after)6788 maybe_optimize_fetch_op (rtx target, rtx mem, rtx val, enum rtx_code code,
6789 enum memmodel model, bool after)
6790 {
6791 /* If the value is prefetched, or not used, it may be possible to replace
6792 the sequence with a native exchange operation. */
6793 if (!after || target == const0_rtx)
6794 {
6795 /* fetch_and (&x, 0, m) can be replaced with exchange (&x, 0, m). */
6796 if (code == AND && val == const0_rtx)
6797 {
6798 if (target == const0_rtx)
6799 target = gen_reg_rtx (GET_MODE (mem));
6800 return maybe_emit_atomic_exchange (target, mem, val, model);
6801 }
6802
6803 /* fetch_or (&x, -1, m) can be replaced with exchange (&x, -1, m). */
6804 if (code == IOR && val == constm1_rtx)
6805 {
6806 if (target == const0_rtx)
6807 target = gen_reg_rtx (GET_MODE (mem));
6808 return maybe_emit_atomic_exchange (target, mem, val, model);
6809 }
6810 }
6811
6812 return NULL_RTX;
6813 }
6814
6815 /* Try to emit an instruction for a specific operation varaition.
6816 OPTAB contains the OP functions.
6817 TARGET is an optional place to return the result. const0_rtx means unused.
6818 MEM is the memory location to operate on.
6819 VAL is the value to use in the operation.
6820 USE_MEMMODEL is TRUE if the variation with a memory model should be tried.
6821 MODEL is the memory model, if used.
6822 AFTER is true if the returned result is the value after the operation. */
6823
6824 static rtx
maybe_emit_op(const struct atomic_op_functions * optab,rtx target,rtx mem,rtx val,bool use_memmodel,enum memmodel model,bool after)6825 maybe_emit_op (const struct atomic_op_functions *optab, rtx target, rtx mem,
6826 rtx val, bool use_memmodel, enum memmodel model, bool after)
6827 {
6828 machine_mode mode = GET_MODE (mem);
6829 struct expand_operand ops[4];
6830 enum insn_code icode;
6831 int op_counter = 0;
6832 int num_ops;
6833
6834 /* Check to see if there is a result returned. */
6835 if (target == const0_rtx)
6836 {
6837 if (use_memmodel)
6838 {
6839 icode = direct_optab_handler (optab->mem_no_result, mode);
6840 create_integer_operand (&ops[2], model);
6841 num_ops = 3;
6842 }
6843 else
6844 {
6845 icode = direct_optab_handler (optab->no_result, mode);
6846 num_ops = 2;
6847 }
6848 }
6849 /* Otherwise, we need to generate a result. */
6850 else
6851 {
6852 if (use_memmodel)
6853 {
6854 icode = direct_optab_handler (after ? optab->mem_fetch_after
6855 : optab->mem_fetch_before, mode);
6856 create_integer_operand (&ops[3], model);
6857 num_ops = 4;
6858 }
6859 else
6860 {
6861 icode = optab_handler (after ? optab->fetch_after
6862 : optab->fetch_before, mode);
6863 num_ops = 3;
6864 }
6865 create_output_operand (&ops[op_counter++], target, mode);
6866 }
6867 if (icode == CODE_FOR_nothing)
6868 return NULL_RTX;
6869
6870 create_fixed_operand (&ops[op_counter++], mem);
6871 /* VAL may have been promoted to a wider mode. Shrink it if so. */
6872 create_convert_operand_to (&ops[op_counter++], val, mode, true);
6873
6874 if (maybe_expand_insn (icode, num_ops, ops))
6875 return (target == const0_rtx ? const0_rtx : ops[0].value);
6876
6877 return NULL_RTX;
6878 }
6879
6880
6881 /* This function expands an atomic fetch_OP or OP_fetch operation:
6882 TARGET is an option place to stick the return value. const0_rtx indicates
6883 the result is unused.
6884 atomically fetch MEM, perform the operation with VAL and return it to MEM.
6885 CODE is the operation being performed (OP)
6886 MEMMODEL is the memory model variant to use.
6887 AFTER is true to return the result of the operation (OP_fetch).
6888 AFTER is false to return the value before the operation (fetch_OP).
6889
6890 This function will *only* generate instructions if there is a direct
6891 optab. No compare and swap loops or libcalls will be generated. */
6892
6893 static rtx
expand_atomic_fetch_op_no_fallback(rtx target,rtx mem,rtx val,enum rtx_code code,enum memmodel model,bool after)6894 expand_atomic_fetch_op_no_fallback (rtx target, rtx mem, rtx val,
6895 enum rtx_code code, enum memmodel model,
6896 bool after)
6897 {
6898 machine_mode mode = GET_MODE (mem);
6899 struct atomic_op_functions optab;
6900 rtx result;
6901 bool unused_result = (target == const0_rtx);
6902
6903 get_atomic_op_for_code (&optab, code);
6904
6905 /* Check to see if there are any better instructions. */
6906 result = maybe_optimize_fetch_op (target, mem, val, code, model, after);
6907 if (result)
6908 return result;
6909
6910 /* Check for the case where the result isn't used and try those patterns. */
6911 if (unused_result)
6912 {
6913 /* Try the memory model variant first. */
6914 result = maybe_emit_op (&optab, target, mem, val, true, model, true);
6915 if (result)
6916 return result;
6917
6918 /* Next try the old style withuot a memory model. */
6919 result = maybe_emit_op (&optab, target, mem, val, false, model, true);
6920 if (result)
6921 return result;
6922
6923 /* There is no no-result pattern, so try patterns with a result. */
6924 target = NULL_RTX;
6925 }
6926
6927 /* Try the __atomic version. */
6928 result = maybe_emit_op (&optab, target, mem, val, true, model, after);
6929 if (result)
6930 return result;
6931
6932 /* Try the older __sync version. */
6933 result = maybe_emit_op (&optab, target, mem, val, false, model, after);
6934 if (result)
6935 return result;
6936
6937 /* If the fetch value can be calculated from the other variation of fetch,
6938 try that operation. */
6939 if (after || unused_result || optab.reverse_code != UNKNOWN)
6940 {
6941 /* Try the __atomic version, then the older __sync version. */
6942 result = maybe_emit_op (&optab, target, mem, val, true, model, !after);
6943 if (!result)
6944 result = maybe_emit_op (&optab, target, mem, val, false, model, !after);
6945
6946 if (result)
6947 {
6948 /* If the result isn't used, no need to do compensation code. */
6949 if (unused_result)
6950 return result;
6951
6952 /* Issue compensation code. Fetch_after == fetch_before OP val.
6953 Fetch_before == after REVERSE_OP val. */
6954 if (!after)
6955 code = optab.reverse_code;
6956 if (code == NOT)
6957 {
6958 result = expand_simple_binop (mode, AND, result, val, NULL_RTX,
6959 true, OPTAB_LIB_WIDEN);
6960 result = expand_simple_unop (mode, NOT, result, target, true);
6961 }
6962 else
6963 result = expand_simple_binop (mode, code, result, val, target,
6964 true, OPTAB_LIB_WIDEN);
6965 return result;
6966 }
6967 }
6968
6969 /* No direct opcode can be generated. */
6970 return NULL_RTX;
6971 }
6972
6973
6974
6975 /* This function expands an atomic fetch_OP or OP_fetch operation:
6976 TARGET is an option place to stick the return value. const0_rtx indicates
6977 the result is unused.
6978 atomically fetch MEM, perform the operation with VAL and return it to MEM.
6979 CODE is the operation being performed (OP)
6980 MEMMODEL is the memory model variant to use.
6981 AFTER is true to return the result of the operation (OP_fetch).
6982 AFTER is false to return the value before the operation (fetch_OP). */
6983 rtx
expand_atomic_fetch_op(rtx target,rtx mem,rtx val,enum rtx_code code,enum memmodel model,bool after)6984 expand_atomic_fetch_op (rtx target, rtx mem, rtx val, enum rtx_code code,
6985 enum memmodel model, bool after)
6986 {
6987 machine_mode mode = GET_MODE (mem);
6988 rtx result;
6989 bool unused_result = (target == const0_rtx);
6990
6991 /* If loads are not atomic for the required size and we are not called to
6992 provide a __sync builtin, do not do anything so that we stay consistent
6993 with atomic loads of the same size. */
6994 if (!can_atomic_load_p (mode) && !is_mm_sync (model))
6995 return NULL_RTX;
6996
6997 result = expand_atomic_fetch_op_no_fallback (target, mem, val, code, model,
6998 after);
6999
7000 if (result)
7001 return result;
7002
7003 /* Add/sub can be implemented by doing the reverse operation with -(val). */
7004 if (code == PLUS || code == MINUS)
7005 {
7006 rtx tmp;
7007 enum rtx_code reverse = (code == PLUS ? MINUS : PLUS);
7008
7009 start_sequence ();
7010 tmp = expand_simple_unop (mode, NEG, val, NULL_RTX, true);
7011 result = expand_atomic_fetch_op_no_fallback (target, mem, tmp, reverse,
7012 model, after);
7013 if (result)
7014 {
7015 /* PLUS worked so emit the insns and return. */
7016 tmp = get_insns ();
7017 end_sequence ();
7018 emit_insn (tmp);
7019 return result;
7020 }
7021
7022 /* PLUS did not work, so throw away the negation code and continue. */
7023 end_sequence ();
7024 }
7025
7026 /* Try the __sync libcalls only if we can't do compare-and-swap inline. */
7027 if (!can_compare_and_swap_p (mode, false))
7028 {
7029 rtx libfunc;
7030 bool fixup = false;
7031 enum rtx_code orig_code = code;
7032 struct atomic_op_functions optab;
7033
7034 get_atomic_op_for_code (&optab, code);
7035 libfunc = optab_libfunc (after ? optab.fetch_after
7036 : optab.fetch_before, mode);
7037 if (libfunc == NULL
7038 && (after || unused_result || optab.reverse_code != UNKNOWN))
7039 {
7040 fixup = true;
7041 if (!after)
7042 code = optab.reverse_code;
7043 libfunc = optab_libfunc (after ? optab.fetch_before
7044 : optab.fetch_after, mode);
7045 }
7046 if (libfunc != NULL)
7047 {
7048 rtx addr = convert_memory_address (ptr_mode, XEXP (mem, 0));
7049 result = emit_library_call_value (libfunc, NULL, LCT_NORMAL, mode,
7050 addr, ptr_mode, val, mode);
7051
7052 if (!unused_result && fixup)
7053 result = expand_simple_binop (mode, code, result, val, target,
7054 true, OPTAB_LIB_WIDEN);
7055 return result;
7056 }
7057
7058 /* We need the original code for any further attempts. */
7059 code = orig_code;
7060 }
7061
7062 /* If nothing else has succeeded, default to a compare and swap loop. */
7063 if (can_compare_and_swap_p (mode, true))
7064 {
7065 rtx_insn *insn;
7066 rtx t0 = gen_reg_rtx (mode), t1;
7067
7068 start_sequence ();
7069
7070 /* If the result is used, get a register for it. */
7071 if (!unused_result)
7072 {
7073 if (!target || !register_operand (target, mode))
7074 target = gen_reg_rtx (mode);
7075 /* If fetch_before, copy the value now. */
7076 if (!after)
7077 emit_move_insn (target, t0);
7078 }
7079 else
7080 target = const0_rtx;
7081
7082 t1 = t0;
7083 if (code == NOT)
7084 {
7085 t1 = expand_simple_binop (mode, AND, t1, val, NULL_RTX,
7086 true, OPTAB_LIB_WIDEN);
7087 t1 = expand_simple_unop (mode, code, t1, NULL_RTX, true);
7088 }
7089 else
7090 t1 = expand_simple_binop (mode, code, t1, val, NULL_RTX, true,
7091 OPTAB_LIB_WIDEN);
7092
7093 /* For after, copy the value now. */
7094 if (!unused_result && after)
7095 emit_move_insn (target, t1);
7096 insn = get_insns ();
7097 end_sequence ();
7098
7099 if (t1 != NULL && expand_compare_and_swap_loop (mem, t0, t1, insn))
7100 return target;
7101 }
7102
7103 return NULL_RTX;
7104 }
7105
7106 /* Return true if OPERAND is suitable for operand number OPNO of
7107 instruction ICODE. */
7108
7109 bool
insn_operand_matches(enum insn_code icode,unsigned int opno,rtx operand)7110 insn_operand_matches (enum insn_code icode, unsigned int opno, rtx operand)
7111 {
7112 return (!insn_data[(int) icode].operand[opno].predicate
7113 || (insn_data[(int) icode].operand[opno].predicate
7114 (operand, insn_data[(int) icode].operand[opno].mode)));
7115 }
7116
7117 /* TARGET is a target of a multiword operation that we are going to
7118 implement as a series of word-mode operations. Return true if
7119 TARGET is suitable for this purpose. */
7120
7121 bool
valid_multiword_target_p(rtx target)7122 valid_multiword_target_p (rtx target)
7123 {
7124 machine_mode mode;
7125 int i, size;
7126
7127 mode = GET_MODE (target);
7128 if (!GET_MODE_SIZE (mode).is_constant (&size))
7129 return false;
7130 for (i = 0; i < size; i += UNITS_PER_WORD)
7131 if (!validate_subreg (word_mode, mode, target, i))
7132 return false;
7133 return true;
7134 }
7135
7136 /* Make OP describe an input operand that has value INTVAL and that has
7137 no inherent mode. This function should only be used for operands that
7138 are always expand-time constants. The backend may request that INTVAL
7139 be copied into a different kind of rtx, but it must specify the mode
7140 of that rtx if so. */
7141
7142 void
create_integer_operand(struct expand_operand * op,poly_int64 intval)7143 create_integer_operand (struct expand_operand *op, poly_int64 intval)
7144 {
7145 create_expand_operand (op, EXPAND_INTEGER,
7146 gen_int_mode (intval, MAX_MODE_INT),
7147 VOIDmode, false, intval);
7148 }
7149
7150 /* Like maybe_legitimize_operand, but do not change the code of the
7151 current rtx value. */
7152
7153 static bool
maybe_legitimize_operand_same_code(enum insn_code icode,unsigned int opno,struct expand_operand * op)7154 maybe_legitimize_operand_same_code (enum insn_code icode, unsigned int opno,
7155 struct expand_operand *op)
7156 {
7157 /* See if the operand matches in its current form. */
7158 if (insn_operand_matches (icode, opno, op->value))
7159 return true;
7160
7161 /* If the operand is a memory whose address has no side effects,
7162 try forcing the address into a non-virtual pseudo register.
7163 The check for side effects is important because copy_to_mode_reg
7164 cannot handle things like auto-modified addresses. */
7165 if (insn_data[(int) icode].operand[opno].allows_mem && MEM_P (op->value))
7166 {
7167 rtx addr, mem;
7168
7169 mem = op->value;
7170 addr = XEXP (mem, 0);
7171 if (!(REG_P (addr) && REGNO (addr) > LAST_VIRTUAL_REGISTER)
7172 && !side_effects_p (addr))
7173 {
7174 rtx_insn *last;
7175 machine_mode mode;
7176
7177 last = get_last_insn ();
7178 mode = get_address_mode (mem);
7179 mem = replace_equiv_address (mem, copy_to_mode_reg (mode, addr));
7180 if (insn_operand_matches (icode, opno, mem))
7181 {
7182 op->value = mem;
7183 return true;
7184 }
7185 delete_insns_since (last);
7186 }
7187 }
7188
7189 return false;
7190 }
7191
7192 /* Try to make OP match operand OPNO of instruction ICODE. Return true
7193 on success, storing the new operand value back in OP. */
7194
7195 static bool
maybe_legitimize_operand(enum insn_code icode,unsigned int opno,struct expand_operand * op)7196 maybe_legitimize_operand (enum insn_code icode, unsigned int opno,
7197 struct expand_operand *op)
7198 {
7199 machine_mode mode, imode;
7200 bool old_volatile_ok, result;
7201
7202 mode = op->mode;
7203 switch (op->type)
7204 {
7205 case EXPAND_FIXED:
7206 old_volatile_ok = volatile_ok;
7207 volatile_ok = true;
7208 result = maybe_legitimize_operand_same_code (icode, opno, op);
7209 volatile_ok = old_volatile_ok;
7210 return result;
7211
7212 case EXPAND_OUTPUT:
7213 gcc_assert (mode != VOIDmode);
7214 if (op->value
7215 && op->value != const0_rtx
7216 && GET_MODE (op->value) == mode
7217 && maybe_legitimize_operand_same_code (icode, opno, op))
7218 return true;
7219
7220 op->value = gen_reg_rtx (mode);
7221 op->target = 0;
7222 break;
7223
7224 case EXPAND_INPUT:
7225 input:
7226 gcc_assert (mode != VOIDmode);
7227 gcc_assert (GET_MODE (op->value) == VOIDmode
7228 || GET_MODE (op->value) == mode);
7229 if (maybe_legitimize_operand_same_code (icode, opno, op))
7230 return true;
7231
7232 op->value = copy_to_mode_reg (mode, op->value);
7233 break;
7234
7235 case EXPAND_CONVERT_TO:
7236 gcc_assert (mode != VOIDmode);
7237 op->value = convert_to_mode (mode, op->value, op->unsigned_p);
7238 goto input;
7239
7240 case EXPAND_CONVERT_FROM:
7241 if (GET_MODE (op->value) != VOIDmode)
7242 mode = GET_MODE (op->value);
7243 else
7244 /* The caller must tell us what mode this value has. */
7245 gcc_assert (mode != VOIDmode);
7246
7247 imode = insn_data[(int) icode].operand[opno].mode;
7248 if (imode != VOIDmode && imode != mode)
7249 {
7250 op->value = convert_modes (imode, mode, op->value, op->unsigned_p);
7251 mode = imode;
7252 }
7253 goto input;
7254
7255 case EXPAND_ADDRESS:
7256 op->value = convert_memory_address (as_a <scalar_int_mode> (mode),
7257 op->value);
7258 goto input;
7259
7260 case EXPAND_INTEGER:
7261 mode = insn_data[(int) icode].operand[opno].mode;
7262 if (mode != VOIDmode
7263 && known_eq (trunc_int_for_mode (op->int_value, mode),
7264 op->int_value))
7265 {
7266 op->value = gen_int_mode (op->int_value, mode);
7267 goto input;
7268 }
7269 break;
7270 }
7271 return insn_operand_matches (icode, opno, op->value);
7272 }
7273
7274 /* Make OP describe an input operand that should have the same value
7275 as VALUE, after any mode conversion that the target might request.
7276 TYPE is the type of VALUE. */
7277
7278 void
create_convert_operand_from_type(struct expand_operand * op,rtx value,tree type)7279 create_convert_operand_from_type (struct expand_operand *op,
7280 rtx value, tree type)
7281 {
7282 create_convert_operand_from (op, value, TYPE_MODE (type),
7283 TYPE_UNSIGNED (type));
7284 }
7285
7286 /* Return true if the requirements on operands OP1 and OP2 of instruction
7287 ICODE are similar enough for the result of legitimizing OP1 to be
7288 reusable for OP2. OPNO1 and OPNO2 are the operand numbers associated
7289 with OP1 and OP2 respectively. */
7290
7291 static inline bool
can_reuse_operands_p(enum insn_code icode,unsigned int opno1,unsigned int opno2,const struct expand_operand * op1,const struct expand_operand * op2)7292 can_reuse_operands_p (enum insn_code icode,
7293 unsigned int opno1, unsigned int opno2,
7294 const struct expand_operand *op1,
7295 const struct expand_operand *op2)
7296 {
7297 /* Check requirements that are common to all types. */
7298 if (op1->type != op2->type
7299 || op1->mode != op2->mode
7300 || (insn_data[(int) icode].operand[opno1].mode
7301 != insn_data[(int) icode].operand[opno2].mode))
7302 return false;
7303
7304 /* Check the requirements for specific types. */
7305 switch (op1->type)
7306 {
7307 case EXPAND_OUTPUT:
7308 /* Outputs must remain distinct. */
7309 return false;
7310
7311 case EXPAND_FIXED:
7312 case EXPAND_INPUT:
7313 case EXPAND_ADDRESS:
7314 case EXPAND_INTEGER:
7315 return true;
7316
7317 case EXPAND_CONVERT_TO:
7318 case EXPAND_CONVERT_FROM:
7319 return op1->unsigned_p == op2->unsigned_p;
7320 }
7321 gcc_unreachable ();
7322 }
7323
7324 /* Try to make operands [OPS, OPS + NOPS) match operands [OPNO, OPNO + NOPS)
7325 of instruction ICODE. Return true on success, leaving the new operand
7326 values in the OPS themselves. Emit no code on failure. */
7327
7328 bool
maybe_legitimize_operands(enum insn_code icode,unsigned int opno,unsigned int nops,struct expand_operand * ops)7329 maybe_legitimize_operands (enum insn_code icode, unsigned int opno,
7330 unsigned int nops, struct expand_operand *ops)
7331 {
7332 rtx_insn *last = get_last_insn ();
7333 rtx *orig_values = XALLOCAVEC (rtx, nops);
7334 for (unsigned int i = 0; i < nops; i++)
7335 {
7336 orig_values[i] = ops[i].value;
7337
7338 /* First try reusing the result of an earlier legitimization.
7339 This avoids duplicate rtl and ensures that tied operands
7340 remain tied.
7341
7342 This search is linear, but NOPS is bounded at compile time
7343 to a small number (current a single digit). */
7344 unsigned int j = 0;
7345 for (; j < i; ++j)
7346 if (can_reuse_operands_p (icode, opno + j, opno + i, &ops[j], &ops[i])
7347 && rtx_equal_p (orig_values[j], orig_values[i])
7348 && ops[j].value
7349 && insn_operand_matches (icode, opno + i, ops[j].value))
7350 {
7351 ops[i].value = copy_rtx (ops[j].value);
7352 break;
7353 }
7354
7355 /* Otherwise try legitimizing the operand on its own. */
7356 if (j == i && !maybe_legitimize_operand (icode, opno + i, &ops[i]))
7357 {
7358 delete_insns_since (last);
7359 return false;
7360 }
7361 }
7362 return true;
7363 }
7364
7365 /* Try to generate instruction ICODE, using operands [OPS, OPS + NOPS)
7366 as its operands. Return the instruction pattern on success,
7367 and emit any necessary set-up code. Return null and emit no
7368 code on failure. */
7369
7370 rtx_insn *
maybe_gen_insn(enum insn_code icode,unsigned int nops,struct expand_operand * ops)7371 maybe_gen_insn (enum insn_code icode, unsigned int nops,
7372 struct expand_operand *ops)
7373 {
7374 gcc_assert (nops == (unsigned int) insn_data[(int) icode].n_generator_args);
7375 if (!maybe_legitimize_operands (icode, 0, nops, ops))
7376 return NULL;
7377
7378 switch (nops)
7379 {
7380 case 1:
7381 return GEN_FCN (icode) (ops[0].value);
7382 case 2:
7383 return GEN_FCN (icode) (ops[0].value, ops[1].value);
7384 case 3:
7385 return GEN_FCN (icode) (ops[0].value, ops[1].value, ops[2].value);
7386 case 4:
7387 return GEN_FCN (icode) (ops[0].value, ops[1].value, ops[2].value,
7388 ops[3].value);
7389 case 5:
7390 return GEN_FCN (icode) (ops[0].value, ops[1].value, ops[2].value,
7391 ops[3].value, ops[4].value);
7392 case 6:
7393 return GEN_FCN (icode) (ops[0].value, ops[1].value, ops[2].value,
7394 ops[3].value, ops[4].value, ops[5].value);
7395 case 7:
7396 return GEN_FCN (icode) (ops[0].value, ops[1].value, ops[2].value,
7397 ops[3].value, ops[4].value, ops[5].value,
7398 ops[6].value);
7399 case 8:
7400 return GEN_FCN (icode) (ops[0].value, ops[1].value, ops[2].value,
7401 ops[3].value, ops[4].value, ops[5].value,
7402 ops[6].value, ops[7].value);
7403 case 9:
7404 return GEN_FCN (icode) (ops[0].value, ops[1].value, ops[2].value,
7405 ops[3].value, ops[4].value, ops[5].value,
7406 ops[6].value, ops[7].value, ops[8].value);
7407 }
7408 gcc_unreachable ();
7409 }
7410
7411 /* Try to emit instruction ICODE, using operands [OPS, OPS + NOPS)
7412 as its operands. Return true on success and emit no code on failure. */
7413
7414 bool
maybe_expand_insn(enum insn_code icode,unsigned int nops,struct expand_operand * ops)7415 maybe_expand_insn (enum insn_code icode, unsigned int nops,
7416 struct expand_operand *ops)
7417 {
7418 rtx_insn *pat = maybe_gen_insn (icode, nops, ops);
7419 if (pat)
7420 {
7421 emit_insn (pat);
7422 return true;
7423 }
7424 return false;
7425 }
7426
7427 /* Like maybe_expand_insn, but for jumps. */
7428
7429 bool
maybe_expand_jump_insn(enum insn_code icode,unsigned int nops,struct expand_operand * ops)7430 maybe_expand_jump_insn (enum insn_code icode, unsigned int nops,
7431 struct expand_operand *ops)
7432 {
7433 rtx_insn *pat = maybe_gen_insn (icode, nops, ops);
7434 if (pat)
7435 {
7436 emit_jump_insn (pat);
7437 return true;
7438 }
7439 return false;
7440 }
7441
7442 /* Emit instruction ICODE, using operands [OPS, OPS + NOPS)
7443 as its operands. */
7444
7445 void
expand_insn(enum insn_code icode,unsigned int nops,struct expand_operand * ops)7446 expand_insn (enum insn_code icode, unsigned int nops,
7447 struct expand_operand *ops)
7448 {
7449 if (!maybe_expand_insn (icode, nops, ops))
7450 gcc_unreachable ();
7451 }
7452
7453 /* Like expand_insn, but for jumps. */
7454
7455 void
expand_jump_insn(enum insn_code icode,unsigned int nops,struct expand_operand * ops)7456 expand_jump_insn (enum insn_code icode, unsigned int nops,
7457 struct expand_operand *ops)
7458 {
7459 if (!maybe_expand_jump_insn (icode, nops, ops))
7460 gcc_unreachable ();
7461 }
7462