1 /* Expand the basic unary and binary arithmetic operations, for GNU compiler.
2 Copyright (C) 1987-2020 Free Software Foundation, Inc.
3
4 This file is part of GCC.
5
6 GCC is free software; you can redistribute it and/or modify it under
7 the terms of the GNU General Public License as published by the Free
8 Software Foundation; either version 3, or (at your option) any later
9 version.
10
11 GCC is distributed in the hope that it will be useful, but WITHOUT ANY
12 WARRANTY; without even the implied warranty of MERCHANTABILITY or
13 FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
14 for more details.
15
16 You should have received a copy of the GNU General Public License
17 along with GCC; see the file COPYING3. If not see
18 <http://www.gnu.org/licenses/>. */
19
20
21 #include "config.h"
22 #include "system.h"
23 #include "coretypes.h"
24 #include "backend.h"
25 #include "target.h"
26 #include "rtl.h"
27 #include "tree.h"
28 #include "memmodel.h"
29 #include "predict.h"
30 #include "tm_p.h"
31 #include "expmed.h"
32 #include "optabs.h"
33 #include "emit-rtl.h"
34 #include "recog.h"
35 #include "diagnostic-core.h"
36 #include "rtx-vector-builder.h"
37
38 /* Include insn-config.h before expr.h so that HAVE_conditional_move
39 is properly defined. */
40 #include "stor-layout.h"
41 #include "except.h"
42 #include "dojump.h"
43 #include "explow.h"
44 #include "expr.h"
45 #include "optabs-tree.h"
46 #include "libfuncs.h"
47
48 static void prepare_float_lib_cmp (rtx, rtx, enum rtx_code, rtx *,
49 machine_mode *);
50 static rtx expand_unop_direct (machine_mode, optab, rtx, rtx, int);
51 static void emit_libcall_block_1 (rtx_insn *, rtx, rtx, rtx, bool);
52
53 /* Debug facility for use in GDB. */
54 void debug_optab_libfuncs (void);
55
56 /* Add a REG_EQUAL note to the last insn in INSNS. TARGET is being set to
57 the result of operation CODE applied to OP0 (and OP1 if it is a binary
58 operation). OP0_MODE is OP0's mode.
59
60 If the last insn does not set TARGET, don't do anything, but return 1.
61
62 If the last insn or a previous insn sets TARGET and TARGET is one of OP0
63 or OP1, don't add the REG_EQUAL note but return 0. Our caller can then
64 try again, ensuring that TARGET is not one of the operands. */
65
66 static int
add_equal_note(rtx_insn * insns,rtx target,enum rtx_code code,rtx op0,rtx op1,machine_mode op0_mode)67 add_equal_note (rtx_insn *insns, rtx target, enum rtx_code code, rtx op0,
68 rtx op1, machine_mode op0_mode)
69 {
70 rtx_insn *last_insn;
71 rtx set;
72 rtx note;
73
74 gcc_assert (insns && INSN_P (insns) && NEXT_INSN (insns));
75
76 if (GET_RTX_CLASS (code) != RTX_COMM_ARITH
77 && GET_RTX_CLASS (code) != RTX_BIN_ARITH
78 && GET_RTX_CLASS (code) != RTX_COMM_COMPARE
79 && GET_RTX_CLASS (code) != RTX_COMPARE
80 && GET_RTX_CLASS (code) != RTX_UNARY)
81 return 1;
82
83 if (GET_CODE (target) == ZERO_EXTRACT)
84 return 1;
85
86 for (last_insn = insns;
87 NEXT_INSN (last_insn) != NULL_RTX;
88 last_insn = NEXT_INSN (last_insn))
89 ;
90
91 /* If TARGET is in OP0 or OP1, punt. We'd end up with a note referencing
92 a value changing in the insn, so the note would be invalid for CSE. */
93 if (reg_overlap_mentioned_p (target, op0)
94 || (op1 && reg_overlap_mentioned_p (target, op1)))
95 {
96 if (MEM_P (target)
97 && (rtx_equal_p (target, op0)
98 || (op1 && rtx_equal_p (target, op1))))
99 {
100 /* For MEM target, with MEM = MEM op X, prefer no REG_EQUAL note
101 over expanding it as temp = MEM op X, MEM = temp. If the target
102 supports MEM = MEM op X instructions, it is sometimes too hard
103 to reconstruct that form later, especially if X is also a memory,
104 and due to multiple occurrences of addresses the address might
105 be forced into register unnecessarily.
106 Note that not emitting the REG_EQUIV note might inhibit
107 CSE in some cases. */
108 set = single_set (last_insn);
109 if (set
110 && GET_CODE (SET_SRC (set)) == code
111 && MEM_P (SET_DEST (set))
112 && (rtx_equal_p (SET_DEST (set), XEXP (SET_SRC (set), 0))
113 || (op1 && rtx_equal_p (SET_DEST (set),
114 XEXP (SET_SRC (set), 1)))))
115 return 1;
116 }
117 return 0;
118 }
119
120 set = set_for_reg_notes (last_insn);
121 if (set == NULL_RTX)
122 return 1;
123
124 if (! rtx_equal_p (SET_DEST (set), target)
125 /* For a STRICT_LOW_PART, the REG_NOTE applies to what is inside it. */
126 && (GET_CODE (SET_DEST (set)) != STRICT_LOW_PART
127 || ! rtx_equal_p (XEXP (SET_DEST (set), 0), target)))
128 return 1;
129
130 if (GET_RTX_CLASS (code) == RTX_UNARY)
131 switch (code)
132 {
133 case FFS:
134 case CLZ:
135 case CTZ:
136 case CLRSB:
137 case POPCOUNT:
138 case PARITY:
139 case BSWAP:
140 if (op0_mode != VOIDmode && GET_MODE (target) != op0_mode)
141 {
142 note = gen_rtx_fmt_e (code, op0_mode, copy_rtx (op0));
143 if (GET_MODE_UNIT_SIZE (op0_mode)
144 > GET_MODE_UNIT_SIZE (GET_MODE (target)))
145 note = simplify_gen_unary (TRUNCATE, GET_MODE (target),
146 note, op0_mode);
147 else
148 note = simplify_gen_unary (ZERO_EXTEND, GET_MODE (target),
149 note, op0_mode);
150 break;
151 }
152 /* FALLTHRU */
153 default:
154 note = gen_rtx_fmt_e (code, GET_MODE (target), copy_rtx (op0));
155 break;
156 }
157 else
158 note = gen_rtx_fmt_ee (code, GET_MODE (target), copy_rtx (op0), copy_rtx (op1));
159
160 set_unique_reg_note (last_insn, REG_EQUAL, note);
161
162 return 1;
163 }
164
165 /* Given two input operands, OP0 and OP1, determine what the correct from_mode
166 for a widening operation would be. In most cases this would be OP0, but if
167 that's a constant it'll be VOIDmode, which isn't useful. */
168
169 static machine_mode
widened_mode(machine_mode to_mode,rtx op0,rtx op1)170 widened_mode (machine_mode to_mode, rtx op0, rtx op1)
171 {
172 machine_mode m0 = GET_MODE (op0);
173 machine_mode m1 = GET_MODE (op1);
174 machine_mode result;
175
176 if (m0 == VOIDmode && m1 == VOIDmode)
177 return to_mode;
178 else if (m0 == VOIDmode || GET_MODE_UNIT_SIZE (m0) < GET_MODE_UNIT_SIZE (m1))
179 result = m1;
180 else
181 result = m0;
182
183 if (GET_MODE_UNIT_SIZE (result) > GET_MODE_UNIT_SIZE (to_mode))
184 return to_mode;
185
186 return result;
187 }
188
189 /* Widen OP to MODE and return the rtx for the widened operand. UNSIGNEDP
190 says whether OP is signed or unsigned. NO_EXTEND is nonzero if we need
191 not actually do a sign-extend or zero-extend, but can leave the
192 higher-order bits of the result rtx undefined, for example, in the case
193 of logical operations, but not right shifts. */
194
195 static rtx
widen_operand(rtx op,machine_mode mode,machine_mode oldmode,int unsignedp,int no_extend)196 widen_operand (rtx op, machine_mode mode, machine_mode oldmode,
197 int unsignedp, int no_extend)
198 {
199 rtx result;
200 scalar_int_mode int_mode;
201
202 /* If we don't have to extend and this is a constant, return it. */
203 if (no_extend && GET_MODE (op) == VOIDmode)
204 return op;
205
206 /* If we must extend do so. If OP is a SUBREG for a promoted object, also
207 extend since it will be more efficient to do so unless the signedness of
208 a promoted object differs from our extension. */
209 if (! no_extend
210 || !is_a <scalar_int_mode> (mode, &int_mode)
211 || (GET_CODE (op) == SUBREG && SUBREG_PROMOTED_VAR_P (op)
212 && SUBREG_CHECK_PROMOTED_SIGN (op, unsignedp)))
213 return convert_modes (mode, oldmode, op, unsignedp);
214
215 /* If MODE is no wider than a single word, we return a lowpart or paradoxical
216 SUBREG. */
217 if (GET_MODE_SIZE (int_mode) <= UNITS_PER_WORD)
218 return gen_lowpart (int_mode, force_reg (GET_MODE (op), op));
219
220 /* Otherwise, get an object of MODE, clobber it, and set the low-order
221 part to OP. */
222
223 result = gen_reg_rtx (int_mode);
224 emit_clobber (result);
225 emit_move_insn (gen_lowpart (GET_MODE (op), result), op);
226 return result;
227 }
228
229 /* Expand vector widening operations.
230
231 There are two different classes of operations handled here:
232 1) Operations whose result is wider than all the arguments to the operation.
233 Examples: VEC_UNPACK_HI/LO_EXPR, VEC_WIDEN_MULT_HI/LO_EXPR
234 In this case OP0 and optionally OP1 would be initialized,
235 but WIDE_OP wouldn't (not relevant for this case).
236 2) Operations whose result is of the same size as the last argument to the
237 operation, but wider than all the other arguments to the operation.
238 Examples: WIDEN_SUM_EXPR, VEC_DOT_PROD_EXPR.
239 In the case WIDE_OP, OP0 and optionally OP1 would be initialized.
240
241 E.g, when called to expand the following operations, this is how
242 the arguments will be initialized:
243 nops OP0 OP1 WIDE_OP
244 widening-sum 2 oprnd0 - oprnd1
245 widening-dot-product 3 oprnd0 oprnd1 oprnd2
246 widening-mult 2 oprnd0 oprnd1 -
247 type-promotion (vec-unpack) 1 oprnd0 - - */
248
249 rtx
expand_widen_pattern_expr(sepops ops,rtx op0,rtx op1,rtx wide_op,rtx target,int unsignedp)250 expand_widen_pattern_expr (sepops ops, rtx op0, rtx op1, rtx wide_op,
251 rtx target, int unsignedp)
252 {
253 class expand_operand eops[4];
254 tree oprnd0, oprnd1, oprnd2;
255 machine_mode wmode = VOIDmode, tmode0, tmode1 = VOIDmode;
256 optab widen_pattern_optab;
257 enum insn_code icode;
258 int nops = TREE_CODE_LENGTH (ops->code);
259 int op;
260 bool sbool = false;
261
262 oprnd0 = ops->op0;
263 tmode0 = TYPE_MODE (TREE_TYPE (oprnd0));
264 if (ops->code == VEC_UNPACK_FIX_TRUNC_HI_EXPR
265 || ops->code == VEC_UNPACK_FIX_TRUNC_LO_EXPR)
266 /* The sign is from the result type rather than operand's type
267 for these ops. */
268 widen_pattern_optab
269 = optab_for_tree_code (ops->code, ops->type, optab_default);
270 else if ((ops->code == VEC_UNPACK_HI_EXPR
271 || ops->code == VEC_UNPACK_LO_EXPR)
272 && VECTOR_BOOLEAN_TYPE_P (ops->type)
273 && VECTOR_BOOLEAN_TYPE_P (TREE_TYPE (oprnd0))
274 && TYPE_MODE (ops->type) == TYPE_MODE (TREE_TYPE (oprnd0))
275 && SCALAR_INT_MODE_P (TYPE_MODE (ops->type)))
276 {
277 /* For VEC_UNPACK_{LO,HI}_EXPR if the mode of op0 and result is
278 the same scalar mode for VECTOR_BOOLEAN_TYPE_P vectors, use
279 vec_unpacks_sbool_{lo,hi}_optab, so that we can pass in
280 the pattern number of elements in the wider vector. */
281 widen_pattern_optab
282 = (ops->code == VEC_UNPACK_HI_EXPR
283 ? vec_unpacks_sbool_hi_optab : vec_unpacks_sbool_lo_optab);
284 sbool = true;
285 }
286 else
287 widen_pattern_optab
288 = optab_for_tree_code (ops->code, TREE_TYPE (oprnd0), optab_default);
289 if (ops->code == WIDEN_MULT_PLUS_EXPR
290 || ops->code == WIDEN_MULT_MINUS_EXPR)
291 icode = find_widening_optab_handler (widen_pattern_optab,
292 TYPE_MODE (TREE_TYPE (ops->op2)),
293 tmode0);
294 else
295 icode = optab_handler (widen_pattern_optab, tmode0);
296 gcc_assert (icode != CODE_FOR_nothing);
297
298 if (nops >= 2)
299 {
300 oprnd1 = ops->op1;
301 tmode1 = TYPE_MODE (TREE_TYPE (oprnd1));
302 }
303 else if (sbool)
304 {
305 nops = 2;
306 op1 = GEN_INT (TYPE_VECTOR_SUBPARTS (TREE_TYPE (oprnd0)).to_constant ());
307 tmode1 = tmode0;
308 }
309
310 /* The last operand is of a wider mode than the rest of the operands. */
311 if (nops == 2)
312 wmode = tmode1;
313 else if (nops == 3)
314 {
315 gcc_assert (tmode1 == tmode0);
316 gcc_assert (op1);
317 oprnd2 = ops->op2;
318 wmode = TYPE_MODE (TREE_TYPE (oprnd2));
319 }
320
321 op = 0;
322 create_output_operand (&eops[op++], target, TYPE_MODE (ops->type));
323 create_convert_operand_from (&eops[op++], op0, tmode0, unsignedp);
324 if (op1)
325 create_convert_operand_from (&eops[op++], op1, tmode1, unsignedp);
326 if (wide_op)
327 create_convert_operand_from (&eops[op++], wide_op, wmode, unsignedp);
328 expand_insn (icode, op, eops);
329 return eops[0].value;
330 }
331
332 /* Generate code to perform an operation specified by TERNARY_OPTAB
333 on operands OP0, OP1 and OP2, with result having machine-mode MODE.
334
335 UNSIGNEDP is for the case where we have to widen the operands
336 to perform the operation. It says to use zero-extension.
337
338 If TARGET is nonzero, the value
339 is generated there, if it is convenient to do so.
340 In all cases an rtx is returned for the locus of the value;
341 this may or may not be TARGET. */
342
343 rtx
expand_ternary_op(machine_mode mode,optab ternary_optab,rtx op0,rtx op1,rtx op2,rtx target,int unsignedp)344 expand_ternary_op (machine_mode mode, optab ternary_optab, rtx op0,
345 rtx op1, rtx op2, rtx target, int unsignedp)
346 {
347 class expand_operand ops[4];
348 enum insn_code icode = optab_handler (ternary_optab, mode);
349
350 gcc_assert (optab_handler (ternary_optab, mode) != CODE_FOR_nothing);
351
352 create_output_operand (&ops[0], target, mode);
353 create_convert_operand_from (&ops[1], op0, mode, unsignedp);
354 create_convert_operand_from (&ops[2], op1, mode, unsignedp);
355 create_convert_operand_from (&ops[3], op2, mode, unsignedp);
356 expand_insn (icode, 4, ops);
357 return ops[0].value;
358 }
359
360
361 /* Like expand_binop, but return a constant rtx if the result can be
362 calculated at compile time. The arguments and return value are
363 otherwise the same as for expand_binop. */
364
365 rtx
simplify_expand_binop(machine_mode mode,optab binoptab,rtx op0,rtx op1,rtx target,int unsignedp,enum optab_methods methods)366 simplify_expand_binop (machine_mode mode, optab binoptab,
367 rtx op0, rtx op1, rtx target, int unsignedp,
368 enum optab_methods methods)
369 {
370 if (CONSTANT_P (op0) && CONSTANT_P (op1))
371 {
372 rtx x = simplify_binary_operation (optab_to_code (binoptab),
373 mode, op0, op1);
374 if (x)
375 return x;
376 }
377
378 return expand_binop (mode, binoptab, op0, op1, target, unsignedp, methods);
379 }
380
381 /* Like simplify_expand_binop, but always put the result in TARGET.
382 Return true if the expansion succeeded. */
383
384 bool
force_expand_binop(machine_mode mode,optab binoptab,rtx op0,rtx op1,rtx target,int unsignedp,enum optab_methods methods)385 force_expand_binop (machine_mode mode, optab binoptab,
386 rtx op0, rtx op1, rtx target, int unsignedp,
387 enum optab_methods methods)
388 {
389 rtx x = simplify_expand_binop (mode, binoptab, op0, op1,
390 target, unsignedp, methods);
391 if (x == 0)
392 return false;
393 if (x != target)
394 emit_move_insn (target, x);
395 return true;
396 }
397
398 /* Create a new vector value in VMODE with all elements set to OP. The
399 mode of OP must be the element mode of VMODE. If OP is a constant,
400 then the return value will be a constant. */
401
402 rtx
expand_vector_broadcast(machine_mode vmode,rtx op)403 expand_vector_broadcast (machine_mode vmode, rtx op)
404 {
405 int n;
406 rtvec vec;
407
408 gcc_checking_assert (VECTOR_MODE_P (vmode));
409
410 if (valid_for_const_vector_p (vmode, op))
411 return gen_const_vec_duplicate (vmode, op);
412
413 insn_code icode = optab_handler (vec_duplicate_optab, vmode);
414 if (icode != CODE_FOR_nothing)
415 {
416 class expand_operand ops[2];
417 create_output_operand (&ops[0], NULL_RTX, vmode);
418 create_input_operand (&ops[1], op, GET_MODE (op));
419 expand_insn (icode, 2, ops);
420 return ops[0].value;
421 }
422
423 if (!GET_MODE_NUNITS (vmode).is_constant (&n))
424 return NULL;
425
426 /* ??? If the target doesn't have a vec_init, then we have no easy way
427 of performing this operation. Most of this sort of generic support
428 is hidden away in the vector lowering support in gimple. */
429 icode = convert_optab_handler (vec_init_optab, vmode,
430 GET_MODE_INNER (vmode));
431 if (icode == CODE_FOR_nothing)
432 return NULL;
433
434 vec = rtvec_alloc (n);
435 for (int i = 0; i < n; ++i)
436 RTVEC_ELT (vec, i) = op;
437 rtx ret = gen_reg_rtx (vmode);
438 emit_insn (GEN_FCN (icode) (ret, gen_rtx_PARALLEL (vmode, vec)));
439
440 return ret;
441 }
442
443 /* This subroutine of expand_doubleword_shift handles the cases in which
444 the effective shift value is >= BITS_PER_WORD. The arguments and return
445 value are the same as for the parent routine, except that SUPERWORD_OP1
446 is the shift count to use when shifting OUTOF_INPUT into INTO_TARGET.
447 INTO_TARGET may be null if the caller has decided to calculate it. */
448
449 static bool
expand_superword_shift(optab binoptab,rtx outof_input,rtx superword_op1,rtx outof_target,rtx into_target,int unsignedp,enum optab_methods methods)450 expand_superword_shift (optab binoptab, rtx outof_input, rtx superword_op1,
451 rtx outof_target, rtx into_target,
452 int unsignedp, enum optab_methods methods)
453 {
454 if (into_target != 0)
455 if (!force_expand_binop (word_mode, binoptab, outof_input, superword_op1,
456 into_target, unsignedp, methods))
457 return false;
458
459 if (outof_target != 0)
460 {
461 /* For a signed right shift, we must fill OUTOF_TARGET with copies
462 of the sign bit, otherwise we must fill it with zeros. */
463 if (binoptab != ashr_optab)
464 emit_move_insn (outof_target, CONST0_RTX (word_mode));
465 else
466 if (!force_expand_binop (word_mode, binoptab, outof_input,
467 gen_int_shift_amount (word_mode,
468 BITS_PER_WORD - 1),
469 outof_target, unsignedp, methods))
470 return false;
471 }
472 return true;
473 }
474
475 /* This subroutine of expand_doubleword_shift handles the cases in which
476 the effective shift value is < BITS_PER_WORD. The arguments and return
477 value are the same as for the parent routine. */
478
479 static bool
expand_subword_shift(scalar_int_mode op1_mode,optab binoptab,rtx outof_input,rtx into_input,rtx op1,rtx outof_target,rtx into_target,int unsignedp,enum optab_methods methods,unsigned HOST_WIDE_INT shift_mask)480 expand_subword_shift (scalar_int_mode op1_mode, optab binoptab,
481 rtx outof_input, rtx into_input, rtx op1,
482 rtx outof_target, rtx into_target,
483 int unsignedp, enum optab_methods methods,
484 unsigned HOST_WIDE_INT shift_mask)
485 {
486 optab reverse_unsigned_shift, unsigned_shift;
487 rtx tmp, carries;
488
489 reverse_unsigned_shift = (binoptab == ashl_optab ? lshr_optab : ashl_optab);
490 unsigned_shift = (binoptab == ashl_optab ? ashl_optab : lshr_optab);
491
492 /* The low OP1 bits of INTO_TARGET come from the high bits of OUTOF_INPUT.
493 We therefore need to shift OUTOF_INPUT by (BITS_PER_WORD - OP1) bits in
494 the opposite direction to BINOPTAB. */
495 if (CONSTANT_P (op1) || shift_mask >= BITS_PER_WORD)
496 {
497 carries = outof_input;
498 tmp = immed_wide_int_const (wi::shwi (BITS_PER_WORD,
499 op1_mode), op1_mode);
500 tmp = simplify_expand_binop (op1_mode, sub_optab, tmp, op1,
501 0, true, methods);
502 }
503 else
504 {
505 /* We must avoid shifting by BITS_PER_WORD bits since that is either
506 the same as a zero shift (if shift_mask == BITS_PER_WORD - 1) or
507 has unknown behavior. Do a single shift first, then shift by the
508 remainder. It's OK to use ~OP1 as the remainder if shift counts
509 are truncated to the mode size. */
510 carries = expand_binop (word_mode, reverse_unsigned_shift,
511 outof_input, const1_rtx, 0, unsignedp, methods);
512 if (shift_mask == BITS_PER_WORD - 1)
513 {
514 tmp = immed_wide_int_const
515 (wi::minus_one (GET_MODE_PRECISION (op1_mode)), op1_mode);
516 tmp = simplify_expand_binop (op1_mode, xor_optab, op1, tmp,
517 0, true, methods);
518 }
519 else
520 {
521 tmp = immed_wide_int_const (wi::shwi (BITS_PER_WORD - 1,
522 op1_mode), op1_mode);
523 tmp = simplify_expand_binop (op1_mode, sub_optab, tmp, op1,
524 0, true, methods);
525 }
526 }
527 if (tmp == 0 || carries == 0)
528 return false;
529 carries = expand_binop (word_mode, reverse_unsigned_shift,
530 carries, tmp, 0, unsignedp, methods);
531 if (carries == 0)
532 return false;
533
534 /* Shift INTO_INPUT logically by OP1. This is the last use of INTO_INPUT
535 so the result can go directly into INTO_TARGET if convenient. */
536 tmp = expand_binop (word_mode, unsigned_shift, into_input, op1,
537 into_target, unsignedp, methods);
538 if (tmp == 0)
539 return false;
540
541 /* Now OR in the bits carried over from OUTOF_INPUT. */
542 if (!force_expand_binop (word_mode, ior_optab, tmp, carries,
543 into_target, unsignedp, methods))
544 return false;
545
546 /* Use a standard word_mode shift for the out-of half. */
547 if (outof_target != 0)
548 if (!force_expand_binop (word_mode, binoptab, outof_input, op1,
549 outof_target, unsignedp, methods))
550 return false;
551
552 return true;
553 }
554
555
556 /* Try implementing expand_doubleword_shift using conditional moves.
557 The shift is by < BITS_PER_WORD if (CMP_CODE CMP1 CMP2) is true,
558 otherwise it is by >= BITS_PER_WORD. SUBWORD_OP1 and SUPERWORD_OP1
559 are the shift counts to use in the former and latter case. All other
560 arguments are the same as the parent routine. */
561
562 static bool
expand_doubleword_shift_condmove(scalar_int_mode op1_mode,optab binoptab,enum rtx_code cmp_code,rtx cmp1,rtx cmp2,rtx outof_input,rtx into_input,rtx subword_op1,rtx superword_op1,rtx outof_target,rtx into_target,int unsignedp,enum optab_methods methods,unsigned HOST_WIDE_INT shift_mask)563 expand_doubleword_shift_condmove (scalar_int_mode op1_mode, optab binoptab,
564 enum rtx_code cmp_code, rtx cmp1, rtx cmp2,
565 rtx outof_input, rtx into_input,
566 rtx subword_op1, rtx superword_op1,
567 rtx outof_target, rtx into_target,
568 int unsignedp, enum optab_methods methods,
569 unsigned HOST_WIDE_INT shift_mask)
570 {
571 rtx outof_superword, into_superword;
572
573 /* Put the superword version of the output into OUTOF_SUPERWORD and
574 INTO_SUPERWORD. */
575 outof_superword = outof_target != 0 ? gen_reg_rtx (word_mode) : 0;
576 if (outof_target != 0 && subword_op1 == superword_op1)
577 {
578 /* The value INTO_TARGET >> SUBWORD_OP1, which we later store in
579 OUTOF_TARGET, is the same as the value of INTO_SUPERWORD. */
580 into_superword = outof_target;
581 if (!expand_superword_shift (binoptab, outof_input, superword_op1,
582 outof_superword, 0, unsignedp, methods))
583 return false;
584 }
585 else
586 {
587 into_superword = gen_reg_rtx (word_mode);
588 if (!expand_superword_shift (binoptab, outof_input, superword_op1,
589 outof_superword, into_superword,
590 unsignedp, methods))
591 return false;
592 }
593
594 /* Put the subword version directly in OUTOF_TARGET and INTO_TARGET. */
595 if (!expand_subword_shift (op1_mode, binoptab,
596 outof_input, into_input, subword_op1,
597 outof_target, into_target,
598 unsignedp, methods, shift_mask))
599 return false;
600
601 /* Select between them. Do the INTO half first because INTO_SUPERWORD
602 might be the current value of OUTOF_TARGET. */
603 if (!emit_conditional_move (into_target, cmp_code, cmp1, cmp2, op1_mode,
604 into_target, into_superword, word_mode, false))
605 return false;
606
607 if (outof_target != 0)
608 if (!emit_conditional_move (outof_target, cmp_code, cmp1, cmp2, op1_mode,
609 outof_target, outof_superword,
610 word_mode, false))
611 return false;
612
613 return true;
614 }
615
616 /* Expand a doubleword shift (ashl, ashr or lshr) using word-mode shifts.
617 OUTOF_INPUT and INTO_INPUT are the two word-sized halves of the first
618 input operand; the shift moves bits in the direction OUTOF_INPUT->
619 INTO_TARGET. OUTOF_TARGET and INTO_TARGET are the equivalent words
620 of the target. OP1 is the shift count and OP1_MODE is its mode.
621 If OP1 is constant, it will have been truncated as appropriate
622 and is known to be nonzero.
623
624 If SHIFT_MASK is zero, the result of word shifts is undefined when the
625 shift count is outside the range [0, BITS_PER_WORD). This routine must
626 avoid generating such shifts for OP1s in the range [0, BITS_PER_WORD * 2).
627
628 If SHIFT_MASK is nonzero, all word-mode shift counts are effectively
629 masked by it and shifts in the range [BITS_PER_WORD, SHIFT_MASK) will
630 fill with zeros or sign bits as appropriate.
631
632 If SHIFT_MASK is BITS_PER_WORD - 1, this routine will synthesize
633 a doubleword shift whose equivalent mask is BITS_PER_WORD * 2 - 1.
634 Doing this preserves semantics required by SHIFT_COUNT_TRUNCATED.
635 In all other cases, shifts by values outside [0, BITS_PER_UNIT * 2)
636 are undefined.
637
638 BINOPTAB, UNSIGNEDP and METHODS are as for expand_binop. This function
639 may not use INTO_INPUT after modifying INTO_TARGET, and similarly for
640 OUTOF_INPUT and OUTOF_TARGET. OUTOF_TARGET can be null if the parent
641 function wants to calculate it itself.
642
643 Return true if the shift could be successfully synthesized. */
644
645 static bool
expand_doubleword_shift(scalar_int_mode op1_mode,optab binoptab,rtx outof_input,rtx into_input,rtx op1,rtx outof_target,rtx into_target,int unsignedp,enum optab_methods methods,unsigned HOST_WIDE_INT shift_mask)646 expand_doubleword_shift (scalar_int_mode op1_mode, optab binoptab,
647 rtx outof_input, rtx into_input, rtx op1,
648 rtx outof_target, rtx into_target,
649 int unsignedp, enum optab_methods methods,
650 unsigned HOST_WIDE_INT shift_mask)
651 {
652 rtx superword_op1, tmp, cmp1, cmp2;
653 enum rtx_code cmp_code;
654
655 /* See if word-mode shifts by BITS_PER_WORD...BITS_PER_WORD * 2 - 1 will
656 fill the result with sign or zero bits as appropriate. If so, the value
657 of OUTOF_TARGET will always be (SHIFT OUTOF_INPUT OP1). Recursively call
658 this routine to calculate INTO_TARGET (which depends on both OUTOF_INPUT
659 and INTO_INPUT), then emit code to set up OUTOF_TARGET.
660
661 This isn't worthwhile for constant shifts since the optimizers will
662 cope better with in-range shift counts. */
663 if (shift_mask >= BITS_PER_WORD
664 && outof_target != 0
665 && !CONSTANT_P (op1))
666 {
667 if (!expand_doubleword_shift (op1_mode, binoptab,
668 outof_input, into_input, op1,
669 0, into_target,
670 unsignedp, methods, shift_mask))
671 return false;
672 if (!force_expand_binop (word_mode, binoptab, outof_input, op1,
673 outof_target, unsignedp, methods))
674 return false;
675 return true;
676 }
677
678 /* Set CMP_CODE, CMP1 and CMP2 so that the rtx (CMP_CODE CMP1 CMP2)
679 is true when the effective shift value is less than BITS_PER_WORD.
680 Set SUPERWORD_OP1 to the shift count that should be used to shift
681 OUTOF_INPUT into INTO_TARGET when the condition is false. */
682 tmp = immed_wide_int_const (wi::shwi (BITS_PER_WORD, op1_mode), op1_mode);
683 if (!CONSTANT_P (op1) && shift_mask == BITS_PER_WORD - 1)
684 {
685 /* Set CMP1 to OP1 & BITS_PER_WORD. The result is zero iff OP1
686 is a subword shift count. */
687 cmp1 = simplify_expand_binop (op1_mode, and_optab, op1, tmp,
688 0, true, methods);
689 cmp2 = CONST0_RTX (op1_mode);
690 cmp_code = EQ;
691 superword_op1 = op1;
692 }
693 else
694 {
695 /* Set CMP1 to OP1 - BITS_PER_WORD. */
696 cmp1 = simplify_expand_binop (op1_mode, sub_optab, op1, tmp,
697 0, true, methods);
698 cmp2 = CONST0_RTX (op1_mode);
699 cmp_code = LT;
700 superword_op1 = cmp1;
701 }
702 if (cmp1 == 0)
703 return false;
704
705 /* If we can compute the condition at compile time, pick the
706 appropriate subroutine. */
707 tmp = simplify_relational_operation (cmp_code, SImode, op1_mode, cmp1, cmp2);
708 if (tmp != 0 && CONST_INT_P (tmp))
709 {
710 if (tmp == const0_rtx)
711 return expand_superword_shift (binoptab, outof_input, superword_op1,
712 outof_target, into_target,
713 unsignedp, methods);
714 else
715 return expand_subword_shift (op1_mode, binoptab,
716 outof_input, into_input, op1,
717 outof_target, into_target,
718 unsignedp, methods, shift_mask);
719 }
720
721 /* Try using conditional moves to generate straight-line code. */
722 if (HAVE_conditional_move)
723 {
724 rtx_insn *start = get_last_insn ();
725 if (expand_doubleword_shift_condmove (op1_mode, binoptab,
726 cmp_code, cmp1, cmp2,
727 outof_input, into_input,
728 op1, superword_op1,
729 outof_target, into_target,
730 unsignedp, methods, shift_mask))
731 return true;
732 delete_insns_since (start);
733 }
734
735 /* As a last resort, use branches to select the correct alternative. */
736 rtx_code_label *subword_label = gen_label_rtx ();
737 rtx_code_label *done_label = gen_label_rtx ();
738
739 NO_DEFER_POP;
740 do_compare_rtx_and_jump (cmp1, cmp2, cmp_code, false, op1_mode,
741 0, 0, subword_label,
742 profile_probability::uninitialized ());
743 OK_DEFER_POP;
744
745 if (!expand_superword_shift (binoptab, outof_input, superword_op1,
746 outof_target, into_target,
747 unsignedp, methods))
748 return false;
749
750 emit_jump_insn (targetm.gen_jump (done_label));
751 emit_barrier ();
752 emit_label (subword_label);
753
754 if (!expand_subword_shift (op1_mode, binoptab,
755 outof_input, into_input, op1,
756 outof_target, into_target,
757 unsignedp, methods, shift_mask))
758 return false;
759
760 emit_label (done_label);
761 return true;
762 }
763
764 /* Subroutine of expand_binop. Perform a double word multiplication of
765 operands OP0 and OP1 both of mode MODE, which is exactly twice as wide
766 as the target's word_mode. This function return NULL_RTX if anything
767 goes wrong, in which case it may have already emitted instructions
768 which need to be deleted.
769
770 If we want to multiply two two-word values and have normal and widening
771 multiplies of single-word values, we can do this with three smaller
772 multiplications.
773
774 The multiplication proceeds as follows:
775 _______________________
776 [__op0_high_|__op0_low__]
777 _______________________
778 * [__op1_high_|__op1_low__]
779 _______________________________________________
780 _______________________
781 (1) [__op0_low__*__op1_low__]
782 _______________________
783 (2a) [__op0_low__*__op1_high_]
784 _______________________
785 (2b) [__op0_high_*__op1_low__]
786 _______________________
787 (3) [__op0_high_*__op1_high_]
788
789
790 This gives a 4-word result. Since we are only interested in the
791 lower 2 words, partial result (3) and the upper words of (2a) and
792 (2b) don't need to be calculated. Hence (2a) and (2b) can be
793 calculated using non-widening multiplication.
794
795 (1), however, needs to be calculated with an unsigned widening
796 multiplication. If this operation is not directly supported we
797 try using a signed widening multiplication and adjust the result.
798 This adjustment works as follows:
799
800 If both operands are positive then no adjustment is needed.
801
802 If the operands have different signs, for example op0_low < 0 and
803 op1_low >= 0, the instruction treats the most significant bit of
804 op0_low as a sign bit instead of a bit with significance
805 2**(BITS_PER_WORD-1), i.e. the instruction multiplies op1_low
806 with 2**BITS_PER_WORD - op0_low, and two's complements the
807 result. Conclusion: We need to add op1_low * 2**BITS_PER_WORD to
808 the result.
809
810 Similarly, if both operands are negative, we need to add
811 (op0_low + op1_low) * 2**BITS_PER_WORD.
812
813 We use a trick to adjust quickly. We logically shift op0_low right
814 (op1_low) BITS_PER_WORD-1 steps to get 0 or 1, and add this to
815 op0_high (op1_high) before it is used to calculate 2b (2a). If no
816 logical shift exists, we do an arithmetic right shift and subtract
817 the 0 or -1. */
818
819 static rtx
expand_doubleword_mult(machine_mode mode,rtx op0,rtx op1,rtx target,bool umulp,enum optab_methods methods)820 expand_doubleword_mult (machine_mode mode, rtx op0, rtx op1, rtx target,
821 bool umulp, enum optab_methods methods)
822 {
823 int low = (WORDS_BIG_ENDIAN ? 1 : 0);
824 int high = (WORDS_BIG_ENDIAN ? 0 : 1);
825 rtx wordm1 = (umulp ? NULL_RTX
826 : gen_int_shift_amount (word_mode, BITS_PER_WORD - 1));
827 rtx product, adjust, product_high, temp;
828
829 rtx op0_high = operand_subword_force (op0, high, mode);
830 rtx op0_low = operand_subword_force (op0, low, mode);
831 rtx op1_high = operand_subword_force (op1, high, mode);
832 rtx op1_low = operand_subword_force (op1, low, mode);
833
834 /* If we're using an unsigned multiply to directly compute the product
835 of the low-order words of the operands and perform any required
836 adjustments of the operands, we begin by trying two more multiplications
837 and then computing the appropriate sum.
838
839 We have checked above that the required addition is provided.
840 Full-word addition will normally always succeed, especially if
841 it is provided at all, so we don't worry about its failure. The
842 multiplication may well fail, however, so we do handle that. */
843
844 if (!umulp)
845 {
846 /* ??? This could be done with emit_store_flag where available. */
847 temp = expand_binop (word_mode, lshr_optab, op0_low, wordm1,
848 NULL_RTX, 1, methods);
849 if (temp)
850 op0_high = expand_binop (word_mode, add_optab, op0_high, temp,
851 NULL_RTX, 0, OPTAB_DIRECT);
852 else
853 {
854 temp = expand_binop (word_mode, ashr_optab, op0_low, wordm1,
855 NULL_RTX, 0, methods);
856 if (!temp)
857 return NULL_RTX;
858 op0_high = expand_binop (word_mode, sub_optab, op0_high, temp,
859 NULL_RTX, 0, OPTAB_DIRECT);
860 }
861
862 if (!op0_high)
863 return NULL_RTX;
864 }
865
866 adjust = expand_binop (word_mode, smul_optab, op0_high, op1_low,
867 NULL_RTX, 0, OPTAB_DIRECT);
868 if (!adjust)
869 return NULL_RTX;
870
871 /* OP0_HIGH should now be dead. */
872
873 if (!umulp)
874 {
875 /* ??? This could be done with emit_store_flag where available. */
876 temp = expand_binop (word_mode, lshr_optab, op1_low, wordm1,
877 NULL_RTX, 1, methods);
878 if (temp)
879 op1_high = expand_binop (word_mode, add_optab, op1_high, temp,
880 NULL_RTX, 0, OPTAB_DIRECT);
881 else
882 {
883 temp = expand_binop (word_mode, ashr_optab, op1_low, wordm1,
884 NULL_RTX, 0, methods);
885 if (!temp)
886 return NULL_RTX;
887 op1_high = expand_binop (word_mode, sub_optab, op1_high, temp,
888 NULL_RTX, 0, OPTAB_DIRECT);
889 }
890
891 if (!op1_high)
892 return NULL_RTX;
893 }
894
895 temp = expand_binop (word_mode, smul_optab, op1_high, op0_low,
896 NULL_RTX, 0, OPTAB_DIRECT);
897 if (!temp)
898 return NULL_RTX;
899
900 /* OP1_HIGH should now be dead. */
901
902 adjust = expand_binop (word_mode, add_optab, adjust, temp,
903 NULL_RTX, 0, OPTAB_DIRECT);
904
905 if (target && !REG_P (target))
906 target = NULL_RTX;
907
908 /* *_widen_optab needs to determine operand mode, make sure at least
909 one operand has non-VOID mode. */
910 if (GET_MODE (op0_low) == VOIDmode && GET_MODE (op1_low) == VOIDmode)
911 op0_low = force_reg (word_mode, op0_low);
912
913 if (umulp)
914 product = expand_binop (mode, umul_widen_optab, op0_low, op1_low,
915 target, 1, OPTAB_DIRECT);
916 else
917 product = expand_binop (mode, smul_widen_optab, op0_low, op1_low,
918 target, 1, OPTAB_DIRECT);
919
920 if (!product)
921 return NULL_RTX;
922
923 product_high = operand_subword (product, high, 1, mode);
924 adjust = expand_binop (word_mode, add_optab, product_high, adjust,
925 NULL_RTX, 0, OPTAB_DIRECT);
926 emit_move_insn (product_high, adjust);
927 return product;
928 }
929
930 /* Wrapper around expand_binop which takes an rtx code to specify
931 the operation to perform, not an optab pointer. All other
932 arguments are the same. */
933 rtx
expand_simple_binop(machine_mode mode,enum rtx_code code,rtx op0,rtx op1,rtx target,int unsignedp,enum optab_methods methods)934 expand_simple_binop (machine_mode mode, enum rtx_code code, rtx op0,
935 rtx op1, rtx target, int unsignedp,
936 enum optab_methods methods)
937 {
938 optab binop = code_to_optab (code);
939 gcc_assert (binop);
940
941 return expand_binop (mode, binop, op0, op1, target, unsignedp, methods);
942 }
943
944 /* Return whether OP0 and OP1 should be swapped when expanding a commutative
945 binop. Order them according to commutative_operand_precedence and, if
946 possible, try to put TARGET or a pseudo first. */
947 static bool
swap_commutative_operands_with_target(rtx target,rtx op0,rtx op1)948 swap_commutative_operands_with_target (rtx target, rtx op0, rtx op1)
949 {
950 int op0_prec = commutative_operand_precedence (op0);
951 int op1_prec = commutative_operand_precedence (op1);
952
953 if (op0_prec < op1_prec)
954 return true;
955
956 if (op0_prec > op1_prec)
957 return false;
958
959 /* With equal precedence, both orders are ok, but it is better if the
960 first operand is TARGET, or if both TARGET and OP0 are pseudos. */
961 if (target == 0 || REG_P (target))
962 return (REG_P (op1) && !REG_P (op0)) || target == op1;
963 else
964 return rtx_equal_p (op1, target);
965 }
966
967 /* Return true if BINOPTAB implements a shift operation. */
968
969 static bool
shift_optab_p(optab binoptab)970 shift_optab_p (optab binoptab)
971 {
972 switch (optab_to_code (binoptab))
973 {
974 case ASHIFT:
975 case SS_ASHIFT:
976 case US_ASHIFT:
977 case ASHIFTRT:
978 case LSHIFTRT:
979 case ROTATE:
980 case ROTATERT:
981 return true;
982
983 default:
984 return false;
985 }
986 }
987
988 /* Return true if BINOPTAB implements a commutative binary operation. */
989
990 static bool
commutative_optab_p(optab binoptab)991 commutative_optab_p (optab binoptab)
992 {
993 return (GET_RTX_CLASS (optab_to_code (binoptab)) == RTX_COMM_ARITH
994 || binoptab == smul_widen_optab
995 || binoptab == umul_widen_optab
996 || binoptab == smul_highpart_optab
997 || binoptab == umul_highpart_optab);
998 }
999
1000 /* X is to be used in mode MODE as operand OPN to BINOPTAB. If we're
1001 optimizing, and if the operand is a constant that costs more than
1002 1 instruction, force the constant into a register and return that
1003 register. Return X otherwise. UNSIGNEDP says whether X is unsigned. */
1004
1005 static rtx
avoid_expensive_constant(machine_mode mode,optab binoptab,int opn,rtx x,bool unsignedp)1006 avoid_expensive_constant (machine_mode mode, optab binoptab,
1007 int opn, rtx x, bool unsignedp)
1008 {
1009 bool speed = optimize_insn_for_speed_p ();
1010
1011 if (mode != VOIDmode
1012 && optimize
1013 && CONSTANT_P (x)
1014 && (rtx_cost (x, mode, optab_to_code (binoptab), opn, speed)
1015 > set_src_cost (x, mode, speed)))
1016 {
1017 if (CONST_INT_P (x))
1018 {
1019 HOST_WIDE_INT intval = trunc_int_for_mode (INTVAL (x), mode);
1020 if (intval != INTVAL (x))
1021 x = GEN_INT (intval);
1022 }
1023 else
1024 x = convert_modes (mode, VOIDmode, x, unsignedp);
1025 x = force_reg (mode, x);
1026 }
1027 return x;
1028 }
1029
1030 /* Helper function for expand_binop: handle the case where there
1031 is an insn ICODE that directly implements the indicated operation.
1032 Returns null if this is not possible. */
1033 static rtx
expand_binop_directly(enum insn_code icode,machine_mode mode,optab binoptab,rtx op0,rtx op1,rtx target,int unsignedp,enum optab_methods methods,rtx_insn * last)1034 expand_binop_directly (enum insn_code icode, machine_mode mode, optab binoptab,
1035 rtx op0, rtx op1,
1036 rtx target, int unsignedp, enum optab_methods methods,
1037 rtx_insn *last)
1038 {
1039 machine_mode xmode0 = insn_data[(int) icode].operand[1].mode;
1040 machine_mode xmode1 = insn_data[(int) icode].operand[2].mode;
1041 machine_mode mode0, mode1, tmp_mode;
1042 class expand_operand ops[3];
1043 bool commutative_p;
1044 rtx_insn *pat;
1045 rtx xop0 = op0, xop1 = op1;
1046 bool canonicalize_op1 = false;
1047
1048 /* If it is a commutative operator and the modes would match
1049 if we would swap the operands, we can save the conversions. */
1050 commutative_p = commutative_optab_p (binoptab);
1051 if (commutative_p
1052 && GET_MODE (xop0) != xmode0 && GET_MODE (xop1) != xmode1
1053 && GET_MODE (xop0) == xmode1 && GET_MODE (xop1) == xmode0)
1054 std::swap (xop0, xop1);
1055
1056 /* If we are optimizing, force expensive constants into a register. */
1057 xop0 = avoid_expensive_constant (xmode0, binoptab, 0, xop0, unsignedp);
1058 if (!shift_optab_p (binoptab))
1059 xop1 = avoid_expensive_constant (xmode1, binoptab, 1, xop1, unsignedp);
1060 else
1061 /* Shifts and rotates often use a different mode for op1 from op0;
1062 for VOIDmode constants we don't know the mode, so force it
1063 to be canonicalized using convert_modes. */
1064 canonicalize_op1 = true;
1065
1066 /* In case the insn wants input operands in modes different from
1067 those of the actual operands, convert the operands. It would
1068 seem that we don't need to convert CONST_INTs, but we do, so
1069 that they're properly zero-extended, sign-extended or truncated
1070 for their mode. */
1071
1072 mode0 = GET_MODE (xop0) != VOIDmode ? GET_MODE (xop0) : mode;
1073 if (xmode0 != VOIDmode && xmode0 != mode0)
1074 {
1075 xop0 = convert_modes (xmode0, mode0, xop0, unsignedp);
1076 mode0 = xmode0;
1077 }
1078
1079 mode1 = ((GET_MODE (xop1) != VOIDmode || canonicalize_op1)
1080 ? GET_MODE (xop1) : mode);
1081 if (xmode1 != VOIDmode && xmode1 != mode1)
1082 {
1083 xop1 = convert_modes (xmode1, mode1, xop1, unsignedp);
1084 mode1 = xmode1;
1085 }
1086
1087 /* If operation is commutative,
1088 try to make the first operand a register.
1089 Even better, try to make it the same as the target.
1090 Also try to make the last operand a constant. */
1091 if (commutative_p
1092 && swap_commutative_operands_with_target (target, xop0, xop1))
1093 std::swap (xop0, xop1);
1094
1095 /* Now, if insn's predicates don't allow our operands, put them into
1096 pseudo regs. */
1097
1098 if (binoptab == vec_pack_trunc_optab
1099 || binoptab == vec_pack_usat_optab
1100 || binoptab == vec_pack_ssat_optab
1101 || binoptab == vec_pack_ufix_trunc_optab
1102 || binoptab == vec_pack_sfix_trunc_optab
1103 || binoptab == vec_packu_float_optab
1104 || binoptab == vec_packs_float_optab)
1105 {
1106 /* The mode of the result is different then the mode of the
1107 arguments. */
1108 tmp_mode = insn_data[(int) icode].operand[0].mode;
1109 if (VECTOR_MODE_P (mode)
1110 && maybe_ne (GET_MODE_NUNITS (tmp_mode), 2 * GET_MODE_NUNITS (mode)))
1111 {
1112 delete_insns_since (last);
1113 return NULL_RTX;
1114 }
1115 }
1116 else
1117 tmp_mode = mode;
1118
1119 create_output_operand (&ops[0], target, tmp_mode);
1120 create_input_operand (&ops[1], xop0, mode0);
1121 create_input_operand (&ops[2], xop1, mode1);
1122 pat = maybe_gen_insn (icode, 3, ops);
1123 if (pat)
1124 {
1125 /* If PAT is composed of more than one insn, try to add an appropriate
1126 REG_EQUAL note to it. If we can't because TEMP conflicts with an
1127 operand, call expand_binop again, this time without a target. */
1128 if (INSN_P (pat) && NEXT_INSN (pat) != NULL_RTX
1129 && ! add_equal_note (pat, ops[0].value,
1130 optab_to_code (binoptab),
1131 ops[1].value, ops[2].value, mode0))
1132 {
1133 delete_insns_since (last);
1134 return expand_binop (mode, binoptab, op0, op1, NULL_RTX,
1135 unsignedp, methods);
1136 }
1137
1138 emit_insn (pat);
1139 return ops[0].value;
1140 }
1141 delete_insns_since (last);
1142 return NULL_RTX;
1143 }
1144
1145 /* Generate code to perform an operation specified by BINOPTAB
1146 on operands OP0 and OP1, with result having machine-mode MODE.
1147
1148 UNSIGNEDP is for the case where we have to widen the operands
1149 to perform the operation. It says to use zero-extension.
1150
1151 If TARGET is nonzero, the value
1152 is generated there, if it is convenient to do so.
1153 In all cases an rtx is returned for the locus of the value;
1154 this may or may not be TARGET. */
1155
1156 rtx
expand_binop(machine_mode mode,optab binoptab,rtx op0,rtx op1,rtx target,int unsignedp,enum optab_methods methods)1157 expand_binop (machine_mode mode, optab binoptab, rtx op0, rtx op1,
1158 rtx target, int unsignedp, enum optab_methods methods)
1159 {
1160 enum optab_methods next_methods
1161 = (methods == OPTAB_LIB || methods == OPTAB_LIB_WIDEN
1162 ? OPTAB_WIDEN : methods);
1163 enum mode_class mclass;
1164 enum insn_code icode;
1165 machine_mode wider_mode;
1166 scalar_int_mode int_mode;
1167 rtx libfunc;
1168 rtx temp;
1169 rtx_insn *entry_last = get_last_insn ();
1170 rtx_insn *last;
1171
1172 mclass = GET_MODE_CLASS (mode);
1173
1174 /* If subtracting an integer constant, convert this into an addition of
1175 the negated constant. */
1176
1177 if (binoptab == sub_optab && CONST_INT_P (op1))
1178 {
1179 op1 = negate_rtx (mode, op1);
1180 binoptab = add_optab;
1181 }
1182 /* For shifts, constant invalid op1 might be expanded from different
1183 mode than MODE. As those are invalid, force them to a register
1184 to avoid further problems during expansion. */
1185 else if (CONST_INT_P (op1)
1186 && shift_optab_p (binoptab)
1187 && UINTVAL (op1) >= GET_MODE_BITSIZE (GET_MODE_INNER (mode)))
1188 {
1189 op1 = gen_int_mode (INTVAL (op1), GET_MODE_INNER (mode));
1190 op1 = force_reg (GET_MODE_INNER (mode), op1);
1191 }
1192
1193 /* Record where to delete back to if we backtrack. */
1194 last = get_last_insn ();
1195
1196 /* If we can do it with a three-operand insn, do so. */
1197
1198 if (methods != OPTAB_MUST_WIDEN)
1199 {
1200 if (convert_optab_p (binoptab))
1201 {
1202 machine_mode from_mode = widened_mode (mode, op0, op1);
1203 icode = find_widening_optab_handler (binoptab, mode, from_mode);
1204 }
1205 else
1206 icode = optab_handler (binoptab, mode);
1207 if (icode != CODE_FOR_nothing)
1208 {
1209 temp = expand_binop_directly (icode, mode, binoptab, op0, op1,
1210 target, unsignedp, methods, last);
1211 if (temp)
1212 return temp;
1213 }
1214 }
1215
1216 /* If we were trying to rotate, and that didn't work, try rotating
1217 the other direction before falling back to shifts and bitwise-or. */
1218 if (((binoptab == rotl_optab
1219 && (icode = optab_handler (rotr_optab, mode)) != CODE_FOR_nothing)
1220 || (binoptab == rotr_optab
1221 && (icode = optab_handler (rotl_optab, mode)) != CODE_FOR_nothing))
1222 && is_int_mode (mode, &int_mode))
1223 {
1224 optab otheroptab = (binoptab == rotl_optab ? rotr_optab : rotl_optab);
1225 rtx newop1;
1226 unsigned int bits = GET_MODE_PRECISION (int_mode);
1227
1228 if (CONST_INT_P (op1))
1229 newop1 = gen_int_shift_amount (int_mode, bits - INTVAL (op1));
1230 else if (targetm.shift_truncation_mask (int_mode) == bits - 1)
1231 newop1 = negate_rtx (GET_MODE (op1), op1);
1232 else
1233 newop1 = expand_binop (GET_MODE (op1), sub_optab,
1234 gen_int_mode (bits, GET_MODE (op1)), op1,
1235 NULL_RTX, unsignedp, OPTAB_DIRECT);
1236
1237 temp = expand_binop_directly (icode, int_mode, otheroptab, op0, newop1,
1238 target, unsignedp, methods, last);
1239 if (temp)
1240 return temp;
1241 }
1242
1243 /* If this is a multiply, see if we can do a widening operation that
1244 takes operands of this mode and makes a wider mode. */
1245
1246 if (binoptab == smul_optab
1247 && GET_MODE_2XWIDER_MODE (mode).exists (&wider_mode)
1248 && (convert_optab_handler ((unsignedp
1249 ? umul_widen_optab
1250 : smul_widen_optab),
1251 wider_mode, mode) != CODE_FOR_nothing))
1252 {
1253 /* *_widen_optab needs to determine operand mode, make sure at least
1254 one operand has non-VOID mode. */
1255 if (GET_MODE (op0) == VOIDmode && GET_MODE (op1) == VOIDmode)
1256 op0 = force_reg (mode, op0);
1257 temp = expand_binop (wider_mode,
1258 unsignedp ? umul_widen_optab : smul_widen_optab,
1259 op0, op1, NULL_RTX, unsignedp, OPTAB_DIRECT);
1260
1261 if (temp != 0)
1262 {
1263 if (GET_MODE_CLASS (mode) == MODE_INT
1264 && TRULY_NOOP_TRUNCATION_MODES_P (mode, GET_MODE (temp)))
1265 return gen_lowpart (mode, temp);
1266 else
1267 return convert_to_mode (mode, temp, unsignedp);
1268 }
1269 }
1270
1271 /* If this is a vector shift by a scalar, see if we can do a vector
1272 shift by a vector. If so, broadcast the scalar into a vector. */
1273 if (mclass == MODE_VECTOR_INT)
1274 {
1275 optab otheroptab = unknown_optab;
1276
1277 if (binoptab == ashl_optab)
1278 otheroptab = vashl_optab;
1279 else if (binoptab == ashr_optab)
1280 otheroptab = vashr_optab;
1281 else if (binoptab == lshr_optab)
1282 otheroptab = vlshr_optab;
1283 else if (binoptab == rotl_optab)
1284 otheroptab = vrotl_optab;
1285 else if (binoptab == rotr_optab)
1286 otheroptab = vrotr_optab;
1287
1288 if (otheroptab
1289 && (icode = optab_handler (otheroptab, mode)) != CODE_FOR_nothing)
1290 {
1291 /* The scalar may have been extended to be too wide. Truncate
1292 it back to the proper size to fit in the broadcast vector. */
1293 scalar_mode inner_mode = GET_MODE_INNER (mode);
1294 if (!CONST_INT_P (op1)
1295 && (GET_MODE_BITSIZE (as_a <scalar_int_mode> (GET_MODE (op1)))
1296 > GET_MODE_BITSIZE (inner_mode)))
1297 op1 = force_reg (inner_mode,
1298 simplify_gen_unary (TRUNCATE, inner_mode, op1,
1299 GET_MODE (op1)));
1300 rtx vop1 = expand_vector_broadcast (mode, op1);
1301 if (vop1)
1302 {
1303 temp = expand_binop_directly (icode, mode, otheroptab, op0, vop1,
1304 target, unsignedp, methods, last);
1305 if (temp)
1306 return temp;
1307 }
1308 }
1309 }
1310
1311 /* Look for a wider mode of the same class for which we think we
1312 can open-code the operation. Check for a widening multiply at the
1313 wider mode as well. */
1314
1315 if (CLASS_HAS_WIDER_MODES_P (mclass)
1316 && methods != OPTAB_DIRECT && methods != OPTAB_LIB)
1317 FOR_EACH_WIDER_MODE (wider_mode, mode)
1318 {
1319 machine_mode next_mode;
1320 if (optab_handler (binoptab, wider_mode) != CODE_FOR_nothing
1321 || (binoptab == smul_optab
1322 && GET_MODE_WIDER_MODE (wider_mode).exists (&next_mode)
1323 && (find_widening_optab_handler ((unsignedp
1324 ? umul_widen_optab
1325 : smul_widen_optab),
1326 next_mode, mode)
1327 != CODE_FOR_nothing)))
1328 {
1329 rtx xop0 = op0, xop1 = op1;
1330 int no_extend = 0;
1331
1332 /* For certain integer operations, we need not actually extend
1333 the narrow operands, as long as we will truncate
1334 the results to the same narrowness. */
1335
1336 if ((binoptab == ior_optab || binoptab == and_optab
1337 || binoptab == xor_optab
1338 || binoptab == add_optab || binoptab == sub_optab
1339 || binoptab == smul_optab || binoptab == ashl_optab)
1340 && mclass == MODE_INT)
1341 {
1342 no_extend = 1;
1343 xop0 = avoid_expensive_constant (mode, binoptab, 0,
1344 xop0, unsignedp);
1345 if (binoptab != ashl_optab)
1346 xop1 = avoid_expensive_constant (mode, binoptab, 1,
1347 xop1, unsignedp);
1348 }
1349
1350 xop0 = widen_operand (xop0, wider_mode, mode, unsignedp, no_extend);
1351
1352 /* The second operand of a shift must always be extended. */
1353 xop1 = widen_operand (xop1, wider_mode, mode, unsignedp,
1354 no_extend && binoptab != ashl_optab);
1355
1356 temp = expand_binop (wider_mode, binoptab, xop0, xop1, NULL_RTX,
1357 unsignedp, OPTAB_DIRECT);
1358 if (temp)
1359 {
1360 if (mclass != MODE_INT
1361 || !TRULY_NOOP_TRUNCATION_MODES_P (mode, wider_mode))
1362 {
1363 if (target == 0)
1364 target = gen_reg_rtx (mode);
1365 convert_move (target, temp, 0);
1366 return target;
1367 }
1368 else
1369 return gen_lowpart (mode, temp);
1370 }
1371 else
1372 delete_insns_since (last);
1373 }
1374 }
1375
1376 /* If operation is commutative,
1377 try to make the first operand a register.
1378 Even better, try to make it the same as the target.
1379 Also try to make the last operand a constant. */
1380 if (commutative_optab_p (binoptab)
1381 && swap_commutative_operands_with_target (target, op0, op1))
1382 std::swap (op0, op1);
1383
1384 /* These can be done a word at a time. */
1385 if ((binoptab == and_optab || binoptab == ior_optab || binoptab == xor_optab)
1386 && is_int_mode (mode, &int_mode)
1387 && GET_MODE_SIZE (int_mode) > UNITS_PER_WORD
1388 && optab_handler (binoptab, word_mode) != CODE_FOR_nothing)
1389 {
1390 int i;
1391 rtx_insn *insns;
1392
1393 /* If TARGET is the same as one of the operands, the REG_EQUAL note
1394 won't be accurate, so use a new target. */
1395 if (target == 0
1396 || target == op0
1397 || target == op1
1398 || reg_overlap_mentioned_p (target, op0)
1399 || reg_overlap_mentioned_p (target, op1)
1400 || !valid_multiword_target_p (target))
1401 target = gen_reg_rtx (int_mode);
1402
1403 start_sequence ();
1404
1405 /* Do the actual arithmetic. */
1406 machine_mode op0_mode = GET_MODE (op0);
1407 machine_mode op1_mode = GET_MODE (op1);
1408 if (op0_mode == VOIDmode)
1409 op0_mode = int_mode;
1410 if (op1_mode == VOIDmode)
1411 op1_mode = int_mode;
1412 for (i = 0; i < GET_MODE_BITSIZE (int_mode) / BITS_PER_WORD; i++)
1413 {
1414 rtx target_piece = operand_subword (target, i, 1, int_mode);
1415 rtx x = expand_binop (word_mode, binoptab,
1416 operand_subword_force (op0, i, op0_mode),
1417 operand_subword_force (op1, i, op1_mode),
1418 target_piece, unsignedp, next_methods);
1419
1420 if (x == 0)
1421 break;
1422
1423 if (target_piece != x)
1424 emit_move_insn (target_piece, x);
1425 }
1426
1427 insns = get_insns ();
1428 end_sequence ();
1429
1430 if (i == GET_MODE_BITSIZE (int_mode) / BITS_PER_WORD)
1431 {
1432 emit_insn (insns);
1433 return target;
1434 }
1435 }
1436
1437 /* Synthesize double word shifts from single word shifts. */
1438 if ((binoptab == lshr_optab || binoptab == ashl_optab
1439 || binoptab == ashr_optab)
1440 && is_int_mode (mode, &int_mode)
1441 && (CONST_INT_P (op1) || optimize_insn_for_speed_p ())
1442 && GET_MODE_SIZE (int_mode) == 2 * UNITS_PER_WORD
1443 && GET_MODE_PRECISION (int_mode) == GET_MODE_BITSIZE (int_mode)
1444 && optab_handler (binoptab, word_mode) != CODE_FOR_nothing
1445 && optab_handler (ashl_optab, word_mode) != CODE_FOR_nothing
1446 && optab_handler (lshr_optab, word_mode) != CODE_FOR_nothing)
1447 {
1448 unsigned HOST_WIDE_INT shift_mask, double_shift_mask;
1449 scalar_int_mode op1_mode;
1450
1451 double_shift_mask = targetm.shift_truncation_mask (int_mode);
1452 shift_mask = targetm.shift_truncation_mask (word_mode);
1453 op1_mode = (GET_MODE (op1) != VOIDmode
1454 ? as_a <scalar_int_mode> (GET_MODE (op1))
1455 : word_mode);
1456
1457 /* Apply the truncation to constant shifts. */
1458 if (double_shift_mask > 0 && CONST_INT_P (op1))
1459 op1 = gen_int_mode (INTVAL (op1) & double_shift_mask, op1_mode);
1460
1461 if (op1 == CONST0_RTX (op1_mode))
1462 return op0;
1463
1464 /* Make sure that this is a combination that expand_doubleword_shift
1465 can handle. See the comments there for details. */
1466 if (double_shift_mask == 0
1467 || (shift_mask == BITS_PER_WORD - 1
1468 && double_shift_mask == BITS_PER_WORD * 2 - 1))
1469 {
1470 rtx_insn *insns;
1471 rtx into_target, outof_target;
1472 rtx into_input, outof_input;
1473 int left_shift, outof_word;
1474
1475 /* If TARGET is the same as one of the operands, the REG_EQUAL note
1476 won't be accurate, so use a new target. */
1477 if (target == 0
1478 || target == op0
1479 || target == op1
1480 || reg_overlap_mentioned_p (target, op0)
1481 || reg_overlap_mentioned_p (target, op1)
1482 || !valid_multiword_target_p (target))
1483 target = gen_reg_rtx (int_mode);
1484
1485 start_sequence ();
1486
1487 /* OUTOF_* is the word we are shifting bits away from, and
1488 INTO_* is the word that we are shifting bits towards, thus
1489 they differ depending on the direction of the shift and
1490 WORDS_BIG_ENDIAN. */
1491
1492 left_shift = binoptab == ashl_optab;
1493 outof_word = left_shift ^ ! WORDS_BIG_ENDIAN;
1494
1495 outof_target = operand_subword (target, outof_word, 1, int_mode);
1496 into_target = operand_subword (target, 1 - outof_word, 1, int_mode);
1497
1498 outof_input = operand_subword_force (op0, outof_word, int_mode);
1499 into_input = operand_subword_force (op0, 1 - outof_word, int_mode);
1500
1501 if (expand_doubleword_shift (op1_mode, binoptab,
1502 outof_input, into_input, op1,
1503 outof_target, into_target,
1504 unsignedp, next_methods, shift_mask))
1505 {
1506 insns = get_insns ();
1507 end_sequence ();
1508
1509 emit_insn (insns);
1510 return target;
1511 }
1512 end_sequence ();
1513 }
1514 }
1515
1516 /* Synthesize double word rotates from single word shifts. */
1517 if ((binoptab == rotl_optab || binoptab == rotr_optab)
1518 && is_int_mode (mode, &int_mode)
1519 && CONST_INT_P (op1)
1520 && GET_MODE_PRECISION (int_mode) == 2 * BITS_PER_WORD
1521 && optab_handler (ashl_optab, word_mode) != CODE_FOR_nothing
1522 && optab_handler (lshr_optab, word_mode) != CODE_FOR_nothing)
1523 {
1524 rtx_insn *insns;
1525 rtx into_target, outof_target;
1526 rtx into_input, outof_input;
1527 rtx inter;
1528 int shift_count, left_shift, outof_word;
1529
1530 /* If TARGET is the same as one of the operands, the REG_EQUAL note
1531 won't be accurate, so use a new target. Do this also if target is not
1532 a REG, first because having a register instead may open optimization
1533 opportunities, and second because if target and op0 happen to be MEMs
1534 designating the same location, we would risk clobbering it too early
1535 in the code sequence we generate below. */
1536 if (target == 0
1537 || target == op0
1538 || target == op1
1539 || !REG_P (target)
1540 || reg_overlap_mentioned_p (target, op0)
1541 || reg_overlap_mentioned_p (target, op1)
1542 || !valid_multiword_target_p (target))
1543 target = gen_reg_rtx (int_mode);
1544
1545 start_sequence ();
1546
1547 shift_count = INTVAL (op1);
1548
1549 /* OUTOF_* is the word we are shifting bits away from, and
1550 INTO_* is the word that we are shifting bits towards, thus
1551 they differ depending on the direction of the shift and
1552 WORDS_BIG_ENDIAN. */
1553
1554 left_shift = (binoptab == rotl_optab);
1555 outof_word = left_shift ^ ! WORDS_BIG_ENDIAN;
1556
1557 outof_target = operand_subword (target, outof_word, 1, int_mode);
1558 into_target = operand_subword (target, 1 - outof_word, 1, int_mode);
1559
1560 outof_input = operand_subword_force (op0, outof_word, int_mode);
1561 into_input = operand_subword_force (op0, 1 - outof_word, int_mode);
1562
1563 if (shift_count == BITS_PER_WORD)
1564 {
1565 /* This is just a word swap. */
1566 emit_move_insn (outof_target, into_input);
1567 emit_move_insn (into_target, outof_input);
1568 inter = const0_rtx;
1569 }
1570 else
1571 {
1572 rtx into_temp1, into_temp2, outof_temp1, outof_temp2;
1573 HOST_WIDE_INT first_shift_count, second_shift_count;
1574 optab reverse_unsigned_shift, unsigned_shift;
1575
1576 reverse_unsigned_shift = (left_shift ^ (shift_count < BITS_PER_WORD)
1577 ? lshr_optab : ashl_optab);
1578
1579 unsigned_shift = (left_shift ^ (shift_count < BITS_PER_WORD)
1580 ? ashl_optab : lshr_optab);
1581
1582 if (shift_count > BITS_PER_WORD)
1583 {
1584 first_shift_count = shift_count - BITS_PER_WORD;
1585 second_shift_count = 2 * BITS_PER_WORD - shift_count;
1586 }
1587 else
1588 {
1589 first_shift_count = BITS_PER_WORD - shift_count;
1590 second_shift_count = shift_count;
1591 }
1592 rtx first_shift_count_rtx
1593 = gen_int_shift_amount (word_mode, first_shift_count);
1594 rtx second_shift_count_rtx
1595 = gen_int_shift_amount (word_mode, second_shift_count);
1596
1597 into_temp1 = expand_binop (word_mode, unsigned_shift,
1598 outof_input, first_shift_count_rtx,
1599 NULL_RTX, unsignedp, next_methods);
1600 into_temp2 = expand_binop (word_mode, reverse_unsigned_shift,
1601 into_input, second_shift_count_rtx,
1602 NULL_RTX, unsignedp, next_methods);
1603
1604 if (into_temp1 != 0 && into_temp2 != 0)
1605 inter = expand_binop (word_mode, ior_optab, into_temp1, into_temp2,
1606 into_target, unsignedp, next_methods);
1607 else
1608 inter = 0;
1609
1610 if (inter != 0 && inter != into_target)
1611 emit_move_insn (into_target, inter);
1612
1613 outof_temp1 = expand_binop (word_mode, unsigned_shift,
1614 into_input, first_shift_count_rtx,
1615 NULL_RTX, unsignedp, next_methods);
1616 outof_temp2 = expand_binop (word_mode, reverse_unsigned_shift,
1617 outof_input, second_shift_count_rtx,
1618 NULL_RTX, unsignedp, next_methods);
1619
1620 if (inter != 0 && outof_temp1 != 0 && outof_temp2 != 0)
1621 inter = expand_binop (word_mode, ior_optab,
1622 outof_temp1, outof_temp2,
1623 outof_target, unsignedp, next_methods);
1624
1625 if (inter != 0 && inter != outof_target)
1626 emit_move_insn (outof_target, inter);
1627 }
1628
1629 insns = get_insns ();
1630 end_sequence ();
1631
1632 if (inter != 0)
1633 {
1634 emit_insn (insns);
1635 return target;
1636 }
1637 }
1638
1639 /* These can be done a word at a time by propagating carries. */
1640 if ((binoptab == add_optab || binoptab == sub_optab)
1641 && is_int_mode (mode, &int_mode)
1642 && GET_MODE_SIZE (int_mode) >= 2 * UNITS_PER_WORD
1643 && optab_handler (binoptab, word_mode) != CODE_FOR_nothing)
1644 {
1645 unsigned int i;
1646 optab otheroptab = binoptab == add_optab ? sub_optab : add_optab;
1647 const unsigned int nwords = GET_MODE_BITSIZE (int_mode) / BITS_PER_WORD;
1648 rtx carry_in = NULL_RTX, carry_out = NULL_RTX;
1649 rtx xop0, xop1, xtarget;
1650
1651 /* We can handle either a 1 or -1 value for the carry. If STORE_FLAG
1652 value is one of those, use it. Otherwise, use 1 since it is the
1653 one easiest to get. */
1654 #if STORE_FLAG_VALUE == 1 || STORE_FLAG_VALUE == -1
1655 int normalizep = STORE_FLAG_VALUE;
1656 #else
1657 int normalizep = 1;
1658 #endif
1659
1660 /* Prepare the operands. */
1661 xop0 = force_reg (int_mode, op0);
1662 xop1 = force_reg (int_mode, op1);
1663
1664 xtarget = gen_reg_rtx (int_mode);
1665
1666 if (target == 0 || !REG_P (target) || !valid_multiword_target_p (target))
1667 target = xtarget;
1668
1669 /* Indicate for flow that the entire target reg is being set. */
1670 if (REG_P (target))
1671 emit_clobber (xtarget);
1672
1673 /* Do the actual arithmetic. */
1674 for (i = 0; i < nwords; i++)
1675 {
1676 int index = (WORDS_BIG_ENDIAN ? nwords - i - 1 : i);
1677 rtx target_piece = operand_subword (xtarget, index, 1, int_mode);
1678 rtx op0_piece = operand_subword_force (xop0, index, int_mode);
1679 rtx op1_piece = operand_subword_force (xop1, index, int_mode);
1680 rtx x;
1681
1682 /* Main add/subtract of the input operands. */
1683 x = expand_binop (word_mode, binoptab,
1684 op0_piece, op1_piece,
1685 target_piece, unsignedp, next_methods);
1686 if (x == 0)
1687 break;
1688
1689 if (i + 1 < nwords)
1690 {
1691 /* Store carry from main add/subtract. */
1692 carry_out = gen_reg_rtx (word_mode);
1693 carry_out = emit_store_flag_force (carry_out,
1694 (binoptab == add_optab
1695 ? LT : GT),
1696 x, op0_piece,
1697 word_mode, 1, normalizep);
1698 }
1699
1700 if (i > 0)
1701 {
1702 rtx newx;
1703
1704 /* Add/subtract previous carry to main result. */
1705 newx = expand_binop (word_mode,
1706 normalizep == 1 ? binoptab : otheroptab,
1707 x, carry_in,
1708 NULL_RTX, 1, next_methods);
1709
1710 if (i + 1 < nwords)
1711 {
1712 /* Get out carry from adding/subtracting carry in. */
1713 rtx carry_tmp = gen_reg_rtx (word_mode);
1714 carry_tmp = emit_store_flag_force (carry_tmp,
1715 (binoptab == add_optab
1716 ? LT : GT),
1717 newx, x,
1718 word_mode, 1, normalizep);
1719
1720 /* Logical-ior the two poss. carry together. */
1721 carry_out = expand_binop (word_mode, ior_optab,
1722 carry_out, carry_tmp,
1723 carry_out, 0, next_methods);
1724 if (carry_out == 0)
1725 break;
1726 }
1727 emit_move_insn (target_piece, newx);
1728 }
1729 else
1730 {
1731 if (x != target_piece)
1732 emit_move_insn (target_piece, x);
1733 }
1734
1735 carry_in = carry_out;
1736 }
1737
1738 if (i == GET_MODE_BITSIZE (int_mode) / (unsigned) BITS_PER_WORD)
1739 {
1740 if (optab_handler (mov_optab, int_mode) != CODE_FOR_nothing
1741 || ! rtx_equal_p (target, xtarget))
1742 {
1743 rtx_insn *temp = emit_move_insn (target, xtarget);
1744
1745 set_dst_reg_note (temp, REG_EQUAL,
1746 gen_rtx_fmt_ee (optab_to_code (binoptab),
1747 int_mode, copy_rtx (xop0),
1748 copy_rtx (xop1)),
1749 target);
1750 }
1751 else
1752 target = xtarget;
1753
1754 return target;
1755 }
1756
1757 else
1758 delete_insns_since (last);
1759 }
1760
1761 /* Attempt to synthesize double word multiplies using a sequence of word
1762 mode multiplications. We first attempt to generate a sequence using a
1763 more efficient unsigned widening multiply, and if that fails we then
1764 try using a signed widening multiply. */
1765
1766 if (binoptab == smul_optab
1767 && is_int_mode (mode, &int_mode)
1768 && GET_MODE_SIZE (int_mode) == 2 * UNITS_PER_WORD
1769 && optab_handler (smul_optab, word_mode) != CODE_FOR_nothing
1770 && optab_handler (add_optab, word_mode) != CODE_FOR_nothing)
1771 {
1772 rtx product = NULL_RTX;
1773 if (convert_optab_handler (umul_widen_optab, int_mode, word_mode)
1774 != CODE_FOR_nothing)
1775 {
1776 product = expand_doubleword_mult (int_mode, op0, op1, target,
1777 true, methods);
1778 if (!product)
1779 delete_insns_since (last);
1780 }
1781
1782 if (product == NULL_RTX
1783 && (convert_optab_handler (smul_widen_optab, int_mode, word_mode)
1784 != CODE_FOR_nothing))
1785 {
1786 product = expand_doubleword_mult (int_mode, op0, op1, target,
1787 false, methods);
1788 if (!product)
1789 delete_insns_since (last);
1790 }
1791
1792 if (product != NULL_RTX)
1793 {
1794 if (optab_handler (mov_optab, int_mode) != CODE_FOR_nothing)
1795 {
1796 rtx_insn *move = emit_move_insn (target ? target : product,
1797 product);
1798 set_dst_reg_note (move,
1799 REG_EQUAL,
1800 gen_rtx_fmt_ee (MULT, int_mode,
1801 copy_rtx (op0),
1802 copy_rtx (op1)),
1803 target ? target : product);
1804 }
1805 return product;
1806 }
1807 }
1808
1809 /* It can't be open-coded in this mode.
1810 Use a library call if one is available and caller says that's ok. */
1811
1812 libfunc = optab_libfunc (binoptab, mode);
1813 if (libfunc
1814 && (methods == OPTAB_LIB || methods == OPTAB_LIB_WIDEN))
1815 {
1816 rtx_insn *insns;
1817 rtx op1x = op1;
1818 machine_mode op1_mode = mode;
1819 rtx value;
1820
1821 start_sequence ();
1822
1823 if (shift_optab_p (binoptab))
1824 {
1825 op1_mode = targetm.libgcc_shift_count_mode ();
1826 /* Specify unsigned here,
1827 since negative shift counts are meaningless. */
1828 op1x = convert_to_mode (op1_mode, op1, 1);
1829 }
1830
1831 if (GET_MODE (op0) != VOIDmode
1832 && GET_MODE (op0) != mode)
1833 op0 = convert_to_mode (mode, op0, unsignedp);
1834
1835 /* Pass 1 for NO_QUEUE so we don't lose any increments
1836 if the libcall is cse'd or moved. */
1837 value = emit_library_call_value (libfunc,
1838 NULL_RTX, LCT_CONST, mode,
1839 op0, mode, op1x, op1_mode);
1840
1841 insns = get_insns ();
1842 end_sequence ();
1843
1844 bool trapv = trapv_binoptab_p (binoptab);
1845 target = gen_reg_rtx (mode);
1846 emit_libcall_block_1 (insns, target, value,
1847 trapv ? NULL_RTX
1848 : gen_rtx_fmt_ee (optab_to_code (binoptab),
1849 mode, op0, op1), trapv);
1850
1851 return target;
1852 }
1853
1854 delete_insns_since (last);
1855
1856 /* It can't be done in this mode. Can we do it in a wider mode? */
1857
1858 if (! (methods == OPTAB_WIDEN || methods == OPTAB_LIB_WIDEN
1859 || methods == OPTAB_MUST_WIDEN))
1860 {
1861 /* Caller says, don't even try. */
1862 delete_insns_since (entry_last);
1863 return 0;
1864 }
1865
1866 /* Compute the value of METHODS to pass to recursive calls.
1867 Don't allow widening to be tried recursively. */
1868
1869 methods = (methods == OPTAB_LIB_WIDEN ? OPTAB_LIB : OPTAB_DIRECT);
1870
1871 /* Look for a wider mode of the same class for which it appears we can do
1872 the operation. */
1873
1874 if (CLASS_HAS_WIDER_MODES_P (mclass))
1875 {
1876 /* This code doesn't make sense for conversion optabs, since we
1877 wouldn't then want to extend the operands to be the same size
1878 as the result. */
1879 gcc_assert (!convert_optab_p (binoptab));
1880 FOR_EACH_WIDER_MODE (wider_mode, mode)
1881 {
1882 if (optab_handler (binoptab, wider_mode)
1883 || (methods == OPTAB_LIB
1884 && optab_libfunc (binoptab, wider_mode)))
1885 {
1886 rtx xop0 = op0, xop1 = op1;
1887 int no_extend = 0;
1888
1889 /* For certain integer operations, we need not actually extend
1890 the narrow operands, as long as we will truncate
1891 the results to the same narrowness. */
1892
1893 if ((binoptab == ior_optab || binoptab == and_optab
1894 || binoptab == xor_optab
1895 || binoptab == add_optab || binoptab == sub_optab
1896 || binoptab == smul_optab || binoptab == ashl_optab)
1897 && mclass == MODE_INT)
1898 no_extend = 1;
1899
1900 xop0 = widen_operand (xop0, wider_mode, mode,
1901 unsignedp, no_extend);
1902
1903 /* The second operand of a shift must always be extended. */
1904 xop1 = widen_operand (xop1, wider_mode, mode, unsignedp,
1905 no_extend && binoptab != ashl_optab);
1906
1907 temp = expand_binop (wider_mode, binoptab, xop0, xop1, NULL_RTX,
1908 unsignedp, methods);
1909 if (temp)
1910 {
1911 if (mclass != MODE_INT
1912 || !TRULY_NOOP_TRUNCATION_MODES_P (mode, wider_mode))
1913 {
1914 if (target == 0)
1915 target = gen_reg_rtx (mode);
1916 convert_move (target, temp, 0);
1917 return target;
1918 }
1919 else
1920 return gen_lowpart (mode, temp);
1921 }
1922 else
1923 delete_insns_since (last);
1924 }
1925 }
1926 }
1927
1928 delete_insns_since (entry_last);
1929 return 0;
1930 }
1931
1932 /* Expand a binary operator which has both signed and unsigned forms.
1933 UOPTAB is the optab for unsigned operations, and SOPTAB is for
1934 signed operations.
1935
1936 If we widen unsigned operands, we may use a signed wider operation instead
1937 of an unsigned wider operation, since the result would be the same. */
1938
1939 rtx
sign_expand_binop(machine_mode mode,optab uoptab,optab soptab,rtx op0,rtx op1,rtx target,int unsignedp,enum optab_methods methods)1940 sign_expand_binop (machine_mode mode, optab uoptab, optab soptab,
1941 rtx op0, rtx op1, rtx target, int unsignedp,
1942 enum optab_methods methods)
1943 {
1944 rtx temp;
1945 optab direct_optab = unsignedp ? uoptab : soptab;
1946 bool save_enable;
1947
1948 /* Do it without widening, if possible. */
1949 temp = expand_binop (mode, direct_optab, op0, op1, target,
1950 unsignedp, OPTAB_DIRECT);
1951 if (temp || methods == OPTAB_DIRECT)
1952 return temp;
1953
1954 /* Try widening to a signed int. Disable any direct use of any
1955 signed insn in the current mode. */
1956 save_enable = swap_optab_enable (soptab, mode, false);
1957
1958 temp = expand_binop (mode, soptab, op0, op1, target,
1959 unsignedp, OPTAB_WIDEN);
1960
1961 /* For unsigned operands, try widening to an unsigned int. */
1962 if (!temp && unsignedp)
1963 temp = expand_binop (mode, uoptab, op0, op1, target,
1964 unsignedp, OPTAB_WIDEN);
1965 if (temp || methods == OPTAB_WIDEN)
1966 goto egress;
1967
1968 /* Use the right width libcall if that exists. */
1969 temp = expand_binop (mode, direct_optab, op0, op1, target,
1970 unsignedp, OPTAB_LIB);
1971 if (temp || methods == OPTAB_LIB)
1972 goto egress;
1973
1974 /* Must widen and use a libcall, use either signed or unsigned. */
1975 temp = expand_binop (mode, soptab, op0, op1, target,
1976 unsignedp, methods);
1977 if (!temp && unsignedp)
1978 temp = expand_binop (mode, uoptab, op0, op1, target,
1979 unsignedp, methods);
1980
1981 egress:
1982 /* Undo the fiddling above. */
1983 if (save_enable)
1984 swap_optab_enable (soptab, mode, true);
1985 return temp;
1986 }
1987
1988 /* Generate code to perform an operation specified by UNOPPTAB
1989 on operand OP0, with two results to TARG0 and TARG1.
1990 We assume that the order of the operands for the instruction
1991 is TARG0, TARG1, OP0.
1992
1993 Either TARG0 or TARG1 may be zero, but what that means is that
1994 the result is not actually wanted. We will generate it into
1995 a dummy pseudo-reg and discard it. They may not both be zero.
1996
1997 Returns 1 if this operation can be performed; 0 if not. */
1998
1999 int
expand_twoval_unop(optab unoptab,rtx op0,rtx targ0,rtx targ1,int unsignedp)2000 expand_twoval_unop (optab unoptab, rtx op0, rtx targ0, rtx targ1,
2001 int unsignedp)
2002 {
2003 machine_mode mode = GET_MODE (targ0 ? targ0 : targ1);
2004 enum mode_class mclass;
2005 machine_mode wider_mode;
2006 rtx_insn *entry_last = get_last_insn ();
2007 rtx_insn *last;
2008
2009 mclass = GET_MODE_CLASS (mode);
2010
2011 if (!targ0)
2012 targ0 = gen_reg_rtx (mode);
2013 if (!targ1)
2014 targ1 = gen_reg_rtx (mode);
2015
2016 /* Record where to go back to if we fail. */
2017 last = get_last_insn ();
2018
2019 if (optab_handler (unoptab, mode) != CODE_FOR_nothing)
2020 {
2021 class expand_operand ops[3];
2022 enum insn_code icode = optab_handler (unoptab, mode);
2023
2024 create_fixed_operand (&ops[0], targ0);
2025 create_fixed_operand (&ops[1], targ1);
2026 create_convert_operand_from (&ops[2], op0, mode, unsignedp);
2027 if (maybe_expand_insn (icode, 3, ops))
2028 return 1;
2029 }
2030
2031 /* It can't be done in this mode. Can we do it in a wider mode? */
2032
2033 if (CLASS_HAS_WIDER_MODES_P (mclass))
2034 {
2035 FOR_EACH_WIDER_MODE (wider_mode, mode)
2036 {
2037 if (optab_handler (unoptab, wider_mode) != CODE_FOR_nothing)
2038 {
2039 rtx t0 = gen_reg_rtx (wider_mode);
2040 rtx t1 = gen_reg_rtx (wider_mode);
2041 rtx cop0 = convert_modes (wider_mode, mode, op0, unsignedp);
2042
2043 if (expand_twoval_unop (unoptab, cop0, t0, t1, unsignedp))
2044 {
2045 convert_move (targ0, t0, unsignedp);
2046 convert_move (targ1, t1, unsignedp);
2047 return 1;
2048 }
2049 else
2050 delete_insns_since (last);
2051 }
2052 }
2053 }
2054
2055 delete_insns_since (entry_last);
2056 return 0;
2057 }
2058
2059 /* Generate code to perform an operation specified by BINOPTAB
2060 on operands OP0 and OP1, with two results to TARG1 and TARG2.
2061 We assume that the order of the operands for the instruction
2062 is TARG0, OP0, OP1, TARG1, which would fit a pattern like
2063 [(set TARG0 (operate OP0 OP1)) (set TARG1 (operate ...))].
2064
2065 Either TARG0 or TARG1 may be zero, but what that means is that
2066 the result is not actually wanted. We will generate it into
2067 a dummy pseudo-reg and discard it. They may not both be zero.
2068
2069 Returns 1 if this operation can be performed; 0 if not. */
2070
2071 int
expand_twoval_binop(optab binoptab,rtx op0,rtx op1,rtx targ0,rtx targ1,int unsignedp)2072 expand_twoval_binop (optab binoptab, rtx op0, rtx op1, rtx targ0, rtx targ1,
2073 int unsignedp)
2074 {
2075 machine_mode mode = GET_MODE (targ0 ? targ0 : targ1);
2076 enum mode_class mclass;
2077 machine_mode wider_mode;
2078 rtx_insn *entry_last = get_last_insn ();
2079 rtx_insn *last;
2080
2081 mclass = GET_MODE_CLASS (mode);
2082
2083 if (!targ0)
2084 targ0 = gen_reg_rtx (mode);
2085 if (!targ1)
2086 targ1 = gen_reg_rtx (mode);
2087
2088 /* Record where to go back to if we fail. */
2089 last = get_last_insn ();
2090
2091 if (optab_handler (binoptab, mode) != CODE_FOR_nothing)
2092 {
2093 class expand_operand ops[4];
2094 enum insn_code icode = optab_handler (binoptab, mode);
2095 machine_mode mode0 = insn_data[icode].operand[1].mode;
2096 machine_mode mode1 = insn_data[icode].operand[2].mode;
2097 rtx xop0 = op0, xop1 = op1;
2098
2099 /* If we are optimizing, force expensive constants into a register. */
2100 xop0 = avoid_expensive_constant (mode0, binoptab, 0, xop0, unsignedp);
2101 xop1 = avoid_expensive_constant (mode1, binoptab, 1, xop1, unsignedp);
2102
2103 create_fixed_operand (&ops[0], targ0);
2104 create_convert_operand_from (&ops[1], xop0, mode, unsignedp);
2105 create_convert_operand_from (&ops[2], xop1, mode, unsignedp);
2106 create_fixed_operand (&ops[3], targ1);
2107 if (maybe_expand_insn (icode, 4, ops))
2108 return 1;
2109 delete_insns_since (last);
2110 }
2111
2112 /* It can't be done in this mode. Can we do it in a wider mode? */
2113
2114 if (CLASS_HAS_WIDER_MODES_P (mclass))
2115 {
2116 FOR_EACH_WIDER_MODE (wider_mode, mode)
2117 {
2118 if (optab_handler (binoptab, wider_mode) != CODE_FOR_nothing)
2119 {
2120 rtx t0 = gen_reg_rtx (wider_mode);
2121 rtx t1 = gen_reg_rtx (wider_mode);
2122 rtx cop0 = convert_modes (wider_mode, mode, op0, unsignedp);
2123 rtx cop1 = convert_modes (wider_mode, mode, op1, unsignedp);
2124
2125 if (expand_twoval_binop (binoptab, cop0, cop1,
2126 t0, t1, unsignedp))
2127 {
2128 convert_move (targ0, t0, unsignedp);
2129 convert_move (targ1, t1, unsignedp);
2130 return 1;
2131 }
2132 else
2133 delete_insns_since (last);
2134 }
2135 }
2136 }
2137
2138 delete_insns_since (entry_last);
2139 return 0;
2140 }
2141
2142 /* Expand the two-valued library call indicated by BINOPTAB, but
2143 preserve only one of the values. If TARG0 is non-NULL, the first
2144 value is placed into TARG0; otherwise the second value is placed
2145 into TARG1. Exactly one of TARG0 and TARG1 must be non-NULL. The
2146 value stored into TARG0 or TARG1 is equivalent to (CODE OP0 OP1).
2147 This routine assumes that the value returned by the library call is
2148 as if the return value was of an integral mode twice as wide as the
2149 mode of OP0. Returns 1 if the call was successful. */
2150
2151 bool
expand_twoval_binop_libfunc(optab binoptab,rtx op0,rtx op1,rtx targ0,rtx targ1,enum rtx_code code)2152 expand_twoval_binop_libfunc (optab binoptab, rtx op0, rtx op1,
2153 rtx targ0, rtx targ1, enum rtx_code code)
2154 {
2155 machine_mode mode;
2156 machine_mode libval_mode;
2157 rtx libval;
2158 rtx_insn *insns;
2159 rtx libfunc;
2160
2161 /* Exactly one of TARG0 or TARG1 should be non-NULL. */
2162 gcc_assert (!targ0 != !targ1);
2163
2164 mode = GET_MODE (op0);
2165 libfunc = optab_libfunc (binoptab, mode);
2166 if (!libfunc)
2167 return false;
2168
2169 /* The value returned by the library function will have twice as
2170 many bits as the nominal MODE. */
2171 libval_mode = smallest_int_mode_for_size (2 * GET_MODE_BITSIZE (mode));
2172 start_sequence ();
2173 libval = emit_library_call_value (libfunc, NULL_RTX, LCT_CONST,
2174 libval_mode,
2175 op0, mode,
2176 op1, mode);
2177 /* Get the part of VAL containing the value that we want. */
2178 libval = simplify_gen_subreg (mode, libval, libval_mode,
2179 targ0 ? 0 : GET_MODE_SIZE (mode));
2180 insns = get_insns ();
2181 end_sequence ();
2182 /* Move the into the desired location. */
2183 emit_libcall_block (insns, targ0 ? targ0 : targ1, libval,
2184 gen_rtx_fmt_ee (code, mode, op0, op1));
2185
2186 return true;
2187 }
2188
2189
2190 /* Wrapper around expand_unop which takes an rtx code to specify
2191 the operation to perform, not an optab pointer. All other
2192 arguments are the same. */
2193 rtx
expand_simple_unop(machine_mode mode,enum rtx_code code,rtx op0,rtx target,int unsignedp)2194 expand_simple_unop (machine_mode mode, enum rtx_code code, rtx op0,
2195 rtx target, int unsignedp)
2196 {
2197 optab unop = code_to_optab (code);
2198 gcc_assert (unop);
2199
2200 return expand_unop (mode, unop, op0, target, unsignedp);
2201 }
2202
2203 /* Try calculating
2204 (clz:narrow x)
2205 as
2206 (clz:wide (zero_extend:wide x)) - ((width wide) - (width narrow)).
2207
2208 A similar operation can be used for clrsb. UNOPTAB says which operation
2209 we are trying to expand. */
2210 static rtx
widen_leading(scalar_int_mode mode,rtx op0,rtx target,optab unoptab)2211 widen_leading (scalar_int_mode mode, rtx op0, rtx target, optab unoptab)
2212 {
2213 opt_scalar_int_mode wider_mode_iter;
2214 FOR_EACH_WIDER_MODE (wider_mode_iter, mode)
2215 {
2216 scalar_int_mode wider_mode = wider_mode_iter.require ();
2217 if (optab_handler (unoptab, wider_mode) != CODE_FOR_nothing)
2218 {
2219 rtx xop0, temp;
2220 rtx_insn *last;
2221
2222 last = get_last_insn ();
2223
2224 if (target == 0)
2225 target = gen_reg_rtx (mode);
2226 xop0 = widen_operand (op0, wider_mode, mode,
2227 unoptab != clrsb_optab, false);
2228 temp = expand_unop (wider_mode, unoptab, xop0, NULL_RTX,
2229 unoptab != clrsb_optab);
2230 if (temp != 0)
2231 temp = expand_binop
2232 (wider_mode, sub_optab, temp,
2233 gen_int_mode (GET_MODE_PRECISION (wider_mode)
2234 - GET_MODE_PRECISION (mode),
2235 wider_mode),
2236 target, true, OPTAB_DIRECT);
2237 if (temp == 0)
2238 delete_insns_since (last);
2239
2240 return temp;
2241 }
2242 }
2243 return 0;
2244 }
2245
2246 /* Try calculating clz of a double-word quantity as two clz's of word-sized
2247 quantities, choosing which based on whether the high word is nonzero. */
2248 static rtx
expand_doubleword_clz(scalar_int_mode mode,rtx op0,rtx target)2249 expand_doubleword_clz (scalar_int_mode mode, rtx op0, rtx target)
2250 {
2251 rtx xop0 = force_reg (mode, op0);
2252 rtx subhi = gen_highpart (word_mode, xop0);
2253 rtx sublo = gen_lowpart (word_mode, xop0);
2254 rtx_code_label *hi0_label = gen_label_rtx ();
2255 rtx_code_label *after_label = gen_label_rtx ();
2256 rtx_insn *seq;
2257 rtx temp, result;
2258
2259 /* If we were not given a target, use a word_mode register, not a
2260 'mode' register. The result will fit, and nobody is expecting
2261 anything bigger (the return type of __builtin_clz* is int). */
2262 if (!target)
2263 target = gen_reg_rtx (word_mode);
2264
2265 /* In any case, write to a word_mode scratch in both branches of the
2266 conditional, so we can ensure there is a single move insn setting
2267 'target' to tag a REG_EQUAL note on. */
2268 result = gen_reg_rtx (word_mode);
2269
2270 start_sequence ();
2271
2272 /* If the high word is not equal to zero,
2273 then clz of the full value is clz of the high word. */
2274 emit_cmp_and_jump_insns (subhi, CONST0_RTX (word_mode), EQ, 0,
2275 word_mode, true, hi0_label);
2276
2277 temp = expand_unop_direct (word_mode, clz_optab, subhi, result, true);
2278 if (!temp)
2279 goto fail;
2280
2281 if (temp != result)
2282 convert_move (result, temp, true);
2283
2284 emit_jump_insn (targetm.gen_jump (after_label));
2285 emit_barrier ();
2286
2287 /* Else clz of the full value is clz of the low word plus the number
2288 of bits in the high word. */
2289 emit_label (hi0_label);
2290
2291 temp = expand_unop_direct (word_mode, clz_optab, sublo, 0, true);
2292 if (!temp)
2293 goto fail;
2294 temp = expand_binop (word_mode, add_optab, temp,
2295 gen_int_mode (GET_MODE_BITSIZE (word_mode), word_mode),
2296 result, true, OPTAB_DIRECT);
2297 if (!temp)
2298 goto fail;
2299 if (temp != result)
2300 convert_move (result, temp, true);
2301
2302 emit_label (after_label);
2303 convert_move (target, result, true);
2304
2305 seq = get_insns ();
2306 end_sequence ();
2307
2308 add_equal_note (seq, target, CLZ, xop0, NULL_RTX, mode);
2309 emit_insn (seq);
2310 return target;
2311
2312 fail:
2313 end_sequence ();
2314 return 0;
2315 }
2316
2317 /* Try calculating popcount of a double-word quantity as two popcount's of
2318 word-sized quantities and summing up the results. */
2319 static rtx
expand_doubleword_popcount(scalar_int_mode mode,rtx op0,rtx target)2320 expand_doubleword_popcount (scalar_int_mode mode, rtx op0, rtx target)
2321 {
2322 rtx t0, t1, t;
2323 rtx_insn *seq;
2324
2325 start_sequence ();
2326
2327 t0 = expand_unop_direct (word_mode, popcount_optab,
2328 operand_subword_force (op0, 0, mode), NULL_RTX,
2329 true);
2330 t1 = expand_unop_direct (word_mode, popcount_optab,
2331 operand_subword_force (op0, 1, mode), NULL_RTX,
2332 true);
2333 if (!t0 || !t1)
2334 {
2335 end_sequence ();
2336 return NULL_RTX;
2337 }
2338
2339 /* If we were not given a target, use a word_mode register, not a
2340 'mode' register. The result will fit, and nobody is expecting
2341 anything bigger (the return type of __builtin_popcount* is int). */
2342 if (!target)
2343 target = gen_reg_rtx (word_mode);
2344
2345 t = expand_binop (word_mode, add_optab, t0, t1, target, 0, OPTAB_DIRECT);
2346
2347 seq = get_insns ();
2348 end_sequence ();
2349
2350 add_equal_note (seq, t, POPCOUNT, op0, NULL_RTX, mode);
2351 emit_insn (seq);
2352 return t;
2353 }
2354
2355 /* Try calculating
2356 (parity:wide x)
2357 as
2358 (parity:narrow (low (x) ^ high (x))) */
2359 static rtx
expand_doubleword_parity(scalar_int_mode mode,rtx op0,rtx target)2360 expand_doubleword_parity (scalar_int_mode mode, rtx op0, rtx target)
2361 {
2362 rtx t = expand_binop (word_mode, xor_optab,
2363 operand_subword_force (op0, 0, mode),
2364 operand_subword_force (op0, 1, mode),
2365 NULL_RTX, 0, OPTAB_DIRECT);
2366 return expand_unop (word_mode, parity_optab, t, target, true);
2367 }
2368
2369 /* Try calculating
2370 (bswap:narrow x)
2371 as
2372 (lshiftrt:wide (bswap:wide x) ((width wide) - (width narrow))). */
2373 static rtx
widen_bswap(scalar_int_mode mode,rtx op0,rtx target)2374 widen_bswap (scalar_int_mode mode, rtx op0, rtx target)
2375 {
2376 rtx x;
2377 rtx_insn *last;
2378 opt_scalar_int_mode wider_mode_iter;
2379
2380 FOR_EACH_WIDER_MODE (wider_mode_iter, mode)
2381 if (optab_handler (bswap_optab, wider_mode_iter.require ())
2382 != CODE_FOR_nothing)
2383 break;
2384
2385 if (!wider_mode_iter.exists ())
2386 return NULL_RTX;
2387
2388 scalar_int_mode wider_mode = wider_mode_iter.require ();
2389 last = get_last_insn ();
2390
2391 x = widen_operand (op0, wider_mode, mode, true, true);
2392 x = expand_unop (wider_mode, bswap_optab, x, NULL_RTX, true);
2393
2394 gcc_assert (GET_MODE_PRECISION (wider_mode) == GET_MODE_BITSIZE (wider_mode)
2395 && GET_MODE_PRECISION (mode) == GET_MODE_BITSIZE (mode));
2396 if (x != 0)
2397 x = expand_shift (RSHIFT_EXPR, wider_mode, x,
2398 GET_MODE_BITSIZE (wider_mode)
2399 - GET_MODE_BITSIZE (mode),
2400 NULL_RTX, true);
2401
2402 if (x != 0)
2403 {
2404 if (target == 0)
2405 target = gen_reg_rtx (mode);
2406 emit_move_insn (target, gen_lowpart (mode, x));
2407 }
2408 else
2409 delete_insns_since (last);
2410
2411 return target;
2412 }
2413
2414 /* Try calculating bswap as two bswaps of two word-sized operands. */
2415
2416 static rtx
expand_doubleword_bswap(machine_mode mode,rtx op,rtx target)2417 expand_doubleword_bswap (machine_mode mode, rtx op, rtx target)
2418 {
2419 rtx t0, t1;
2420
2421 t1 = expand_unop (word_mode, bswap_optab,
2422 operand_subword_force (op, 0, mode), NULL_RTX, true);
2423 t0 = expand_unop (word_mode, bswap_optab,
2424 operand_subword_force (op, 1, mode), NULL_RTX, true);
2425
2426 if (target == 0 || !valid_multiword_target_p (target))
2427 target = gen_reg_rtx (mode);
2428 if (REG_P (target))
2429 emit_clobber (target);
2430 emit_move_insn (operand_subword (target, 0, 1, mode), t0);
2431 emit_move_insn (operand_subword (target, 1, 1, mode), t1);
2432
2433 return target;
2434 }
2435
2436 /* Try calculating (parity x) as (and (popcount x) 1), where
2437 popcount can also be done in a wider mode. */
2438 static rtx
expand_parity(scalar_int_mode mode,rtx op0,rtx target)2439 expand_parity (scalar_int_mode mode, rtx op0, rtx target)
2440 {
2441 enum mode_class mclass = GET_MODE_CLASS (mode);
2442 opt_scalar_int_mode wider_mode_iter;
2443 FOR_EACH_MODE_FROM (wider_mode_iter, mode)
2444 {
2445 scalar_int_mode wider_mode = wider_mode_iter.require ();
2446 if (optab_handler (popcount_optab, wider_mode) != CODE_FOR_nothing)
2447 {
2448 rtx xop0, temp;
2449 rtx_insn *last;
2450
2451 last = get_last_insn ();
2452
2453 if (target == 0 || GET_MODE (target) != wider_mode)
2454 target = gen_reg_rtx (wider_mode);
2455
2456 xop0 = widen_operand (op0, wider_mode, mode, true, false);
2457 temp = expand_unop (wider_mode, popcount_optab, xop0, NULL_RTX,
2458 true);
2459 if (temp != 0)
2460 temp = expand_binop (wider_mode, and_optab, temp, const1_rtx,
2461 target, true, OPTAB_DIRECT);
2462
2463 if (temp)
2464 {
2465 if (mclass != MODE_INT
2466 || !TRULY_NOOP_TRUNCATION_MODES_P (mode, wider_mode))
2467 return convert_to_mode (mode, temp, 0);
2468 else
2469 return gen_lowpart (mode, temp);
2470 }
2471 else
2472 delete_insns_since (last);
2473 }
2474 }
2475 return 0;
2476 }
2477
2478 /* Try calculating ctz(x) as K - clz(x & -x) ,
2479 where K is GET_MODE_PRECISION(mode) - 1.
2480
2481 Both __builtin_ctz and __builtin_clz are undefined at zero, so we
2482 don't have to worry about what the hardware does in that case. (If
2483 the clz instruction produces the usual value at 0, which is K, the
2484 result of this code sequence will be -1; expand_ffs, below, relies
2485 on this. It might be nice to have it be K instead, for consistency
2486 with the (very few) processors that provide a ctz with a defined
2487 value, but that would take one more instruction, and it would be
2488 less convenient for expand_ffs anyway. */
2489
2490 static rtx
expand_ctz(scalar_int_mode mode,rtx op0,rtx target)2491 expand_ctz (scalar_int_mode mode, rtx op0, rtx target)
2492 {
2493 rtx_insn *seq;
2494 rtx temp;
2495
2496 if (optab_handler (clz_optab, mode) == CODE_FOR_nothing)
2497 return 0;
2498
2499 start_sequence ();
2500
2501 temp = expand_unop_direct (mode, neg_optab, op0, NULL_RTX, true);
2502 if (temp)
2503 temp = expand_binop (mode, and_optab, op0, temp, NULL_RTX,
2504 true, OPTAB_DIRECT);
2505 if (temp)
2506 temp = expand_unop_direct (mode, clz_optab, temp, NULL_RTX, true);
2507 if (temp)
2508 temp = expand_binop (mode, sub_optab,
2509 gen_int_mode (GET_MODE_PRECISION (mode) - 1, mode),
2510 temp, target,
2511 true, OPTAB_DIRECT);
2512 if (temp == 0)
2513 {
2514 end_sequence ();
2515 return 0;
2516 }
2517
2518 seq = get_insns ();
2519 end_sequence ();
2520
2521 add_equal_note (seq, temp, CTZ, op0, NULL_RTX, mode);
2522 emit_insn (seq);
2523 return temp;
2524 }
2525
2526
2527 /* Try calculating ffs(x) using ctz(x) if we have that instruction, or
2528 else with the sequence used by expand_clz.
2529
2530 The ffs builtin promises to return zero for a zero value and ctz/clz
2531 may have an undefined value in that case. If they do not give us a
2532 convenient value, we have to generate a test and branch. */
2533 static rtx
expand_ffs(scalar_int_mode mode,rtx op0,rtx target)2534 expand_ffs (scalar_int_mode mode, rtx op0, rtx target)
2535 {
2536 HOST_WIDE_INT val = 0;
2537 bool defined_at_zero = false;
2538 rtx temp;
2539 rtx_insn *seq;
2540
2541 if (optab_handler (ctz_optab, mode) != CODE_FOR_nothing)
2542 {
2543 start_sequence ();
2544
2545 temp = expand_unop_direct (mode, ctz_optab, op0, 0, true);
2546 if (!temp)
2547 goto fail;
2548
2549 defined_at_zero = (CTZ_DEFINED_VALUE_AT_ZERO (mode, val) == 2);
2550 }
2551 else if (optab_handler (clz_optab, mode) != CODE_FOR_nothing)
2552 {
2553 start_sequence ();
2554 temp = expand_ctz (mode, op0, 0);
2555 if (!temp)
2556 goto fail;
2557
2558 if (CLZ_DEFINED_VALUE_AT_ZERO (mode, val) == 2)
2559 {
2560 defined_at_zero = true;
2561 val = (GET_MODE_PRECISION (mode) - 1) - val;
2562 }
2563 }
2564 else
2565 return 0;
2566
2567 if (defined_at_zero && val == -1)
2568 /* No correction needed at zero. */;
2569 else
2570 {
2571 /* We don't try to do anything clever with the situation found
2572 on some processors (eg Alpha) where ctz(0:mode) ==
2573 bitsize(mode). If someone can think of a way to send N to -1
2574 and leave alone all values in the range 0..N-1 (where N is a
2575 power of two), cheaper than this test-and-branch, please add it.
2576
2577 The test-and-branch is done after the operation itself, in case
2578 the operation sets condition codes that can be recycled for this.
2579 (This is true on i386, for instance.) */
2580
2581 rtx_code_label *nonzero_label = gen_label_rtx ();
2582 emit_cmp_and_jump_insns (op0, CONST0_RTX (mode), NE, 0,
2583 mode, true, nonzero_label);
2584
2585 convert_move (temp, GEN_INT (-1), false);
2586 emit_label (nonzero_label);
2587 }
2588
2589 /* temp now has a value in the range -1..bitsize-1. ffs is supposed
2590 to produce a value in the range 0..bitsize. */
2591 temp = expand_binop (mode, add_optab, temp, gen_int_mode (1, mode),
2592 target, false, OPTAB_DIRECT);
2593 if (!temp)
2594 goto fail;
2595
2596 seq = get_insns ();
2597 end_sequence ();
2598
2599 add_equal_note (seq, temp, FFS, op0, NULL_RTX, mode);
2600 emit_insn (seq);
2601 return temp;
2602
2603 fail:
2604 end_sequence ();
2605 return 0;
2606 }
2607
2608 /* Extract the OMODE lowpart from VAL, which has IMODE. Under certain
2609 conditions, VAL may already be a SUBREG against which we cannot generate
2610 a further SUBREG. In this case, we expect forcing the value into a
2611 register will work around the situation. */
2612
2613 static rtx
lowpart_subreg_maybe_copy(machine_mode omode,rtx val,machine_mode imode)2614 lowpart_subreg_maybe_copy (machine_mode omode, rtx val,
2615 machine_mode imode)
2616 {
2617 rtx ret;
2618 ret = lowpart_subreg (omode, val, imode);
2619 if (ret == NULL)
2620 {
2621 val = force_reg (imode, val);
2622 ret = lowpart_subreg (omode, val, imode);
2623 gcc_assert (ret != NULL);
2624 }
2625 return ret;
2626 }
2627
2628 /* Expand a floating point absolute value or negation operation via a
2629 logical operation on the sign bit. */
2630
2631 static rtx
expand_absneg_bit(enum rtx_code code,scalar_float_mode mode,rtx op0,rtx target)2632 expand_absneg_bit (enum rtx_code code, scalar_float_mode mode,
2633 rtx op0, rtx target)
2634 {
2635 const struct real_format *fmt;
2636 int bitpos, word, nwords, i;
2637 scalar_int_mode imode;
2638 rtx temp;
2639 rtx_insn *insns;
2640
2641 /* The format has to have a simple sign bit. */
2642 fmt = REAL_MODE_FORMAT (mode);
2643 if (fmt == NULL)
2644 return NULL_RTX;
2645
2646 bitpos = fmt->signbit_rw;
2647 if (bitpos < 0)
2648 return NULL_RTX;
2649
2650 /* Don't create negative zeros if the format doesn't support them. */
2651 if (code == NEG && !fmt->has_signed_zero)
2652 return NULL_RTX;
2653
2654 if (GET_MODE_SIZE (mode) <= UNITS_PER_WORD)
2655 {
2656 if (!int_mode_for_mode (mode).exists (&imode))
2657 return NULL_RTX;
2658 word = 0;
2659 nwords = 1;
2660 }
2661 else
2662 {
2663 imode = word_mode;
2664
2665 if (FLOAT_WORDS_BIG_ENDIAN)
2666 word = (GET_MODE_BITSIZE (mode) - bitpos) / BITS_PER_WORD;
2667 else
2668 word = bitpos / BITS_PER_WORD;
2669 bitpos = bitpos % BITS_PER_WORD;
2670 nwords = (GET_MODE_BITSIZE (mode) + BITS_PER_WORD - 1) / BITS_PER_WORD;
2671 }
2672
2673 wide_int mask = wi::set_bit_in_zero (bitpos, GET_MODE_PRECISION (imode));
2674 if (code == ABS)
2675 mask = ~mask;
2676
2677 if (target == 0
2678 || target == op0
2679 || reg_overlap_mentioned_p (target, op0)
2680 || (nwords > 1 && !valid_multiword_target_p (target)))
2681 target = gen_reg_rtx (mode);
2682
2683 if (nwords > 1)
2684 {
2685 start_sequence ();
2686
2687 for (i = 0; i < nwords; ++i)
2688 {
2689 rtx targ_piece = operand_subword (target, i, 1, mode);
2690 rtx op0_piece = operand_subword_force (op0, i, mode);
2691
2692 if (i == word)
2693 {
2694 temp = expand_binop (imode, code == ABS ? and_optab : xor_optab,
2695 op0_piece,
2696 immed_wide_int_const (mask, imode),
2697 targ_piece, 1, OPTAB_LIB_WIDEN);
2698 if (temp != targ_piece)
2699 emit_move_insn (targ_piece, temp);
2700 }
2701 else
2702 emit_move_insn (targ_piece, op0_piece);
2703 }
2704
2705 insns = get_insns ();
2706 end_sequence ();
2707
2708 emit_insn (insns);
2709 }
2710 else
2711 {
2712 temp = expand_binop (imode, code == ABS ? and_optab : xor_optab,
2713 gen_lowpart (imode, op0),
2714 immed_wide_int_const (mask, imode),
2715 gen_lowpart (imode, target), 1, OPTAB_LIB_WIDEN);
2716 target = lowpart_subreg_maybe_copy (mode, temp, imode);
2717
2718 set_dst_reg_note (get_last_insn (), REG_EQUAL,
2719 gen_rtx_fmt_e (code, mode, copy_rtx (op0)),
2720 target);
2721 }
2722
2723 return target;
2724 }
2725
2726 /* As expand_unop, but will fail rather than attempt the operation in a
2727 different mode or with a libcall. */
2728 static rtx
expand_unop_direct(machine_mode mode,optab unoptab,rtx op0,rtx target,int unsignedp)2729 expand_unop_direct (machine_mode mode, optab unoptab, rtx op0, rtx target,
2730 int unsignedp)
2731 {
2732 if (optab_handler (unoptab, mode) != CODE_FOR_nothing)
2733 {
2734 class expand_operand ops[2];
2735 enum insn_code icode = optab_handler (unoptab, mode);
2736 rtx_insn *last = get_last_insn ();
2737 rtx_insn *pat;
2738
2739 create_output_operand (&ops[0], target, mode);
2740 create_convert_operand_from (&ops[1], op0, mode, unsignedp);
2741 pat = maybe_gen_insn (icode, 2, ops);
2742 if (pat)
2743 {
2744 if (INSN_P (pat) && NEXT_INSN (pat) != NULL_RTX
2745 && ! add_equal_note (pat, ops[0].value,
2746 optab_to_code (unoptab),
2747 ops[1].value, NULL_RTX, mode))
2748 {
2749 delete_insns_since (last);
2750 return expand_unop (mode, unoptab, op0, NULL_RTX, unsignedp);
2751 }
2752
2753 emit_insn (pat);
2754
2755 return ops[0].value;
2756 }
2757 }
2758 return 0;
2759 }
2760
2761 /* Generate code to perform an operation specified by UNOPTAB
2762 on operand OP0, with result having machine-mode MODE.
2763
2764 UNSIGNEDP is for the case where we have to widen the operands
2765 to perform the operation. It says to use zero-extension.
2766
2767 If TARGET is nonzero, the value
2768 is generated there, if it is convenient to do so.
2769 In all cases an rtx is returned for the locus of the value;
2770 this may or may not be TARGET. */
2771
2772 rtx
expand_unop(machine_mode mode,optab unoptab,rtx op0,rtx target,int unsignedp)2773 expand_unop (machine_mode mode, optab unoptab, rtx op0, rtx target,
2774 int unsignedp)
2775 {
2776 enum mode_class mclass = GET_MODE_CLASS (mode);
2777 machine_mode wider_mode;
2778 scalar_int_mode int_mode;
2779 scalar_float_mode float_mode;
2780 rtx temp;
2781 rtx libfunc;
2782
2783 temp = expand_unop_direct (mode, unoptab, op0, target, unsignedp);
2784 if (temp)
2785 return temp;
2786
2787 /* It can't be done in this mode. Can we open-code it in a wider mode? */
2788
2789 /* Widening (or narrowing) clz needs special treatment. */
2790 if (unoptab == clz_optab)
2791 {
2792 if (is_a <scalar_int_mode> (mode, &int_mode))
2793 {
2794 temp = widen_leading (int_mode, op0, target, unoptab);
2795 if (temp)
2796 return temp;
2797
2798 if (GET_MODE_SIZE (int_mode) == 2 * UNITS_PER_WORD
2799 && optab_handler (unoptab, word_mode) != CODE_FOR_nothing)
2800 {
2801 temp = expand_doubleword_clz (int_mode, op0, target);
2802 if (temp)
2803 return temp;
2804 }
2805 }
2806
2807 goto try_libcall;
2808 }
2809
2810 if (unoptab == clrsb_optab)
2811 {
2812 if (is_a <scalar_int_mode> (mode, &int_mode))
2813 {
2814 temp = widen_leading (int_mode, op0, target, unoptab);
2815 if (temp)
2816 return temp;
2817 }
2818 goto try_libcall;
2819 }
2820
2821 if (unoptab == popcount_optab
2822 && is_a <scalar_int_mode> (mode, &int_mode)
2823 && GET_MODE_SIZE (int_mode) == 2 * UNITS_PER_WORD
2824 && optab_handler (unoptab, word_mode) != CODE_FOR_nothing
2825 && optimize_insn_for_speed_p ())
2826 {
2827 temp = expand_doubleword_popcount (int_mode, op0, target);
2828 if (temp)
2829 return temp;
2830 }
2831
2832 if (unoptab == parity_optab
2833 && is_a <scalar_int_mode> (mode, &int_mode)
2834 && GET_MODE_SIZE (int_mode) == 2 * UNITS_PER_WORD
2835 && (optab_handler (unoptab, word_mode) != CODE_FOR_nothing
2836 || optab_handler (popcount_optab, word_mode) != CODE_FOR_nothing)
2837 && optimize_insn_for_speed_p ())
2838 {
2839 temp = expand_doubleword_parity (int_mode, op0, target);
2840 if (temp)
2841 return temp;
2842 }
2843
2844 /* Widening (or narrowing) bswap needs special treatment. */
2845 if (unoptab == bswap_optab)
2846 {
2847 /* HImode is special because in this mode BSWAP is equivalent to ROTATE
2848 or ROTATERT. First try these directly; if this fails, then try the
2849 obvious pair of shifts with allowed widening, as this will probably
2850 be always more efficient than the other fallback methods. */
2851 if (mode == HImode)
2852 {
2853 rtx_insn *last;
2854 rtx temp1, temp2;
2855
2856 if (optab_handler (rotl_optab, mode) != CODE_FOR_nothing)
2857 {
2858 temp = expand_binop (mode, rotl_optab, op0,
2859 gen_int_shift_amount (mode, 8),
2860 target, unsignedp, OPTAB_DIRECT);
2861 if (temp)
2862 return temp;
2863 }
2864
2865 if (optab_handler (rotr_optab, mode) != CODE_FOR_nothing)
2866 {
2867 temp = expand_binop (mode, rotr_optab, op0,
2868 gen_int_shift_amount (mode, 8),
2869 target, unsignedp, OPTAB_DIRECT);
2870 if (temp)
2871 return temp;
2872 }
2873
2874 last = get_last_insn ();
2875
2876 temp1 = expand_binop (mode, ashl_optab, op0,
2877 gen_int_shift_amount (mode, 8), NULL_RTX,
2878 unsignedp, OPTAB_WIDEN);
2879 temp2 = expand_binop (mode, lshr_optab, op0,
2880 gen_int_shift_amount (mode, 8), NULL_RTX,
2881 unsignedp, OPTAB_WIDEN);
2882 if (temp1 && temp2)
2883 {
2884 temp = expand_binop (mode, ior_optab, temp1, temp2, target,
2885 unsignedp, OPTAB_WIDEN);
2886 if (temp)
2887 return temp;
2888 }
2889
2890 delete_insns_since (last);
2891 }
2892
2893 if (is_a <scalar_int_mode> (mode, &int_mode))
2894 {
2895 temp = widen_bswap (int_mode, op0, target);
2896 if (temp)
2897 return temp;
2898
2899 if (GET_MODE_SIZE (int_mode) == 2 * UNITS_PER_WORD
2900 && optab_handler (unoptab, word_mode) != CODE_FOR_nothing)
2901 {
2902 temp = expand_doubleword_bswap (mode, op0, target);
2903 if (temp)
2904 return temp;
2905 }
2906 }
2907
2908 goto try_libcall;
2909 }
2910
2911 if (CLASS_HAS_WIDER_MODES_P (mclass))
2912 FOR_EACH_WIDER_MODE (wider_mode, mode)
2913 {
2914 if (optab_handler (unoptab, wider_mode) != CODE_FOR_nothing)
2915 {
2916 rtx xop0 = op0;
2917 rtx_insn *last = get_last_insn ();
2918
2919 /* For certain operations, we need not actually extend
2920 the narrow operand, as long as we will truncate the
2921 results to the same narrowness. */
2922
2923 xop0 = widen_operand (xop0, wider_mode, mode, unsignedp,
2924 (unoptab == neg_optab
2925 || unoptab == one_cmpl_optab)
2926 && mclass == MODE_INT);
2927
2928 temp = expand_unop (wider_mode, unoptab, xop0, NULL_RTX,
2929 unsignedp);
2930
2931 if (temp)
2932 {
2933 if (mclass != MODE_INT
2934 || !TRULY_NOOP_TRUNCATION_MODES_P (mode, wider_mode))
2935 {
2936 if (target == 0)
2937 target = gen_reg_rtx (mode);
2938 convert_move (target, temp, 0);
2939 return target;
2940 }
2941 else
2942 return gen_lowpart (mode, temp);
2943 }
2944 else
2945 delete_insns_since (last);
2946 }
2947 }
2948
2949 /* These can be done a word at a time. */
2950 if (unoptab == one_cmpl_optab
2951 && is_int_mode (mode, &int_mode)
2952 && GET_MODE_SIZE (int_mode) > UNITS_PER_WORD
2953 && optab_handler (unoptab, word_mode) != CODE_FOR_nothing)
2954 {
2955 int i;
2956 rtx_insn *insns;
2957
2958 if (target == 0
2959 || target == op0
2960 || reg_overlap_mentioned_p (target, op0)
2961 || !valid_multiword_target_p (target))
2962 target = gen_reg_rtx (int_mode);
2963
2964 start_sequence ();
2965
2966 /* Do the actual arithmetic. */
2967 for (i = 0; i < GET_MODE_BITSIZE (int_mode) / BITS_PER_WORD; i++)
2968 {
2969 rtx target_piece = operand_subword (target, i, 1, int_mode);
2970 rtx x = expand_unop (word_mode, unoptab,
2971 operand_subword_force (op0, i, int_mode),
2972 target_piece, unsignedp);
2973
2974 if (target_piece != x)
2975 emit_move_insn (target_piece, x);
2976 }
2977
2978 insns = get_insns ();
2979 end_sequence ();
2980
2981 emit_insn (insns);
2982 return target;
2983 }
2984
2985 /* Emit ~op0 as op0 ^ -1. */
2986 if (unoptab == one_cmpl_optab
2987 && (SCALAR_INT_MODE_P (mode) || GET_MODE_CLASS (mode) == MODE_VECTOR_INT)
2988 && optab_handler (xor_optab, mode) != CODE_FOR_nothing)
2989 {
2990 temp = expand_binop (mode, xor_optab, op0, CONSTM1_RTX (mode),
2991 target, unsignedp, OPTAB_DIRECT);
2992 if (temp)
2993 return temp;
2994 }
2995
2996 if (optab_to_code (unoptab) == NEG)
2997 {
2998 /* Try negating floating point values by flipping the sign bit. */
2999 if (is_a <scalar_float_mode> (mode, &float_mode))
3000 {
3001 temp = expand_absneg_bit (NEG, float_mode, op0, target);
3002 if (temp)
3003 return temp;
3004 }
3005
3006 /* If there is no negation pattern, and we have no negative zero,
3007 try subtracting from zero. */
3008 if (!HONOR_SIGNED_ZEROS (mode))
3009 {
3010 temp = expand_binop (mode, (unoptab == negv_optab
3011 ? subv_optab : sub_optab),
3012 CONST0_RTX (mode), op0, target,
3013 unsignedp, OPTAB_DIRECT);
3014 if (temp)
3015 return temp;
3016 }
3017 }
3018
3019 /* Try calculating parity (x) as popcount (x) % 2. */
3020 if (unoptab == parity_optab && is_a <scalar_int_mode> (mode, &int_mode))
3021 {
3022 temp = expand_parity (int_mode, op0, target);
3023 if (temp)
3024 return temp;
3025 }
3026
3027 /* Try implementing ffs (x) in terms of clz (x). */
3028 if (unoptab == ffs_optab && is_a <scalar_int_mode> (mode, &int_mode))
3029 {
3030 temp = expand_ffs (int_mode, op0, target);
3031 if (temp)
3032 return temp;
3033 }
3034
3035 /* Try implementing ctz (x) in terms of clz (x). */
3036 if (unoptab == ctz_optab && is_a <scalar_int_mode> (mode, &int_mode))
3037 {
3038 temp = expand_ctz (int_mode, op0, target);
3039 if (temp)
3040 return temp;
3041 }
3042
3043 try_libcall:
3044 /* Now try a library call in this mode. */
3045 libfunc = optab_libfunc (unoptab, mode);
3046 if (libfunc)
3047 {
3048 rtx_insn *insns;
3049 rtx value;
3050 rtx eq_value;
3051 machine_mode outmode = mode;
3052
3053 /* All of these functions return small values. Thus we choose to
3054 have them return something that isn't a double-word. */
3055 if (unoptab == ffs_optab || unoptab == clz_optab || unoptab == ctz_optab
3056 || unoptab == clrsb_optab || unoptab == popcount_optab
3057 || unoptab == parity_optab)
3058 outmode
3059 = GET_MODE (hard_libcall_value (TYPE_MODE (integer_type_node),
3060 optab_libfunc (unoptab, mode)));
3061
3062 start_sequence ();
3063
3064 /* Pass 1 for NO_QUEUE so we don't lose any increments
3065 if the libcall is cse'd or moved. */
3066 value = emit_library_call_value (libfunc, NULL_RTX, LCT_CONST, outmode,
3067 op0, mode);
3068 insns = get_insns ();
3069 end_sequence ();
3070
3071 target = gen_reg_rtx (outmode);
3072 bool trapv = trapv_unoptab_p (unoptab);
3073 if (trapv)
3074 eq_value = NULL_RTX;
3075 else
3076 {
3077 eq_value = gen_rtx_fmt_e (optab_to_code (unoptab), mode, op0);
3078 if (GET_MODE_UNIT_SIZE (outmode) < GET_MODE_UNIT_SIZE (mode))
3079 eq_value = simplify_gen_unary (TRUNCATE, outmode, eq_value, mode);
3080 else if (GET_MODE_UNIT_SIZE (outmode) > GET_MODE_UNIT_SIZE (mode))
3081 eq_value = simplify_gen_unary (ZERO_EXTEND,
3082 outmode, eq_value, mode);
3083 }
3084 emit_libcall_block_1 (insns, target, value, eq_value, trapv);
3085
3086 return target;
3087 }
3088
3089 /* It can't be done in this mode. Can we do it in a wider mode? */
3090
3091 if (CLASS_HAS_WIDER_MODES_P (mclass))
3092 {
3093 FOR_EACH_WIDER_MODE (wider_mode, mode)
3094 {
3095 if (optab_handler (unoptab, wider_mode) != CODE_FOR_nothing
3096 || optab_libfunc (unoptab, wider_mode))
3097 {
3098 rtx xop0 = op0;
3099 rtx_insn *last = get_last_insn ();
3100
3101 /* For certain operations, we need not actually extend
3102 the narrow operand, as long as we will truncate the
3103 results to the same narrowness. */
3104 xop0 = widen_operand (xop0, wider_mode, mode, unsignedp,
3105 (unoptab == neg_optab
3106 || unoptab == one_cmpl_optab
3107 || unoptab == bswap_optab)
3108 && mclass == MODE_INT);
3109
3110 temp = expand_unop (wider_mode, unoptab, xop0, NULL_RTX,
3111 unsignedp);
3112
3113 /* If we are generating clz using wider mode, adjust the
3114 result. Similarly for clrsb. */
3115 if ((unoptab == clz_optab || unoptab == clrsb_optab)
3116 && temp != 0)
3117 {
3118 scalar_int_mode wider_int_mode
3119 = as_a <scalar_int_mode> (wider_mode);
3120 int_mode = as_a <scalar_int_mode> (mode);
3121 temp = expand_binop
3122 (wider_mode, sub_optab, temp,
3123 gen_int_mode (GET_MODE_PRECISION (wider_int_mode)
3124 - GET_MODE_PRECISION (int_mode),
3125 wider_int_mode),
3126 target, true, OPTAB_DIRECT);
3127 }
3128
3129 /* Likewise for bswap. */
3130 if (unoptab == bswap_optab && temp != 0)
3131 {
3132 scalar_int_mode wider_int_mode
3133 = as_a <scalar_int_mode> (wider_mode);
3134 int_mode = as_a <scalar_int_mode> (mode);
3135 gcc_assert (GET_MODE_PRECISION (wider_int_mode)
3136 == GET_MODE_BITSIZE (wider_int_mode)
3137 && GET_MODE_PRECISION (int_mode)
3138 == GET_MODE_BITSIZE (int_mode));
3139
3140 temp = expand_shift (RSHIFT_EXPR, wider_int_mode, temp,
3141 GET_MODE_BITSIZE (wider_int_mode)
3142 - GET_MODE_BITSIZE (int_mode),
3143 NULL_RTX, true);
3144 }
3145
3146 if (temp)
3147 {
3148 if (mclass != MODE_INT)
3149 {
3150 if (target == 0)
3151 target = gen_reg_rtx (mode);
3152 convert_move (target, temp, 0);
3153 return target;
3154 }
3155 else
3156 return gen_lowpart (mode, temp);
3157 }
3158 else
3159 delete_insns_since (last);
3160 }
3161 }
3162 }
3163
3164 /* One final attempt at implementing negation via subtraction,
3165 this time allowing widening of the operand. */
3166 if (optab_to_code (unoptab) == NEG && !HONOR_SIGNED_ZEROS (mode))
3167 {
3168 rtx temp;
3169 temp = expand_binop (mode,
3170 unoptab == negv_optab ? subv_optab : sub_optab,
3171 CONST0_RTX (mode), op0,
3172 target, unsignedp, OPTAB_LIB_WIDEN);
3173 if (temp)
3174 return temp;
3175 }
3176
3177 return 0;
3178 }
3179
3180 /* Emit code to compute the absolute value of OP0, with result to
3181 TARGET if convenient. (TARGET may be 0.) The return value says
3182 where the result actually is to be found.
3183
3184 MODE is the mode of the operand; the mode of the result is
3185 different but can be deduced from MODE.
3186
3187 */
3188
3189 rtx
expand_abs_nojump(machine_mode mode,rtx op0,rtx target,int result_unsignedp)3190 expand_abs_nojump (machine_mode mode, rtx op0, rtx target,
3191 int result_unsignedp)
3192 {
3193 rtx temp;
3194
3195 if (GET_MODE_CLASS (mode) != MODE_INT
3196 || ! flag_trapv)
3197 result_unsignedp = 1;
3198
3199 /* First try to do it with a special abs instruction. */
3200 temp = expand_unop (mode, result_unsignedp ? abs_optab : absv_optab,
3201 op0, target, 0);
3202 if (temp != 0)
3203 return temp;
3204
3205 /* For floating point modes, try clearing the sign bit. */
3206 scalar_float_mode float_mode;
3207 if (is_a <scalar_float_mode> (mode, &float_mode))
3208 {
3209 temp = expand_absneg_bit (ABS, float_mode, op0, target);
3210 if (temp)
3211 return temp;
3212 }
3213
3214 /* If we have a MAX insn, we can do this as MAX (x, -x). */
3215 if (optab_handler (smax_optab, mode) != CODE_FOR_nothing
3216 && !HONOR_SIGNED_ZEROS (mode))
3217 {
3218 rtx_insn *last = get_last_insn ();
3219
3220 temp = expand_unop (mode, result_unsignedp ? neg_optab : negv_optab,
3221 op0, NULL_RTX, 0);
3222 if (temp != 0)
3223 temp = expand_binop (mode, smax_optab, op0, temp, target, 0,
3224 OPTAB_WIDEN);
3225
3226 if (temp != 0)
3227 return temp;
3228
3229 delete_insns_since (last);
3230 }
3231
3232 /* If this machine has expensive jumps, we can do integer absolute
3233 value of X as (((signed) x >> (W-1)) ^ x) - ((signed) x >> (W-1)),
3234 where W is the width of MODE. */
3235
3236 scalar_int_mode int_mode;
3237 if (is_int_mode (mode, &int_mode)
3238 && BRANCH_COST (optimize_insn_for_speed_p (),
3239 false) >= 2)
3240 {
3241 rtx extended = expand_shift (RSHIFT_EXPR, int_mode, op0,
3242 GET_MODE_PRECISION (int_mode) - 1,
3243 NULL_RTX, 0);
3244
3245 temp = expand_binop (int_mode, xor_optab, extended, op0, target, 0,
3246 OPTAB_LIB_WIDEN);
3247 if (temp != 0)
3248 temp = expand_binop (int_mode,
3249 result_unsignedp ? sub_optab : subv_optab,
3250 temp, extended, target, 0, OPTAB_LIB_WIDEN);
3251
3252 if (temp != 0)
3253 return temp;
3254 }
3255
3256 return NULL_RTX;
3257 }
3258
3259 rtx
expand_abs(machine_mode mode,rtx op0,rtx target,int result_unsignedp,int safe)3260 expand_abs (machine_mode mode, rtx op0, rtx target,
3261 int result_unsignedp, int safe)
3262 {
3263 rtx temp;
3264 rtx_code_label *op1;
3265
3266 if (GET_MODE_CLASS (mode) != MODE_INT
3267 || ! flag_trapv)
3268 result_unsignedp = 1;
3269
3270 temp = expand_abs_nojump (mode, op0, target, result_unsignedp);
3271 if (temp != 0)
3272 return temp;
3273
3274 /* If that does not win, use conditional jump and negate. */
3275
3276 /* It is safe to use the target if it is the same
3277 as the source if this is also a pseudo register */
3278 if (op0 == target && REG_P (op0)
3279 && REGNO (op0) >= FIRST_PSEUDO_REGISTER)
3280 safe = 1;
3281
3282 op1 = gen_label_rtx ();
3283 if (target == 0 || ! safe
3284 || GET_MODE (target) != mode
3285 || (MEM_P (target) && MEM_VOLATILE_P (target))
3286 || (REG_P (target)
3287 && REGNO (target) < FIRST_PSEUDO_REGISTER))
3288 target = gen_reg_rtx (mode);
3289
3290 emit_move_insn (target, op0);
3291 NO_DEFER_POP;
3292
3293 do_compare_rtx_and_jump (target, CONST0_RTX (mode), GE, 0, mode,
3294 NULL_RTX, NULL, op1,
3295 profile_probability::uninitialized ());
3296
3297 op0 = expand_unop (mode, result_unsignedp ? neg_optab : negv_optab,
3298 target, target, 0);
3299 if (op0 != target)
3300 emit_move_insn (target, op0);
3301 emit_label (op1);
3302 OK_DEFER_POP;
3303 return target;
3304 }
3305
3306 /* Emit code to compute the one's complement absolute value of OP0
3307 (if (OP0 < 0) OP0 = ~OP0), with result to TARGET if convenient.
3308 (TARGET may be NULL_RTX.) The return value says where the result
3309 actually is to be found.
3310
3311 MODE is the mode of the operand; the mode of the result is
3312 different but can be deduced from MODE. */
3313
3314 rtx
expand_one_cmpl_abs_nojump(machine_mode mode,rtx op0,rtx target)3315 expand_one_cmpl_abs_nojump (machine_mode mode, rtx op0, rtx target)
3316 {
3317 rtx temp;
3318
3319 /* Not applicable for floating point modes. */
3320 if (FLOAT_MODE_P (mode))
3321 return NULL_RTX;
3322
3323 /* If we have a MAX insn, we can do this as MAX (x, ~x). */
3324 if (optab_handler (smax_optab, mode) != CODE_FOR_nothing)
3325 {
3326 rtx_insn *last = get_last_insn ();
3327
3328 temp = expand_unop (mode, one_cmpl_optab, op0, NULL_RTX, 0);
3329 if (temp != 0)
3330 temp = expand_binop (mode, smax_optab, op0, temp, target, 0,
3331 OPTAB_WIDEN);
3332
3333 if (temp != 0)
3334 return temp;
3335
3336 delete_insns_since (last);
3337 }
3338
3339 /* If this machine has expensive jumps, we can do one's complement
3340 absolute value of X as (((signed) x >> (W-1)) ^ x). */
3341
3342 scalar_int_mode int_mode;
3343 if (is_int_mode (mode, &int_mode)
3344 && BRANCH_COST (optimize_insn_for_speed_p (),
3345 false) >= 2)
3346 {
3347 rtx extended = expand_shift (RSHIFT_EXPR, int_mode, op0,
3348 GET_MODE_PRECISION (int_mode) - 1,
3349 NULL_RTX, 0);
3350
3351 temp = expand_binop (int_mode, xor_optab, extended, op0, target, 0,
3352 OPTAB_LIB_WIDEN);
3353
3354 if (temp != 0)
3355 return temp;
3356 }
3357
3358 return NULL_RTX;
3359 }
3360
3361 /* A subroutine of expand_copysign, perform the copysign operation using the
3362 abs and neg primitives advertised to exist on the target. The assumption
3363 is that we have a split register file, and leaving op0 in fp registers,
3364 and not playing with subregs so much, will help the register allocator. */
3365
3366 static rtx
expand_copysign_absneg(scalar_float_mode mode,rtx op0,rtx op1,rtx target,int bitpos,bool op0_is_abs)3367 expand_copysign_absneg (scalar_float_mode mode, rtx op0, rtx op1, rtx target,
3368 int bitpos, bool op0_is_abs)
3369 {
3370 scalar_int_mode imode;
3371 enum insn_code icode;
3372 rtx sign;
3373 rtx_code_label *label;
3374
3375 if (target == op1)
3376 target = NULL_RTX;
3377
3378 /* Check if the back end provides an insn that handles signbit for the
3379 argument's mode. */
3380 icode = optab_handler (signbit_optab, mode);
3381 if (icode != CODE_FOR_nothing)
3382 {
3383 imode = as_a <scalar_int_mode> (insn_data[(int) icode].operand[0].mode);
3384 sign = gen_reg_rtx (imode);
3385 emit_unop_insn (icode, sign, op1, UNKNOWN);
3386 }
3387 else
3388 {
3389 if (GET_MODE_SIZE (mode) <= UNITS_PER_WORD)
3390 {
3391 if (!int_mode_for_mode (mode).exists (&imode))
3392 return NULL_RTX;
3393 op1 = gen_lowpart (imode, op1);
3394 }
3395 else
3396 {
3397 int word;
3398
3399 imode = word_mode;
3400 if (FLOAT_WORDS_BIG_ENDIAN)
3401 word = (GET_MODE_BITSIZE (mode) - bitpos) / BITS_PER_WORD;
3402 else
3403 word = bitpos / BITS_PER_WORD;
3404 bitpos = bitpos % BITS_PER_WORD;
3405 op1 = operand_subword_force (op1, word, mode);
3406 }
3407
3408 wide_int mask = wi::set_bit_in_zero (bitpos, GET_MODE_PRECISION (imode));
3409 sign = expand_binop (imode, and_optab, op1,
3410 immed_wide_int_const (mask, imode),
3411 NULL_RTX, 1, OPTAB_LIB_WIDEN);
3412 }
3413
3414 if (!op0_is_abs)
3415 {
3416 op0 = expand_unop (mode, abs_optab, op0, target, 0);
3417 if (op0 == NULL)
3418 return NULL_RTX;
3419 target = op0;
3420 }
3421 else
3422 {
3423 if (target == NULL_RTX)
3424 target = copy_to_reg (op0);
3425 else
3426 emit_move_insn (target, op0);
3427 }
3428
3429 label = gen_label_rtx ();
3430 emit_cmp_and_jump_insns (sign, const0_rtx, EQ, NULL_RTX, imode, 1, label);
3431
3432 if (CONST_DOUBLE_AS_FLOAT_P (op0))
3433 op0 = simplify_unary_operation (NEG, mode, op0, mode);
3434 else
3435 op0 = expand_unop (mode, neg_optab, op0, target, 0);
3436 if (op0 != target)
3437 emit_move_insn (target, op0);
3438
3439 emit_label (label);
3440
3441 return target;
3442 }
3443
3444
3445 /* A subroutine of expand_copysign, perform the entire copysign operation
3446 with integer bitmasks. BITPOS is the position of the sign bit; OP0_IS_ABS
3447 is true if op0 is known to have its sign bit clear. */
3448
3449 static rtx
expand_copysign_bit(scalar_float_mode mode,rtx op0,rtx op1,rtx target,int bitpos,bool op0_is_abs)3450 expand_copysign_bit (scalar_float_mode mode, rtx op0, rtx op1, rtx target,
3451 int bitpos, bool op0_is_abs)
3452 {
3453 scalar_int_mode imode;
3454 int word, nwords, i;
3455 rtx temp;
3456 rtx_insn *insns;
3457
3458 if (GET_MODE_SIZE (mode) <= UNITS_PER_WORD)
3459 {
3460 if (!int_mode_for_mode (mode).exists (&imode))
3461 return NULL_RTX;
3462 word = 0;
3463 nwords = 1;
3464 }
3465 else
3466 {
3467 imode = word_mode;
3468
3469 if (FLOAT_WORDS_BIG_ENDIAN)
3470 word = (GET_MODE_BITSIZE (mode) - bitpos) / BITS_PER_WORD;
3471 else
3472 word = bitpos / BITS_PER_WORD;
3473 bitpos = bitpos % BITS_PER_WORD;
3474 nwords = (GET_MODE_BITSIZE (mode) + BITS_PER_WORD - 1) / BITS_PER_WORD;
3475 }
3476
3477 wide_int mask = wi::set_bit_in_zero (bitpos, GET_MODE_PRECISION (imode));
3478
3479 if (target == 0
3480 || target == op0
3481 || target == op1
3482 || reg_overlap_mentioned_p (target, op0)
3483 || reg_overlap_mentioned_p (target, op1)
3484 || (nwords > 1 && !valid_multiword_target_p (target)))
3485 target = gen_reg_rtx (mode);
3486
3487 if (nwords > 1)
3488 {
3489 start_sequence ();
3490
3491 for (i = 0; i < nwords; ++i)
3492 {
3493 rtx targ_piece = operand_subword (target, i, 1, mode);
3494 rtx op0_piece = operand_subword_force (op0, i, mode);
3495
3496 if (i == word)
3497 {
3498 if (!op0_is_abs)
3499 op0_piece
3500 = expand_binop (imode, and_optab, op0_piece,
3501 immed_wide_int_const (~mask, imode),
3502 NULL_RTX, 1, OPTAB_LIB_WIDEN);
3503 op1 = expand_binop (imode, and_optab,
3504 operand_subword_force (op1, i, mode),
3505 immed_wide_int_const (mask, imode),
3506 NULL_RTX, 1, OPTAB_LIB_WIDEN);
3507
3508 temp = expand_binop (imode, ior_optab, op0_piece, op1,
3509 targ_piece, 1, OPTAB_LIB_WIDEN);
3510 if (temp != targ_piece)
3511 emit_move_insn (targ_piece, temp);
3512 }
3513 else
3514 emit_move_insn (targ_piece, op0_piece);
3515 }
3516
3517 insns = get_insns ();
3518 end_sequence ();
3519
3520 emit_insn (insns);
3521 }
3522 else
3523 {
3524 op1 = expand_binop (imode, and_optab, gen_lowpart (imode, op1),
3525 immed_wide_int_const (mask, imode),
3526 NULL_RTX, 1, OPTAB_LIB_WIDEN);
3527
3528 op0 = gen_lowpart (imode, op0);
3529 if (!op0_is_abs)
3530 op0 = expand_binop (imode, and_optab, op0,
3531 immed_wide_int_const (~mask, imode),
3532 NULL_RTX, 1, OPTAB_LIB_WIDEN);
3533
3534 temp = expand_binop (imode, ior_optab, op0, op1,
3535 gen_lowpart (imode, target), 1, OPTAB_LIB_WIDEN);
3536 target = lowpart_subreg_maybe_copy (mode, temp, imode);
3537 }
3538
3539 return target;
3540 }
3541
3542 /* Expand the C99 copysign operation. OP0 and OP1 must be the same
3543 scalar floating point mode. Return NULL if we do not know how to
3544 expand the operation inline. */
3545
3546 rtx
expand_copysign(rtx op0,rtx op1,rtx target)3547 expand_copysign (rtx op0, rtx op1, rtx target)
3548 {
3549 scalar_float_mode mode;
3550 const struct real_format *fmt;
3551 bool op0_is_abs;
3552 rtx temp;
3553
3554 mode = as_a <scalar_float_mode> (GET_MODE (op0));
3555 gcc_assert (GET_MODE (op1) == mode);
3556
3557 /* First try to do it with a special instruction. */
3558 temp = expand_binop (mode, copysign_optab, op0, op1,
3559 target, 0, OPTAB_DIRECT);
3560 if (temp)
3561 return temp;
3562
3563 fmt = REAL_MODE_FORMAT (mode);
3564 if (fmt == NULL || !fmt->has_signed_zero)
3565 return NULL_RTX;
3566
3567 op0_is_abs = false;
3568 if (CONST_DOUBLE_AS_FLOAT_P (op0))
3569 {
3570 if (real_isneg (CONST_DOUBLE_REAL_VALUE (op0)))
3571 op0 = simplify_unary_operation (ABS, mode, op0, mode);
3572 op0_is_abs = true;
3573 }
3574
3575 if (fmt->signbit_ro >= 0
3576 && (CONST_DOUBLE_AS_FLOAT_P (op0)
3577 || (optab_handler (neg_optab, mode) != CODE_FOR_nothing
3578 && optab_handler (abs_optab, mode) != CODE_FOR_nothing)))
3579 {
3580 temp = expand_copysign_absneg (mode, op0, op1, target,
3581 fmt->signbit_ro, op0_is_abs);
3582 if (temp)
3583 return temp;
3584 }
3585
3586 if (fmt->signbit_rw < 0)
3587 return NULL_RTX;
3588 return expand_copysign_bit (mode, op0, op1, target,
3589 fmt->signbit_rw, op0_is_abs);
3590 }
3591
3592 /* Generate an instruction whose insn-code is INSN_CODE,
3593 with two operands: an output TARGET and an input OP0.
3594 TARGET *must* be nonzero, and the output is always stored there.
3595 CODE is an rtx code such that (CODE OP0) is an rtx that describes
3596 the value that is stored into TARGET.
3597
3598 Return false if expansion failed. */
3599
3600 bool
maybe_emit_unop_insn(enum insn_code icode,rtx target,rtx op0,enum rtx_code code)3601 maybe_emit_unop_insn (enum insn_code icode, rtx target, rtx op0,
3602 enum rtx_code code)
3603 {
3604 class expand_operand ops[2];
3605 rtx_insn *pat;
3606
3607 create_output_operand (&ops[0], target, GET_MODE (target));
3608 create_input_operand (&ops[1], op0, GET_MODE (op0));
3609 pat = maybe_gen_insn (icode, 2, ops);
3610 if (!pat)
3611 return false;
3612
3613 if (INSN_P (pat) && NEXT_INSN (pat) != NULL_RTX
3614 && code != UNKNOWN)
3615 add_equal_note (pat, ops[0].value, code, ops[1].value, NULL_RTX,
3616 GET_MODE (op0));
3617
3618 emit_insn (pat);
3619
3620 if (ops[0].value != target)
3621 emit_move_insn (target, ops[0].value);
3622 return true;
3623 }
3624 /* Generate an instruction whose insn-code is INSN_CODE,
3625 with two operands: an output TARGET and an input OP0.
3626 TARGET *must* be nonzero, and the output is always stored there.
3627 CODE is an rtx code such that (CODE OP0) is an rtx that describes
3628 the value that is stored into TARGET. */
3629
3630 void
emit_unop_insn(enum insn_code icode,rtx target,rtx op0,enum rtx_code code)3631 emit_unop_insn (enum insn_code icode, rtx target, rtx op0, enum rtx_code code)
3632 {
3633 bool ok = maybe_emit_unop_insn (icode, target, op0, code);
3634 gcc_assert (ok);
3635 }
3636
3637 struct no_conflict_data
3638 {
3639 rtx target;
3640 rtx_insn *first, *insn;
3641 bool must_stay;
3642 };
3643
3644 /* Called via note_stores by emit_libcall_block. Set P->must_stay if
3645 the currently examined clobber / store has to stay in the list of
3646 insns that constitute the actual libcall block. */
3647 static void
no_conflict_move_test(rtx dest,const_rtx set,void * p0)3648 no_conflict_move_test (rtx dest, const_rtx set, void *p0)
3649 {
3650 struct no_conflict_data *p= (struct no_conflict_data *) p0;
3651
3652 /* If this inns directly contributes to setting the target, it must stay. */
3653 if (reg_overlap_mentioned_p (p->target, dest))
3654 p->must_stay = true;
3655 /* If we haven't committed to keeping any other insns in the list yet,
3656 there is nothing more to check. */
3657 else if (p->insn == p->first)
3658 return;
3659 /* If this insn sets / clobbers a register that feeds one of the insns
3660 already in the list, this insn has to stay too. */
3661 else if (reg_overlap_mentioned_p (dest, PATTERN (p->first))
3662 || (CALL_P (p->first) && (find_reg_fusage (p->first, USE, dest)))
3663 || reg_used_between_p (dest, p->first, p->insn)
3664 /* Likewise if this insn depends on a register set by a previous
3665 insn in the list, or if it sets a result (presumably a hard
3666 register) that is set or clobbered by a previous insn.
3667 N.B. the modified_*_p (SET_DEST...) tests applied to a MEM
3668 SET_DEST perform the former check on the address, and the latter
3669 check on the MEM. */
3670 || (GET_CODE (set) == SET
3671 && (modified_in_p (SET_SRC (set), p->first)
3672 || modified_in_p (SET_DEST (set), p->first)
3673 || modified_between_p (SET_SRC (set), p->first, p->insn)
3674 || modified_between_p (SET_DEST (set), p->first, p->insn))))
3675 p->must_stay = true;
3676 }
3677
3678
3679 /* Emit code to make a call to a constant function or a library call.
3680
3681 INSNS is a list containing all insns emitted in the call.
3682 These insns leave the result in RESULT. Our block is to copy RESULT
3683 to TARGET, which is logically equivalent to EQUIV.
3684
3685 We first emit any insns that set a pseudo on the assumption that these are
3686 loading constants into registers; doing so allows them to be safely cse'ed
3687 between blocks. Then we emit all the other insns in the block, followed by
3688 an insn to move RESULT to TARGET. This last insn will have a REQ_EQUAL
3689 note with an operand of EQUIV. */
3690
3691 static void
emit_libcall_block_1(rtx_insn * insns,rtx target,rtx result,rtx equiv,bool equiv_may_trap)3692 emit_libcall_block_1 (rtx_insn *insns, rtx target, rtx result, rtx equiv,
3693 bool equiv_may_trap)
3694 {
3695 rtx final_dest = target;
3696 rtx_insn *next, *last, *insn;
3697
3698 /* If this is a reg with REG_USERVAR_P set, then it could possibly turn
3699 into a MEM later. Protect the libcall block from this change. */
3700 if (! REG_P (target) || REG_USERVAR_P (target))
3701 target = gen_reg_rtx (GET_MODE (target));
3702
3703 /* If we're using non-call exceptions, a libcall corresponding to an
3704 operation that may trap may also trap. */
3705 /* ??? See the comment in front of make_reg_eh_region_note. */
3706 if (cfun->can_throw_non_call_exceptions
3707 && (equiv_may_trap || may_trap_p (equiv)))
3708 {
3709 for (insn = insns; insn; insn = NEXT_INSN (insn))
3710 if (CALL_P (insn))
3711 {
3712 rtx note = find_reg_note (insn, REG_EH_REGION, NULL_RTX);
3713 if (note)
3714 {
3715 int lp_nr = INTVAL (XEXP (note, 0));
3716 if (lp_nr == 0 || lp_nr == INT_MIN)
3717 remove_note (insn, note);
3718 }
3719 }
3720 }
3721 else
3722 {
3723 /* Look for any CALL_INSNs in this sequence, and attach a REG_EH_REGION
3724 reg note to indicate that this call cannot throw or execute a nonlocal
3725 goto (unless there is already a REG_EH_REGION note, in which case
3726 we update it). */
3727 for (insn = insns; insn; insn = NEXT_INSN (insn))
3728 if (CALL_P (insn))
3729 make_reg_eh_region_note_nothrow_nononlocal (insn);
3730 }
3731
3732 /* First emit all insns that set pseudos. Remove them from the list as
3733 we go. Avoid insns that set pseudos which were referenced in previous
3734 insns. These can be generated by move_by_pieces, for example,
3735 to update an address. Similarly, avoid insns that reference things
3736 set in previous insns. */
3737
3738 for (insn = insns; insn; insn = next)
3739 {
3740 rtx set = single_set (insn);
3741
3742 next = NEXT_INSN (insn);
3743
3744 if (set != 0 && REG_P (SET_DEST (set))
3745 && REGNO (SET_DEST (set)) >= FIRST_PSEUDO_REGISTER)
3746 {
3747 struct no_conflict_data data;
3748
3749 data.target = const0_rtx;
3750 data.first = insns;
3751 data.insn = insn;
3752 data.must_stay = 0;
3753 note_stores (insn, no_conflict_move_test, &data);
3754 if (! data.must_stay)
3755 {
3756 if (PREV_INSN (insn))
3757 SET_NEXT_INSN (PREV_INSN (insn)) = next;
3758 else
3759 insns = next;
3760
3761 if (next)
3762 SET_PREV_INSN (next) = PREV_INSN (insn);
3763
3764 add_insn (insn);
3765 }
3766 }
3767
3768 /* Some ports use a loop to copy large arguments onto the stack.
3769 Don't move anything outside such a loop. */
3770 if (LABEL_P (insn))
3771 break;
3772 }
3773
3774 /* Write the remaining insns followed by the final copy. */
3775 for (insn = insns; insn; insn = next)
3776 {
3777 next = NEXT_INSN (insn);
3778
3779 add_insn (insn);
3780 }
3781
3782 last = emit_move_insn (target, result);
3783 if (equiv)
3784 set_dst_reg_note (last, REG_EQUAL, copy_rtx (equiv), target);
3785
3786 if (final_dest != target)
3787 emit_move_insn (final_dest, target);
3788 }
3789
3790 void
emit_libcall_block(rtx_insn * insns,rtx target,rtx result,rtx equiv)3791 emit_libcall_block (rtx_insn *insns, rtx target, rtx result, rtx equiv)
3792 {
3793 emit_libcall_block_1 (insns, target, result, equiv, false);
3794 }
3795
3796 /* Nonzero if we can perform a comparison of mode MODE straightforwardly.
3797 PURPOSE describes how this comparison will be used. CODE is the rtx
3798 comparison code we will be using.
3799
3800 ??? Actually, CODE is slightly weaker than that. A target is still
3801 required to implement all of the normal bcc operations, but not
3802 required to implement all (or any) of the unordered bcc operations. */
3803
3804 int
can_compare_p(enum rtx_code code,machine_mode mode,enum can_compare_purpose purpose)3805 can_compare_p (enum rtx_code code, machine_mode mode,
3806 enum can_compare_purpose purpose)
3807 {
3808 rtx test;
3809 test = gen_rtx_fmt_ee (code, mode, const0_rtx, const0_rtx);
3810 do
3811 {
3812 enum insn_code icode;
3813
3814 if (purpose == ccp_jump
3815 && (icode = optab_handler (cbranch_optab, mode)) != CODE_FOR_nothing
3816 && insn_operand_matches (icode, 0, test))
3817 return 1;
3818 if (purpose == ccp_store_flag
3819 && (icode = optab_handler (cstore_optab, mode)) != CODE_FOR_nothing
3820 && insn_operand_matches (icode, 1, test))
3821 return 1;
3822 if (purpose == ccp_cmov
3823 && optab_handler (cmov_optab, mode) != CODE_FOR_nothing)
3824 return 1;
3825
3826 mode = GET_MODE_WIDER_MODE (mode).else_void ();
3827 PUT_MODE (test, mode);
3828 }
3829 while (mode != VOIDmode);
3830
3831 return 0;
3832 }
3833
3834 /* Return whether the backend can emit a vector comparison for code CODE,
3835 comparing operands of mode CMP_OP_MODE and producing a result with
3836 VALUE_MODE. */
3837
3838 bool
can_vcond_compare_p(enum rtx_code code,machine_mode value_mode,machine_mode cmp_op_mode)3839 can_vcond_compare_p (enum rtx_code code, machine_mode value_mode,
3840 machine_mode cmp_op_mode)
3841 {
3842 enum insn_code icode;
3843 bool unsigned_p = (code == LTU || code == LEU || code == GTU || code == GEU);
3844 rtx reg1 = alloca_raw_REG (cmp_op_mode, LAST_VIRTUAL_REGISTER + 1);
3845 rtx reg2 = alloca_raw_REG (cmp_op_mode, LAST_VIRTUAL_REGISTER + 2);
3846 rtx test = alloca_rtx_fmt_ee (code, value_mode, reg1, reg2);
3847
3848 return (icode = get_vcond_icode (value_mode, cmp_op_mode, unsigned_p))
3849 != CODE_FOR_nothing
3850 && insn_operand_matches (icode, 3, test);
3851 }
3852
3853 /* This function is called when we are going to emit a compare instruction that
3854 compares the values found in X and Y, using the rtl operator COMPARISON.
3855
3856 If they have mode BLKmode, then SIZE specifies the size of both operands.
3857
3858 UNSIGNEDP nonzero says that the operands are unsigned;
3859 this matters if they need to be widened (as given by METHODS).
3860
3861 *PTEST is where the resulting comparison RTX is returned or NULL_RTX
3862 if we failed to produce one.
3863
3864 *PMODE is the mode of the inputs (in case they are const_int).
3865
3866 This function performs all the setup necessary so that the caller only has
3867 to emit a single comparison insn. This setup can involve doing a BLKmode
3868 comparison or emitting a library call to perform the comparison if no insn
3869 is available to handle it.
3870 The values which are passed in through pointers can be modified; the caller
3871 should perform the comparison on the modified values. Constant
3872 comparisons must have already been folded. */
3873
3874 static void
prepare_cmp_insn(rtx x,rtx y,enum rtx_code comparison,rtx size,int unsignedp,enum optab_methods methods,rtx * ptest,machine_mode * pmode)3875 prepare_cmp_insn (rtx x, rtx y, enum rtx_code comparison, rtx size,
3876 int unsignedp, enum optab_methods methods,
3877 rtx *ptest, machine_mode *pmode)
3878 {
3879 machine_mode mode = *pmode;
3880 rtx libfunc, test;
3881 machine_mode cmp_mode;
3882 enum mode_class mclass;
3883
3884 /* The other methods are not needed. */
3885 gcc_assert (methods == OPTAB_DIRECT || methods == OPTAB_WIDEN
3886 || methods == OPTAB_LIB_WIDEN);
3887
3888 if (CONST_SCALAR_INT_P (y))
3889 canonicalize_comparison (mode, &comparison, &y);
3890
3891 /* If we are optimizing, force expensive constants into a register. */
3892 if (CONSTANT_P (x) && optimize
3893 && (rtx_cost (x, mode, COMPARE, 0, optimize_insn_for_speed_p ())
3894 > COSTS_N_INSNS (1)))
3895 x = force_reg (mode, x);
3896
3897 if (CONSTANT_P (y) && optimize
3898 && (rtx_cost (y, mode, COMPARE, 1, optimize_insn_for_speed_p ())
3899 > COSTS_N_INSNS (1)))
3900 y = force_reg (mode, y);
3901
3902 #if HAVE_cc0
3903 /* Make sure if we have a canonical comparison. The RTL
3904 documentation states that canonical comparisons are required only
3905 for targets which have cc0. */
3906 gcc_assert (!CONSTANT_P (x) || CONSTANT_P (y));
3907 #endif
3908
3909 /* Don't let both operands fail to indicate the mode. */
3910 if (GET_MODE (x) == VOIDmode && GET_MODE (y) == VOIDmode)
3911 x = force_reg (mode, x);
3912 if (mode == VOIDmode)
3913 mode = GET_MODE (x) != VOIDmode ? GET_MODE (x) : GET_MODE (y);
3914
3915 /* Handle all BLKmode compares. */
3916
3917 if (mode == BLKmode)
3918 {
3919 machine_mode result_mode;
3920 enum insn_code cmp_code;
3921 rtx result;
3922 rtx opalign
3923 = GEN_INT (MIN (MEM_ALIGN (x), MEM_ALIGN (y)) / BITS_PER_UNIT);
3924
3925 gcc_assert (size);
3926
3927 /* Try to use a memory block compare insn - either cmpstr
3928 or cmpmem will do. */
3929 opt_scalar_int_mode cmp_mode_iter;
3930 FOR_EACH_MODE_IN_CLASS (cmp_mode_iter, MODE_INT)
3931 {
3932 scalar_int_mode cmp_mode = cmp_mode_iter.require ();
3933 cmp_code = direct_optab_handler (cmpmem_optab, cmp_mode);
3934 if (cmp_code == CODE_FOR_nothing)
3935 cmp_code = direct_optab_handler (cmpstr_optab, cmp_mode);
3936 if (cmp_code == CODE_FOR_nothing)
3937 cmp_code = direct_optab_handler (cmpstrn_optab, cmp_mode);
3938 if (cmp_code == CODE_FOR_nothing)
3939 continue;
3940
3941 /* Must make sure the size fits the insn's mode. */
3942 if (CONST_INT_P (size)
3943 ? UINTVAL (size) > GET_MODE_MASK (cmp_mode)
3944 : (GET_MODE_BITSIZE (as_a <scalar_int_mode> (GET_MODE (size)))
3945 > GET_MODE_BITSIZE (cmp_mode)))
3946 continue;
3947
3948 result_mode = insn_data[cmp_code].operand[0].mode;
3949 result = gen_reg_rtx (result_mode);
3950 size = convert_to_mode (cmp_mode, size, 1);
3951 emit_insn (GEN_FCN (cmp_code) (result, x, y, size, opalign));
3952
3953 *ptest = gen_rtx_fmt_ee (comparison, VOIDmode, result, const0_rtx);
3954 *pmode = result_mode;
3955 return;
3956 }
3957
3958 if (methods != OPTAB_LIB && methods != OPTAB_LIB_WIDEN)
3959 goto fail;
3960
3961 /* Otherwise call a library function. */
3962 result = emit_block_comp_via_libcall (x, y, size);
3963
3964 x = result;
3965 y = const0_rtx;
3966 mode = TYPE_MODE (integer_type_node);
3967 methods = OPTAB_LIB_WIDEN;
3968 unsignedp = false;
3969 }
3970
3971 /* Don't allow operands to the compare to trap, as that can put the
3972 compare and branch in different basic blocks. */
3973 if (cfun->can_throw_non_call_exceptions)
3974 {
3975 if (may_trap_p (x))
3976 x = copy_to_reg (x);
3977 if (may_trap_p (y))
3978 y = copy_to_reg (y);
3979 }
3980
3981 if (GET_MODE_CLASS (mode) == MODE_CC)
3982 {
3983 enum insn_code icode = optab_handler (cbranch_optab, CCmode);
3984 test = gen_rtx_fmt_ee (comparison, VOIDmode, x, y);
3985 gcc_assert (icode != CODE_FOR_nothing
3986 && insn_operand_matches (icode, 0, test));
3987 *ptest = test;
3988 return;
3989 }
3990
3991 mclass = GET_MODE_CLASS (mode);
3992 test = gen_rtx_fmt_ee (comparison, VOIDmode, x, y);
3993 FOR_EACH_MODE_FROM (cmp_mode, mode)
3994 {
3995 enum insn_code icode;
3996 icode = optab_handler (cbranch_optab, cmp_mode);
3997 if (icode != CODE_FOR_nothing
3998 && insn_operand_matches (icode, 0, test))
3999 {
4000 rtx_insn *last = get_last_insn ();
4001 rtx op0 = prepare_operand (icode, x, 1, mode, cmp_mode, unsignedp);
4002 rtx op1 = prepare_operand (icode, y, 2, mode, cmp_mode, unsignedp);
4003 if (op0 && op1
4004 && insn_operand_matches (icode, 1, op0)
4005 && insn_operand_matches (icode, 2, op1))
4006 {
4007 XEXP (test, 0) = op0;
4008 XEXP (test, 1) = op1;
4009 *ptest = test;
4010 *pmode = cmp_mode;
4011 return;
4012 }
4013 delete_insns_since (last);
4014 }
4015
4016 if (methods == OPTAB_DIRECT || !CLASS_HAS_WIDER_MODES_P (mclass))
4017 break;
4018 }
4019
4020 if (methods != OPTAB_LIB_WIDEN)
4021 goto fail;
4022
4023 if (SCALAR_FLOAT_MODE_P (mode))
4024 {
4025 /* Small trick if UNORDERED isn't implemented by the hardware. */
4026 if (comparison == UNORDERED && rtx_equal_p (x, y))
4027 {
4028 prepare_cmp_insn (x, y, UNLT, NULL_RTX, unsignedp, OPTAB_WIDEN,
4029 ptest, pmode);
4030 if (*ptest)
4031 return;
4032 }
4033
4034 prepare_float_lib_cmp (x, y, comparison, ptest, pmode);
4035 }
4036 else
4037 {
4038 rtx result;
4039 machine_mode ret_mode;
4040
4041 /* Handle a libcall just for the mode we are using. */
4042 libfunc = optab_libfunc (cmp_optab, mode);
4043 gcc_assert (libfunc);
4044
4045 /* If we want unsigned, and this mode has a distinct unsigned
4046 comparison routine, use that. */
4047 if (unsignedp)
4048 {
4049 rtx ulibfunc = optab_libfunc (ucmp_optab, mode);
4050 if (ulibfunc)
4051 libfunc = ulibfunc;
4052 }
4053
4054 ret_mode = targetm.libgcc_cmp_return_mode ();
4055 result = emit_library_call_value (libfunc, NULL_RTX, LCT_CONST,
4056 ret_mode, x, mode, y, mode);
4057
4058 /* There are two kinds of comparison routines. Biased routines
4059 return 0/1/2, and unbiased routines return -1/0/1. Other parts
4060 of gcc expect that the comparison operation is equivalent
4061 to the modified comparison. For signed comparisons compare the
4062 result against 1 in the biased case, and zero in the unbiased
4063 case. For unsigned comparisons always compare against 1 after
4064 biasing the unbiased result by adding 1. This gives us a way to
4065 represent LTU.
4066 The comparisons in the fixed-point helper library are always
4067 biased. */
4068 x = result;
4069 y = const1_rtx;
4070
4071 if (!TARGET_LIB_INT_CMP_BIASED && !ALL_FIXED_POINT_MODE_P (mode))
4072 {
4073 if (unsignedp)
4074 x = plus_constant (ret_mode, result, 1);
4075 else
4076 y = const0_rtx;
4077 }
4078
4079 *pmode = ret_mode;
4080 prepare_cmp_insn (x, y, comparison, NULL_RTX, unsignedp, methods,
4081 ptest, pmode);
4082 }
4083
4084 return;
4085
4086 fail:
4087 *ptest = NULL_RTX;
4088 }
4089
4090 /* Before emitting an insn with code ICODE, make sure that X, which is going
4091 to be used for operand OPNUM of the insn, is converted from mode MODE to
4092 WIDER_MODE (UNSIGNEDP determines whether it is an unsigned conversion), and
4093 that it is accepted by the operand predicate. Return the new value. */
4094
4095 rtx
prepare_operand(enum insn_code icode,rtx x,int opnum,machine_mode mode,machine_mode wider_mode,int unsignedp)4096 prepare_operand (enum insn_code icode, rtx x, int opnum, machine_mode mode,
4097 machine_mode wider_mode, int unsignedp)
4098 {
4099 if (mode != wider_mode)
4100 x = convert_modes (wider_mode, mode, x, unsignedp);
4101
4102 if (!insn_operand_matches (icode, opnum, x))
4103 {
4104 machine_mode op_mode = insn_data[(int) icode].operand[opnum].mode;
4105 if (reload_completed)
4106 return NULL_RTX;
4107 if (GET_MODE (x) != op_mode && GET_MODE (x) != VOIDmode)
4108 return NULL_RTX;
4109 x = copy_to_mode_reg (op_mode, x);
4110 }
4111
4112 return x;
4113 }
4114
4115 /* Subroutine of emit_cmp_and_jump_insns; this function is called when we know
4116 we can do the branch. */
4117
4118 static void
emit_cmp_and_jump_insn_1(rtx test,machine_mode mode,rtx label,profile_probability prob)4119 emit_cmp_and_jump_insn_1 (rtx test, machine_mode mode, rtx label,
4120 profile_probability prob)
4121 {
4122 machine_mode optab_mode;
4123 enum mode_class mclass;
4124 enum insn_code icode;
4125 rtx_insn *insn;
4126
4127 mclass = GET_MODE_CLASS (mode);
4128 optab_mode = (mclass == MODE_CC) ? CCmode : mode;
4129 icode = optab_handler (cbranch_optab, optab_mode);
4130
4131 gcc_assert (icode != CODE_FOR_nothing);
4132 gcc_assert (insn_operand_matches (icode, 0, test));
4133 insn = emit_jump_insn (GEN_FCN (icode) (test, XEXP (test, 0),
4134 XEXP (test, 1), label));
4135 if (prob.initialized_p ()
4136 && profile_status_for_fn (cfun) != PROFILE_ABSENT
4137 && insn
4138 && JUMP_P (insn)
4139 && any_condjump_p (insn)
4140 && !find_reg_note (insn, REG_BR_PROB, 0))
4141 add_reg_br_prob_note (insn, prob);
4142 }
4143
4144 /* Generate code to compare X with Y so that the condition codes are
4145 set and to jump to LABEL if the condition is true. If X is a
4146 constant and Y is not a constant, then the comparison is swapped to
4147 ensure that the comparison RTL has the canonical form.
4148
4149 UNSIGNEDP nonzero says that X and Y are unsigned; this matters if they
4150 need to be widened. UNSIGNEDP is also used to select the proper
4151 branch condition code.
4152
4153 If X and Y have mode BLKmode, then SIZE specifies the size of both X and Y.
4154
4155 MODE is the mode of the inputs (in case they are const_int).
4156
4157 COMPARISON is the rtl operator to compare with (EQ, NE, GT, etc.).
4158 It will be potentially converted into an unsigned variant based on
4159 UNSIGNEDP to select a proper jump instruction.
4160
4161 PROB is the probability of jumping to LABEL. */
4162
4163 void
emit_cmp_and_jump_insns(rtx x,rtx y,enum rtx_code comparison,rtx size,machine_mode mode,int unsignedp,rtx label,profile_probability prob)4164 emit_cmp_and_jump_insns (rtx x, rtx y, enum rtx_code comparison, rtx size,
4165 machine_mode mode, int unsignedp, rtx label,
4166 profile_probability prob)
4167 {
4168 rtx op0 = x, op1 = y;
4169 rtx test;
4170
4171 /* Swap operands and condition to ensure canonical RTL. */
4172 if (swap_commutative_operands_p (x, y)
4173 && can_compare_p (swap_condition (comparison), mode, ccp_jump))
4174 {
4175 op0 = y, op1 = x;
4176 comparison = swap_condition (comparison);
4177 }
4178
4179 /* If OP0 is still a constant, then both X and Y must be constants
4180 or the opposite comparison is not supported. Force X into a register
4181 to create canonical RTL. */
4182 if (CONSTANT_P (op0))
4183 op0 = force_reg (mode, op0);
4184
4185 if (unsignedp)
4186 comparison = unsigned_condition (comparison);
4187
4188 prepare_cmp_insn (op0, op1, comparison, size, unsignedp, OPTAB_LIB_WIDEN,
4189 &test, &mode);
4190 emit_cmp_and_jump_insn_1 (test, mode, label, prob);
4191 }
4192
4193
4194 /* Emit a library call comparison between floating point X and Y.
4195 COMPARISON is the rtl operator to compare with (EQ, NE, GT, etc.). */
4196
4197 static void
prepare_float_lib_cmp(rtx x,rtx y,enum rtx_code comparison,rtx * ptest,machine_mode * pmode)4198 prepare_float_lib_cmp (rtx x, rtx y, enum rtx_code comparison,
4199 rtx *ptest, machine_mode *pmode)
4200 {
4201 enum rtx_code swapped = swap_condition (comparison);
4202 enum rtx_code reversed = reverse_condition_maybe_unordered (comparison);
4203 machine_mode orig_mode = GET_MODE (x);
4204 machine_mode mode;
4205 rtx true_rtx, false_rtx;
4206 rtx value, target, equiv;
4207 rtx_insn *insns;
4208 rtx libfunc = 0;
4209 bool reversed_p = false;
4210 scalar_int_mode cmp_mode = targetm.libgcc_cmp_return_mode ();
4211
4212 FOR_EACH_MODE_FROM (mode, orig_mode)
4213 {
4214 if (code_to_optab (comparison)
4215 && (libfunc = optab_libfunc (code_to_optab (comparison), mode)))
4216 break;
4217
4218 if (code_to_optab (swapped)
4219 && (libfunc = optab_libfunc (code_to_optab (swapped), mode)))
4220 {
4221 std::swap (x, y);
4222 comparison = swapped;
4223 break;
4224 }
4225
4226 if (code_to_optab (reversed)
4227 && (libfunc = optab_libfunc (code_to_optab (reversed), mode)))
4228 {
4229 comparison = reversed;
4230 reversed_p = true;
4231 break;
4232 }
4233 }
4234
4235 gcc_assert (mode != VOIDmode);
4236
4237 if (mode != orig_mode)
4238 {
4239 x = convert_to_mode (mode, x, 0);
4240 y = convert_to_mode (mode, y, 0);
4241 }
4242
4243 /* Attach a REG_EQUAL note describing the semantics of the libcall to
4244 the RTL. The allows the RTL optimizers to delete the libcall if the
4245 condition can be determined at compile-time. */
4246 if (comparison == UNORDERED
4247 || FLOAT_LIB_COMPARE_RETURNS_BOOL (mode, comparison))
4248 {
4249 true_rtx = const_true_rtx;
4250 false_rtx = const0_rtx;
4251 }
4252 else
4253 {
4254 switch (comparison)
4255 {
4256 case EQ:
4257 true_rtx = const0_rtx;
4258 false_rtx = const_true_rtx;
4259 break;
4260
4261 case NE:
4262 true_rtx = const_true_rtx;
4263 false_rtx = const0_rtx;
4264 break;
4265
4266 case GT:
4267 true_rtx = const1_rtx;
4268 false_rtx = const0_rtx;
4269 break;
4270
4271 case GE:
4272 true_rtx = const0_rtx;
4273 false_rtx = constm1_rtx;
4274 break;
4275
4276 case LT:
4277 true_rtx = constm1_rtx;
4278 false_rtx = const0_rtx;
4279 break;
4280
4281 case LE:
4282 true_rtx = const0_rtx;
4283 false_rtx = const1_rtx;
4284 break;
4285
4286 default:
4287 gcc_unreachable ();
4288 }
4289 }
4290
4291 if (comparison == UNORDERED)
4292 {
4293 rtx temp = simplify_gen_relational (NE, cmp_mode, mode, x, x);
4294 equiv = simplify_gen_relational (NE, cmp_mode, mode, y, y);
4295 equiv = simplify_gen_ternary (IF_THEN_ELSE, cmp_mode, cmp_mode,
4296 temp, const_true_rtx, equiv);
4297 }
4298 else
4299 {
4300 equiv = simplify_gen_relational (comparison, cmp_mode, mode, x, y);
4301 if (! FLOAT_LIB_COMPARE_RETURNS_BOOL (mode, comparison))
4302 equiv = simplify_gen_ternary (IF_THEN_ELSE, cmp_mode, cmp_mode,
4303 equiv, true_rtx, false_rtx);
4304 }
4305
4306 start_sequence ();
4307 value = emit_library_call_value (libfunc, NULL_RTX, LCT_CONST,
4308 cmp_mode, x, mode, y, mode);
4309 insns = get_insns ();
4310 end_sequence ();
4311
4312 target = gen_reg_rtx (cmp_mode);
4313 emit_libcall_block (insns, target, value, equiv);
4314
4315 if (comparison == UNORDERED
4316 || FLOAT_LIB_COMPARE_RETURNS_BOOL (mode, comparison)
4317 || reversed_p)
4318 *ptest = gen_rtx_fmt_ee (reversed_p ? EQ : NE, VOIDmode, target, false_rtx);
4319 else
4320 *ptest = gen_rtx_fmt_ee (comparison, VOIDmode, target, const0_rtx);
4321
4322 *pmode = cmp_mode;
4323 }
4324
4325 /* Generate code to indirectly jump to a location given in the rtx LOC. */
4326
4327 void
emit_indirect_jump(rtx loc)4328 emit_indirect_jump (rtx loc)
4329 {
4330 if (!targetm.have_indirect_jump ())
4331 sorry ("indirect jumps are not available on this target");
4332 else
4333 {
4334 class expand_operand ops[1];
4335 create_address_operand (&ops[0], loc);
4336 expand_jump_insn (targetm.code_for_indirect_jump, 1, ops);
4337 emit_barrier ();
4338 }
4339 }
4340
4341
4342 /* Emit a conditional move instruction if the machine supports one for that
4343 condition and machine mode.
4344
4345 OP0 and OP1 are the operands that should be compared using CODE. CMODE is
4346 the mode to use should they be constants. If it is VOIDmode, they cannot
4347 both be constants.
4348
4349 OP2 should be stored in TARGET if the comparison is true, otherwise OP3
4350 should be stored there. MODE is the mode to use should they be constants.
4351 If it is VOIDmode, they cannot both be constants.
4352
4353 The result is either TARGET (perhaps modified) or NULL_RTX if the operation
4354 is not supported. */
4355
4356 rtx
emit_conditional_move(rtx target,enum rtx_code code,rtx op0,rtx op1,machine_mode cmode,rtx op2,rtx op3,machine_mode mode,int unsignedp)4357 emit_conditional_move (rtx target, enum rtx_code code, rtx op0, rtx op1,
4358 machine_mode cmode, rtx op2, rtx op3,
4359 machine_mode mode, int unsignedp)
4360 {
4361 rtx comparison;
4362 rtx_insn *last;
4363 enum insn_code icode;
4364 enum rtx_code reversed;
4365
4366 /* If the two source operands are identical, that's just a move. */
4367
4368 if (rtx_equal_p (op2, op3))
4369 {
4370 if (!target)
4371 target = gen_reg_rtx (mode);
4372
4373 emit_move_insn (target, op3);
4374 return target;
4375 }
4376
4377 /* If one operand is constant, make it the second one. Only do this
4378 if the other operand is not constant as well. */
4379
4380 if (swap_commutative_operands_p (op0, op1))
4381 {
4382 std::swap (op0, op1);
4383 code = swap_condition (code);
4384 }
4385
4386 /* get_condition will prefer to generate LT and GT even if the old
4387 comparison was against zero, so undo that canonicalization here since
4388 comparisons against zero are cheaper. */
4389 if (code == LT && op1 == const1_rtx)
4390 code = LE, op1 = const0_rtx;
4391 else if (code == GT && op1 == constm1_rtx)
4392 code = GE, op1 = const0_rtx;
4393
4394 if (cmode == VOIDmode)
4395 cmode = GET_MODE (op0);
4396
4397 enum rtx_code orig_code = code;
4398 bool swapped = false;
4399 if (swap_commutative_operands_p (op2, op3)
4400 && ((reversed = reversed_comparison_code_parts (code, op0, op1, NULL))
4401 != UNKNOWN))
4402 {
4403 std::swap (op2, op3);
4404 code = reversed;
4405 swapped = true;
4406 }
4407
4408 if (mode == VOIDmode)
4409 mode = GET_MODE (op2);
4410
4411 icode = direct_optab_handler (movcc_optab, mode);
4412
4413 if (icode == CODE_FOR_nothing)
4414 return NULL_RTX;
4415
4416 if (!target)
4417 target = gen_reg_rtx (mode);
4418
4419 for (int pass = 0; ; pass++)
4420 {
4421 code = unsignedp ? unsigned_condition (code) : code;
4422 comparison = simplify_gen_relational (code, VOIDmode, cmode, op0, op1);
4423
4424 /* We can get const0_rtx or const_true_rtx in some circumstances. Just
4425 punt and let the caller figure out how best to deal with this
4426 situation. */
4427 if (COMPARISON_P (comparison))
4428 {
4429 saved_pending_stack_adjust save;
4430 save_pending_stack_adjust (&save);
4431 last = get_last_insn ();
4432 do_pending_stack_adjust ();
4433 machine_mode cmpmode = cmode;
4434 prepare_cmp_insn (XEXP (comparison, 0), XEXP (comparison, 1),
4435 GET_CODE (comparison), NULL_RTX, unsignedp,
4436 OPTAB_WIDEN, &comparison, &cmpmode);
4437 if (comparison)
4438 {
4439 class expand_operand ops[4];
4440
4441 create_output_operand (&ops[0], target, mode);
4442 create_fixed_operand (&ops[1], comparison);
4443 create_input_operand (&ops[2], op2, mode);
4444 create_input_operand (&ops[3], op3, mode);
4445 if (maybe_expand_insn (icode, 4, ops))
4446 {
4447 if (ops[0].value != target)
4448 convert_move (target, ops[0].value, false);
4449 return target;
4450 }
4451 }
4452 delete_insns_since (last);
4453 restore_pending_stack_adjust (&save);
4454 }
4455
4456 if (pass == 1)
4457 return NULL_RTX;
4458
4459 /* If the preferred op2/op3 order is not usable, retry with other
4460 operand order, perhaps it will expand successfully. */
4461 if (swapped)
4462 code = orig_code;
4463 else if ((reversed = reversed_comparison_code_parts (orig_code, op0, op1,
4464 NULL))
4465 != UNKNOWN)
4466 code = reversed;
4467 else
4468 return NULL_RTX;
4469 std::swap (op2, op3);
4470 }
4471 }
4472
4473
4474 /* Emit a conditional negate or bitwise complement using the
4475 negcc or notcc optabs if available. Return NULL_RTX if such operations
4476 are not available. Otherwise return the RTX holding the result.
4477 TARGET is the desired destination of the result. COMP is the comparison
4478 on which to negate. If COND is true move into TARGET the negation
4479 or bitwise complement of OP1. Otherwise move OP2 into TARGET.
4480 CODE is either NEG or NOT. MODE is the machine mode in which the
4481 operation is performed. */
4482
4483 rtx
emit_conditional_neg_or_complement(rtx target,rtx_code code,machine_mode mode,rtx cond,rtx op1,rtx op2)4484 emit_conditional_neg_or_complement (rtx target, rtx_code code,
4485 machine_mode mode, rtx cond, rtx op1,
4486 rtx op2)
4487 {
4488 optab op = unknown_optab;
4489 if (code == NEG)
4490 op = negcc_optab;
4491 else if (code == NOT)
4492 op = notcc_optab;
4493 else
4494 gcc_unreachable ();
4495
4496 insn_code icode = direct_optab_handler (op, mode);
4497
4498 if (icode == CODE_FOR_nothing)
4499 return NULL_RTX;
4500
4501 if (!target)
4502 target = gen_reg_rtx (mode);
4503
4504 rtx_insn *last = get_last_insn ();
4505 class expand_operand ops[4];
4506
4507 create_output_operand (&ops[0], target, mode);
4508 create_fixed_operand (&ops[1], cond);
4509 create_input_operand (&ops[2], op1, mode);
4510 create_input_operand (&ops[3], op2, mode);
4511
4512 if (maybe_expand_insn (icode, 4, ops))
4513 {
4514 if (ops[0].value != target)
4515 convert_move (target, ops[0].value, false);
4516
4517 return target;
4518 }
4519 delete_insns_since (last);
4520 return NULL_RTX;
4521 }
4522
4523 /* Emit a conditional addition instruction if the machine supports one for that
4524 condition and machine mode.
4525
4526 OP0 and OP1 are the operands that should be compared using CODE. CMODE is
4527 the mode to use should they be constants. If it is VOIDmode, they cannot
4528 both be constants.
4529
4530 OP2 should be stored in TARGET if the comparison is false, otherwise OP2+OP3
4531 should be stored there. MODE is the mode to use should they be constants.
4532 If it is VOIDmode, they cannot both be constants.
4533
4534 The result is either TARGET (perhaps modified) or NULL_RTX if the operation
4535 is not supported. */
4536
4537 rtx
emit_conditional_add(rtx target,enum rtx_code code,rtx op0,rtx op1,machine_mode cmode,rtx op2,rtx op3,machine_mode mode,int unsignedp)4538 emit_conditional_add (rtx target, enum rtx_code code, rtx op0, rtx op1,
4539 machine_mode cmode, rtx op2, rtx op3,
4540 machine_mode mode, int unsignedp)
4541 {
4542 rtx comparison;
4543 rtx_insn *last;
4544 enum insn_code icode;
4545
4546 /* If one operand is constant, make it the second one. Only do this
4547 if the other operand is not constant as well. */
4548
4549 if (swap_commutative_operands_p (op0, op1))
4550 {
4551 std::swap (op0, op1);
4552 code = swap_condition (code);
4553 }
4554
4555 /* get_condition will prefer to generate LT and GT even if the old
4556 comparison was against zero, so undo that canonicalization here since
4557 comparisons against zero are cheaper. */
4558 if (code == LT && op1 == const1_rtx)
4559 code = LE, op1 = const0_rtx;
4560 else if (code == GT && op1 == constm1_rtx)
4561 code = GE, op1 = const0_rtx;
4562
4563 if (cmode == VOIDmode)
4564 cmode = GET_MODE (op0);
4565
4566 if (mode == VOIDmode)
4567 mode = GET_MODE (op2);
4568
4569 icode = optab_handler (addcc_optab, mode);
4570
4571 if (icode == CODE_FOR_nothing)
4572 return 0;
4573
4574 if (!target)
4575 target = gen_reg_rtx (mode);
4576
4577 code = unsignedp ? unsigned_condition (code) : code;
4578 comparison = simplify_gen_relational (code, VOIDmode, cmode, op0, op1);
4579
4580 /* We can get const0_rtx or const_true_rtx in some circumstances. Just
4581 return NULL and let the caller figure out how best to deal with this
4582 situation. */
4583 if (!COMPARISON_P (comparison))
4584 return NULL_RTX;
4585
4586 do_pending_stack_adjust ();
4587 last = get_last_insn ();
4588 prepare_cmp_insn (XEXP (comparison, 0), XEXP (comparison, 1),
4589 GET_CODE (comparison), NULL_RTX, unsignedp, OPTAB_WIDEN,
4590 &comparison, &cmode);
4591 if (comparison)
4592 {
4593 class expand_operand ops[4];
4594
4595 create_output_operand (&ops[0], target, mode);
4596 create_fixed_operand (&ops[1], comparison);
4597 create_input_operand (&ops[2], op2, mode);
4598 create_input_operand (&ops[3], op3, mode);
4599 if (maybe_expand_insn (icode, 4, ops))
4600 {
4601 if (ops[0].value != target)
4602 convert_move (target, ops[0].value, false);
4603 return target;
4604 }
4605 }
4606 delete_insns_since (last);
4607 return NULL_RTX;
4608 }
4609
4610 /* These functions attempt to generate an insn body, rather than
4611 emitting the insn, but if the gen function already emits them, we
4612 make no attempt to turn them back into naked patterns. */
4613
4614 /* Generate and return an insn body to add Y to X. */
4615
4616 rtx_insn *
gen_add2_insn(rtx x,rtx y)4617 gen_add2_insn (rtx x, rtx y)
4618 {
4619 enum insn_code icode = optab_handler (add_optab, GET_MODE (x));
4620
4621 gcc_assert (insn_operand_matches (icode, 0, x));
4622 gcc_assert (insn_operand_matches (icode, 1, x));
4623 gcc_assert (insn_operand_matches (icode, 2, y));
4624
4625 return GEN_FCN (icode) (x, x, y);
4626 }
4627
4628 /* Generate and return an insn body to add r1 and c,
4629 storing the result in r0. */
4630
4631 rtx_insn *
gen_add3_insn(rtx r0,rtx r1,rtx c)4632 gen_add3_insn (rtx r0, rtx r1, rtx c)
4633 {
4634 enum insn_code icode = optab_handler (add_optab, GET_MODE (r0));
4635
4636 if (icode == CODE_FOR_nothing
4637 || !insn_operand_matches (icode, 0, r0)
4638 || !insn_operand_matches (icode, 1, r1)
4639 || !insn_operand_matches (icode, 2, c))
4640 return NULL;
4641
4642 return GEN_FCN (icode) (r0, r1, c);
4643 }
4644
4645 int
have_add2_insn(rtx x,rtx y)4646 have_add2_insn (rtx x, rtx y)
4647 {
4648 enum insn_code icode;
4649
4650 gcc_assert (GET_MODE (x) != VOIDmode);
4651
4652 icode = optab_handler (add_optab, GET_MODE (x));
4653
4654 if (icode == CODE_FOR_nothing)
4655 return 0;
4656
4657 if (!insn_operand_matches (icode, 0, x)
4658 || !insn_operand_matches (icode, 1, x)
4659 || !insn_operand_matches (icode, 2, y))
4660 return 0;
4661
4662 return 1;
4663 }
4664
4665 /* Generate and return an insn body to add Y to X. */
4666
4667 rtx_insn *
gen_addptr3_insn(rtx x,rtx y,rtx z)4668 gen_addptr3_insn (rtx x, rtx y, rtx z)
4669 {
4670 enum insn_code icode = optab_handler (addptr3_optab, GET_MODE (x));
4671
4672 gcc_assert (insn_operand_matches (icode, 0, x));
4673 gcc_assert (insn_operand_matches (icode, 1, y));
4674 gcc_assert (insn_operand_matches (icode, 2, z));
4675
4676 return GEN_FCN (icode) (x, y, z);
4677 }
4678
4679 /* Return true if the target implements an addptr pattern and X, Y,
4680 and Z are valid for the pattern predicates. */
4681
4682 int
have_addptr3_insn(rtx x,rtx y,rtx z)4683 have_addptr3_insn (rtx x, rtx y, rtx z)
4684 {
4685 enum insn_code icode;
4686
4687 gcc_assert (GET_MODE (x) != VOIDmode);
4688
4689 icode = optab_handler (addptr3_optab, GET_MODE (x));
4690
4691 if (icode == CODE_FOR_nothing)
4692 return 0;
4693
4694 if (!insn_operand_matches (icode, 0, x)
4695 || !insn_operand_matches (icode, 1, y)
4696 || !insn_operand_matches (icode, 2, z))
4697 return 0;
4698
4699 return 1;
4700 }
4701
4702 /* Generate and return an insn body to subtract Y from X. */
4703
4704 rtx_insn *
gen_sub2_insn(rtx x,rtx y)4705 gen_sub2_insn (rtx x, rtx y)
4706 {
4707 enum insn_code icode = optab_handler (sub_optab, GET_MODE (x));
4708
4709 gcc_assert (insn_operand_matches (icode, 0, x));
4710 gcc_assert (insn_operand_matches (icode, 1, x));
4711 gcc_assert (insn_operand_matches (icode, 2, y));
4712
4713 return GEN_FCN (icode) (x, x, y);
4714 }
4715
4716 /* Generate and return an insn body to subtract r1 and c,
4717 storing the result in r0. */
4718
4719 rtx_insn *
gen_sub3_insn(rtx r0,rtx r1,rtx c)4720 gen_sub3_insn (rtx r0, rtx r1, rtx c)
4721 {
4722 enum insn_code icode = optab_handler (sub_optab, GET_MODE (r0));
4723
4724 if (icode == CODE_FOR_nothing
4725 || !insn_operand_matches (icode, 0, r0)
4726 || !insn_operand_matches (icode, 1, r1)
4727 || !insn_operand_matches (icode, 2, c))
4728 return NULL;
4729
4730 return GEN_FCN (icode) (r0, r1, c);
4731 }
4732
4733 int
have_sub2_insn(rtx x,rtx y)4734 have_sub2_insn (rtx x, rtx y)
4735 {
4736 enum insn_code icode;
4737
4738 gcc_assert (GET_MODE (x) != VOIDmode);
4739
4740 icode = optab_handler (sub_optab, GET_MODE (x));
4741
4742 if (icode == CODE_FOR_nothing)
4743 return 0;
4744
4745 if (!insn_operand_matches (icode, 0, x)
4746 || !insn_operand_matches (icode, 1, x)
4747 || !insn_operand_matches (icode, 2, y))
4748 return 0;
4749
4750 return 1;
4751 }
4752
4753 /* Generate the body of an insn to extend Y (with mode MFROM)
4754 into X (with mode MTO). Do zero-extension if UNSIGNEDP is nonzero. */
4755
4756 rtx_insn *
gen_extend_insn(rtx x,rtx y,machine_mode mto,machine_mode mfrom,int unsignedp)4757 gen_extend_insn (rtx x, rtx y, machine_mode mto,
4758 machine_mode mfrom, int unsignedp)
4759 {
4760 enum insn_code icode = can_extend_p (mto, mfrom, unsignedp);
4761 return GEN_FCN (icode) (x, y);
4762 }
4763
4764 /* Generate code to convert FROM to floating point
4765 and store in TO. FROM must be fixed point and not VOIDmode.
4766 UNSIGNEDP nonzero means regard FROM as unsigned.
4767 Normally this is done by correcting the final value
4768 if it is negative. */
4769
4770 void
expand_float(rtx to,rtx from,int unsignedp)4771 expand_float (rtx to, rtx from, int unsignedp)
4772 {
4773 enum insn_code icode;
4774 rtx target = to;
4775 scalar_mode from_mode, to_mode;
4776 machine_mode fmode, imode;
4777 bool can_do_signed = false;
4778
4779 /* Crash now, because we won't be able to decide which mode to use. */
4780 gcc_assert (GET_MODE (from) != VOIDmode);
4781
4782 /* Look for an insn to do the conversion. Do it in the specified
4783 modes if possible; otherwise convert either input, output or both to
4784 wider mode. If the integer mode is wider than the mode of FROM,
4785 we can do the conversion signed even if the input is unsigned. */
4786
4787 FOR_EACH_MODE_FROM (fmode, GET_MODE (to))
4788 FOR_EACH_MODE_FROM (imode, GET_MODE (from))
4789 {
4790 int doing_unsigned = unsignedp;
4791
4792 if (fmode != GET_MODE (to)
4793 && (significand_size (fmode)
4794 < GET_MODE_UNIT_PRECISION (GET_MODE (from))))
4795 continue;
4796
4797 icode = can_float_p (fmode, imode, unsignedp);
4798 if (icode == CODE_FOR_nothing && unsignedp)
4799 {
4800 enum insn_code scode = can_float_p (fmode, imode, 0);
4801 if (scode != CODE_FOR_nothing)
4802 can_do_signed = true;
4803 if (imode != GET_MODE (from))
4804 icode = scode, doing_unsigned = 0;
4805 }
4806
4807 if (icode != CODE_FOR_nothing)
4808 {
4809 if (imode != GET_MODE (from))
4810 from = convert_to_mode (imode, from, unsignedp);
4811
4812 if (fmode != GET_MODE (to))
4813 target = gen_reg_rtx (fmode);
4814
4815 emit_unop_insn (icode, target, from,
4816 doing_unsigned ? UNSIGNED_FLOAT : FLOAT);
4817
4818 if (target != to)
4819 convert_move (to, target, 0);
4820 return;
4821 }
4822 }
4823
4824 /* Unsigned integer, and no way to convert directly. Convert as signed,
4825 then unconditionally adjust the result. */
4826 if (unsignedp
4827 && can_do_signed
4828 && is_a <scalar_mode> (GET_MODE (to), &to_mode)
4829 && is_a <scalar_mode> (GET_MODE (from), &from_mode))
4830 {
4831 opt_scalar_mode fmode_iter;
4832 rtx_code_label *label = gen_label_rtx ();
4833 rtx temp;
4834 REAL_VALUE_TYPE offset;
4835
4836 /* Look for a usable floating mode FMODE wider than the source and at
4837 least as wide as the target. Using FMODE will avoid rounding woes
4838 with unsigned values greater than the signed maximum value. */
4839
4840 FOR_EACH_MODE_FROM (fmode_iter, to_mode)
4841 {
4842 scalar_mode fmode = fmode_iter.require ();
4843 if (GET_MODE_PRECISION (from_mode) < GET_MODE_BITSIZE (fmode)
4844 && can_float_p (fmode, from_mode, 0) != CODE_FOR_nothing)
4845 break;
4846 }
4847
4848 if (!fmode_iter.exists (&fmode))
4849 {
4850 /* There is no such mode. Pretend the target is wide enough. */
4851 fmode = to_mode;
4852
4853 /* Avoid double-rounding when TO is narrower than FROM. */
4854 if ((significand_size (fmode) + 1)
4855 < GET_MODE_PRECISION (from_mode))
4856 {
4857 rtx temp1;
4858 rtx_code_label *neglabel = gen_label_rtx ();
4859
4860 /* Don't use TARGET if it isn't a register, is a hard register,
4861 or is the wrong mode. */
4862 if (!REG_P (target)
4863 || REGNO (target) < FIRST_PSEUDO_REGISTER
4864 || GET_MODE (target) != fmode)
4865 target = gen_reg_rtx (fmode);
4866
4867 imode = from_mode;
4868 do_pending_stack_adjust ();
4869
4870 /* Test whether the sign bit is set. */
4871 emit_cmp_and_jump_insns (from, const0_rtx, LT, NULL_RTX, imode,
4872 0, neglabel);
4873
4874 /* The sign bit is not set. Convert as signed. */
4875 expand_float (target, from, 0);
4876 emit_jump_insn (targetm.gen_jump (label));
4877 emit_barrier ();
4878
4879 /* The sign bit is set.
4880 Convert to a usable (positive signed) value by shifting right
4881 one bit, while remembering if a nonzero bit was shifted
4882 out; i.e., compute (from & 1) | (from >> 1). */
4883
4884 emit_label (neglabel);
4885 temp = expand_binop (imode, and_optab, from, const1_rtx,
4886 NULL_RTX, 1, OPTAB_LIB_WIDEN);
4887 temp1 = expand_shift (RSHIFT_EXPR, imode, from, 1, NULL_RTX, 1);
4888 temp = expand_binop (imode, ior_optab, temp, temp1, temp, 1,
4889 OPTAB_LIB_WIDEN);
4890 expand_float (target, temp, 0);
4891
4892 /* Multiply by 2 to undo the shift above. */
4893 temp = expand_binop (fmode, add_optab, target, target,
4894 target, 0, OPTAB_LIB_WIDEN);
4895 if (temp != target)
4896 emit_move_insn (target, temp);
4897
4898 do_pending_stack_adjust ();
4899 emit_label (label);
4900 goto done;
4901 }
4902 }
4903
4904 /* If we are about to do some arithmetic to correct for an
4905 unsigned operand, do it in a pseudo-register. */
4906
4907 if (to_mode != fmode
4908 || !REG_P (to) || REGNO (to) < FIRST_PSEUDO_REGISTER)
4909 target = gen_reg_rtx (fmode);
4910
4911 /* Convert as signed integer to floating. */
4912 expand_float (target, from, 0);
4913
4914 /* If FROM is negative (and therefore TO is negative),
4915 correct its value by 2**bitwidth. */
4916
4917 do_pending_stack_adjust ();
4918 emit_cmp_and_jump_insns (from, const0_rtx, GE, NULL_RTX, from_mode,
4919 0, label);
4920
4921
4922 real_2expN (&offset, GET_MODE_PRECISION (from_mode), fmode);
4923 temp = expand_binop (fmode, add_optab, target,
4924 const_double_from_real_value (offset, fmode),
4925 target, 0, OPTAB_LIB_WIDEN);
4926 if (temp != target)
4927 emit_move_insn (target, temp);
4928
4929 do_pending_stack_adjust ();
4930 emit_label (label);
4931 goto done;
4932 }
4933
4934 /* No hardware instruction available; call a library routine. */
4935 {
4936 rtx libfunc;
4937 rtx_insn *insns;
4938 rtx value;
4939 convert_optab tab = unsignedp ? ufloat_optab : sfloat_optab;
4940
4941 if (is_narrower_int_mode (GET_MODE (from), SImode))
4942 from = convert_to_mode (SImode, from, unsignedp);
4943
4944 libfunc = convert_optab_libfunc (tab, GET_MODE (to), GET_MODE (from));
4945 gcc_assert (libfunc);
4946
4947 start_sequence ();
4948
4949 value = emit_library_call_value (libfunc, NULL_RTX, LCT_CONST,
4950 GET_MODE (to), from, GET_MODE (from));
4951 insns = get_insns ();
4952 end_sequence ();
4953
4954 emit_libcall_block (insns, target, value,
4955 gen_rtx_fmt_e (unsignedp ? UNSIGNED_FLOAT : FLOAT,
4956 GET_MODE (to), from));
4957 }
4958
4959 done:
4960
4961 /* Copy result to requested destination
4962 if we have been computing in a temp location. */
4963
4964 if (target != to)
4965 {
4966 if (GET_MODE (target) == GET_MODE (to))
4967 emit_move_insn (to, target);
4968 else
4969 convert_move (to, target, 0);
4970 }
4971 }
4972
4973 /* Generate code to convert FROM to fixed point and store in TO. FROM
4974 must be floating point. */
4975
4976 void
expand_fix(rtx to,rtx from,int unsignedp)4977 expand_fix (rtx to, rtx from, int unsignedp)
4978 {
4979 enum insn_code icode;
4980 rtx target = to;
4981 machine_mode fmode, imode;
4982 opt_scalar_mode fmode_iter;
4983 bool must_trunc = false;
4984
4985 /* We first try to find a pair of modes, one real and one integer, at
4986 least as wide as FROM and TO, respectively, in which we can open-code
4987 this conversion. If the integer mode is wider than the mode of TO,
4988 we can do the conversion either signed or unsigned. */
4989
4990 FOR_EACH_MODE_FROM (fmode, GET_MODE (from))
4991 FOR_EACH_MODE_FROM (imode, GET_MODE (to))
4992 {
4993 int doing_unsigned = unsignedp;
4994
4995 icode = can_fix_p (imode, fmode, unsignedp, &must_trunc);
4996 if (icode == CODE_FOR_nothing && imode != GET_MODE (to) && unsignedp)
4997 icode = can_fix_p (imode, fmode, 0, &must_trunc), doing_unsigned = 0;
4998
4999 if (icode != CODE_FOR_nothing)
5000 {
5001 rtx_insn *last = get_last_insn ();
5002 if (fmode != GET_MODE (from))
5003 from = convert_to_mode (fmode, from, 0);
5004
5005 if (must_trunc)
5006 {
5007 rtx temp = gen_reg_rtx (GET_MODE (from));
5008 from = expand_unop (GET_MODE (from), ftrunc_optab, from,
5009 temp, 0);
5010 }
5011
5012 if (imode != GET_MODE (to))
5013 target = gen_reg_rtx (imode);
5014
5015 if (maybe_emit_unop_insn (icode, target, from,
5016 doing_unsigned ? UNSIGNED_FIX : FIX))
5017 {
5018 if (target != to)
5019 convert_move (to, target, unsignedp);
5020 return;
5021 }
5022 delete_insns_since (last);
5023 }
5024 }
5025
5026 /* For an unsigned conversion, there is one more way to do it.
5027 If we have a signed conversion, we generate code that compares
5028 the real value to the largest representable positive number. If if
5029 is smaller, the conversion is done normally. Otherwise, subtract
5030 one plus the highest signed number, convert, and add it back.
5031
5032 We only need to check all real modes, since we know we didn't find
5033 anything with a wider integer mode.
5034
5035 This code used to extend FP value into mode wider than the destination.
5036 This is needed for decimal float modes which cannot accurately
5037 represent one plus the highest signed number of the same size, but
5038 not for binary modes. Consider, for instance conversion from SFmode
5039 into DImode.
5040
5041 The hot path through the code is dealing with inputs smaller than 2^63
5042 and doing just the conversion, so there is no bits to lose.
5043
5044 In the other path we know the value is positive in the range 2^63..2^64-1
5045 inclusive. (as for other input overflow happens and result is undefined)
5046 So we know that the most important bit set in mantissa corresponds to
5047 2^63. The subtraction of 2^63 should not generate any rounding as it
5048 simply clears out that bit. The rest is trivial. */
5049
5050 scalar_int_mode to_mode;
5051 if (unsignedp
5052 && is_a <scalar_int_mode> (GET_MODE (to), &to_mode)
5053 && HWI_COMPUTABLE_MODE_P (to_mode))
5054 FOR_EACH_MODE_FROM (fmode_iter, as_a <scalar_mode> (GET_MODE (from)))
5055 {
5056 scalar_mode fmode = fmode_iter.require ();
5057 if (CODE_FOR_nothing != can_fix_p (to_mode, fmode,
5058 0, &must_trunc)
5059 && (!DECIMAL_FLOAT_MODE_P (fmode)
5060 || (GET_MODE_BITSIZE (fmode) > GET_MODE_PRECISION (to_mode))))
5061 {
5062 int bitsize;
5063 REAL_VALUE_TYPE offset;
5064 rtx limit;
5065 rtx_code_label *lab1, *lab2;
5066 rtx_insn *insn;
5067
5068 bitsize = GET_MODE_PRECISION (to_mode);
5069 real_2expN (&offset, bitsize - 1, fmode);
5070 limit = const_double_from_real_value (offset, fmode);
5071 lab1 = gen_label_rtx ();
5072 lab2 = gen_label_rtx ();
5073
5074 if (fmode != GET_MODE (from))
5075 from = convert_to_mode (fmode, from, 0);
5076
5077 /* See if we need to do the subtraction. */
5078 do_pending_stack_adjust ();
5079 emit_cmp_and_jump_insns (from, limit, GE, NULL_RTX,
5080 GET_MODE (from), 0, lab1);
5081
5082 /* If not, do the signed "fix" and branch around fixup code. */
5083 expand_fix (to, from, 0);
5084 emit_jump_insn (targetm.gen_jump (lab2));
5085 emit_barrier ();
5086
5087 /* Otherwise, subtract 2**(N-1), convert to signed number,
5088 then add 2**(N-1). Do the addition using XOR since this
5089 will often generate better code. */
5090 emit_label (lab1);
5091 target = expand_binop (GET_MODE (from), sub_optab, from, limit,
5092 NULL_RTX, 0, OPTAB_LIB_WIDEN);
5093 expand_fix (to, target, 0);
5094 target = expand_binop (to_mode, xor_optab, to,
5095 gen_int_mode
5096 (HOST_WIDE_INT_1 << (bitsize - 1),
5097 to_mode),
5098 to, 1, OPTAB_LIB_WIDEN);
5099
5100 if (target != to)
5101 emit_move_insn (to, target);
5102
5103 emit_label (lab2);
5104
5105 if (optab_handler (mov_optab, to_mode) != CODE_FOR_nothing)
5106 {
5107 /* Make a place for a REG_NOTE and add it. */
5108 insn = emit_move_insn (to, to);
5109 set_dst_reg_note (insn, REG_EQUAL,
5110 gen_rtx_fmt_e (UNSIGNED_FIX, to_mode,
5111 copy_rtx (from)),
5112 to);
5113 }
5114
5115 return;
5116 }
5117 }
5118
5119 /* We can't do it with an insn, so use a library call. But first ensure
5120 that the mode of TO is at least as wide as SImode, since those are the
5121 only library calls we know about. */
5122
5123 if (is_narrower_int_mode (GET_MODE (to), SImode))
5124 {
5125 target = gen_reg_rtx (SImode);
5126
5127 expand_fix (target, from, unsignedp);
5128 }
5129 else
5130 {
5131 rtx_insn *insns;
5132 rtx value;
5133 rtx libfunc;
5134
5135 convert_optab tab = unsignedp ? ufix_optab : sfix_optab;
5136 libfunc = convert_optab_libfunc (tab, GET_MODE (to), GET_MODE (from));
5137 gcc_assert (libfunc);
5138
5139 start_sequence ();
5140
5141 value = emit_library_call_value (libfunc, NULL_RTX, LCT_CONST,
5142 GET_MODE (to), from, GET_MODE (from));
5143 insns = get_insns ();
5144 end_sequence ();
5145
5146 emit_libcall_block (insns, target, value,
5147 gen_rtx_fmt_e (unsignedp ? UNSIGNED_FIX : FIX,
5148 GET_MODE (to), from));
5149 }
5150
5151 if (target != to)
5152 {
5153 if (GET_MODE (to) == GET_MODE (target))
5154 emit_move_insn (to, target);
5155 else
5156 convert_move (to, target, 0);
5157 }
5158 }
5159
5160
5161 /* Promote integer arguments for a libcall if necessary.
5162 emit_library_call_value cannot do the promotion because it does not
5163 know if it should do a signed or unsigned promotion. This is because
5164 there are no tree types defined for libcalls. */
5165
5166 static rtx
prepare_libcall_arg(rtx arg,int uintp)5167 prepare_libcall_arg (rtx arg, int uintp)
5168 {
5169 scalar_int_mode mode;
5170 machine_mode arg_mode;
5171 if (is_a <scalar_int_mode> (GET_MODE (arg), &mode))
5172 {
5173 /* If we need to promote the integer function argument we need to do
5174 it here instead of inside emit_library_call_value because in
5175 emit_library_call_value we don't know if we should do a signed or
5176 unsigned promotion. */
5177
5178 int unsigned_p = 0;
5179 arg_mode = promote_function_mode (NULL_TREE, mode,
5180 &unsigned_p, NULL_TREE, 0);
5181 if (arg_mode != mode)
5182 return convert_to_mode (arg_mode, arg, uintp);
5183 }
5184 return arg;
5185 }
5186
5187 /* Generate code to convert FROM or TO a fixed-point.
5188 If UINTP is true, either TO or FROM is an unsigned integer.
5189 If SATP is true, we need to saturate the result. */
5190
5191 void
expand_fixed_convert(rtx to,rtx from,int uintp,int satp)5192 expand_fixed_convert (rtx to, rtx from, int uintp, int satp)
5193 {
5194 machine_mode to_mode = GET_MODE (to);
5195 machine_mode from_mode = GET_MODE (from);
5196 convert_optab tab;
5197 enum rtx_code this_code;
5198 enum insn_code code;
5199 rtx_insn *insns;
5200 rtx value;
5201 rtx libfunc;
5202
5203 if (to_mode == from_mode)
5204 {
5205 emit_move_insn (to, from);
5206 return;
5207 }
5208
5209 if (uintp)
5210 {
5211 tab = satp ? satfractuns_optab : fractuns_optab;
5212 this_code = satp ? UNSIGNED_SAT_FRACT : UNSIGNED_FRACT_CONVERT;
5213 }
5214 else
5215 {
5216 tab = satp ? satfract_optab : fract_optab;
5217 this_code = satp ? SAT_FRACT : FRACT_CONVERT;
5218 }
5219 code = convert_optab_handler (tab, to_mode, from_mode);
5220 if (code != CODE_FOR_nothing)
5221 {
5222 emit_unop_insn (code, to, from, this_code);
5223 return;
5224 }
5225
5226 libfunc = convert_optab_libfunc (tab, to_mode, from_mode);
5227 gcc_assert (libfunc);
5228
5229 from = prepare_libcall_arg (from, uintp);
5230 from_mode = GET_MODE (from);
5231
5232 start_sequence ();
5233 value = emit_library_call_value (libfunc, NULL_RTX, LCT_CONST, to_mode,
5234 from, from_mode);
5235 insns = get_insns ();
5236 end_sequence ();
5237
5238 emit_libcall_block (insns, to, value,
5239 gen_rtx_fmt_e (optab_to_code (tab), to_mode, from));
5240 }
5241
5242 /* Generate code to convert FROM to fixed point and store in TO. FROM
5243 must be floating point, TO must be signed. Use the conversion optab
5244 TAB to do the conversion. */
5245
5246 bool
expand_sfix_optab(rtx to,rtx from,convert_optab tab)5247 expand_sfix_optab (rtx to, rtx from, convert_optab tab)
5248 {
5249 enum insn_code icode;
5250 rtx target = to;
5251 machine_mode fmode, imode;
5252
5253 /* We first try to find a pair of modes, one real and one integer, at
5254 least as wide as FROM and TO, respectively, in which we can open-code
5255 this conversion. If the integer mode is wider than the mode of TO,
5256 we can do the conversion either signed or unsigned. */
5257
5258 FOR_EACH_MODE_FROM (fmode, GET_MODE (from))
5259 FOR_EACH_MODE_FROM (imode, GET_MODE (to))
5260 {
5261 icode = convert_optab_handler (tab, imode, fmode);
5262 if (icode != CODE_FOR_nothing)
5263 {
5264 rtx_insn *last = get_last_insn ();
5265 if (fmode != GET_MODE (from))
5266 from = convert_to_mode (fmode, from, 0);
5267
5268 if (imode != GET_MODE (to))
5269 target = gen_reg_rtx (imode);
5270
5271 if (!maybe_emit_unop_insn (icode, target, from, UNKNOWN))
5272 {
5273 delete_insns_since (last);
5274 continue;
5275 }
5276 if (target != to)
5277 convert_move (to, target, 0);
5278 return true;
5279 }
5280 }
5281
5282 return false;
5283 }
5284
5285 /* Report whether we have an instruction to perform the operation
5286 specified by CODE on operands of mode MODE. */
5287 int
have_insn_for(enum rtx_code code,machine_mode mode)5288 have_insn_for (enum rtx_code code, machine_mode mode)
5289 {
5290 return (code_to_optab (code)
5291 && (optab_handler (code_to_optab (code), mode)
5292 != CODE_FOR_nothing));
5293 }
5294
5295 /* Print information about the current contents of the optabs on
5296 STDERR. */
5297
5298 DEBUG_FUNCTION void
debug_optab_libfuncs(void)5299 debug_optab_libfuncs (void)
5300 {
5301 int i, j, k;
5302
5303 /* Dump the arithmetic optabs. */
5304 for (i = FIRST_NORM_OPTAB; i <= LAST_NORMLIB_OPTAB; ++i)
5305 for (j = 0; j < NUM_MACHINE_MODES; ++j)
5306 {
5307 rtx l = optab_libfunc ((optab) i, (machine_mode) j);
5308 if (l)
5309 {
5310 gcc_assert (GET_CODE (l) == SYMBOL_REF);
5311 fprintf (stderr, "%s\t%s:\t%s\n",
5312 GET_RTX_NAME (optab_to_code ((optab) i)),
5313 GET_MODE_NAME (j),
5314 XSTR (l, 0));
5315 }
5316 }
5317
5318 /* Dump the conversion optabs. */
5319 for (i = FIRST_CONV_OPTAB; i <= LAST_CONVLIB_OPTAB; ++i)
5320 for (j = 0; j < NUM_MACHINE_MODES; ++j)
5321 for (k = 0; k < NUM_MACHINE_MODES; ++k)
5322 {
5323 rtx l = convert_optab_libfunc ((optab) i, (machine_mode) j,
5324 (machine_mode) k);
5325 if (l)
5326 {
5327 gcc_assert (GET_CODE (l) == SYMBOL_REF);
5328 fprintf (stderr, "%s\t%s\t%s:\t%s\n",
5329 GET_RTX_NAME (optab_to_code ((optab) i)),
5330 GET_MODE_NAME (j),
5331 GET_MODE_NAME (k),
5332 XSTR (l, 0));
5333 }
5334 }
5335 }
5336
5337 /* Generate insns to trap with code TCODE if OP1 and OP2 satisfy condition
5338 CODE. Return 0 on failure. */
5339
5340 rtx_insn *
gen_cond_trap(enum rtx_code code,rtx op1,rtx op2,rtx tcode)5341 gen_cond_trap (enum rtx_code code, rtx op1, rtx op2, rtx tcode)
5342 {
5343 machine_mode mode = GET_MODE (op1);
5344 enum insn_code icode;
5345 rtx_insn *insn;
5346 rtx trap_rtx;
5347
5348 if (mode == VOIDmode)
5349 return 0;
5350
5351 icode = optab_handler (ctrap_optab, mode);
5352 if (icode == CODE_FOR_nothing)
5353 return 0;
5354
5355 /* Some targets only accept a zero trap code. */
5356 if (!insn_operand_matches (icode, 3, tcode))
5357 return 0;
5358
5359 do_pending_stack_adjust ();
5360 start_sequence ();
5361 prepare_cmp_insn (op1, op2, code, NULL_RTX, false, OPTAB_DIRECT,
5362 &trap_rtx, &mode);
5363 if (!trap_rtx)
5364 insn = NULL;
5365 else
5366 insn = GEN_FCN (icode) (trap_rtx, XEXP (trap_rtx, 0), XEXP (trap_rtx, 1),
5367 tcode);
5368
5369 /* If that failed, then give up. */
5370 if (insn == 0)
5371 {
5372 end_sequence ();
5373 return 0;
5374 }
5375
5376 emit_insn (insn);
5377 insn = get_insns ();
5378 end_sequence ();
5379 return insn;
5380 }
5381
5382 /* Return rtx code for TCODE. Use UNSIGNEDP to select signed
5383 or unsigned operation code. */
5384
5385 enum rtx_code
get_rtx_code(enum tree_code tcode,bool unsignedp)5386 get_rtx_code (enum tree_code tcode, bool unsignedp)
5387 {
5388 enum rtx_code code;
5389 switch (tcode)
5390 {
5391 case EQ_EXPR:
5392 code = EQ;
5393 break;
5394 case NE_EXPR:
5395 code = NE;
5396 break;
5397 case LT_EXPR:
5398 code = unsignedp ? LTU : LT;
5399 break;
5400 case LE_EXPR:
5401 code = unsignedp ? LEU : LE;
5402 break;
5403 case GT_EXPR:
5404 code = unsignedp ? GTU : GT;
5405 break;
5406 case GE_EXPR:
5407 code = unsignedp ? GEU : GE;
5408 break;
5409
5410 case UNORDERED_EXPR:
5411 code = UNORDERED;
5412 break;
5413 case ORDERED_EXPR:
5414 code = ORDERED;
5415 break;
5416 case UNLT_EXPR:
5417 code = UNLT;
5418 break;
5419 case UNLE_EXPR:
5420 code = UNLE;
5421 break;
5422 case UNGT_EXPR:
5423 code = UNGT;
5424 break;
5425 case UNGE_EXPR:
5426 code = UNGE;
5427 break;
5428 case UNEQ_EXPR:
5429 code = UNEQ;
5430 break;
5431 case LTGT_EXPR:
5432 code = LTGT;
5433 break;
5434
5435 case BIT_AND_EXPR:
5436 code = AND;
5437 break;
5438
5439 case BIT_IOR_EXPR:
5440 code = IOR;
5441 break;
5442
5443 default:
5444 gcc_unreachable ();
5445 }
5446 return code;
5447 }
5448
5449 /* Return a comparison rtx of mode CMP_MODE for COND. Use UNSIGNEDP to
5450 select signed or unsigned operators. OPNO holds the index of the
5451 first comparison operand for insn ICODE. Do not generate the
5452 compare instruction itself. */
5453
5454 static rtx
vector_compare_rtx(machine_mode cmp_mode,enum tree_code tcode,tree t_op0,tree t_op1,bool unsignedp,enum insn_code icode,unsigned int opno)5455 vector_compare_rtx (machine_mode cmp_mode, enum tree_code tcode,
5456 tree t_op0, tree t_op1, bool unsignedp,
5457 enum insn_code icode, unsigned int opno)
5458 {
5459 class expand_operand ops[2];
5460 rtx rtx_op0, rtx_op1;
5461 machine_mode m0, m1;
5462 enum rtx_code rcode = get_rtx_code (tcode, unsignedp);
5463
5464 gcc_assert (TREE_CODE_CLASS (tcode) == tcc_comparison);
5465
5466 /* Expand operands. For vector types with scalar modes, e.g. where int64x1_t
5467 has mode DImode, this can produce a constant RTX of mode VOIDmode; in such
5468 cases, use the original mode. */
5469 rtx_op0 = expand_expr (t_op0, NULL_RTX, TYPE_MODE (TREE_TYPE (t_op0)),
5470 EXPAND_STACK_PARM);
5471 m0 = GET_MODE (rtx_op0);
5472 if (m0 == VOIDmode)
5473 m0 = TYPE_MODE (TREE_TYPE (t_op0));
5474
5475 rtx_op1 = expand_expr (t_op1, NULL_RTX, TYPE_MODE (TREE_TYPE (t_op1)),
5476 EXPAND_STACK_PARM);
5477 m1 = GET_MODE (rtx_op1);
5478 if (m1 == VOIDmode)
5479 m1 = TYPE_MODE (TREE_TYPE (t_op1));
5480
5481 create_input_operand (&ops[0], rtx_op0, m0);
5482 create_input_operand (&ops[1], rtx_op1, m1);
5483 if (!maybe_legitimize_operands (icode, opno, 2, ops))
5484 gcc_unreachable ();
5485 return gen_rtx_fmt_ee (rcode, cmp_mode, ops[0].value, ops[1].value);
5486 }
5487
5488 /* Check if vec_perm mask SEL is a constant equivalent to a shift of
5489 the first vec_perm operand, assuming the second operand (for left shift
5490 first operand) is a constant vector of zeros. Return the shift distance
5491 in bits if so, or NULL_RTX if the vec_perm is not a shift. MODE is the
5492 mode of the value being shifted. SHIFT_OPTAB is vec_shr_optab for right
5493 shift or vec_shl_optab for left shift. */
5494 static rtx
shift_amt_for_vec_perm_mask(machine_mode mode,const vec_perm_indices & sel,optab shift_optab)5495 shift_amt_for_vec_perm_mask (machine_mode mode, const vec_perm_indices &sel,
5496 optab shift_optab)
5497 {
5498 unsigned int bitsize = GET_MODE_UNIT_BITSIZE (mode);
5499 poly_int64 first = sel[0];
5500 if (maybe_ge (sel[0], GET_MODE_NUNITS (mode)))
5501 return NULL_RTX;
5502
5503 if (shift_optab == vec_shl_optab)
5504 {
5505 unsigned int nelt;
5506 if (!GET_MODE_NUNITS (mode).is_constant (&nelt))
5507 return NULL_RTX;
5508 unsigned firstidx = 0;
5509 for (unsigned int i = 0; i < nelt; i++)
5510 {
5511 if (known_eq (sel[i], nelt))
5512 {
5513 if (i == 0 || firstidx)
5514 return NULL_RTX;
5515 firstidx = i;
5516 }
5517 else if (firstidx
5518 ? maybe_ne (sel[i], nelt + i - firstidx)
5519 : maybe_ge (sel[i], nelt))
5520 return NULL_RTX;
5521 }
5522
5523 if (firstidx == 0)
5524 return NULL_RTX;
5525 first = firstidx;
5526 }
5527 else if (!sel.series_p (0, 1, first, 1))
5528 {
5529 unsigned int nelt;
5530 if (!GET_MODE_NUNITS (mode).is_constant (&nelt))
5531 return NULL_RTX;
5532 for (unsigned int i = 1; i < nelt; i++)
5533 {
5534 poly_int64 expected = i + first;
5535 /* Indices into the second vector are all equivalent. */
5536 if (maybe_lt (sel[i], nelt)
5537 ? maybe_ne (sel[i], expected)
5538 : maybe_lt (expected, nelt))
5539 return NULL_RTX;
5540 }
5541 }
5542
5543 return gen_int_shift_amount (mode, first * bitsize);
5544 }
5545
5546 /* A subroutine of expand_vec_perm_var for expanding one vec_perm insn. */
5547
5548 static rtx
expand_vec_perm_1(enum insn_code icode,rtx target,rtx v0,rtx v1,rtx sel)5549 expand_vec_perm_1 (enum insn_code icode, rtx target,
5550 rtx v0, rtx v1, rtx sel)
5551 {
5552 machine_mode tmode = GET_MODE (target);
5553 machine_mode smode = GET_MODE (sel);
5554 class expand_operand ops[4];
5555
5556 gcc_assert (GET_MODE_CLASS (smode) == MODE_VECTOR_INT
5557 || related_int_vector_mode (tmode).require () == smode);
5558 create_output_operand (&ops[0], target, tmode);
5559 create_input_operand (&ops[3], sel, smode);
5560
5561 /* Make an effort to preserve v0 == v1. The target expander is able to
5562 rely on this to determine if we're permuting a single input operand. */
5563 if (rtx_equal_p (v0, v1))
5564 {
5565 if (!insn_operand_matches (icode, 1, v0))
5566 v0 = force_reg (tmode, v0);
5567 gcc_checking_assert (insn_operand_matches (icode, 1, v0));
5568 gcc_checking_assert (insn_operand_matches (icode, 2, v0));
5569
5570 create_fixed_operand (&ops[1], v0);
5571 create_fixed_operand (&ops[2], v0);
5572 }
5573 else
5574 {
5575 create_input_operand (&ops[1], v0, tmode);
5576 create_input_operand (&ops[2], v1, tmode);
5577 }
5578
5579 if (maybe_expand_insn (icode, 4, ops))
5580 return ops[0].value;
5581 return NULL_RTX;
5582 }
5583
5584 /* Implement a permutation of vectors v0 and v1 using the permutation
5585 vector in SEL and return the result. Use TARGET to hold the result
5586 if nonnull and convenient.
5587
5588 MODE is the mode of the vectors being permuted (V0 and V1). SEL_MODE
5589 is the TYPE_MODE associated with SEL, or BLKmode if SEL isn't known
5590 to have a particular mode. */
5591
5592 rtx
expand_vec_perm_const(machine_mode mode,rtx v0,rtx v1,const vec_perm_builder & sel,machine_mode sel_mode,rtx target)5593 expand_vec_perm_const (machine_mode mode, rtx v0, rtx v1,
5594 const vec_perm_builder &sel, machine_mode sel_mode,
5595 rtx target)
5596 {
5597 if (!target || !register_operand (target, mode))
5598 target = gen_reg_rtx (mode);
5599
5600 /* Set QIMODE to a different vector mode with byte elements.
5601 If no such mode, or if MODE already has byte elements, use VOIDmode. */
5602 machine_mode qimode;
5603 if (!qimode_for_vec_perm (mode).exists (&qimode))
5604 qimode = VOIDmode;
5605
5606 rtx_insn *last = get_last_insn ();
5607
5608 bool single_arg_p = rtx_equal_p (v0, v1);
5609 /* Always specify two input vectors here and leave the target to handle
5610 cases in which the inputs are equal. Not all backends can cope with
5611 the single-input representation when testing for a double-input
5612 target instruction. */
5613 vec_perm_indices indices (sel, 2, GET_MODE_NUNITS (mode));
5614
5615 /* See if this can be handled with a vec_shr or vec_shl. We only do this
5616 if the second (for vec_shr) or first (for vec_shl) vector is all
5617 zeroes. */
5618 insn_code shift_code = CODE_FOR_nothing;
5619 insn_code shift_code_qi = CODE_FOR_nothing;
5620 optab shift_optab = unknown_optab;
5621 rtx v2 = v0;
5622 if (v1 == CONST0_RTX (GET_MODE (v1)))
5623 shift_optab = vec_shr_optab;
5624 else if (v0 == CONST0_RTX (GET_MODE (v0)))
5625 {
5626 shift_optab = vec_shl_optab;
5627 v2 = v1;
5628 }
5629 if (shift_optab != unknown_optab)
5630 {
5631 shift_code = optab_handler (shift_optab, mode);
5632 shift_code_qi = ((qimode != VOIDmode && qimode != mode)
5633 ? optab_handler (shift_optab, qimode)
5634 : CODE_FOR_nothing);
5635 }
5636 if (shift_code != CODE_FOR_nothing || shift_code_qi != CODE_FOR_nothing)
5637 {
5638 rtx shift_amt = shift_amt_for_vec_perm_mask (mode, indices, shift_optab);
5639 if (shift_amt)
5640 {
5641 class expand_operand ops[3];
5642 if (shift_amt == const0_rtx)
5643 return v2;
5644 if (shift_code != CODE_FOR_nothing)
5645 {
5646 create_output_operand (&ops[0], target, mode);
5647 create_input_operand (&ops[1], v2, mode);
5648 create_convert_operand_from_type (&ops[2], shift_amt, sizetype);
5649 if (maybe_expand_insn (shift_code, 3, ops))
5650 return ops[0].value;
5651 }
5652 if (shift_code_qi != CODE_FOR_nothing)
5653 {
5654 rtx tmp = gen_reg_rtx (qimode);
5655 create_output_operand (&ops[0], tmp, qimode);
5656 create_input_operand (&ops[1], gen_lowpart (qimode, v2), qimode);
5657 create_convert_operand_from_type (&ops[2], shift_amt, sizetype);
5658 if (maybe_expand_insn (shift_code_qi, 3, ops))
5659 return gen_lowpart (mode, ops[0].value);
5660 }
5661 }
5662 }
5663
5664 if (targetm.vectorize.vec_perm_const != NULL)
5665 {
5666 v0 = force_reg (mode, v0);
5667 if (single_arg_p)
5668 v1 = v0;
5669 else
5670 v1 = force_reg (mode, v1);
5671
5672 if (targetm.vectorize.vec_perm_const (mode, target, v0, v1, indices))
5673 return target;
5674 }
5675
5676 /* Fall back to a constant byte-based permutation. */
5677 vec_perm_indices qimode_indices;
5678 rtx target_qi = NULL_RTX, v0_qi = NULL_RTX, v1_qi = NULL_RTX;
5679 if (qimode != VOIDmode)
5680 {
5681 qimode_indices.new_expanded_vector (indices, GET_MODE_UNIT_SIZE (mode));
5682 target_qi = gen_reg_rtx (qimode);
5683 v0_qi = gen_lowpart (qimode, v0);
5684 v1_qi = gen_lowpart (qimode, v1);
5685 if (targetm.vectorize.vec_perm_const != NULL
5686 && targetm.vectorize.vec_perm_const (qimode, target_qi, v0_qi,
5687 v1_qi, qimode_indices))
5688 return gen_lowpart (mode, target_qi);
5689 }
5690
5691 /* Otherwise expand as a fully variable permuation. */
5692
5693 /* The optabs are only defined for selectors with the same width
5694 as the values being permuted. */
5695 machine_mode required_sel_mode;
5696 if (!related_int_vector_mode (mode).exists (&required_sel_mode))
5697 {
5698 delete_insns_since (last);
5699 return NULL_RTX;
5700 }
5701
5702 /* We know that it is semantically valid to treat SEL as having SEL_MODE.
5703 If that isn't the mode we want then we need to prove that using
5704 REQUIRED_SEL_MODE is OK. */
5705 if (sel_mode != required_sel_mode)
5706 {
5707 if (!selector_fits_mode_p (required_sel_mode, indices))
5708 {
5709 delete_insns_since (last);
5710 return NULL_RTX;
5711 }
5712 sel_mode = required_sel_mode;
5713 }
5714
5715 insn_code icode = direct_optab_handler (vec_perm_optab, mode);
5716 if (icode != CODE_FOR_nothing)
5717 {
5718 rtx sel_rtx = vec_perm_indices_to_rtx (sel_mode, indices);
5719 rtx tmp = expand_vec_perm_1 (icode, target, v0, v1, sel_rtx);
5720 if (tmp)
5721 return tmp;
5722 }
5723
5724 if (qimode != VOIDmode
5725 && selector_fits_mode_p (qimode, qimode_indices))
5726 {
5727 icode = direct_optab_handler (vec_perm_optab, qimode);
5728 if (icode != CODE_FOR_nothing)
5729 {
5730 rtx sel_qi = vec_perm_indices_to_rtx (qimode, qimode_indices);
5731 rtx tmp = expand_vec_perm_1 (icode, target_qi, v0_qi, v1_qi, sel_qi);
5732 if (tmp)
5733 return gen_lowpart (mode, tmp);
5734 }
5735 }
5736
5737 delete_insns_since (last);
5738 return NULL_RTX;
5739 }
5740
5741 /* Implement a permutation of vectors v0 and v1 using the permutation
5742 vector in SEL and return the result. Use TARGET to hold the result
5743 if nonnull and convenient.
5744
5745 MODE is the mode of the vectors being permuted (V0 and V1).
5746 SEL must have the integer equivalent of MODE and is known to be
5747 unsuitable for permutes with a constant permutation vector. */
5748
5749 rtx
expand_vec_perm_var(machine_mode mode,rtx v0,rtx v1,rtx sel,rtx target)5750 expand_vec_perm_var (machine_mode mode, rtx v0, rtx v1, rtx sel, rtx target)
5751 {
5752 enum insn_code icode;
5753 unsigned int i, u;
5754 rtx tmp, sel_qi;
5755
5756 u = GET_MODE_UNIT_SIZE (mode);
5757
5758 if (!target || GET_MODE (target) != mode)
5759 target = gen_reg_rtx (mode);
5760
5761 icode = direct_optab_handler (vec_perm_optab, mode);
5762 if (icode != CODE_FOR_nothing)
5763 {
5764 tmp = expand_vec_perm_1 (icode, target, v0, v1, sel);
5765 if (tmp)
5766 return tmp;
5767 }
5768
5769 /* As a special case to aid several targets, lower the element-based
5770 permutation to a byte-based permutation and try again. */
5771 machine_mode qimode;
5772 if (!qimode_for_vec_perm (mode).exists (&qimode)
5773 || maybe_gt (GET_MODE_NUNITS (qimode), GET_MODE_MASK (QImode) + 1))
5774 return NULL_RTX;
5775 icode = direct_optab_handler (vec_perm_optab, qimode);
5776 if (icode == CODE_FOR_nothing)
5777 return NULL_RTX;
5778
5779 /* Multiply each element by its byte size. */
5780 machine_mode selmode = GET_MODE (sel);
5781 if (u == 2)
5782 sel = expand_simple_binop (selmode, PLUS, sel, sel,
5783 NULL, 0, OPTAB_DIRECT);
5784 else
5785 sel = expand_simple_binop (selmode, ASHIFT, sel,
5786 gen_int_shift_amount (selmode, exact_log2 (u)),
5787 NULL, 0, OPTAB_DIRECT);
5788 gcc_assert (sel != NULL);
5789
5790 /* Broadcast the low byte each element into each of its bytes.
5791 The encoding has U interleaved stepped patterns, one for each
5792 byte of an element. */
5793 vec_perm_builder const_sel (GET_MODE_SIZE (mode), u, 3);
5794 unsigned int low_byte_in_u = BYTES_BIG_ENDIAN ? u - 1 : 0;
5795 for (i = 0; i < 3; ++i)
5796 for (unsigned int j = 0; j < u; ++j)
5797 const_sel.quick_push (i * u + low_byte_in_u);
5798 sel = gen_lowpart (qimode, sel);
5799 sel = expand_vec_perm_const (qimode, sel, sel, const_sel, qimode, NULL);
5800 gcc_assert (sel != NULL);
5801
5802 /* Add the byte offset to each byte element. */
5803 /* Note that the definition of the indicies here is memory ordering,
5804 so there should be no difference between big and little endian. */
5805 rtx_vector_builder byte_indices (qimode, u, 1);
5806 for (i = 0; i < u; ++i)
5807 byte_indices.quick_push (GEN_INT (i));
5808 tmp = byte_indices.build ();
5809 sel_qi = expand_simple_binop (qimode, PLUS, sel, tmp,
5810 sel, 0, OPTAB_DIRECT);
5811 gcc_assert (sel_qi != NULL);
5812
5813 tmp = mode != qimode ? gen_reg_rtx (qimode) : target;
5814 tmp = expand_vec_perm_1 (icode, tmp, gen_lowpart (qimode, v0),
5815 gen_lowpart (qimode, v1), sel_qi);
5816 if (tmp)
5817 tmp = gen_lowpart (mode, tmp);
5818 return tmp;
5819 }
5820
5821 /* Generate insns for a VEC_COND_EXPR with mask, given its TYPE and its
5822 three operands. */
5823
5824 rtx
expand_vec_cond_mask_expr(tree vec_cond_type,tree op0,tree op1,tree op2,rtx target)5825 expand_vec_cond_mask_expr (tree vec_cond_type, tree op0, tree op1, tree op2,
5826 rtx target)
5827 {
5828 class expand_operand ops[4];
5829 machine_mode mode = TYPE_MODE (vec_cond_type);
5830 machine_mode mask_mode = TYPE_MODE (TREE_TYPE (op0));
5831 enum insn_code icode = get_vcond_mask_icode (mode, mask_mode);
5832 rtx mask, rtx_op1, rtx_op2;
5833
5834 if (icode == CODE_FOR_nothing)
5835 return 0;
5836
5837 mask = expand_normal (op0);
5838 rtx_op1 = expand_normal (op1);
5839 rtx_op2 = expand_normal (op2);
5840
5841 mask = force_reg (mask_mode, mask);
5842 rtx_op1 = force_reg (GET_MODE (rtx_op1), rtx_op1);
5843
5844 create_output_operand (&ops[0], target, mode);
5845 create_input_operand (&ops[1], rtx_op1, mode);
5846 create_input_operand (&ops[2], rtx_op2, mode);
5847 create_input_operand (&ops[3], mask, mask_mode);
5848 expand_insn (icode, 4, ops);
5849
5850 return ops[0].value;
5851 }
5852
5853 /* Generate insns for a VEC_COND_EXPR, given its TYPE and its
5854 three operands. */
5855
5856 rtx
expand_vec_cond_expr(tree vec_cond_type,tree op0,tree op1,tree op2,rtx target)5857 expand_vec_cond_expr (tree vec_cond_type, tree op0, tree op1, tree op2,
5858 rtx target)
5859 {
5860 class expand_operand ops[6];
5861 enum insn_code icode;
5862 rtx comparison, rtx_op1, rtx_op2;
5863 machine_mode mode = TYPE_MODE (vec_cond_type);
5864 machine_mode cmp_op_mode;
5865 bool unsignedp;
5866 tree op0a, op0b;
5867 enum tree_code tcode;
5868
5869 if (COMPARISON_CLASS_P (op0))
5870 {
5871 op0a = TREE_OPERAND (op0, 0);
5872 op0b = TREE_OPERAND (op0, 1);
5873 tcode = TREE_CODE (op0);
5874 }
5875 else
5876 {
5877 gcc_assert (VECTOR_BOOLEAN_TYPE_P (TREE_TYPE (op0)));
5878 if (get_vcond_mask_icode (mode, TYPE_MODE (TREE_TYPE (op0)))
5879 != CODE_FOR_nothing)
5880 return expand_vec_cond_mask_expr (vec_cond_type, op0, op1,
5881 op2, target);
5882 /* Fake op0 < 0. */
5883 else
5884 {
5885 gcc_assert (GET_MODE_CLASS (TYPE_MODE (TREE_TYPE (op0)))
5886 == MODE_VECTOR_INT);
5887 op0a = op0;
5888 op0b = build_zero_cst (TREE_TYPE (op0));
5889 tcode = LT_EXPR;
5890 }
5891 }
5892 cmp_op_mode = TYPE_MODE (TREE_TYPE (op0a));
5893 unsignedp = TYPE_UNSIGNED (TREE_TYPE (op0a));
5894
5895
5896 gcc_assert (known_eq (GET_MODE_SIZE (mode), GET_MODE_SIZE (cmp_op_mode))
5897 && known_eq (GET_MODE_NUNITS (mode),
5898 GET_MODE_NUNITS (cmp_op_mode)));
5899
5900 icode = get_vcond_icode (mode, cmp_op_mode, unsignedp);
5901 if (icode == CODE_FOR_nothing)
5902 {
5903 if (tcode == LT_EXPR
5904 && op0a == op0
5905 && TREE_CODE (op0) == VECTOR_CST)
5906 {
5907 /* A VEC_COND_EXPR condition could be folded from EQ_EXPR/NE_EXPR
5908 into a constant when only get_vcond_eq_icode is supported.
5909 Verify < 0 and != 0 behave the same and change it to NE_EXPR. */
5910 unsigned HOST_WIDE_INT nelts;
5911 if (!VECTOR_CST_NELTS (op0).is_constant (&nelts))
5912 {
5913 if (VECTOR_CST_STEPPED_P (op0))
5914 return 0;
5915 nelts = vector_cst_encoded_nelts (op0);
5916 }
5917 for (unsigned int i = 0; i < nelts; ++i)
5918 if (tree_int_cst_sgn (vector_cst_elt (op0, i)) == 1)
5919 return 0;
5920 tcode = NE_EXPR;
5921 }
5922 if (tcode == EQ_EXPR || tcode == NE_EXPR)
5923 icode = get_vcond_eq_icode (mode, cmp_op_mode);
5924 if (icode == CODE_FOR_nothing)
5925 return 0;
5926 }
5927
5928 comparison = vector_compare_rtx (VOIDmode, tcode, op0a, op0b, unsignedp,
5929 icode, 4);
5930 rtx_op1 = expand_normal (op1);
5931 rtx_op2 = expand_normal (op2);
5932
5933 create_output_operand (&ops[0], target, mode);
5934 create_input_operand (&ops[1], rtx_op1, mode);
5935 create_input_operand (&ops[2], rtx_op2, mode);
5936 create_fixed_operand (&ops[3], comparison);
5937 create_fixed_operand (&ops[4], XEXP (comparison, 0));
5938 create_fixed_operand (&ops[5], XEXP (comparison, 1));
5939 expand_insn (icode, 6, ops);
5940 return ops[0].value;
5941 }
5942
5943 /* Generate VEC_SERIES_EXPR <OP0, OP1>, returning a value of mode VMODE.
5944 Use TARGET for the result if nonnull and convenient. */
5945
5946 rtx
expand_vec_series_expr(machine_mode vmode,rtx op0,rtx op1,rtx target)5947 expand_vec_series_expr (machine_mode vmode, rtx op0, rtx op1, rtx target)
5948 {
5949 class expand_operand ops[3];
5950 enum insn_code icode;
5951 machine_mode emode = GET_MODE_INNER (vmode);
5952
5953 icode = direct_optab_handler (vec_series_optab, vmode);
5954 gcc_assert (icode != CODE_FOR_nothing);
5955
5956 create_output_operand (&ops[0], target, vmode);
5957 create_input_operand (&ops[1], op0, emode);
5958 create_input_operand (&ops[2], op1, emode);
5959
5960 expand_insn (icode, 3, ops);
5961 return ops[0].value;
5962 }
5963
5964 /* Generate insns for a vector comparison into a mask. */
5965
5966 rtx
expand_vec_cmp_expr(tree type,tree exp,rtx target)5967 expand_vec_cmp_expr (tree type, tree exp, rtx target)
5968 {
5969 class expand_operand ops[4];
5970 enum insn_code icode;
5971 rtx comparison;
5972 machine_mode mask_mode = TYPE_MODE (type);
5973 machine_mode vmode;
5974 bool unsignedp;
5975 tree op0a, op0b;
5976 enum tree_code tcode;
5977
5978 op0a = TREE_OPERAND (exp, 0);
5979 op0b = TREE_OPERAND (exp, 1);
5980 tcode = TREE_CODE (exp);
5981
5982 unsignedp = TYPE_UNSIGNED (TREE_TYPE (op0a));
5983 vmode = TYPE_MODE (TREE_TYPE (op0a));
5984
5985 icode = get_vec_cmp_icode (vmode, mask_mode, unsignedp);
5986 if (icode == CODE_FOR_nothing)
5987 {
5988 if (tcode == EQ_EXPR || tcode == NE_EXPR)
5989 icode = get_vec_cmp_eq_icode (vmode, mask_mode);
5990 if (icode == CODE_FOR_nothing)
5991 return 0;
5992 }
5993
5994 comparison = vector_compare_rtx (mask_mode, tcode, op0a, op0b,
5995 unsignedp, icode, 2);
5996 create_output_operand (&ops[0], target, mask_mode);
5997 create_fixed_operand (&ops[1], comparison);
5998 create_fixed_operand (&ops[2], XEXP (comparison, 0));
5999 create_fixed_operand (&ops[3], XEXP (comparison, 1));
6000 expand_insn (icode, 4, ops);
6001 return ops[0].value;
6002 }
6003
6004 /* Expand a highpart multiply. */
6005
6006 rtx
expand_mult_highpart(machine_mode mode,rtx op0,rtx op1,rtx target,bool uns_p)6007 expand_mult_highpart (machine_mode mode, rtx op0, rtx op1,
6008 rtx target, bool uns_p)
6009 {
6010 class expand_operand eops[3];
6011 enum insn_code icode;
6012 int method, i;
6013 machine_mode wmode;
6014 rtx m1, m2;
6015 optab tab1, tab2;
6016
6017 method = can_mult_highpart_p (mode, uns_p);
6018 switch (method)
6019 {
6020 case 0:
6021 return NULL_RTX;
6022 case 1:
6023 tab1 = uns_p ? umul_highpart_optab : smul_highpart_optab;
6024 return expand_binop (mode, tab1, op0, op1, target, uns_p,
6025 OPTAB_LIB_WIDEN);
6026 case 2:
6027 tab1 = uns_p ? vec_widen_umult_even_optab : vec_widen_smult_even_optab;
6028 tab2 = uns_p ? vec_widen_umult_odd_optab : vec_widen_smult_odd_optab;
6029 break;
6030 case 3:
6031 tab1 = uns_p ? vec_widen_umult_lo_optab : vec_widen_smult_lo_optab;
6032 tab2 = uns_p ? vec_widen_umult_hi_optab : vec_widen_smult_hi_optab;
6033 if (BYTES_BIG_ENDIAN)
6034 std::swap (tab1, tab2);
6035 break;
6036 default:
6037 gcc_unreachable ();
6038 }
6039
6040 icode = optab_handler (tab1, mode);
6041 wmode = insn_data[icode].operand[0].mode;
6042 gcc_checking_assert (known_eq (2 * GET_MODE_NUNITS (wmode),
6043 GET_MODE_NUNITS (mode)));
6044 gcc_checking_assert (known_eq (GET_MODE_SIZE (wmode), GET_MODE_SIZE (mode)));
6045
6046 create_output_operand (&eops[0], gen_reg_rtx (wmode), wmode);
6047 create_input_operand (&eops[1], op0, mode);
6048 create_input_operand (&eops[2], op1, mode);
6049 expand_insn (icode, 3, eops);
6050 m1 = gen_lowpart (mode, eops[0].value);
6051
6052 create_output_operand (&eops[0], gen_reg_rtx (wmode), wmode);
6053 create_input_operand (&eops[1], op0, mode);
6054 create_input_operand (&eops[2], op1, mode);
6055 expand_insn (optab_handler (tab2, mode), 3, eops);
6056 m2 = gen_lowpart (mode, eops[0].value);
6057
6058 vec_perm_builder sel;
6059 if (method == 2)
6060 {
6061 /* The encoding has 2 interleaved stepped patterns. */
6062 sel.new_vector (GET_MODE_NUNITS (mode), 2, 3);
6063 for (i = 0; i < 6; ++i)
6064 sel.quick_push (!BYTES_BIG_ENDIAN + (i & ~1)
6065 + ((i & 1) ? GET_MODE_NUNITS (mode) : 0));
6066 }
6067 else
6068 {
6069 /* The encoding has a single interleaved stepped pattern. */
6070 sel.new_vector (GET_MODE_NUNITS (mode), 1, 3);
6071 for (i = 0; i < 3; ++i)
6072 sel.quick_push (2 * i + (BYTES_BIG_ENDIAN ? 0 : 1));
6073 }
6074
6075 return expand_vec_perm_const (mode, m1, m2, sel, BLKmode, target);
6076 }
6077
6078 /* Helper function to find the MODE_CC set in a sync_compare_and_swap
6079 pattern. */
6080
6081 static void
find_cc_set(rtx x,const_rtx pat,void * data)6082 find_cc_set (rtx x, const_rtx pat, void *data)
6083 {
6084 if (REG_P (x) && GET_MODE_CLASS (GET_MODE (x)) == MODE_CC
6085 && GET_CODE (pat) == SET)
6086 {
6087 rtx *p_cc_reg = (rtx *) data;
6088 gcc_assert (!*p_cc_reg);
6089 *p_cc_reg = x;
6090 }
6091 }
6092
6093 /* This is a helper function for the other atomic operations. This function
6094 emits a loop that contains SEQ that iterates until a compare-and-swap
6095 operation at the end succeeds. MEM is the memory to be modified. SEQ is
6096 a set of instructions that takes a value from OLD_REG as an input and
6097 produces a value in NEW_REG as an output. Before SEQ, OLD_REG will be
6098 set to the current contents of MEM. After SEQ, a compare-and-swap will
6099 attempt to update MEM with NEW_REG. The function returns true when the
6100 loop was generated successfully. */
6101
6102 static bool
expand_compare_and_swap_loop(rtx mem,rtx old_reg,rtx new_reg,rtx seq)6103 expand_compare_and_swap_loop (rtx mem, rtx old_reg, rtx new_reg, rtx seq)
6104 {
6105 machine_mode mode = GET_MODE (mem);
6106 rtx_code_label *label;
6107 rtx cmp_reg, success, oldval;
6108
6109 /* The loop we want to generate looks like
6110
6111 cmp_reg = mem;
6112 label:
6113 old_reg = cmp_reg;
6114 seq;
6115 (success, cmp_reg) = compare-and-swap(mem, old_reg, new_reg)
6116 if (success)
6117 goto label;
6118
6119 Note that we only do the plain load from memory once. Subsequent
6120 iterations use the value loaded by the compare-and-swap pattern. */
6121
6122 label = gen_label_rtx ();
6123 cmp_reg = gen_reg_rtx (mode);
6124
6125 emit_move_insn (cmp_reg, mem);
6126 emit_label (label);
6127 emit_move_insn (old_reg, cmp_reg);
6128 if (seq)
6129 emit_insn (seq);
6130
6131 success = NULL_RTX;
6132 oldval = cmp_reg;
6133 if (!expand_atomic_compare_and_swap (&success, &oldval, mem, old_reg,
6134 new_reg, false, MEMMODEL_SYNC_SEQ_CST,
6135 MEMMODEL_RELAXED))
6136 return false;
6137
6138 if (oldval != cmp_reg)
6139 emit_move_insn (cmp_reg, oldval);
6140
6141 /* Mark this jump predicted not taken. */
6142 emit_cmp_and_jump_insns (success, const0_rtx, EQ, const0_rtx,
6143 GET_MODE (success), 1, label,
6144 profile_probability::guessed_never ());
6145 return true;
6146 }
6147
6148
6149 /* This function tries to emit an atomic_exchange intruction. VAL is written
6150 to *MEM using memory model MODEL. The previous contents of *MEM are returned,
6151 using TARGET if possible. */
6152
6153 static rtx
maybe_emit_atomic_exchange(rtx target,rtx mem,rtx val,enum memmodel model)6154 maybe_emit_atomic_exchange (rtx target, rtx mem, rtx val, enum memmodel model)
6155 {
6156 machine_mode mode = GET_MODE (mem);
6157 enum insn_code icode;
6158
6159 /* If the target supports the exchange directly, great. */
6160 icode = direct_optab_handler (atomic_exchange_optab, mode);
6161 if (icode != CODE_FOR_nothing)
6162 {
6163 class expand_operand ops[4];
6164
6165 create_output_operand (&ops[0], target, mode);
6166 create_fixed_operand (&ops[1], mem);
6167 create_input_operand (&ops[2], val, mode);
6168 create_integer_operand (&ops[3], model);
6169 if (maybe_expand_insn (icode, 4, ops))
6170 return ops[0].value;
6171 }
6172
6173 return NULL_RTX;
6174 }
6175
6176 /* This function tries to implement an atomic exchange operation using
6177 __sync_lock_test_and_set. VAL is written to *MEM using memory model MODEL.
6178 The previous contents of *MEM are returned, using TARGET if possible.
6179 Since this instructionn is an acquire barrier only, stronger memory
6180 models may require additional barriers to be emitted. */
6181
6182 static rtx
maybe_emit_sync_lock_test_and_set(rtx target,rtx mem,rtx val,enum memmodel model)6183 maybe_emit_sync_lock_test_and_set (rtx target, rtx mem, rtx val,
6184 enum memmodel model)
6185 {
6186 machine_mode mode = GET_MODE (mem);
6187 enum insn_code icode;
6188 rtx_insn *last_insn = get_last_insn ();
6189
6190 icode = optab_handler (sync_lock_test_and_set_optab, mode);
6191
6192 /* Legacy sync_lock_test_and_set is an acquire barrier. If the pattern
6193 exists, and the memory model is stronger than acquire, add a release
6194 barrier before the instruction. */
6195
6196 if (is_mm_seq_cst (model) || is_mm_release (model) || is_mm_acq_rel (model))
6197 expand_mem_thread_fence (model);
6198
6199 if (icode != CODE_FOR_nothing)
6200 {
6201 class expand_operand ops[3];
6202 create_output_operand (&ops[0], target, mode);
6203 create_fixed_operand (&ops[1], mem);
6204 create_input_operand (&ops[2], val, mode);
6205 if (maybe_expand_insn (icode, 3, ops))
6206 return ops[0].value;
6207 }
6208
6209 /* If an external test-and-set libcall is provided, use that instead of
6210 any external compare-and-swap that we might get from the compare-and-
6211 swap-loop expansion later. */
6212 if (!can_compare_and_swap_p (mode, false))
6213 {
6214 rtx libfunc = optab_libfunc (sync_lock_test_and_set_optab, mode);
6215 if (libfunc != NULL)
6216 {
6217 rtx addr;
6218
6219 addr = convert_memory_address (ptr_mode, XEXP (mem, 0));
6220 return emit_library_call_value (libfunc, NULL_RTX, LCT_NORMAL,
6221 mode, addr, ptr_mode,
6222 val, mode);
6223 }
6224 }
6225
6226 /* If the test_and_set can't be emitted, eliminate any barrier that might
6227 have been emitted. */
6228 delete_insns_since (last_insn);
6229 return NULL_RTX;
6230 }
6231
6232 /* This function tries to implement an atomic exchange operation using a
6233 compare_and_swap loop. VAL is written to *MEM. The previous contents of
6234 *MEM are returned, using TARGET if possible. No memory model is required
6235 since a compare_and_swap loop is seq-cst. */
6236
6237 static rtx
maybe_emit_compare_and_swap_exchange_loop(rtx target,rtx mem,rtx val)6238 maybe_emit_compare_and_swap_exchange_loop (rtx target, rtx mem, rtx val)
6239 {
6240 machine_mode mode = GET_MODE (mem);
6241
6242 if (can_compare_and_swap_p (mode, true))
6243 {
6244 if (!target || !register_operand (target, mode))
6245 target = gen_reg_rtx (mode);
6246 if (expand_compare_and_swap_loop (mem, target, val, NULL_RTX))
6247 return target;
6248 }
6249
6250 return NULL_RTX;
6251 }
6252
6253 /* This function tries to implement an atomic test-and-set operation
6254 using the atomic_test_and_set instruction pattern. A boolean value
6255 is returned from the operation, using TARGET if possible. */
6256
6257 static rtx
maybe_emit_atomic_test_and_set(rtx target,rtx mem,enum memmodel model)6258 maybe_emit_atomic_test_and_set (rtx target, rtx mem, enum memmodel model)
6259 {
6260 machine_mode pat_bool_mode;
6261 class expand_operand ops[3];
6262
6263 if (!targetm.have_atomic_test_and_set ())
6264 return NULL_RTX;
6265
6266 /* While we always get QImode from __atomic_test_and_set, we get
6267 other memory modes from __sync_lock_test_and_set. Note that we
6268 use no endian adjustment here. This matches the 4.6 behavior
6269 in the Sparc backend. */
6270 enum insn_code icode = targetm.code_for_atomic_test_and_set;
6271 gcc_checking_assert (insn_data[icode].operand[1].mode == QImode);
6272 if (GET_MODE (mem) != QImode)
6273 mem = adjust_address_nv (mem, QImode, 0);
6274
6275 pat_bool_mode = insn_data[icode].operand[0].mode;
6276 create_output_operand (&ops[0], target, pat_bool_mode);
6277 create_fixed_operand (&ops[1], mem);
6278 create_integer_operand (&ops[2], model);
6279
6280 if (maybe_expand_insn (icode, 3, ops))
6281 return ops[0].value;
6282 return NULL_RTX;
6283 }
6284
6285 /* This function expands the legacy _sync_lock test_and_set operation which is
6286 generally an atomic exchange. Some limited targets only allow the
6287 constant 1 to be stored. This is an ACQUIRE operation.
6288
6289 TARGET is an optional place to stick the return value.
6290 MEM is where VAL is stored. */
6291
6292 rtx
expand_sync_lock_test_and_set(rtx target,rtx mem,rtx val)6293 expand_sync_lock_test_and_set (rtx target, rtx mem, rtx val)
6294 {
6295 rtx ret;
6296
6297 /* Try an atomic_exchange first. */
6298 ret = maybe_emit_atomic_exchange (target, mem, val, MEMMODEL_SYNC_ACQUIRE);
6299 if (ret)
6300 return ret;
6301
6302 ret = maybe_emit_sync_lock_test_and_set (target, mem, val,
6303 MEMMODEL_SYNC_ACQUIRE);
6304 if (ret)
6305 return ret;
6306
6307 ret = maybe_emit_compare_and_swap_exchange_loop (target, mem, val);
6308 if (ret)
6309 return ret;
6310
6311 /* If there are no other options, try atomic_test_and_set if the value
6312 being stored is 1. */
6313 if (val == const1_rtx)
6314 ret = maybe_emit_atomic_test_and_set (target, mem, MEMMODEL_SYNC_ACQUIRE);
6315
6316 return ret;
6317 }
6318
6319 /* This function expands the atomic test_and_set operation:
6320 atomically store a boolean TRUE into MEM and return the previous value.
6321
6322 MEMMODEL is the memory model variant to use.
6323 TARGET is an optional place to stick the return value. */
6324
6325 rtx
expand_atomic_test_and_set(rtx target,rtx mem,enum memmodel model)6326 expand_atomic_test_and_set (rtx target, rtx mem, enum memmodel model)
6327 {
6328 machine_mode mode = GET_MODE (mem);
6329 rtx ret, trueval, subtarget;
6330
6331 ret = maybe_emit_atomic_test_and_set (target, mem, model);
6332 if (ret)
6333 return ret;
6334
6335 /* Be binary compatible with non-default settings of trueval, and different
6336 cpu revisions. E.g. one revision may have atomic-test-and-set, but
6337 another only has atomic-exchange. */
6338 if (targetm.atomic_test_and_set_trueval == 1)
6339 {
6340 trueval = const1_rtx;
6341 subtarget = target ? target : gen_reg_rtx (mode);
6342 }
6343 else
6344 {
6345 trueval = gen_int_mode (targetm.atomic_test_and_set_trueval, mode);
6346 subtarget = gen_reg_rtx (mode);
6347 }
6348
6349 /* Try the atomic-exchange optab... */
6350 ret = maybe_emit_atomic_exchange (subtarget, mem, trueval, model);
6351
6352 /* ... then an atomic-compare-and-swap loop ... */
6353 if (!ret)
6354 ret = maybe_emit_compare_and_swap_exchange_loop (subtarget, mem, trueval);
6355
6356 /* ... before trying the vaguely defined legacy lock_test_and_set. */
6357 if (!ret)
6358 ret = maybe_emit_sync_lock_test_and_set (subtarget, mem, trueval, model);
6359
6360 /* Recall that the legacy lock_test_and_set optab was allowed to do magic
6361 things with the value 1. Thus we try again without trueval. */
6362 if (!ret && targetm.atomic_test_and_set_trueval != 1)
6363 ret = maybe_emit_sync_lock_test_and_set (subtarget, mem, const1_rtx, model);
6364
6365 /* Failing all else, assume a single threaded environment and simply
6366 perform the operation. */
6367 if (!ret)
6368 {
6369 /* If the result is ignored skip the move to target. */
6370 if (subtarget != const0_rtx)
6371 emit_move_insn (subtarget, mem);
6372
6373 emit_move_insn (mem, trueval);
6374 ret = subtarget;
6375 }
6376
6377 /* Recall that have to return a boolean value; rectify if trueval
6378 is not exactly one. */
6379 if (targetm.atomic_test_and_set_trueval != 1)
6380 ret = emit_store_flag_force (target, NE, ret, const0_rtx, mode, 0, 1);
6381
6382 return ret;
6383 }
6384
6385 /* This function expands the atomic exchange operation:
6386 atomically store VAL in MEM and return the previous value in MEM.
6387
6388 MEMMODEL is the memory model variant to use.
6389 TARGET is an optional place to stick the return value. */
6390
6391 rtx
expand_atomic_exchange(rtx target,rtx mem,rtx val,enum memmodel model)6392 expand_atomic_exchange (rtx target, rtx mem, rtx val, enum memmodel model)
6393 {
6394 machine_mode mode = GET_MODE (mem);
6395 rtx ret;
6396
6397 /* If loads are not atomic for the required size and we are not called to
6398 provide a __sync builtin, do not do anything so that we stay consistent
6399 with atomic loads of the same size. */
6400 if (!can_atomic_load_p (mode) && !is_mm_sync (model))
6401 return NULL_RTX;
6402
6403 ret = maybe_emit_atomic_exchange (target, mem, val, model);
6404
6405 /* Next try a compare-and-swap loop for the exchange. */
6406 if (!ret)
6407 ret = maybe_emit_compare_and_swap_exchange_loop (target, mem, val);
6408
6409 return ret;
6410 }
6411
6412 /* This function expands the atomic compare exchange operation:
6413
6414 *PTARGET_BOOL is an optional place to store the boolean success/failure.
6415 *PTARGET_OVAL is an optional place to store the old value from memory.
6416 Both target parameters may be NULL or const0_rtx to indicate that we do
6417 not care about that return value. Both target parameters are updated on
6418 success to the actual location of the corresponding result.
6419
6420 MEMMODEL is the memory model variant to use.
6421
6422 The return value of the function is true for success. */
6423
6424 bool
expand_atomic_compare_and_swap(rtx * ptarget_bool,rtx * ptarget_oval,rtx mem,rtx expected,rtx desired,bool is_weak,enum memmodel succ_model,enum memmodel fail_model)6425 expand_atomic_compare_and_swap (rtx *ptarget_bool, rtx *ptarget_oval,
6426 rtx mem, rtx expected, rtx desired,
6427 bool is_weak, enum memmodel succ_model,
6428 enum memmodel fail_model)
6429 {
6430 machine_mode mode = GET_MODE (mem);
6431 class expand_operand ops[8];
6432 enum insn_code icode;
6433 rtx target_oval, target_bool = NULL_RTX;
6434 rtx libfunc;
6435
6436 /* If loads are not atomic for the required size and we are not called to
6437 provide a __sync builtin, do not do anything so that we stay consistent
6438 with atomic loads of the same size. */
6439 if (!can_atomic_load_p (mode) && !is_mm_sync (succ_model))
6440 return false;
6441
6442 /* Load expected into a register for the compare and swap. */
6443 if (MEM_P (expected))
6444 expected = copy_to_reg (expected);
6445
6446 /* Make sure we always have some place to put the return oldval.
6447 Further, make sure that place is distinct from the input expected,
6448 just in case we need that path down below. */
6449 if (ptarget_oval && *ptarget_oval == const0_rtx)
6450 ptarget_oval = NULL;
6451
6452 if (ptarget_oval == NULL
6453 || (target_oval = *ptarget_oval) == NULL
6454 || reg_overlap_mentioned_p (expected, target_oval))
6455 target_oval = gen_reg_rtx (mode);
6456
6457 icode = direct_optab_handler (atomic_compare_and_swap_optab, mode);
6458 if (icode != CODE_FOR_nothing)
6459 {
6460 machine_mode bool_mode = insn_data[icode].operand[0].mode;
6461
6462 if (ptarget_bool && *ptarget_bool == const0_rtx)
6463 ptarget_bool = NULL;
6464
6465 /* Make sure we always have a place for the bool operand. */
6466 if (ptarget_bool == NULL
6467 || (target_bool = *ptarget_bool) == NULL
6468 || GET_MODE (target_bool) != bool_mode)
6469 target_bool = gen_reg_rtx (bool_mode);
6470
6471 /* Emit the compare_and_swap. */
6472 create_output_operand (&ops[0], target_bool, bool_mode);
6473 create_output_operand (&ops[1], target_oval, mode);
6474 create_fixed_operand (&ops[2], mem);
6475 create_input_operand (&ops[3], expected, mode);
6476 create_input_operand (&ops[4], desired, mode);
6477 create_integer_operand (&ops[5], is_weak);
6478 create_integer_operand (&ops[6], succ_model);
6479 create_integer_operand (&ops[7], fail_model);
6480 if (maybe_expand_insn (icode, 8, ops))
6481 {
6482 /* Return success/failure. */
6483 target_bool = ops[0].value;
6484 target_oval = ops[1].value;
6485 goto success;
6486 }
6487 }
6488
6489 /* Otherwise fall back to the original __sync_val_compare_and_swap
6490 which is always seq-cst. */
6491 icode = optab_handler (sync_compare_and_swap_optab, mode);
6492 if (icode != CODE_FOR_nothing)
6493 {
6494 rtx cc_reg;
6495
6496 create_output_operand (&ops[0], target_oval, mode);
6497 create_fixed_operand (&ops[1], mem);
6498 create_input_operand (&ops[2], expected, mode);
6499 create_input_operand (&ops[3], desired, mode);
6500 if (!maybe_expand_insn (icode, 4, ops))
6501 return false;
6502
6503 target_oval = ops[0].value;
6504
6505 /* If the caller isn't interested in the boolean return value,
6506 skip the computation of it. */
6507 if (ptarget_bool == NULL)
6508 goto success;
6509
6510 /* Otherwise, work out if the compare-and-swap succeeded. */
6511 cc_reg = NULL_RTX;
6512 if (have_insn_for (COMPARE, CCmode))
6513 note_stores (get_last_insn (), find_cc_set, &cc_reg);
6514 if (cc_reg)
6515 {
6516 target_bool = emit_store_flag_force (target_bool, EQ, cc_reg,
6517 const0_rtx, VOIDmode, 0, 1);
6518 goto success;
6519 }
6520 goto success_bool_from_val;
6521 }
6522
6523 /* Also check for library support for __sync_val_compare_and_swap. */
6524 libfunc = optab_libfunc (sync_compare_and_swap_optab, mode);
6525 if (libfunc != NULL)
6526 {
6527 rtx addr = convert_memory_address (ptr_mode, XEXP (mem, 0));
6528 rtx target = emit_library_call_value (libfunc, NULL_RTX, LCT_NORMAL,
6529 mode, addr, ptr_mode,
6530 expected, mode, desired, mode);
6531 emit_move_insn (target_oval, target);
6532
6533 /* Compute the boolean return value only if requested. */
6534 if (ptarget_bool)
6535 goto success_bool_from_val;
6536 else
6537 goto success;
6538 }
6539
6540 /* Failure. */
6541 return false;
6542
6543 success_bool_from_val:
6544 target_bool = emit_store_flag_force (target_bool, EQ, target_oval,
6545 expected, VOIDmode, 1, 1);
6546 success:
6547 /* Make sure that the oval output winds up where the caller asked. */
6548 if (ptarget_oval)
6549 *ptarget_oval = target_oval;
6550 if (ptarget_bool)
6551 *ptarget_bool = target_bool;
6552 return true;
6553 }
6554
6555 /* Generate asm volatile("" : : : "memory") as the memory blockage. */
6556
6557 static void
expand_asm_memory_blockage(void)6558 expand_asm_memory_blockage (void)
6559 {
6560 rtx asm_op, clob;
6561
6562 asm_op = gen_rtx_ASM_OPERANDS (VOIDmode, "", "", 0,
6563 rtvec_alloc (0), rtvec_alloc (0),
6564 rtvec_alloc (0), UNKNOWN_LOCATION);
6565 MEM_VOLATILE_P (asm_op) = 1;
6566
6567 clob = gen_rtx_SCRATCH (VOIDmode);
6568 clob = gen_rtx_MEM (BLKmode, clob);
6569 clob = gen_rtx_CLOBBER (VOIDmode, clob);
6570
6571 emit_insn (gen_rtx_PARALLEL (VOIDmode, gen_rtvec (2, asm_op, clob)));
6572 }
6573
6574 /* Do not propagate memory accesses across this point. */
6575
6576 static void
expand_memory_blockage(void)6577 expand_memory_blockage (void)
6578 {
6579 if (targetm.have_memory_blockage ())
6580 emit_insn (targetm.gen_memory_blockage ());
6581 else
6582 expand_asm_memory_blockage ();
6583 }
6584
6585 /* This routine will either emit the mem_thread_fence pattern or issue a
6586 sync_synchronize to generate a fence for memory model MEMMODEL. */
6587
6588 void
expand_mem_thread_fence(enum memmodel model)6589 expand_mem_thread_fence (enum memmodel model)
6590 {
6591 if (is_mm_relaxed (model))
6592 return;
6593 if (targetm.have_mem_thread_fence ())
6594 {
6595 emit_insn (targetm.gen_mem_thread_fence (GEN_INT (model)));
6596 expand_memory_blockage ();
6597 }
6598 else if (targetm.have_memory_barrier ())
6599 emit_insn (targetm.gen_memory_barrier ());
6600 else if (synchronize_libfunc != NULL_RTX)
6601 emit_library_call (synchronize_libfunc, LCT_NORMAL, VOIDmode);
6602 else
6603 expand_memory_blockage ();
6604 }
6605
6606 /* Emit a signal fence with given memory model. */
6607
6608 void
expand_mem_signal_fence(enum memmodel model)6609 expand_mem_signal_fence (enum memmodel model)
6610 {
6611 /* No machine barrier is required to implement a signal fence, but
6612 a compiler memory barrier must be issued, except for relaxed MM. */
6613 if (!is_mm_relaxed (model))
6614 expand_memory_blockage ();
6615 }
6616
6617 /* This function expands the atomic load operation:
6618 return the atomically loaded value in MEM.
6619
6620 MEMMODEL is the memory model variant to use.
6621 TARGET is an option place to stick the return value. */
6622
6623 rtx
expand_atomic_load(rtx target,rtx mem,enum memmodel model)6624 expand_atomic_load (rtx target, rtx mem, enum memmodel model)
6625 {
6626 machine_mode mode = GET_MODE (mem);
6627 enum insn_code icode;
6628
6629 /* If the target supports the load directly, great. */
6630 icode = direct_optab_handler (atomic_load_optab, mode);
6631 if (icode != CODE_FOR_nothing)
6632 {
6633 class expand_operand ops[3];
6634 rtx_insn *last = get_last_insn ();
6635 if (is_mm_seq_cst (model))
6636 expand_memory_blockage ();
6637
6638 create_output_operand (&ops[0], target, mode);
6639 create_fixed_operand (&ops[1], mem);
6640 create_integer_operand (&ops[2], model);
6641 if (maybe_expand_insn (icode, 3, ops))
6642 {
6643 if (!is_mm_relaxed (model))
6644 expand_memory_blockage ();
6645 return ops[0].value;
6646 }
6647 delete_insns_since (last);
6648 }
6649
6650 /* If the size of the object is greater than word size on this target,
6651 then we assume that a load will not be atomic. We could try to
6652 emulate a load with a compare-and-swap operation, but the store that
6653 doing this could result in would be incorrect if this is a volatile
6654 atomic load or targetting read-only-mapped memory. */
6655 if (maybe_gt (GET_MODE_PRECISION (mode), BITS_PER_WORD))
6656 /* If there is no atomic load, leave the library call. */
6657 return NULL_RTX;
6658
6659 /* Otherwise assume loads are atomic, and emit the proper barriers. */
6660 if (!target || target == const0_rtx)
6661 target = gen_reg_rtx (mode);
6662
6663 /* For SEQ_CST, emit a barrier before the load. */
6664 if (is_mm_seq_cst (model))
6665 expand_mem_thread_fence (model);
6666
6667 emit_move_insn (target, mem);
6668
6669 /* Emit the appropriate barrier after the load. */
6670 expand_mem_thread_fence (model);
6671
6672 return target;
6673 }
6674
6675 /* This function expands the atomic store operation:
6676 Atomically store VAL in MEM.
6677 MEMMODEL is the memory model variant to use.
6678 USE_RELEASE is true if __sync_lock_release can be used as a fall back.
6679 function returns const0_rtx if a pattern was emitted. */
6680
6681 rtx
expand_atomic_store(rtx mem,rtx val,enum memmodel model,bool use_release)6682 expand_atomic_store (rtx mem, rtx val, enum memmodel model, bool use_release)
6683 {
6684 machine_mode mode = GET_MODE (mem);
6685 enum insn_code icode;
6686 class expand_operand ops[3];
6687
6688 /* If the target supports the store directly, great. */
6689 icode = direct_optab_handler (atomic_store_optab, mode);
6690 if (icode != CODE_FOR_nothing)
6691 {
6692 rtx_insn *last = get_last_insn ();
6693 if (!is_mm_relaxed (model))
6694 expand_memory_blockage ();
6695 create_fixed_operand (&ops[0], mem);
6696 create_input_operand (&ops[1], val, mode);
6697 create_integer_operand (&ops[2], model);
6698 if (maybe_expand_insn (icode, 3, ops))
6699 {
6700 if (is_mm_seq_cst (model))
6701 expand_memory_blockage ();
6702 return const0_rtx;
6703 }
6704 delete_insns_since (last);
6705 }
6706
6707 /* If using __sync_lock_release is a viable alternative, try it.
6708 Note that this will not be set to true if we are expanding a generic
6709 __atomic_store_n. */
6710 if (use_release)
6711 {
6712 icode = direct_optab_handler (sync_lock_release_optab, mode);
6713 if (icode != CODE_FOR_nothing)
6714 {
6715 create_fixed_operand (&ops[0], mem);
6716 create_input_operand (&ops[1], const0_rtx, mode);
6717 if (maybe_expand_insn (icode, 2, ops))
6718 {
6719 /* lock_release is only a release barrier. */
6720 if (is_mm_seq_cst (model))
6721 expand_mem_thread_fence (model);
6722 return const0_rtx;
6723 }
6724 }
6725 }
6726
6727 /* If the size of the object is greater than word size on this target,
6728 a default store will not be atomic. */
6729 if (maybe_gt (GET_MODE_PRECISION (mode), BITS_PER_WORD))
6730 {
6731 /* If loads are atomic or we are called to provide a __sync builtin,
6732 we can try a atomic_exchange and throw away the result. Otherwise,
6733 don't do anything so that we do not create an inconsistency between
6734 loads and stores. */
6735 if (can_atomic_load_p (mode) || is_mm_sync (model))
6736 {
6737 rtx target = maybe_emit_atomic_exchange (NULL_RTX, mem, val, model);
6738 if (!target)
6739 target = maybe_emit_compare_and_swap_exchange_loop (NULL_RTX, mem,
6740 val);
6741 if (target)
6742 return const0_rtx;
6743 }
6744 return NULL_RTX;
6745 }
6746
6747 /* Otherwise assume stores are atomic, and emit the proper barriers. */
6748 expand_mem_thread_fence (model);
6749
6750 emit_move_insn (mem, val);
6751
6752 /* For SEQ_CST, also emit a barrier after the store. */
6753 if (is_mm_seq_cst (model))
6754 expand_mem_thread_fence (model);
6755
6756 return const0_rtx;
6757 }
6758
6759
6760 /* Structure containing the pointers and values required to process the
6761 various forms of the atomic_fetch_op and atomic_op_fetch builtins. */
6762
6763 struct atomic_op_functions
6764 {
6765 direct_optab mem_fetch_before;
6766 direct_optab mem_fetch_after;
6767 direct_optab mem_no_result;
6768 optab fetch_before;
6769 optab fetch_after;
6770 direct_optab no_result;
6771 enum rtx_code reverse_code;
6772 };
6773
6774
6775 /* Fill in structure pointed to by OP with the various optab entries for an
6776 operation of type CODE. */
6777
6778 static void
get_atomic_op_for_code(struct atomic_op_functions * op,enum rtx_code code)6779 get_atomic_op_for_code (struct atomic_op_functions *op, enum rtx_code code)
6780 {
6781 gcc_assert (op!= NULL);
6782
6783 /* If SWITCHABLE_TARGET is defined, then subtargets can be switched
6784 in the source code during compilation, and the optab entries are not
6785 computable until runtime. Fill in the values at runtime. */
6786 switch (code)
6787 {
6788 case PLUS:
6789 op->mem_fetch_before = atomic_fetch_add_optab;
6790 op->mem_fetch_after = atomic_add_fetch_optab;
6791 op->mem_no_result = atomic_add_optab;
6792 op->fetch_before = sync_old_add_optab;
6793 op->fetch_after = sync_new_add_optab;
6794 op->no_result = sync_add_optab;
6795 op->reverse_code = MINUS;
6796 break;
6797 case MINUS:
6798 op->mem_fetch_before = atomic_fetch_sub_optab;
6799 op->mem_fetch_after = atomic_sub_fetch_optab;
6800 op->mem_no_result = atomic_sub_optab;
6801 op->fetch_before = sync_old_sub_optab;
6802 op->fetch_after = sync_new_sub_optab;
6803 op->no_result = sync_sub_optab;
6804 op->reverse_code = PLUS;
6805 break;
6806 case XOR:
6807 op->mem_fetch_before = atomic_fetch_xor_optab;
6808 op->mem_fetch_after = atomic_xor_fetch_optab;
6809 op->mem_no_result = atomic_xor_optab;
6810 op->fetch_before = sync_old_xor_optab;
6811 op->fetch_after = sync_new_xor_optab;
6812 op->no_result = sync_xor_optab;
6813 op->reverse_code = XOR;
6814 break;
6815 case AND:
6816 op->mem_fetch_before = atomic_fetch_and_optab;
6817 op->mem_fetch_after = atomic_and_fetch_optab;
6818 op->mem_no_result = atomic_and_optab;
6819 op->fetch_before = sync_old_and_optab;
6820 op->fetch_after = sync_new_and_optab;
6821 op->no_result = sync_and_optab;
6822 op->reverse_code = UNKNOWN;
6823 break;
6824 case IOR:
6825 op->mem_fetch_before = atomic_fetch_or_optab;
6826 op->mem_fetch_after = atomic_or_fetch_optab;
6827 op->mem_no_result = atomic_or_optab;
6828 op->fetch_before = sync_old_ior_optab;
6829 op->fetch_after = sync_new_ior_optab;
6830 op->no_result = sync_ior_optab;
6831 op->reverse_code = UNKNOWN;
6832 break;
6833 case NOT:
6834 op->mem_fetch_before = atomic_fetch_nand_optab;
6835 op->mem_fetch_after = atomic_nand_fetch_optab;
6836 op->mem_no_result = atomic_nand_optab;
6837 op->fetch_before = sync_old_nand_optab;
6838 op->fetch_after = sync_new_nand_optab;
6839 op->no_result = sync_nand_optab;
6840 op->reverse_code = UNKNOWN;
6841 break;
6842 default:
6843 gcc_unreachable ();
6844 }
6845 }
6846
6847 /* See if there is a more optimal way to implement the operation "*MEM CODE VAL"
6848 using memory order MODEL. If AFTER is true the operation needs to return
6849 the value of *MEM after the operation, otherwise the previous value.
6850 TARGET is an optional place to place the result. The result is unused if
6851 it is const0_rtx.
6852 Return the result if there is a better sequence, otherwise NULL_RTX. */
6853
6854 static rtx
maybe_optimize_fetch_op(rtx target,rtx mem,rtx val,enum rtx_code code,enum memmodel model,bool after)6855 maybe_optimize_fetch_op (rtx target, rtx mem, rtx val, enum rtx_code code,
6856 enum memmodel model, bool after)
6857 {
6858 /* If the value is prefetched, or not used, it may be possible to replace
6859 the sequence with a native exchange operation. */
6860 if (!after || target == const0_rtx)
6861 {
6862 /* fetch_and (&x, 0, m) can be replaced with exchange (&x, 0, m). */
6863 if (code == AND && val == const0_rtx)
6864 {
6865 if (target == const0_rtx)
6866 target = gen_reg_rtx (GET_MODE (mem));
6867 return maybe_emit_atomic_exchange (target, mem, val, model);
6868 }
6869
6870 /* fetch_or (&x, -1, m) can be replaced with exchange (&x, -1, m). */
6871 if (code == IOR && val == constm1_rtx)
6872 {
6873 if (target == const0_rtx)
6874 target = gen_reg_rtx (GET_MODE (mem));
6875 return maybe_emit_atomic_exchange (target, mem, val, model);
6876 }
6877 }
6878
6879 return NULL_RTX;
6880 }
6881
6882 /* Try to emit an instruction for a specific operation varaition.
6883 OPTAB contains the OP functions.
6884 TARGET is an optional place to return the result. const0_rtx means unused.
6885 MEM is the memory location to operate on.
6886 VAL is the value to use in the operation.
6887 USE_MEMMODEL is TRUE if the variation with a memory model should be tried.
6888 MODEL is the memory model, if used.
6889 AFTER is true if the returned result is the value after the operation. */
6890
6891 static rtx
maybe_emit_op(const struct atomic_op_functions * optab,rtx target,rtx mem,rtx val,bool use_memmodel,enum memmodel model,bool after)6892 maybe_emit_op (const struct atomic_op_functions *optab, rtx target, rtx mem,
6893 rtx val, bool use_memmodel, enum memmodel model, bool after)
6894 {
6895 machine_mode mode = GET_MODE (mem);
6896 class expand_operand ops[4];
6897 enum insn_code icode;
6898 int op_counter = 0;
6899 int num_ops;
6900
6901 /* Check to see if there is a result returned. */
6902 if (target == const0_rtx)
6903 {
6904 if (use_memmodel)
6905 {
6906 icode = direct_optab_handler (optab->mem_no_result, mode);
6907 create_integer_operand (&ops[2], model);
6908 num_ops = 3;
6909 }
6910 else
6911 {
6912 icode = direct_optab_handler (optab->no_result, mode);
6913 num_ops = 2;
6914 }
6915 }
6916 /* Otherwise, we need to generate a result. */
6917 else
6918 {
6919 if (use_memmodel)
6920 {
6921 icode = direct_optab_handler (after ? optab->mem_fetch_after
6922 : optab->mem_fetch_before, mode);
6923 create_integer_operand (&ops[3], model);
6924 num_ops = 4;
6925 }
6926 else
6927 {
6928 icode = optab_handler (after ? optab->fetch_after
6929 : optab->fetch_before, mode);
6930 num_ops = 3;
6931 }
6932 create_output_operand (&ops[op_counter++], target, mode);
6933 }
6934 if (icode == CODE_FOR_nothing)
6935 return NULL_RTX;
6936
6937 create_fixed_operand (&ops[op_counter++], mem);
6938 /* VAL may have been promoted to a wider mode. Shrink it if so. */
6939 create_convert_operand_to (&ops[op_counter++], val, mode, true);
6940
6941 if (maybe_expand_insn (icode, num_ops, ops))
6942 return (target == const0_rtx ? const0_rtx : ops[0].value);
6943
6944 return NULL_RTX;
6945 }
6946
6947
6948 /* This function expands an atomic fetch_OP or OP_fetch operation:
6949 TARGET is an option place to stick the return value. const0_rtx indicates
6950 the result is unused.
6951 atomically fetch MEM, perform the operation with VAL and return it to MEM.
6952 CODE is the operation being performed (OP)
6953 MEMMODEL is the memory model variant to use.
6954 AFTER is true to return the result of the operation (OP_fetch).
6955 AFTER is false to return the value before the operation (fetch_OP).
6956
6957 This function will *only* generate instructions if there is a direct
6958 optab. No compare and swap loops or libcalls will be generated. */
6959
6960 static rtx
expand_atomic_fetch_op_no_fallback(rtx target,rtx mem,rtx val,enum rtx_code code,enum memmodel model,bool after)6961 expand_atomic_fetch_op_no_fallback (rtx target, rtx mem, rtx val,
6962 enum rtx_code code, enum memmodel model,
6963 bool after)
6964 {
6965 machine_mode mode = GET_MODE (mem);
6966 struct atomic_op_functions optab;
6967 rtx result;
6968 bool unused_result = (target == const0_rtx);
6969
6970 get_atomic_op_for_code (&optab, code);
6971
6972 /* Check to see if there are any better instructions. */
6973 result = maybe_optimize_fetch_op (target, mem, val, code, model, after);
6974 if (result)
6975 return result;
6976
6977 /* Check for the case where the result isn't used and try those patterns. */
6978 if (unused_result)
6979 {
6980 /* Try the memory model variant first. */
6981 result = maybe_emit_op (&optab, target, mem, val, true, model, true);
6982 if (result)
6983 return result;
6984
6985 /* Next try the old style withuot a memory model. */
6986 result = maybe_emit_op (&optab, target, mem, val, false, model, true);
6987 if (result)
6988 return result;
6989
6990 /* There is no no-result pattern, so try patterns with a result. */
6991 target = NULL_RTX;
6992 }
6993
6994 /* Try the __atomic version. */
6995 result = maybe_emit_op (&optab, target, mem, val, true, model, after);
6996 if (result)
6997 return result;
6998
6999 /* Try the older __sync version. */
7000 result = maybe_emit_op (&optab, target, mem, val, false, model, after);
7001 if (result)
7002 return result;
7003
7004 /* If the fetch value can be calculated from the other variation of fetch,
7005 try that operation. */
7006 if (after || unused_result || optab.reverse_code != UNKNOWN)
7007 {
7008 /* Try the __atomic version, then the older __sync version. */
7009 result = maybe_emit_op (&optab, target, mem, val, true, model, !after);
7010 if (!result)
7011 result = maybe_emit_op (&optab, target, mem, val, false, model, !after);
7012
7013 if (result)
7014 {
7015 /* If the result isn't used, no need to do compensation code. */
7016 if (unused_result)
7017 return result;
7018
7019 /* Issue compensation code. Fetch_after == fetch_before OP val.
7020 Fetch_before == after REVERSE_OP val. */
7021 if (!after)
7022 code = optab.reverse_code;
7023 if (code == NOT)
7024 {
7025 result = expand_simple_binop (mode, AND, result, val, NULL_RTX,
7026 true, OPTAB_LIB_WIDEN);
7027 result = expand_simple_unop (mode, NOT, result, target, true);
7028 }
7029 else
7030 result = expand_simple_binop (mode, code, result, val, target,
7031 true, OPTAB_LIB_WIDEN);
7032 return result;
7033 }
7034 }
7035
7036 /* No direct opcode can be generated. */
7037 return NULL_RTX;
7038 }
7039
7040
7041
7042 /* This function expands an atomic fetch_OP or OP_fetch operation:
7043 TARGET is an option place to stick the return value. const0_rtx indicates
7044 the result is unused.
7045 atomically fetch MEM, perform the operation with VAL and return it to MEM.
7046 CODE is the operation being performed (OP)
7047 MEMMODEL is the memory model variant to use.
7048 AFTER is true to return the result of the operation (OP_fetch).
7049 AFTER is false to return the value before the operation (fetch_OP). */
7050 rtx
expand_atomic_fetch_op(rtx target,rtx mem,rtx val,enum rtx_code code,enum memmodel model,bool after)7051 expand_atomic_fetch_op (rtx target, rtx mem, rtx val, enum rtx_code code,
7052 enum memmodel model, bool after)
7053 {
7054 machine_mode mode = GET_MODE (mem);
7055 rtx result;
7056 bool unused_result = (target == const0_rtx);
7057
7058 /* If loads are not atomic for the required size and we are not called to
7059 provide a __sync builtin, do not do anything so that we stay consistent
7060 with atomic loads of the same size. */
7061 if (!can_atomic_load_p (mode) && !is_mm_sync (model))
7062 return NULL_RTX;
7063
7064 result = expand_atomic_fetch_op_no_fallback (target, mem, val, code, model,
7065 after);
7066
7067 if (result)
7068 return result;
7069
7070 /* Add/sub can be implemented by doing the reverse operation with -(val). */
7071 if (code == PLUS || code == MINUS)
7072 {
7073 rtx tmp;
7074 enum rtx_code reverse = (code == PLUS ? MINUS : PLUS);
7075
7076 start_sequence ();
7077 tmp = expand_simple_unop (mode, NEG, val, NULL_RTX, true);
7078 result = expand_atomic_fetch_op_no_fallback (target, mem, tmp, reverse,
7079 model, after);
7080 if (result)
7081 {
7082 /* PLUS worked so emit the insns and return. */
7083 tmp = get_insns ();
7084 end_sequence ();
7085 emit_insn (tmp);
7086 return result;
7087 }
7088
7089 /* PLUS did not work, so throw away the negation code and continue. */
7090 end_sequence ();
7091 }
7092
7093 /* Try the __sync libcalls only if we can't do compare-and-swap inline. */
7094 if (!can_compare_and_swap_p (mode, false))
7095 {
7096 rtx libfunc;
7097 bool fixup = false;
7098 enum rtx_code orig_code = code;
7099 struct atomic_op_functions optab;
7100
7101 get_atomic_op_for_code (&optab, code);
7102 libfunc = optab_libfunc (after ? optab.fetch_after
7103 : optab.fetch_before, mode);
7104 if (libfunc == NULL
7105 && (after || unused_result || optab.reverse_code != UNKNOWN))
7106 {
7107 fixup = true;
7108 if (!after)
7109 code = optab.reverse_code;
7110 libfunc = optab_libfunc (after ? optab.fetch_before
7111 : optab.fetch_after, mode);
7112 }
7113 if (libfunc != NULL)
7114 {
7115 rtx addr = convert_memory_address (ptr_mode, XEXP (mem, 0));
7116 result = emit_library_call_value (libfunc, NULL, LCT_NORMAL, mode,
7117 addr, ptr_mode, val, mode);
7118
7119 if (!unused_result && fixup)
7120 result = expand_simple_binop (mode, code, result, val, target,
7121 true, OPTAB_LIB_WIDEN);
7122 return result;
7123 }
7124
7125 /* We need the original code for any further attempts. */
7126 code = orig_code;
7127 }
7128
7129 /* If nothing else has succeeded, default to a compare and swap loop. */
7130 if (can_compare_and_swap_p (mode, true))
7131 {
7132 rtx_insn *insn;
7133 rtx t0 = gen_reg_rtx (mode), t1;
7134
7135 start_sequence ();
7136
7137 /* If the result is used, get a register for it. */
7138 if (!unused_result)
7139 {
7140 if (!target || !register_operand (target, mode))
7141 target = gen_reg_rtx (mode);
7142 /* If fetch_before, copy the value now. */
7143 if (!after)
7144 emit_move_insn (target, t0);
7145 }
7146 else
7147 target = const0_rtx;
7148
7149 t1 = t0;
7150 if (code == NOT)
7151 {
7152 t1 = expand_simple_binop (mode, AND, t1, val, NULL_RTX,
7153 true, OPTAB_LIB_WIDEN);
7154 t1 = expand_simple_unop (mode, code, t1, NULL_RTX, true);
7155 }
7156 else
7157 t1 = expand_simple_binop (mode, code, t1, val, NULL_RTX, true,
7158 OPTAB_LIB_WIDEN);
7159
7160 /* For after, copy the value now. */
7161 if (!unused_result && after)
7162 emit_move_insn (target, t1);
7163 insn = get_insns ();
7164 end_sequence ();
7165
7166 if (t1 != NULL && expand_compare_and_swap_loop (mem, t0, t1, insn))
7167 return target;
7168 }
7169
7170 return NULL_RTX;
7171 }
7172
7173 /* Return true if OPERAND is suitable for operand number OPNO of
7174 instruction ICODE. */
7175
7176 bool
insn_operand_matches(enum insn_code icode,unsigned int opno,rtx operand)7177 insn_operand_matches (enum insn_code icode, unsigned int opno, rtx operand)
7178 {
7179 return (!insn_data[(int) icode].operand[opno].predicate
7180 || (insn_data[(int) icode].operand[opno].predicate
7181 (operand, insn_data[(int) icode].operand[opno].mode)));
7182 }
7183
7184 /* TARGET is a target of a multiword operation that we are going to
7185 implement as a series of word-mode operations. Return true if
7186 TARGET is suitable for this purpose. */
7187
7188 bool
valid_multiword_target_p(rtx target)7189 valid_multiword_target_p (rtx target)
7190 {
7191 machine_mode mode;
7192 int i, size;
7193
7194 mode = GET_MODE (target);
7195 if (!GET_MODE_SIZE (mode).is_constant (&size))
7196 return false;
7197 for (i = 0; i < size; i += UNITS_PER_WORD)
7198 if (!validate_subreg (word_mode, mode, target, i))
7199 return false;
7200 return true;
7201 }
7202
7203 /* Make OP describe an input operand that has value INTVAL and that has
7204 no inherent mode. This function should only be used for operands that
7205 are always expand-time constants. The backend may request that INTVAL
7206 be copied into a different kind of rtx, but it must specify the mode
7207 of that rtx if so. */
7208
7209 void
create_integer_operand(class expand_operand * op,poly_int64 intval)7210 create_integer_operand (class expand_operand *op, poly_int64 intval)
7211 {
7212 create_expand_operand (op, EXPAND_INTEGER,
7213 gen_int_mode (intval, MAX_MODE_INT),
7214 VOIDmode, false, intval);
7215 }
7216
7217 /* Like maybe_legitimize_operand, but do not change the code of the
7218 current rtx value. */
7219
7220 static bool
maybe_legitimize_operand_same_code(enum insn_code icode,unsigned int opno,class expand_operand * op)7221 maybe_legitimize_operand_same_code (enum insn_code icode, unsigned int opno,
7222 class expand_operand *op)
7223 {
7224 /* See if the operand matches in its current form. */
7225 if (insn_operand_matches (icode, opno, op->value))
7226 return true;
7227
7228 /* If the operand is a memory whose address has no side effects,
7229 try forcing the address into a non-virtual pseudo register.
7230 The check for side effects is important because copy_to_mode_reg
7231 cannot handle things like auto-modified addresses. */
7232 if (insn_data[(int) icode].operand[opno].allows_mem && MEM_P (op->value))
7233 {
7234 rtx addr, mem;
7235
7236 mem = op->value;
7237 addr = XEXP (mem, 0);
7238 if (!(REG_P (addr) && REGNO (addr) > LAST_VIRTUAL_REGISTER)
7239 && !side_effects_p (addr))
7240 {
7241 rtx_insn *last;
7242 machine_mode mode;
7243
7244 last = get_last_insn ();
7245 mode = get_address_mode (mem);
7246 mem = replace_equiv_address (mem, copy_to_mode_reg (mode, addr));
7247 if (insn_operand_matches (icode, opno, mem))
7248 {
7249 op->value = mem;
7250 return true;
7251 }
7252 delete_insns_since (last);
7253 }
7254 }
7255
7256 return false;
7257 }
7258
7259 /* Try to make OP match operand OPNO of instruction ICODE. Return true
7260 on success, storing the new operand value back in OP. */
7261
7262 static bool
maybe_legitimize_operand(enum insn_code icode,unsigned int opno,class expand_operand * op)7263 maybe_legitimize_operand (enum insn_code icode, unsigned int opno,
7264 class expand_operand *op)
7265 {
7266 machine_mode mode, imode, tmode;
7267
7268 mode = op->mode;
7269 switch (op->type)
7270 {
7271 case EXPAND_FIXED:
7272 {
7273 temporary_volatile_ok v (true);
7274 return maybe_legitimize_operand_same_code (icode, opno, op);
7275 }
7276
7277 case EXPAND_OUTPUT:
7278 gcc_assert (mode != VOIDmode);
7279 if (op->value
7280 && op->value != const0_rtx
7281 && GET_MODE (op->value) == mode
7282 && maybe_legitimize_operand_same_code (icode, opno, op))
7283 return true;
7284
7285 op->value = gen_reg_rtx (mode);
7286 op->target = 0;
7287 break;
7288
7289 case EXPAND_INPUT:
7290 input:
7291 gcc_assert (mode != VOIDmode);
7292 gcc_assert (GET_MODE (op->value) == VOIDmode
7293 || GET_MODE (op->value) == mode);
7294 if (maybe_legitimize_operand_same_code (icode, opno, op))
7295 return true;
7296
7297 op->value = copy_to_mode_reg (mode, op->value);
7298 break;
7299
7300 case EXPAND_CONVERT_TO:
7301 gcc_assert (mode != VOIDmode);
7302 op->value = convert_to_mode (mode, op->value, op->unsigned_p);
7303 goto input;
7304
7305 case EXPAND_CONVERT_FROM:
7306 if (GET_MODE (op->value) != VOIDmode)
7307 mode = GET_MODE (op->value);
7308 else
7309 /* The caller must tell us what mode this value has. */
7310 gcc_assert (mode != VOIDmode);
7311
7312 imode = insn_data[(int) icode].operand[opno].mode;
7313 tmode = (VECTOR_MODE_P (imode) && !VECTOR_MODE_P (mode)
7314 ? GET_MODE_INNER (imode) : imode);
7315 if (tmode != VOIDmode && tmode != mode)
7316 {
7317 op->value = convert_modes (tmode, mode, op->value, op->unsigned_p);
7318 mode = tmode;
7319 }
7320 if (imode != VOIDmode && imode != mode)
7321 {
7322 gcc_assert (VECTOR_MODE_P (imode) && !VECTOR_MODE_P (mode));
7323 op->value = expand_vector_broadcast (imode, op->value);
7324 mode = imode;
7325 }
7326 goto input;
7327
7328 case EXPAND_ADDRESS:
7329 op->value = convert_memory_address (as_a <scalar_int_mode> (mode),
7330 op->value);
7331 goto input;
7332
7333 case EXPAND_INTEGER:
7334 mode = insn_data[(int) icode].operand[opno].mode;
7335 if (mode != VOIDmode
7336 && known_eq (trunc_int_for_mode (op->int_value, mode),
7337 op->int_value))
7338 {
7339 op->value = gen_int_mode (op->int_value, mode);
7340 goto input;
7341 }
7342 break;
7343 }
7344 return insn_operand_matches (icode, opno, op->value);
7345 }
7346
7347 /* Make OP describe an input operand that should have the same value
7348 as VALUE, after any mode conversion that the target might request.
7349 TYPE is the type of VALUE. */
7350
7351 void
create_convert_operand_from_type(class expand_operand * op,rtx value,tree type)7352 create_convert_operand_from_type (class expand_operand *op,
7353 rtx value, tree type)
7354 {
7355 create_convert_operand_from (op, value, TYPE_MODE (type),
7356 TYPE_UNSIGNED (type));
7357 }
7358
7359 /* Return true if the requirements on operands OP1 and OP2 of instruction
7360 ICODE are similar enough for the result of legitimizing OP1 to be
7361 reusable for OP2. OPNO1 and OPNO2 are the operand numbers associated
7362 with OP1 and OP2 respectively. */
7363
7364 static inline bool
can_reuse_operands_p(enum insn_code icode,unsigned int opno1,unsigned int opno2,const class expand_operand * op1,const class expand_operand * op2)7365 can_reuse_operands_p (enum insn_code icode,
7366 unsigned int opno1, unsigned int opno2,
7367 const class expand_operand *op1,
7368 const class expand_operand *op2)
7369 {
7370 /* Check requirements that are common to all types. */
7371 if (op1->type != op2->type
7372 || op1->mode != op2->mode
7373 || (insn_data[(int) icode].operand[opno1].mode
7374 != insn_data[(int) icode].operand[opno2].mode))
7375 return false;
7376
7377 /* Check the requirements for specific types. */
7378 switch (op1->type)
7379 {
7380 case EXPAND_OUTPUT:
7381 /* Outputs must remain distinct. */
7382 return false;
7383
7384 case EXPAND_FIXED:
7385 case EXPAND_INPUT:
7386 case EXPAND_ADDRESS:
7387 case EXPAND_INTEGER:
7388 return true;
7389
7390 case EXPAND_CONVERT_TO:
7391 case EXPAND_CONVERT_FROM:
7392 return op1->unsigned_p == op2->unsigned_p;
7393 }
7394 gcc_unreachable ();
7395 }
7396
7397 /* Try to make operands [OPS, OPS + NOPS) match operands [OPNO, OPNO + NOPS)
7398 of instruction ICODE. Return true on success, leaving the new operand
7399 values in the OPS themselves. Emit no code on failure. */
7400
7401 bool
maybe_legitimize_operands(enum insn_code icode,unsigned int opno,unsigned int nops,class expand_operand * ops)7402 maybe_legitimize_operands (enum insn_code icode, unsigned int opno,
7403 unsigned int nops, class expand_operand *ops)
7404 {
7405 rtx_insn *last = get_last_insn ();
7406 rtx *orig_values = XALLOCAVEC (rtx, nops);
7407 for (unsigned int i = 0; i < nops; i++)
7408 {
7409 orig_values[i] = ops[i].value;
7410
7411 /* First try reusing the result of an earlier legitimization.
7412 This avoids duplicate rtl and ensures that tied operands
7413 remain tied.
7414
7415 This search is linear, but NOPS is bounded at compile time
7416 to a small number (current a single digit). */
7417 unsigned int j = 0;
7418 for (; j < i; ++j)
7419 if (can_reuse_operands_p (icode, opno + j, opno + i, &ops[j], &ops[i])
7420 && rtx_equal_p (orig_values[j], orig_values[i])
7421 && ops[j].value
7422 && insn_operand_matches (icode, opno + i, ops[j].value))
7423 {
7424 ops[i].value = copy_rtx (ops[j].value);
7425 break;
7426 }
7427
7428 /* Otherwise try legitimizing the operand on its own. */
7429 if (j == i && !maybe_legitimize_operand (icode, opno + i, &ops[i]))
7430 {
7431 delete_insns_since (last);
7432 return false;
7433 }
7434 }
7435 return true;
7436 }
7437
7438 /* Try to generate instruction ICODE, using operands [OPS, OPS + NOPS)
7439 as its operands. Return the instruction pattern on success,
7440 and emit any necessary set-up code. Return null and emit no
7441 code on failure. */
7442
7443 rtx_insn *
maybe_gen_insn(enum insn_code icode,unsigned int nops,class expand_operand * ops)7444 maybe_gen_insn (enum insn_code icode, unsigned int nops,
7445 class expand_operand *ops)
7446 {
7447 gcc_assert (nops == (unsigned int) insn_data[(int) icode].n_generator_args);
7448 if (!maybe_legitimize_operands (icode, 0, nops, ops))
7449 return NULL;
7450
7451 switch (nops)
7452 {
7453 case 1:
7454 return GEN_FCN (icode) (ops[0].value);
7455 case 2:
7456 return GEN_FCN (icode) (ops[0].value, ops[1].value);
7457 case 3:
7458 return GEN_FCN (icode) (ops[0].value, ops[1].value, ops[2].value);
7459 case 4:
7460 return GEN_FCN (icode) (ops[0].value, ops[1].value, ops[2].value,
7461 ops[3].value);
7462 case 5:
7463 return GEN_FCN (icode) (ops[0].value, ops[1].value, ops[2].value,
7464 ops[3].value, ops[4].value);
7465 case 6:
7466 return GEN_FCN (icode) (ops[0].value, ops[1].value, ops[2].value,
7467 ops[3].value, ops[4].value, ops[5].value);
7468 case 7:
7469 return GEN_FCN (icode) (ops[0].value, ops[1].value, ops[2].value,
7470 ops[3].value, ops[4].value, ops[5].value,
7471 ops[6].value);
7472 case 8:
7473 return GEN_FCN (icode) (ops[0].value, ops[1].value, ops[2].value,
7474 ops[3].value, ops[4].value, ops[5].value,
7475 ops[6].value, ops[7].value);
7476 case 9:
7477 return GEN_FCN (icode) (ops[0].value, ops[1].value, ops[2].value,
7478 ops[3].value, ops[4].value, ops[5].value,
7479 ops[6].value, ops[7].value, ops[8].value);
7480 }
7481 gcc_unreachable ();
7482 }
7483
7484 /* Try to emit instruction ICODE, using operands [OPS, OPS + NOPS)
7485 as its operands. Return true on success and emit no code on failure. */
7486
7487 bool
maybe_expand_insn(enum insn_code icode,unsigned int nops,class expand_operand * ops)7488 maybe_expand_insn (enum insn_code icode, unsigned int nops,
7489 class expand_operand *ops)
7490 {
7491 rtx_insn *pat = maybe_gen_insn (icode, nops, ops);
7492 if (pat)
7493 {
7494 emit_insn (pat);
7495 return true;
7496 }
7497 return false;
7498 }
7499
7500 /* Like maybe_expand_insn, but for jumps. */
7501
7502 bool
maybe_expand_jump_insn(enum insn_code icode,unsigned int nops,class expand_operand * ops)7503 maybe_expand_jump_insn (enum insn_code icode, unsigned int nops,
7504 class expand_operand *ops)
7505 {
7506 rtx_insn *pat = maybe_gen_insn (icode, nops, ops);
7507 if (pat)
7508 {
7509 emit_jump_insn (pat);
7510 return true;
7511 }
7512 return false;
7513 }
7514
7515 /* Emit instruction ICODE, using operands [OPS, OPS + NOPS)
7516 as its operands. */
7517
7518 void
expand_insn(enum insn_code icode,unsigned int nops,class expand_operand * ops)7519 expand_insn (enum insn_code icode, unsigned int nops,
7520 class expand_operand *ops)
7521 {
7522 if (!maybe_expand_insn (icode, nops, ops))
7523 gcc_unreachable ();
7524 }
7525
7526 /* Like expand_insn, but for jumps. */
7527
7528 void
expand_jump_insn(enum insn_code icode,unsigned int nops,class expand_operand * ops)7529 expand_jump_insn (enum insn_code icode, unsigned int nops,
7530 class expand_operand *ops)
7531 {
7532 if (!maybe_expand_jump_insn (icode, nops, ops))
7533 gcc_unreachable ();
7534 }
7535