1 /* Expand the basic unary and binary arithmetic operations, for GNU compiler.
2    Copyright (C) 1987-2018 Free Software Foundation, Inc.
3 
4 This file is part of GCC.
5 
6 GCC is free software; you can redistribute it and/or modify it under
7 the terms of the GNU General Public License as published by the Free
8 Software Foundation; either version 3, or (at your option) any later
9 version.
10 
11 GCC is distributed in the hope that it will be useful, but WITHOUT ANY
12 WARRANTY; without even the implied warranty of MERCHANTABILITY or
13 FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
14 for more details.
15 
16 You should have received a copy of the GNU General Public License
17 along with GCC; see the file COPYING3.  If not see
18 <http://www.gnu.org/licenses/>.  */
19 
20 
21 #include "config.h"
22 #include "system.h"
23 #include "coretypes.h"
24 #include "backend.h"
25 #include "target.h"
26 #include "rtl.h"
27 #include "tree.h"
28 #include "memmodel.h"
29 #include "predict.h"
30 #include "tm_p.h"
31 #include "expmed.h"
32 #include "optabs.h"
33 #include "emit-rtl.h"
34 #include "recog.h"
35 #include "diagnostic-core.h"
36 #include "rtx-vector-builder.h"
37 
38 /* Include insn-config.h before expr.h so that HAVE_conditional_move
39    is properly defined.  */
40 #include "stor-layout.h"
41 #include "except.h"
42 #include "dojump.h"
43 #include "explow.h"
44 #include "expr.h"
45 #include "optabs-tree.h"
46 #include "libfuncs.h"
47 
48 static void prepare_float_lib_cmp (rtx, rtx, enum rtx_code, rtx *,
49 				   machine_mode *);
50 static rtx expand_unop_direct (machine_mode, optab, rtx, rtx, int);
51 static void emit_libcall_block_1 (rtx_insn *, rtx, rtx, rtx, bool);
52 
53 /* Debug facility for use in GDB.  */
54 void debug_optab_libfuncs (void);
55 
56 /* Add a REG_EQUAL note to the last insn in INSNS.  TARGET is being set to
57    the result of operation CODE applied to OP0 (and OP1 if it is a binary
58    operation).  OP0_MODE is OP0's mode.
59 
60    If the last insn does not set TARGET, don't do anything, but return 1.
61 
62    If the last insn or a previous insn sets TARGET and TARGET is one of OP0
63    or OP1, don't add the REG_EQUAL note but return 0.  Our caller can then
64    try again, ensuring that TARGET is not one of the operands.  */
65 
66 static int
add_equal_note(rtx_insn * insns,rtx target,enum rtx_code code,rtx op0,rtx op1,machine_mode op0_mode)67 add_equal_note (rtx_insn *insns, rtx target, enum rtx_code code, rtx op0,
68 		rtx op1, machine_mode op0_mode)
69 {
70   rtx_insn *last_insn;
71   rtx set;
72   rtx note;
73 
74   gcc_assert (insns && INSN_P (insns) && NEXT_INSN (insns));
75 
76   if (GET_RTX_CLASS (code) != RTX_COMM_ARITH
77       && GET_RTX_CLASS (code) != RTX_BIN_ARITH
78       && GET_RTX_CLASS (code) != RTX_COMM_COMPARE
79       && GET_RTX_CLASS (code) != RTX_COMPARE
80       && GET_RTX_CLASS (code) != RTX_UNARY)
81     return 1;
82 
83   if (GET_CODE (target) == ZERO_EXTRACT)
84     return 1;
85 
86   for (last_insn = insns;
87        NEXT_INSN (last_insn) != NULL_RTX;
88        last_insn = NEXT_INSN (last_insn))
89     ;
90 
91   /* If TARGET is in OP0 or OP1, punt.  We'd end up with a note referencing
92      a value changing in the insn, so the note would be invalid for CSE.  */
93   if (reg_overlap_mentioned_p (target, op0)
94       || (op1 && reg_overlap_mentioned_p (target, op1)))
95     {
96       if (MEM_P (target)
97 	  && (rtx_equal_p (target, op0)
98 	      || (op1 && rtx_equal_p (target, op1))))
99 	{
100 	  /* For MEM target, with MEM = MEM op X, prefer no REG_EQUAL note
101 	     over expanding it as temp = MEM op X, MEM = temp.  If the target
102 	     supports MEM = MEM op X instructions, it is sometimes too hard
103 	     to reconstruct that form later, especially if X is also a memory,
104 	     and due to multiple occurrences of addresses the address might
105 	     be forced into register unnecessarily.
106 	     Note that not emitting the REG_EQUIV note might inhibit
107 	     CSE in some cases.  */
108 	  set = single_set (last_insn);
109 	  if (set
110 	      && GET_CODE (SET_SRC (set)) == code
111 	      && MEM_P (SET_DEST (set))
112 	      && (rtx_equal_p (SET_DEST (set), XEXP (SET_SRC (set), 0))
113 		  || (op1 && rtx_equal_p (SET_DEST (set),
114 					  XEXP (SET_SRC (set), 1)))))
115 	    return 1;
116 	}
117       return 0;
118     }
119 
120   set = set_for_reg_notes (last_insn);
121   if (set == NULL_RTX)
122     return 1;
123 
124   if (! rtx_equal_p (SET_DEST (set), target)
125       /* For a STRICT_LOW_PART, the REG_NOTE applies to what is inside it.  */
126       && (GET_CODE (SET_DEST (set)) != STRICT_LOW_PART
127 	  || ! rtx_equal_p (XEXP (SET_DEST (set), 0), target)))
128     return 1;
129 
130   if (GET_RTX_CLASS (code) == RTX_UNARY)
131     switch (code)
132       {
133       case FFS:
134       case CLZ:
135       case CTZ:
136       case CLRSB:
137       case POPCOUNT:
138       case PARITY:
139       case BSWAP:
140 	if (op0_mode != VOIDmode && GET_MODE (target) != op0_mode)
141 	  {
142 	    note = gen_rtx_fmt_e (code, op0_mode, copy_rtx (op0));
143 	    if (GET_MODE_UNIT_SIZE (op0_mode)
144 		> GET_MODE_UNIT_SIZE (GET_MODE (target)))
145 	      note = simplify_gen_unary (TRUNCATE, GET_MODE (target),
146 					 note, op0_mode);
147 	    else
148 	      note = simplify_gen_unary (ZERO_EXTEND, GET_MODE (target),
149 					 note, op0_mode);
150 	    break;
151 	  }
152 	/* FALLTHRU */
153       default:
154 	note = gen_rtx_fmt_e (code, GET_MODE (target), copy_rtx (op0));
155 	break;
156       }
157   else
158     note = gen_rtx_fmt_ee (code, GET_MODE (target), copy_rtx (op0), copy_rtx (op1));
159 
160   set_unique_reg_note (last_insn, REG_EQUAL, note);
161 
162   return 1;
163 }
164 
165 /* Given two input operands, OP0 and OP1, determine what the correct from_mode
166    for a widening operation would be.  In most cases this would be OP0, but if
167    that's a constant it'll be VOIDmode, which isn't useful.  */
168 
169 static machine_mode
widened_mode(machine_mode to_mode,rtx op0,rtx op1)170 widened_mode (machine_mode to_mode, rtx op0, rtx op1)
171 {
172   machine_mode m0 = GET_MODE (op0);
173   machine_mode m1 = GET_MODE (op1);
174   machine_mode result;
175 
176   if (m0 == VOIDmode && m1 == VOIDmode)
177     return to_mode;
178   else if (m0 == VOIDmode || GET_MODE_UNIT_SIZE (m0) < GET_MODE_UNIT_SIZE (m1))
179     result = m1;
180   else
181     result = m0;
182 
183   if (GET_MODE_UNIT_SIZE (result) > GET_MODE_UNIT_SIZE (to_mode))
184     return to_mode;
185 
186   return result;
187 }
188 
189 /* Widen OP to MODE and return the rtx for the widened operand.  UNSIGNEDP
190    says whether OP is signed or unsigned.  NO_EXTEND is nonzero if we need
191    not actually do a sign-extend or zero-extend, but can leave the
192    higher-order bits of the result rtx undefined, for example, in the case
193    of logical operations, but not right shifts.  */
194 
195 static rtx
widen_operand(rtx op,machine_mode mode,machine_mode oldmode,int unsignedp,int no_extend)196 widen_operand (rtx op, machine_mode mode, machine_mode oldmode,
197 	       int unsignedp, int no_extend)
198 {
199   rtx result;
200   scalar_int_mode int_mode;
201 
202   /* If we don't have to extend and this is a constant, return it.  */
203   if (no_extend && GET_MODE (op) == VOIDmode)
204     return op;
205 
206   /* If we must extend do so.  If OP is a SUBREG for a promoted object, also
207      extend since it will be more efficient to do so unless the signedness of
208      a promoted object differs from our extension.  */
209   if (! no_extend
210       || !is_a <scalar_int_mode> (mode, &int_mode)
211       || (GET_CODE (op) == SUBREG && SUBREG_PROMOTED_VAR_P (op)
212 	  && SUBREG_CHECK_PROMOTED_SIGN (op, unsignedp)))
213     return convert_modes (mode, oldmode, op, unsignedp);
214 
215   /* If MODE is no wider than a single word, we return a lowpart or paradoxical
216      SUBREG.  */
217   if (GET_MODE_SIZE (int_mode) <= UNITS_PER_WORD)
218     return gen_lowpart (int_mode, force_reg (GET_MODE (op), op));
219 
220   /* Otherwise, get an object of MODE, clobber it, and set the low-order
221      part to OP.  */
222 
223   result = gen_reg_rtx (int_mode);
224   emit_clobber (result);
225   emit_move_insn (gen_lowpart (GET_MODE (op), result), op);
226   return result;
227 }
228 
229 /* Expand vector widening operations.
230 
231    There are two different classes of operations handled here:
232    1) Operations whose result is wider than all the arguments to the operation.
233       Examples: VEC_UNPACK_HI/LO_EXPR, VEC_WIDEN_MULT_HI/LO_EXPR
234       In this case OP0 and optionally OP1 would be initialized,
235       but WIDE_OP wouldn't (not relevant for this case).
236    2) Operations whose result is of the same size as the last argument to the
237       operation, but wider than all the other arguments to the operation.
238       Examples: WIDEN_SUM_EXPR, VEC_DOT_PROD_EXPR.
239       In the case WIDE_OP, OP0 and optionally OP1 would be initialized.
240 
241    E.g, when called to expand the following operations, this is how
242    the arguments will be initialized:
243                                 nops    OP0     OP1     WIDE_OP
244    widening-sum                 2       oprnd0  -       oprnd1
245    widening-dot-product         3       oprnd0  oprnd1  oprnd2
246    widening-mult                2       oprnd0  oprnd1  -
247    type-promotion (vec-unpack)  1       oprnd0  -       -  */
248 
249 rtx
expand_widen_pattern_expr(sepops ops,rtx op0,rtx op1,rtx wide_op,rtx target,int unsignedp)250 expand_widen_pattern_expr (sepops ops, rtx op0, rtx op1, rtx wide_op,
251 			   rtx target, int unsignedp)
252 {
253   struct expand_operand eops[4];
254   tree oprnd0, oprnd1, oprnd2;
255   machine_mode wmode = VOIDmode, tmode0, tmode1 = VOIDmode;
256   optab widen_pattern_optab;
257   enum insn_code icode;
258   int nops = TREE_CODE_LENGTH (ops->code);
259   int op;
260 
261   oprnd0 = ops->op0;
262   tmode0 = TYPE_MODE (TREE_TYPE (oprnd0));
263   widen_pattern_optab =
264     optab_for_tree_code (ops->code, TREE_TYPE (oprnd0), optab_default);
265   if (ops->code == WIDEN_MULT_PLUS_EXPR
266       || ops->code == WIDEN_MULT_MINUS_EXPR)
267     icode = find_widening_optab_handler (widen_pattern_optab,
268 					 TYPE_MODE (TREE_TYPE (ops->op2)),
269 					 tmode0);
270   else
271     icode = optab_handler (widen_pattern_optab, tmode0);
272   gcc_assert (icode != CODE_FOR_nothing);
273 
274   if (nops >= 2)
275     {
276       oprnd1 = ops->op1;
277       tmode1 = TYPE_MODE (TREE_TYPE (oprnd1));
278     }
279 
280   /* The last operand is of a wider mode than the rest of the operands.  */
281   if (nops == 2)
282     wmode = tmode1;
283   else if (nops == 3)
284     {
285       gcc_assert (tmode1 == tmode0);
286       gcc_assert (op1);
287       oprnd2 = ops->op2;
288       wmode = TYPE_MODE (TREE_TYPE (oprnd2));
289     }
290 
291   op = 0;
292   create_output_operand (&eops[op++], target, TYPE_MODE (ops->type));
293   create_convert_operand_from (&eops[op++], op0, tmode0, unsignedp);
294   if (op1)
295     create_convert_operand_from (&eops[op++], op1, tmode1, unsignedp);
296   if (wide_op)
297     create_convert_operand_from (&eops[op++], wide_op, wmode, unsignedp);
298   expand_insn (icode, op, eops);
299   return eops[0].value;
300 }
301 
302 /* Generate code to perform an operation specified by TERNARY_OPTAB
303    on operands OP0, OP1 and OP2, with result having machine-mode MODE.
304 
305    UNSIGNEDP is for the case where we have to widen the operands
306    to perform the operation.  It says to use zero-extension.
307 
308    If TARGET is nonzero, the value
309    is generated there, if it is convenient to do so.
310    In all cases an rtx is returned for the locus of the value;
311    this may or may not be TARGET.  */
312 
313 rtx
expand_ternary_op(machine_mode mode,optab ternary_optab,rtx op0,rtx op1,rtx op2,rtx target,int unsignedp)314 expand_ternary_op (machine_mode mode, optab ternary_optab, rtx op0,
315 		   rtx op1, rtx op2, rtx target, int unsignedp)
316 {
317   struct expand_operand ops[4];
318   enum insn_code icode = optab_handler (ternary_optab, mode);
319 
320   gcc_assert (optab_handler (ternary_optab, mode) != CODE_FOR_nothing);
321 
322   create_output_operand (&ops[0], target, mode);
323   create_convert_operand_from (&ops[1], op0, mode, unsignedp);
324   create_convert_operand_from (&ops[2], op1, mode, unsignedp);
325   create_convert_operand_from (&ops[3], op2, mode, unsignedp);
326   expand_insn (icode, 4, ops);
327   return ops[0].value;
328 }
329 
330 
331 /* Like expand_binop, but return a constant rtx if the result can be
332    calculated at compile time.  The arguments and return value are
333    otherwise the same as for expand_binop.  */
334 
335 rtx
simplify_expand_binop(machine_mode mode,optab binoptab,rtx op0,rtx op1,rtx target,int unsignedp,enum optab_methods methods)336 simplify_expand_binop (machine_mode mode, optab binoptab,
337 		       rtx op0, rtx op1, rtx target, int unsignedp,
338 		       enum optab_methods methods)
339 {
340   if (CONSTANT_P (op0) && CONSTANT_P (op1))
341     {
342       rtx x = simplify_binary_operation (optab_to_code (binoptab),
343 					 mode, op0, op1);
344       if (x)
345 	return x;
346     }
347 
348   return expand_binop (mode, binoptab, op0, op1, target, unsignedp, methods);
349 }
350 
351 /* Like simplify_expand_binop, but always put the result in TARGET.
352    Return true if the expansion succeeded.  */
353 
354 bool
force_expand_binop(machine_mode mode,optab binoptab,rtx op0,rtx op1,rtx target,int unsignedp,enum optab_methods methods)355 force_expand_binop (machine_mode mode, optab binoptab,
356 		    rtx op0, rtx op1, rtx target, int unsignedp,
357 		    enum optab_methods methods)
358 {
359   rtx x = simplify_expand_binop (mode, binoptab, op0, op1,
360 				 target, unsignedp, methods);
361   if (x == 0)
362     return false;
363   if (x != target)
364     emit_move_insn (target, x);
365   return true;
366 }
367 
368 /* Create a new vector value in VMODE with all elements set to OP.  The
369    mode of OP must be the element mode of VMODE.  If OP is a constant,
370    then the return value will be a constant.  */
371 
372 rtx
expand_vector_broadcast(machine_mode vmode,rtx op)373 expand_vector_broadcast (machine_mode vmode, rtx op)
374 {
375   int n;
376   rtvec vec;
377 
378   gcc_checking_assert (VECTOR_MODE_P (vmode));
379 
380   if (valid_for_const_vector_p (vmode, op))
381     return gen_const_vec_duplicate (vmode, op);
382 
383   insn_code icode = optab_handler (vec_duplicate_optab, vmode);
384   if (icode != CODE_FOR_nothing)
385     {
386       struct expand_operand ops[2];
387       create_output_operand (&ops[0], NULL_RTX, vmode);
388       create_input_operand (&ops[1], op, GET_MODE (op));
389       expand_insn (icode, 2, ops);
390       return ops[0].value;
391     }
392 
393   if (!GET_MODE_NUNITS (vmode).is_constant (&n))
394     return NULL;
395 
396   /* ??? If the target doesn't have a vec_init, then we have no easy way
397      of performing this operation.  Most of this sort of generic support
398      is hidden away in the vector lowering support in gimple.  */
399   icode = convert_optab_handler (vec_init_optab, vmode,
400 				 GET_MODE_INNER (vmode));
401   if (icode == CODE_FOR_nothing)
402     return NULL;
403 
404   vec = rtvec_alloc (n);
405   for (int i = 0; i < n; ++i)
406     RTVEC_ELT (vec, i) = op;
407   rtx ret = gen_reg_rtx (vmode);
408   emit_insn (GEN_FCN (icode) (ret, gen_rtx_PARALLEL (vmode, vec)));
409 
410   return ret;
411 }
412 
413 /* This subroutine of expand_doubleword_shift handles the cases in which
414    the effective shift value is >= BITS_PER_WORD.  The arguments and return
415    value are the same as for the parent routine, except that SUPERWORD_OP1
416    is the shift count to use when shifting OUTOF_INPUT into INTO_TARGET.
417    INTO_TARGET may be null if the caller has decided to calculate it.  */
418 
419 static bool
expand_superword_shift(optab binoptab,rtx outof_input,rtx superword_op1,rtx outof_target,rtx into_target,int unsignedp,enum optab_methods methods)420 expand_superword_shift (optab binoptab, rtx outof_input, rtx superword_op1,
421 			rtx outof_target, rtx into_target,
422 			int unsignedp, enum optab_methods methods)
423 {
424   if (into_target != 0)
425     if (!force_expand_binop (word_mode, binoptab, outof_input, superword_op1,
426 			     into_target, unsignedp, methods))
427       return false;
428 
429   if (outof_target != 0)
430     {
431       /* For a signed right shift, we must fill OUTOF_TARGET with copies
432 	 of the sign bit, otherwise we must fill it with zeros.  */
433       if (binoptab != ashr_optab)
434 	emit_move_insn (outof_target, CONST0_RTX (word_mode));
435       else
436 	if (!force_expand_binop (word_mode, binoptab, outof_input,
437 				 gen_int_shift_amount (word_mode,
438 						       BITS_PER_WORD - 1),
439 				 outof_target, unsignedp, methods))
440 	  return false;
441     }
442   return true;
443 }
444 
445 /* This subroutine of expand_doubleword_shift handles the cases in which
446    the effective shift value is < BITS_PER_WORD.  The arguments and return
447    value are the same as for the parent routine.  */
448 
449 static bool
expand_subword_shift(scalar_int_mode op1_mode,optab binoptab,rtx outof_input,rtx into_input,rtx op1,rtx outof_target,rtx into_target,int unsignedp,enum optab_methods methods,unsigned HOST_WIDE_INT shift_mask)450 expand_subword_shift (scalar_int_mode op1_mode, optab binoptab,
451 		      rtx outof_input, rtx into_input, rtx op1,
452 		      rtx outof_target, rtx into_target,
453 		      int unsignedp, enum optab_methods methods,
454 		      unsigned HOST_WIDE_INT shift_mask)
455 {
456   optab reverse_unsigned_shift, unsigned_shift;
457   rtx tmp, carries;
458 
459   reverse_unsigned_shift = (binoptab == ashl_optab ? lshr_optab : ashl_optab);
460   unsigned_shift = (binoptab == ashl_optab ? ashl_optab : lshr_optab);
461 
462   /* The low OP1 bits of INTO_TARGET come from the high bits of OUTOF_INPUT.
463      We therefore need to shift OUTOF_INPUT by (BITS_PER_WORD - OP1) bits in
464      the opposite direction to BINOPTAB.  */
465   if (CONSTANT_P (op1) || shift_mask >= BITS_PER_WORD)
466     {
467       carries = outof_input;
468       tmp = immed_wide_int_const (wi::shwi (BITS_PER_WORD,
469 					    op1_mode), op1_mode);
470       tmp = simplify_expand_binop (op1_mode, sub_optab, tmp, op1,
471 				   0, true, methods);
472     }
473   else
474     {
475       /* We must avoid shifting by BITS_PER_WORD bits since that is either
476 	 the same as a zero shift (if shift_mask == BITS_PER_WORD - 1) or
477 	 has unknown behavior.  Do a single shift first, then shift by the
478 	 remainder.  It's OK to use ~OP1 as the remainder if shift counts
479 	 are truncated to the mode size.  */
480       carries = expand_binop (word_mode, reverse_unsigned_shift,
481 			      outof_input, const1_rtx, 0, unsignedp, methods);
482       if (shift_mask == BITS_PER_WORD - 1)
483 	{
484 	  tmp = immed_wide_int_const
485 	    (wi::minus_one (GET_MODE_PRECISION (op1_mode)), op1_mode);
486 	  tmp = simplify_expand_binop (op1_mode, xor_optab, op1, tmp,
487 				       0, true, methods);
488 	}
489       else
490 	{
491 	  tmp = immed_wide_int_const (wi::shwi (BITS_PER_WORD - 1,
492 						op1_mode), op1_mode);
493 	  tmp = simplify_expand_binop (op1_mode, sub_optab, tmp, op1,
494 				       0, true, methods);
495 	}
496     }
497   if (tmp == 0 || carries == 0)
498     return false;
499   carries = expand_binop (word_mode, reverse_unsigned_shift,
500 			  carries, tmp, 0, unsignedp, methods);
501   if (carries == 0)
502     return false;
503 
504   /* Shift INTO_INPUT logically by OP1.  This is the last use of INTO_INPUT
505      so the result can go directly into INTO_TARGET if convenient.  */
506   tmp = expand_binop (word_mode, unsigned_shift, into_input, op1,
507 		      into_target, unsignedp, methods);
508   if (tmp == 0)
509     return false;
510 
511   /* Now OR in the bits carried over from OUTOF_INPUT.  */
512   if (!force_expand_binop (word_mode, ior_optab, tmp, carries,
513 			   into_target, unsignedp, methods))
514     return false;
515 
516   /* Use a standard word_mode shift for the out-of half.  */
517   if (outof_target != 0)
518     if (!force_expand_binop (word_mode, binoptab, outof_input, op1,
519 			     outof_target, unsignedp, methods))
520       return false;
521 
522   return true;
523 }
524 
525 
526 /* Try implementing expand_doubleword_shift using conditional moves.
527    The shift is by < BITS_PER_WORD if (CMP_CODE CMP1 CMP2) is true,
528    otherwise it is by >= BITS_PER_WORD.  SUBWORD_OP1 and SUPERWORD_OP1
529    are the shift counts to use in the former and latter case.  All other
530    arguments are the same as the parent routine.  */
531 
532 static bool
expand_doubleword_shift_condmove(scalar_int_mode op1_mode,optab binoptab,enum rtx_code cmp_code,rtx cmp1,rtx cmp2,rtx outof_input,rtx into_input,rtx subword_op1,rtx superword_op1,rtx outof_target,rtx into_target,int unsignedp,enum optab_methods methods,unsigned HOST_WIDE_INT shift_mask)533 expand_doubleword_shift_condmove (scalar_int_mode op1_mode, optab binoptab,
534 				  enum rtx_code cmp_code, rtx cmp1, rtx cmp2,
535 				  rtx outof_input, rtx into_input,
536 				  rtx subword_op1, rtx superword_op1,
537 				  rtx outof_target, rtx into_target,
538 				  int unsignedp, enum optab_methods methods,
539 				  unsigned HOST_WIDE_INT shift_mask)
540 {
541   rtx outof_superword, into_superword;
542 
543   /* Put the superword version of the output into OUTOF_SUPERWORD and
544      INTO_SUPERWORD.  */
545   outof_superword = outof_target != 0 ? gen_reg_rtx (word_mode) : 0;
546   if (outof_target != 0 && subword_op1 == superword_op1)
547     {
548       /* The value INTO_TARGET >> SUBWORD_OP1, which we later store in
549 	 OUTOF_TARGET, is the same as the value of INTO_SUPERWORD.  */
550       into_superword = outof_target;
551       if (!expand_superword_shift (binoptab, outof_input, superword_op1,
552 				   outof_superword, 0, unsignedp, methods))
553 	return false;
554     }
555   else
556     {
557       into_superword = gen_reg_rtx (word_mode);
558       if (!expand_superword_shift (binoptab, outof_input, superword_op1,
559 				   outof_superword, into_superword,
560 				   unsignedp, methods))
561 	return false;
562     }
563 
564   /* Put the subword version directly in OUTOF_TARGET and INTO_TARGET.  */
565   if (!expand_subword_shift (op1_mode, binoptab,
566 			     outof_input, into_input, subword_op1,
567 			     outof_target, into_target,
568 			     unsignedp, methods, shift_mask))
569     return false;
570 
571   /* Select between them.  Do the INTO half first because INTO_SUPERWORD
572      might be the current value of OUTOF_TARGET.  */
573   if (!emit_conditional_move (into_target, cmp_code, cmp1, cmp2, op1_mode,
574 			      into_target, into_superword, word_mode, false))
575     return false;
576 
577   if (outof_target != 0)
578     if (!emit_conditional_move (outof_target, cmp_code, cmp1, cmp2, op1_mode,
579 				outof_target, outof_superword,
580 				word_mode, false))
581       return false;
582 
583   return true;
584 }
585 
586 /* Expand a doubleword shift (ashl, ashr or lshr) using word-mode shifts.
587    OUTOF_INPUT and INTO_INPUT are the two word-sized halves of the first
588    input operand; the shift moves bits in the direction OUTOF_INPUT->
589    INTO_TARGET.  OUTOF_TARGET and INTO_TARGET are the equivalent words
590    of the target.  OP1 is the shift count and OP1_MODE is its mode.
591    If OP1 is constant, it will have been truncated as appropriate
592    and is known to be nonzero.
593 
594    If SHIFT_MASK is zero, the result of word shifts is undefined when the
595    shift count is outside the range [0, BITS_PER_WORD).  This routine must
596    avoid generating such shifts for OP1s in the range [0, BITS_PER_WORD * 2).
597 
598    If SHIFT_MASK is nonzero, all word-mode shift counts are effectively
599    masked by it and shifts in the range [BITS_PER_WORD, SHIFT_MASK) will
600    fill with zeros or sign bits as appropriate.
601 
602    If SHIFT_MASK is BITS_PER_WORD - 1, this routine will synthesize
603    a doubleword shift whose equivalent mask is BITS_PER_WORD * 2 - 1.
604    Doing this preserves semantics required by SHIFT_COUNT_TRUNCATED.
605    In all other cases, shifts by values outside [0, BITS_PER_UNIT * 2)
606    are undefined.
607 
608    BINOPTAB, UNSIGNEDP and METHODS are as for expand_binop.  This function
609    may not use INTO_INPUT after modifying INTO_TARGET, and similarly for
610    OUTOF_INPUT and OUTOF_TARGET.  OUTOF_TARGET can be null if the parent
611    function wants to calculate it itself.
612 
613    Return true if the shift could be successfully synthesized.  */
614 
615 static bool
expand_doubleword_shift(scalar_int_mode op1_mode,optab binoptab,rtx outof_input,rtx into_input,rtx op1,rtx outof_target,rtx into_target,int unsignedp,enum optab_methods methods,unsigned HOST_WIDE_INT shift_mask)616 expand_doubleword_shift (scalar_int_mode op1_mode, optab binoptab,
617 			 rtx outof_input, rtx into_input, rtx op1,
618 			 rtx outof_target, rtx into_target,
619 			 int unsignedp, enum optab_methods methods,
620 			 unsigned HOST_WIDE_INT shift_mask)
621 {
622   rtx superword_op1, tmp, cmp1, cmp2;
623   enum rtx_code cmp_code;
624 
625   /* See if word-mode shifts by BITS_PER_WORD...BITS_PER_WORD * 2 - 1 will
626      fill the result with sign or zero bits as appropriate.  If so, the value
627      of OUTOF_TARGET will always be (SHIFT OUTOF_INPUT OP1).   Recursively call
628      this routine to calculate INTO_TARGET (which depends on both OUTOF_INPUT
629      and INTO_INPUT), then emit code to set up OUTOF_TARGET.
630 
631      This isn't worthwhile for constant shifts since the optimizers will
632      cope better with in-range shift counts.  */
633   if (shift_mask >= BITS_PER_WORD
634       && outof_target != 0
635       && !CONSTANT_P (op1))
636     {
637       if (!expand_doubleword_shift (op1_mode, binoptab,
638 				    outof_input, into_input, op1,
639 				    0, into_target,
640 				    unsignedp, methods, shift_mask))
641 	return false;
642       if (!force_expand_binop (word_mode, binoptab, outof_input, op1,
643 			       outof_target, unsignedp, methods))
644 	return false;
645       return true;
646     }
647 
648   /* Set CMP_CODE, CMP1 and CMP2 so that the rtx (CMP_CODE CMP1 CMP2)
649      is true when the effective shift value is less than BITS_PER_WORD.
650      Set SUPERWORD_OP1 to the shift count that should be used to shift
651      OUTOF_INPUT into INTO_TARGET when the condition is false.  */
652   tmp = immed_wide_int_const (wi::shwi (BITS_PER_WORD, op1_mode), op1_mode);
653   if (!CONSTANT_P (op1) && shift_mask == BITS_PER_WORD - 1)
654     {
655       /* Set CMP1 to OP1 & BITS_PER_WORD.  The result is zero iff OP1
656 	 is a subword shift count.  */
657       cmp1 = simplify_expand_binop (op1_mode, and_optab, op1, tmp,
658 				    0, true, methods);
659       cmp2 = CONST0_RTX (op1_mode);
660       cmp_code = EQ;
661       superword_op1 = op1;
662     }
663   else
664     {
665       /* Set CMP1 to OP1 - BITS_PER_WORD.  */
666       cmp1 = simplify_expand_binop (op1_mode, sub_optab, op1, tmp,
667 				    0, true, methods);
668       cmp2 = CONST0_RTX (op1_mode);
669       cmp_code = LT;
670       superword_op1 = cmp1;
671     }
672   if (cmp1 == 0)
673     return false;
674 
675   /* If we can compute the condition at compile time, pick the
676      appropriate subroutine.  */
677   tmp = simplify_relational_operation (cmp_code, SImode, op1_mode, cmp1, cmp2);
678   if (tmp != 0 && CONST_INT_P (tmp))
679     {
680       if (tmp == const0_rtx)
681 	return expand_superword_shift (binoptab, outof_input, superword_op1,
682 				       outof_target, into_target,
683 				       unsignedp, methods);
684       else
685 	return expand_subword_shift (op1_mode, binoptab,
686 				     outof_input, into_input, op1,
687 				     outof_target, into_target,
688 				     unsignedp, methods, shift_mask);
689     }
690 
691   /* Try using conditional moves to generate straight-line code.  */
692   if (HAVE_conditional_move)
693     {
694       rtx_insn *start = get_last_insn ();
695       if (expand_doubleword_shift_condmove (op1_mode, binoptab,
696 					    cmp_code, cmp1, cmp2,
697 					    outof_input, into_input,
698 					    op1, superword_op1,
699 					    outof_target, into_target,
700 					    unsignedp, methods, shift_mask))
701 	return true;
702       delete_insns_since (start);
703     }
704 
705   /* As a last resort, use branches to select the correct alternative.  */
706   rtx_code_label *subword_label = gen_label_rtx ();
707   rtx_code_label *done_label = gen_label_rtx ();
708 
709   NO_DEFER_POP;
710   do_compare_rtx_and_jump (cmp1, cmp2, cmp_code, false, op1_mode,
711 			   0, 0, subword_label,
712 			   profile_probability::uninitialized ());
713   OK_DEFER_POP;
714 
715   if (!expand_superword_shift (binoptab, outof_input, superword_op1,
716 			       outof_target, into_target,
717 			       unsignedp, methods))
718     return false;
719 
720   emit_jump_insn (targetm.gen_jump (done_label));
721   emit_barrier ();
722   emit_label (subword_label);
723 
724   if (!expand_subword_shift (op1_mode, binoptab,
725 			     outof_input, into_input, op1,
726 			     outof_target, into_target,
727 			     unsignedp, methods, shift_mask))
728     return false;
729 
730   emit_label (done_label);
731   return true;
732 }
733 
734 /* Subroutine of expand_binop.  Perform a double word multiplication of
735    operands OP0 and OP1 both of mode MODE, which is exactly twice as wide
736    as the target's word_mode.  This function return NULL_RTX if anything
737    goes wrong, in which case it may have already emitted instructions
738    which need to be deleted.
739 
740    If we want to multiply two two-word values and have normal and widening
741    multiplies of single-word values, we can do this with three smaller
742    multiplications.
743 
744    The multiplication proceeds as follows:
745 			         _______________________
746 			        [__op0_high_|__op0_low__]
747 			         _______________________
748         *			[__op1_high_|__op1_low__]
749         _______________________________________________
750 			         _______________________
751     (1)				[__op0_low__*__op1_low__]
752 		     _______________________
753     (2a)	    [__op0_low__*__op1_high_]
754 		     _______________________
755     (2b)	    [__op0_high_*__op1_low__]
756          _______________________
757     (3) [__op0_high_*__op1_high_]
758 
759 
760   This gives a 4-word result.  Since we are only interested in the
761   lower 2 words, partial result (3) and the upper words of (2a) and
762   (2b) don't need to be calculated.  Hence (2a) and (2b) can be
763   calculated using non-widening multiplication.
764 
765   (1), however, needs to be calculated with an unsigned widening
766   multiplication.  If this operation is not directly supported we
767   try using a signed widening multiplication and adjust the result.
768   This adjustment works as follows:
769 
770       If both operands are positive then no adjustment is needed.
771 
772       If the operands have different signs, for example op0_low < 0 and
773       op1_low >= 0, the instruction treats the most significant bit of
774       op0_low as a sign bit instead of a bit with significance
775       2**(BITS_PER_WORD-1), i.e. the instruction multiplies op1_low
776       with 2**BITS_PER_WORD - op0_low, and two's complements the
777       result.  Conclusion: We need to add op1_low * 2**BITS_PER_WORD to
778       the result.
779 
780       Similarly, if both operands are negative, we need to add
781       (op0_low + op1_low) * 2**BITS_PER_WORD.
782 
783       We use a trick to adjust quickly.  We logically shift op0_low right
784       (op1_low) BITS_PER_WORD-1 steps to get 0 or 1, and add this to
785       op0_high (op1_high) before it is used to calculate 2b (2a).  If no
786       logical shift exists, we do an arithmetic right shift and subtract
787       the 0 or -1.  */
788 
789 static rtx
expand_doubleword_mult(machine_mode mode,rtx op0,rtx op1,rtx target,bool umulp,enum optab_methods methods)790 expand_doubleword_mult (machine_mode mode, rtx op0, rtx op1, rtx target,
791 		       bool umulp, enum optab_methods methods)
792 {
793   int low = (WORDS_BIG_ENDIAN ? 1 : 0);
794   int high = (WORDS_BIG_ENDIAN ? 0 : 1);
795   rtx wordm1 = (umulp ? NULL_RTX
796 		: gen_int_shift_amount (word_mode, BITS_PER_WORD - 1));
797   rtx product, adjust, product_high, temp;
798 
799   rtx op0_high = operand_subword_force (op0, high, mode);
800   rtx op0_low = operand_subword_force (op0, low, mode);
801   rtx op1_high = operand_subword_force (op1, high, mode);
802   rtx op1_low = operand_subword_force (op1, low, mode);
803 
804   /* If we're using an unsigned multiply to directly compute the product
805      of the low-order words of the operands and perform any required
806      adjustments of the operands, we begin by trying two more multiplications
807      and then computing the appropriate sum.
808 
809      We have checked above that the required addition is provided.
810      Full-word addition will normally always succeed, especially if
811      it is provided at all, so we don't worry about its failure.  The
812      multiplication may well fail, however, so we do handle that.  */
813 
814   if (!umulp)
815     {
816       /* ??? This could be done with emit_store_flag where available.  */
817       temp = expand_binop (word_mode, lshr_optab, op0_low, wordm1,
818 			   NULL_RTX, 1, methods);
819       if (temp)
820 	op0_high = expand_binop (word_mode, add_optab, op0_high, temp,
821 				 NULL_RTX, 0, OPTAB_DIRECT);
822       else
823 	{
824 	  temp = expand_binop (word_mode, ashr_optab, op0_low, wordm1,
825 			       NULL_RTX, 0, methods);
826 	  if (!temp)
827 	    return NULL_RTX;
828 	  op0_high = expand_binop (word_mode, sub_optab, op0_high, temp,
829 				   NULL_RTX, 0, OPTAB_DIRECT);
830 	}
831 
832       if (!op0_high)
833 	return NULL_RTX;
834     }
835 
836   adjust = expand_binop (word_mode, smul_optab, op0_high, op1_low,
837 			 NULL_RTX, 0, OPTAB_DIRECT);
838   if (!adjust)
839     return NULL_RTX;
840 
841   /* OP0_HIGH should now be dead.  */
842 
843   if (!umulp)
844     {
845       /* ??? This could be done with emit_store_flag where available.  */
846       temp = expand_binop (word_mode, lshr_optab, op1_low, wordm1,
847 			   NULL_RTX, 1, methods);
848       if (temp)
849 	op1_high = expand_binop (word_mode, add_optab, op1_high, temp,
850 				 NULL_RTX, 0, OPTAB_DIRECT);
851       else
852 	{
853 	  temp = expand_binop (word_mode, ashr_optab, op1_low, wordm1,
854 			       NULL_RTX, 0, methods);
855 	  if (!temp)
856 	    return NULL_RTX;
857 	  op1_high = expand_binop (word_mode, sub_optab, op1_high, temp,
858 				   NULL_RTX, 0, OPTAB_DIRECT);
859 	}
860 
861       if (!op1_high)
862 	return NULL_RTX;
863     }
864 
865   temp = expand_binop (word_mode, smul_optab, op1_high, op0_low,
866 		       NULL_RTX, 0, OPTAB_DIRECT);
867   if (!temp)
868     return NULL_RTX;
869 
870   /* OP1_HIGH should now be dead.  */
871 
872   adjust = expand_binop (word_mode, add_optab, adjust, temp,
873 			 NULL_RTX, 0, OPTAB_DIRECT);
874 
875   if (target && !REG_P (target))
876     target = NULL_RTX;
877 
878   /* *_widen_optab needs to determine operand mode, make sure at least
879      one operand has non-VOID mode.  */
880   if (GET_MODE (op0_low) == VOIDmode && GET_MODE (op1_low) == VOIDmode)
881     op0_low = force_reg (word_mode, op0_low);
882 
883   if (umulp)
884     product = expand_binop (mode, umul_widen_optab, op0_low, op1_low,
885 			    target, 1, OPTAB_DIRECT);
886   else
887     product = expand_binop (mode, smul_widen_optab, op0_low, op1_low,
888 			    target, 1, OPTAB_DIRECT);
889 
890   if (!product)
891     return NULL_RTX;
892 
893   product_high = operand_subword (product, high, 1, mode);
894   adjust = expand_binop (word_mode, add_optab, product_high, adjust,
895 			 NULL_RTX, 0, OPTAB_DIRECT);
896   emit_move_insn (product_high, adjust);
897   return product;
898 }
899 
900 /* Wrapper around expand_binop which takes an rtx code to specify
901    the operation to perform, not an optab pointer.  All other
902    arguments are the same.  */
903 rtx
expand_simple_binop(machine_mode mode,enum rtx_code code,rtx op0,rtx op1,rtx target,int unsignedp,enum optab_methods methods)904 expand_simple_binop (machine_mode mode, enum rtx_code code, rtx op0,
905 		     rtx op1, rtx target, int unsignedp,
906 		     enum optab_methods methods)
907 {
908   optab binop = code_to_optab (code);
909   gcc_assert (binop);
910 
911   return expand_binop (mode, binop, op0, op1, target, unsignedp, methods);
912 }
913 
914 /* Return whether OP0 and OP1 should be swapped when expanding a commutative
915    binop.  Order them according to commutative_operand_precedence and, if
916    possible, try to put TARGET or a pseudo first.  */
917 static bool
swap_commutative_operands_with_target(rtx target,rtx op0,rtx op1)918 swap_commutative_operands_with_target (rtx target, rtx op0, rtx op1)
919 {
920   int op0_prec = commutative_operand_precedence (op0);
921   int op1_prec = commutative_operand_precedence (op1);
922 
923   if (op0_prec < op1_prec)
924     return true;
925 
926   if (op0_prec > op1_prec)
927     return false;
928 
929   /* With equal precedence, both orders are ok, but it is better if the
930      first operand is TARGET, or if both TARGET and OP0 are pseudos.  */
931   if (target == 0 || REG_P (target))
932     return (REG_P (op1) && !REG_P (op0)) || target == op1;
933   else
934     return rtx_equal_p (op1, target);
935 }
936 
937 /* Return true if BINOPTAB implements a shift operation.  */
938 
939 static bool
shift_optab_p(optab binoptab)940 shift_optab_p (optab binoptab)
941 {
942   switch (optab_to_code (binoptab))
943     {
944     case ASHIFT:
945     case SS_ASHIFT:
946     case US_ASHIFT:
947     case ASHIFTRT:
948     case LSHIFTRT:
949     case ROTATE:
950     case ROTATERT:
951       return true;
952 
953     default:
954       return false;
955     }
956 }
957 
958 /* Return true if BINOPTAB implements a commutative binary operation.  */
959 
960 static bool
commutative_optab_p(optab binoptab)961 commutative_optab_p (optab binoptab)
962 {
963   return (GET_RTX_CLASS (optab_to_code (binoptab)) == RTX_COMM_ARITH
964 	  || binoptab == smul_widen_optab
965 	  || binoptab == umul_widen_optab
966 	  || binoptab == smul_highpart_optab
967 	  || binoptab == umul_highpart_optab);
968 }
969 
970 /* X is to be used in mode MODE as operand OPN to BINOPTAB.  If we're
971    optimizing, and if the operand is a constant that costs more than
972    1 instruction, force the constant into a register and return that
973    register.  Return X otherwise.  UNSIGNEDP says whether X is unsigned.  */
974 
975 static rtx
avoid_expensive_constant(machine_mode mode,optab binoptab,int opn,rtx x,bool unsignedp)976 avoid_expensive_constant (machine_mode mode, optab binoptab,
977 			  int opn, rtx x, bool unsignedp)
978 {
979   bool speed = optimize_insn_for_speed_p ();
980 
981   if (mode != VOIDmode
982       && optimize
983       && CONSTANT_P (x)
984       && (rtx_cost (x, mode, optab_to_code (binoptab), opn, speed)
985 	  > set_src_cost (x, mode, speed)))
986     {
987       if (CONST_INT_P (x))
988 	{
989 	  HOST_WIDE_INT intval = trunc_int_for_mode (INTVAL (x), mode);
990 	  if (intval != INTVAL (x))
991 	    x = GEN_INT (intval);
992 	}
993       else
994 	x = convert_modes (mode, VOIDmode, x, unsignedp);
995       x = force_reg (mode, x);
996     }
997   return x;
998 }
999 
1000 /* Helper function for expand_binop: handle the case where there
1001    is an insn ICODE that directly implements the indicated operation.
1002    Returns null if this is not possible.  */
1003 static rtx
expand_binop_directly(enum insn_code icode,machine_mode mode,optab binoptab,rtx op0,rtx op1,rtx target,int unsignedp,enum optab_methods methods,rtx_insn * last)1004 expand_binop_directly (enum insn_code icode, machine_mode mode, optab binoptab,
1005 		       rtx op0, rtx op1,
1006 		       rtx target, int unsignedp, enum optab_methods methods,
1007 		       rtx_insn *last)
1008 {
1009   machine_mode xmode0 = insn_data[(int) icode].operand[1].mode;
1010   machine_mode xmode1 = insn_data[(int) icode].operand[2].mode;
1011   machine_mode mode0, mode1, tmp_mode;
1012   struct expand_operand ops[3];
1013   bool commutative_p;
1014   rtx_insn *pat;
1015   rtx xop0 = op0, xop1 = op1;
1016   bool canonicalize_op1 = false;
1017 
1018   /* If it is a commutative operator and the modes would match
1019      if we would swap the operands, we can save the conversions.  */
1020   commutative_p = commutative_optab_p (binoptab);
1021   if (commutative_p
1022       && GET_MODE (xop0) != xmode0 && GET_MODE (xop1) != xmode1
1023       && GET_MODE (xop0) == xmode1 && GET_MODE (xop1) == xmode1)
1024     std::swap (xop0, xop1);
1025 
1026   /* If we are optimizing, force expensive constants into a register.  */
1027   xop0 = avoid_expensive_constant (xmode0, binoptab, 0, xop0, unsignedp);
1028   if (!shift_optab_p (binoptab))
1029     xop1 = avoid_expensive_constant (xmode1, binoptab, 1, xop1, unsignedp);
1030   else
1031     /* Shifts and rotates often use a different mode for op1 from op0;
1032        for VOIDmode constants we don't know the mode, so force it
1033        to be canonicalized using convert_modes.  */
1034     canonicalize_op1 = true;
1035 
1036   /* In case the insn wants input operands in modes different from
1037      those of the actual operands, convert the operands.  It would
1038      seem that we don't need to convert CONST_INTs, but we do, so
1039      that they're properly zero-extended, sign-extended or truncated
1040      for their mode.  */
1041 
1042   mode0 = GET_MODE (xop0) != VOIDmode ? GET_MODE (xop0) : mode;
1043   if (xmode0 != VOIDmode && xmode0 != mode0)
1044     {
1045       xop0 = convert_modes (xmode0, mode0, xop0, unsignedp);
1046       mode0 = xmode0;
1047     }
1048 
1049   mode1 = ((GET_MODE (xop1) != VOIDmode || canonicalize_op1)
1050 	   ? GET_MODE (xop1) : mode);
1051   if (xmode1 != VOIDmode && xmode1 != mode1)
1052     {
1053       xop1 = convert_modes (xmode1, mode1, xop1, unsignedp);
1054       mode1 = xmode1;
1055     }
1056 
1057   /* If operation is commutative,
1058      try to make the first operand a register.
1059      Even better, try to make it the same as the target.
1060      Also try to make the last operand a constant.  */
1061   if (commutative_p
1062       && swap_commutative_operands_with_target (target, xop0, xop1))
1063     std::swap (xop0, xop1);
1064 
1065   /* Now, if insn's predicates don't allow our operands, put them into
1066      pseudo regs.  */
1067 
1068   if (binoptab == vec_pack_trunc_optab
1069       || binoptab == vec_pack_usat_optab
1070       || binoptab == vec_pack_ssat_optab
1071       || binoptab == vec_pack_ufix_trunc_optab
1072       || binoptab == vec_pack_sfix_trunc_optab)
1073     {
1074       /* The mode of the result is different then the mode of the
1075 	 arguments.  */
1076       tmp_mode = insn_data[(int) icode].operand[0].mode;
1077       if (VECTOR_MODE_P (mode)
1078 	  && maybe_ne (GET_MODE_NUNITS (tmp_mode), 2 * GET_MODE_NUNITS (mode)))
1079 	{
1080 	  delete_insns_since (last);
1081 	  return NULL_RTX;
1082 	}
1083     }
1084   else
1085     tmp_mode = mode;
1086 
1087   create_output_operand (&ops[0], target, tmp_mode);
1088   create_input_operand (&ops[1], xop0, mode0);
1089   create_input_operand (&ops[2], xop1, mode1);
1090   pat = maybe_gen_insn (icode, 3, ops);
1091   if (pat)
1092     {
1093       /* If PAT is composed of more than one insn, try to add an appropriate
1094 	 REG_EQUAL note to it.  If we can't because TEMP conflicts with an
1095 	 operand, call expand_binop again, this time without a target.  */
1096       if (INSN_P (pat) && NEXT_INSN (pat) != NULL_RTX
1097 	  && ! add_equal_note (pat, ops[0].value,
1098 			       optab_to_code (binoptab),
1099 			       ops[1].value, ops[2].value, mode0))
1100 	{
1101 	  delete_insns_since (last);
1102 	  return expand_binop (mode, binoptab, op0, op1, NULL_RTX,
1103 			       unsignedp, methods);
1104 	}
1105 
1106       emit_insn (pat);
1107       return ops[0].value;
1108     }
1109   delete_insns_since (last);
1110   return NULL_RTX;
1111 }
1112 
1113 /* Generate code to perform an operation specified by BINOPTAB
1114    on operands OP0 and OP1, with result having machine-mode MODE.
1115 
1116    UNSIGNEDP is for the case where we have to widen the operands
1117    to perform the operation.  It says to use zero-extension.
1118 
1119    If TARGET is nonzero, the value
1120    is generated there, if it is convenient to do so.
1121    In all cases an rtx is returned for the locus of the value;
1122    this may or may not be TARGET.  */
1123 
1124 rtx
expand_binop(machine_mode mode,optab binoptab,rtx op0,rtx op1,rtx target,int unsignedp,enum optab_methods methods)1125 expand_binop (machine_mode mode, optab binoptab, rtx op0, rtx op1,
1126 	      rtx target, int unsignedp, enum optab_methods methods)
1127 {
1128   enum optab_methods next_methods
1129     = (methods == OPTAB_LIB || methods == OPTAB_LIB_WIDEN
1130        ? OPTAB_WIDEN : methods);
1131   enum mode_class mclass;
1132   enum insn_code icode;
1133   machine_mode wider_mode;
1134   scalar_int_mode int_mode;
1135   rtx libfunc;
1136   rtx temp;
1137   rtx_insn *entry_last = get_last_insn ();
1138   rtx_insn *last;
1139 
1140   mclass = GET_MODE_CLASS (mode);
1141 
1142   /* If subtracting an integer constant, convert this into an addition of
1143      the negated constant.  */
1144 
1145   if (binoptab == sub_optab && CONST_INT_P (op1))
1146     {
1147       op1 = negate_rtx (mode, op1);
1148       binoptab = add_optab;
1149     }
1150   /* For shifts, constant invalid op1 might be expanded from different
1151      mode than MODE.  As those are invalid, force them to a register
1152      to avoid further problems during expansion.  */
1153   else if (CONST_INT_P (op1)
1154 	   && shift_optab_p (binoptab)
1155 	   && UINTVAL (op1) >= GET_MODE_BITSIZE (GET_MODE_INNER (mode)))
1156     {
1157       op1 = gen_int_mode (INTVAL (op1), GET_MODE_INNER (mode));
1158       op1 = force_reg (GET_MODE_INNER (mode), op1);
1159     }
1160 
1161   /* Record where to delete back to if we backtrack.  */
1162   last = get_last_insn ();
1163 
1164   /* If we can do it with a three-operand insn, do so.  */
1165 
1166   if (methods != OPTAB_MUST_WIDEN)
1167     {
1168       if (convert_optab_p (binoptab))
1169 	{
1170 	  machine_mode from_mode = widened_mode (mode, op0, op1);
1171 	  icode = find_widening_optab_handler (binoptab, mode, from_mode);
1172 	}
1173       else
1174 	icode = optab_handler (binoptab, mode);
1175       if (icode != CODE_FOR_nothing)
1176 	{
1177 	  temp = expand_binop_directly (icode, mode, binoptab, op0, op1,
1178 					target, unsignedp, methods, last);
1179 	  if (temp)
1180 	    return temp;
1181 	}
1182     }
1183 
1184   /* If we were trying to rotate, and that didn't work, try rotating
1185      the other direction before falling back to shifts and bitwise-or.  */
1186   if (((binoptab == rotl_optab
1187 	&& (icode = optab_handler (rotr_optab, mode)) != CODE_FOR_nothing)
1188        || (binoptab == rotr_optab
1189 	   && (icode = optab_handler (rotl_optab, mode)) != CODE_FOR_nothing))
1190       && is_int_mode (mode, &int_mode))
1191     {
1192       optab otheroptab = (binoptab == rotl_optab ? rotr_optab : rotl_optab);
1193       rtx newop1;
1194       unsigned int bits = GET_MODE_PRECISION (int_mode);
1195 
1196       if (CONST_INT_P (op1))
1197 	newop1 = gen_int_shift_amount (int_mode, bits - INTVAL (op1));
1198       else if (targetm.shift_truncation_mask (int_mode) == bits - 1)
1199         newop1 = negate_rtx (GET_MODE (op1), op1);
1200       else
1201         newop1 = expand_binop (GET_MODE (op1), sub_optab,
1202 			       gen_int_mode (bits, GET_MODE (op1)), op1,
1203 			       NULL_RTX, unsignedp, OPTAB_DIRECT);
1204 
1205       temp = expand_binop_directly (icode, int_mode, otheroptab, op0, newop1,
1206 				    target, unsignedp, methods, last);
1207       if (temp)
1208 	return temp;
1209     }
1210 
1211   /* If this is a multiply, see if we can do a widening operation that
1212      takes operands of this mode and makes a wider mode.  */
1213 
1214   if (binoptab == smul_optab
1215       && GET_MODE_2XWIDER_MODE (mode).exists (&wider_mode)
1216       && (convert_optab_handler ((unsignedp
1217 				  ? umul_widen_optab
1218 				  : smul_widen_optab),
1219 				 wider_mode, mode) != CODE_FOR_nothing))
1220     {
1221       /* *_widen_optab needs to determine operand mode, make sure at least
1222 	 one operand has non-VOID mode.  */
1223       if (GET_MODE (op0) == VOIDmode && GET_MODE (op1) == VOIDmode)
1224 	op0 = force_reg (mode, op0);
1225       temp = expand_binop (wider_mode,
1226 			   unsignedp ? umul_widen_optab : smul_widen_optab,
1227 			   op0, op1, NULL_RTX, unsignedp, OPTAB_DIRECT);
1228 
1229       if (temp != 0)
1230 	{
1231 	  if (GET_MODE_CLASS (mode) == MODE_INT
1232 	      && TRULY_NOOP_TRUNCATION_MODES_P (mode, GET_MODE (temp)))
1233 	    return gen_lowpart (mode, temp);
1234 	  else
1235 	    return convert_to_mode (mode, temp, unsignedp);
1236 	}
1237     }
1238 
1239   /* If this is a vector shift by a scalar, see if we can do a vector
1240      shift by a vector.  If so, broadcast the scalar into a vector.  */
1241   if (mclass == MODE_VECTOR_INT)
1242     {
1243       optab otheroptab = unknown_optab;
1244 
1245       if (binoptab == ashl_optab)
1246 	otheroptab = vashl_optab;
1247       else if (binoptab == ashr_optab)
1248 	otheroptab = vashr_optab;
1249       else if (binoptab == lshr_optab)
1250 	otheroptab = vlshr_optab;
1251       else if (binoptab == rotl_optab)
1252 	otheroptab = vrotl_optab;
1253       else if (binoptab == rotr_optab)
1254 	otheroptab = vrotr_optab;
1255 
1256       if (otheroptab
1257 	  && (icode = optab_handler (otheroptab, mode)) != CODE_FOR_nothing)
1258 	{
1259 	  /* The scalar may have been extended to be too wide.  Truncate
1260 	     it back to the proper size to fit in the broadcast vector.  */
1261 	  scalar_mode inner_mode = GET_MODE_INNER (mode);
1262 	  if (!CONST_INT_P (op1)
1263 	      && (GET_MODE_BITSIZE (as_a <scalar_int_mode> (GET_MODE (op1)))
1264 		  > GET_MODE_BITSIZE (inner_mode)))
1265 	    op1 = force_reg (inner_mode,
1266 			     simplify_gen_unary (TRUNCATE, inner_mode, op1,
1267 						 GET_MODE (op1)));
1268 	  rtx vop1 = expand_vector_broadcast (mode, op1);
1269 	  if (vop1)
1270 	    {
1271 	      temp = expand_binop_directly (icode, mode, otheroptab, op0, vop1,
1272 					    target, unsignedp, methods, last);
1273 	      if (temp)
1274 		return temp;
1275 	    }
1276 	}
1277     }
1278 
1279   /* Look for a wider mode of the same class for which we think we
1280      can open-code the operation.  Check for a widening multiply at the
1281      wider mode as well.  */
1282 
1283   if (CLASS_HAS_WIDER_MODES_P (mclass)
1284       && methods != OPTAB_DIRECT && methods != OPTAB_LIB)
1285     FOR_EACH_WIDER_MODE (wider_mode, mode)
1286       {
1287 	machine_mode next_mode;
1288 	if (optab_handler (binoptab, wider_mode) != CODE_FOR_nothing
1289 	    || (binoptab == smul_optab
1290 		&& GET_MODE_WIDER_MODE (wider_mode).exists (&next_mode)
1291 		&& (find_widening_optab_handler ((unsignedp
1292 						  ? umul_widen_optab
1293 						  : smul_widen_optab),
1294 						 next_mode, mode)
1295 		    != CODE_FOR_nothing)))
1296 	  {
1297 	    rtx xop0 = op0, xop1 = op1;
1298 	    int no_extend = 0;
1299 
1300 	    /* For certain integer operations, we need not actually extend
1301 	       the narrow operands, as long as we will truncate
1302 	       the results to the same narrowness.  */
1303 
1304 	    if ((binoptab == ior_optab || binoptab == and_optab
1305 		 || binoptab == xor_optab
1306 		 || binoptab == add_optab || binoptab == sub_optab
1307 		 || binoptab == smul_optab || binoptab == ashl_optab)
1308 		&& mclass == MODE_INT)
1309 	      {
1310 		no_extend = 1;
1311 		xop0 = avoid_expensive_constant (mode, binoptab, 0,
1312 						 xop0, unsignedp);
1313 		if (binoptab != ashl_optab)
1314 		  xop1 = avoid_expensive_constant (mode, binoptab, 1,
1315 						   xop1, unsignedp);
1316 	      }
1317 
1318 	    xop0 = widen_operand (xop0, wider_mode, mode, unsignedp, no_extend);
1319 
1320 	    /* The second operand of a shift must always be extended.  */
1321 	    xop1 = widen_operand (xop1, wider_mode, mode, unsignedp,
1322 				  no_extend && binoptab != ashl_optab);
1323 
1324 	    temp = expand_binop (wider_mode, binoptab, xop0, xop1, NULL_RTX,
1325 				 unsignedp, OPTAB_DIRECT);
1326 	    if (temp)
1327 	      {
1328 		if (mclass != MODE_INT
1329                     || !TRULY_NOOP_TRUNCATION_MODES_P (mode, wider_mode))
1330 		  {
1331 		    if (target == 0)
1332 		      target = gen_reg_rtx (mode);
1333 		    convert_move (target, temp, 0);
1334 		    return target;
1335 		  }
1336 		else
1337 		  return gen_lowpart (mode, temp);
1338 	      }
1339 	    else
1340 	      delete_insns_since (last);
1341 	  }
1342       }
1343 
1344   /* If operation is commutative,
1345      try to make the first operand a register.
1346      Even better, try to make it the same as the target.
1347      Also try to make the last operand a constant.  */
1348   if (commutative_optab_p (binoptab)
1349       && swap_commutative_operands_with_target (target, op0, op1))
1350     std::swap (op0, op1);
1351 
1352   /* These can be done a word at a time.  */
1353   if ((binoptab == and_optab || binoptab == ior_optab || binoptab == xor_optab)
1354       && is_int_mode (mode, &int_mode)
1355       && GET_MODE_SIZE (int_mode) > UNITS_PER_WORD
1356       && optab_handler (binoptab, word_mode) != CODE_FOR_nothing)
1357     {
1358       int i;
1359       rtx_insn *insns;
1360 
1361       /* If TARGET is the same as one of the operands, the REG_EQUAL note
1362 	 won't be accurate, so use a new target.  */
1363       if (target == 0
1364 	  || target == op0
1365 	  || target == op1
1366 	  || !valid_multiword_target_p (target))
1367 	target = gen_reg_rtx (int_mode);
1368 
1369       start_sequence ();
1370 
1371       /* Do the actual arithmetic.  */
1372       for (i = 0; i < GET_MODE_BITSIZE (int_mode) / BITS_PER_WORD; i++)
1373 	{
1374 	  rtx target_piece = operand_subword (target, i, 1, int_mode);
1375 	  rtx x = expand_binop (word_mode, binoptab,
1376 				operand_subword_force (op0, i, int_mode),
1377 				operand_subword_force (op1, i, int_mode),
1378 				target_piece, unsignedp, next_methods);
1379 
1380 	  if (x == 0)
1381 	    break;
1382 
1383 	  if (target_piece != x)
1384 	    emit_move_insn (target_piece, x);
1385 	}
1386 
1387       insns = get_insns ();
1388       end_sequence ();
1389 
1390       if (i == GET_MODE_BITSIZE (int_mode) / BITS_PER_WORD)
1391 	{
1392 	  emit_insn (insns);
1393 	  return target;
1394 	}
1395     }
1396 
1397   /* Synthesize double word shifts from single word shifts.  */
1398   if ((binoptab == lshr_optab || binoptab == ashl_optab
1399        || binoptab == ashr_optab)
1400       && is_int_mode (mode, &int_mode)
1401       && (CONST_INT_P (op1) || optimize_insn_for_speed_p ())
1402       && GET_MODE_SIZE (int_mode) == 2 * UNITS_PER_WORD
1403       && GET_MODE_PRECISION (int_mode) == GET_MODE_BITSIZE (int_mode)
1404       && optab_handler (binoptab, word_mode) != CODE_FOR_nothing
1405       && optab_handler (ashl_optab, word_mode) != CODE_FOR_nothing
1406       && optab_handler (lshr_optab, word_mode) != CODE_FOR_nothing)
1407     {
1408       unsigned HOST_WIDE_INT shift_mask, double_shift_mask;
1409       scalar_int_mode op1_mode;
1410 
1411       double_shift_mask = targetm.shift_truncation_mask (int_mode);
1412       shift_mask = targetm.shift_truncation_mask (word_mode);
1413       op1_mode = (GET_MODE (op1) != VOIDmode
1414 		  ? as_a <scalar_int_mode> (GET_MODE (op1))
1415 		  : word_mode);
1416 
1417       /* Apply the truncation to constant shifts.  */
1418       if (double_shift_mask > 0 && CONST_INT_P (op1))
1419 	op1 = gen_int_mode (INTVAL (op1) & double_shift_mask, op1_mode);
1420 
1421       if (op1 == CONST0_RTX (op1_mode))
1422 	return op0;
1423 
1424       /* Make sure that this is a combination that expand_doubleword_shift
1425 	 can handle.  See the comments there for details.  */
1426       if (double_shift_mask == 0
1427 	  || (shift_mask == BITS_PER_WORD - 1
1428 	      && double_shift_mask == BITS_PER_WORD * 2 - 1))
1429 	{
1430 	  rtx_insn *insns;
1431 	  rtx into_target, outof_target;
1432 	  rtx into_input, outof_input;
1433 	  int left_shift, outof_word;
1434 
1435 	  /* If TARGET is the same as one of the operands, the REG_EQUAL note
1436 	     won't be accurate, so use a new target.  */
1437 	  if (target == 0
1438 	      || target == op0
1439 	      || target == op1
1440 	      || !valid_multiword_target_p (target))
1441 	    target = gen_reg_rtx (int_mode);
1442 
1443 	  start_sequence ();
1444 
1445 	  /* OUTOF_* is the word we are shifting bits away from, and
1446 	     INTO_* is the word that we are shifting bits towards, thus
1447 	     they differ depending on the direction of the shift and
1448 	     WORDS_BIG_ENDIAN.  */
1449 
1450 	  left_shift = binoptab == ashl_optab;
1451 	  outof_word = left_shift ^ ! WORDS_BIG_ENDIAN;
1452 
1453 	  outof_target = operand_subword (target, outof_word, 1, int_mode);
1454 	  into_target = operand_subword (target, 1 - outof_word, 1, int_mode);
1455 
1456 	  outof_input = operand_subword_force (op0, outof_word, int_mode);
1457 	  into_input = operand_subword_force (op0, 1 - outof_word, int_mode);
1458 
1459 	  if (expand_doubleword_shift (op1_mode, binoptab,
1460 				       outof_input, into_input, op1,
1461 				       outof_target, into_target,
1462 				       unsignedp, next_methods, shift_mask))
1463 	    {
1464 	      insns = get_insns ();
1465 	      end_sequence ();
1466 
1467 	      emit_insn (insns);
1468 	      return target;
1469 	    }
1470 	  end_sequence ();
1471 	}
1472     }
1473 
1474   /* Synthesize double word rotates from single word shifts.  */
1475   if ((binoptab == rotl_optab || binoptab == rotr_optab)
1476       && is_int_mode (mode, &int_mode)
1477       && CONST_INT_P (op1)
1478       && GET_MODE_PRECISION (int_mode) == 2 * BITS_PER_WORD
1479       && optab_handler (ashl_optab, word_mode) != CODE_FOR_nothing
1480       && optab_handler (lshr_optab, word_mode) != CODE_FOR_nothing)
1481     {
1482       rtx_insn *insns;
1483       rtx into_target, outof_target;
1484       rtx into_input, outof_input;
1485       rtx inter;
1486       int shift_count, left_shift, outof_word;
1487 
1488       /* If TARGET is the same as one of the operands, the REG_EQUAL note
1489 	 won't be accurate, so use a new target. Do this also if target is not
1490 	 a REG, first because having a register instead may open optimization
1491 	 opportunities, and second because if target and op0 happen to be MEMs
1492 	 designating the same location, we would risk clobbering it too early
1493 	 in the code sequence we generate below.  */
1494       if (target == 0
1495 	  || target == op0
1496 	  || target == op1
1497 	  || !REG_P (target)
1498 	  || !valid_multiword_target_p (target))
1499 	target = gen_reg_rtx (int_mode);
1500 
1501       start_sequence ();
1502 
1503       shift_count = INTVAL (op1);
1504 
1505       /* OUTOF_* is the word we are shifting bits away from, and
1506 	 INTO_* is the word that we are shifting bits towards, thus
1507 	 they differ depending on the direction of the shift and
1508 	 WORDS_BIG_ENDIAN.  */
1509 
1510       left_shift = (binoptab == rotl_optab);
1511       outof_word = left_shift ^ ! WORDS_BIG_ENDIAN;
1512 
1513       outof_target = operand_subword (target, outof_word, 1, int_mode);
1514       into_target = operand_subword (target, 1 - outof_word, 1, int_mode);
1515 
1516       outof_input = operand_subword_force (op0, outof_word, int_mode);
1517       into_input = operand_subword_force (op0, 1 - outof_word, int_mode);
1518 
1519       if (shift_count == BITS_PER_WORD)
1520 	{
1521 	  /* This is just a word swap.  */
1522 	  emit_move_insn (outof_target, into_input);
1523 	  emit_move_insn (into_target, outof_input);
1524 	  inter = const0_rtx;
1525 	}
1526       else
1527 	{
1528 	  rtx into_temp1, into_temp2, outof_temp1, outof_temp2;
1529 	  HOST_WIDE_INT first_shift_count, second_shift_count;
1530 	  optab reverse_unsigned_shift, unsigned_shift;
1531 
1532 	  reverse_unsigned_shift = (left_shift ^ (shift_count < BITS_PER_WORD)
1533 				    ? lshr_optab : ashl_optab);
1534 
1535 	  unsigned_shift = (left_shift ^ (shift_count < BITS_PER_WORD)
1536 			    ? ashl_optab : lshr_optab);
1537 
1538 	  if (shift_count > BITS_PER_WORD)
1539 	    {
1540 	      first_shift_count = shift_count - BITS_PER_WORD;
1541 	      second_shift_count = 2 * BITS_PER_WORD - shift_count;
1542 	    }
1543 	  else
1544 	    {
1545 	      first_shift_count = BITS_PER_WORD - shift_count;
1546 	      second_shift_count = shift_count;
1547 	    }
1548 	  rtx first_shift_count_rtx
1549 	    = gen_int_shift_amount (word_mode, first_shift_count);
1550 	  rtx second_shift_count_rtx
1551 	    = gen_int_shift_amount (word_mode, second_shift_count);
1552 
1553 	  into_temp1 = expand_binop (word_mode, unsigned_shift,
1554 				     outof_input, first_shift_count_rtx,
1555 				     NULL_RTX, unsignedp, next_methods);
1556 	  into_temp2 = expand_binop (word_mode, reverse_unsigned_shift,
1557 				     into_input, second_shift_count_rtx,
1558 				     NULL_RTX, unsignedp, next_methods);
1559 
1560 	  if (into_temp1 != 0 && into_temp2 != 0)
1561 	    inter = expand_binop (word_mode, ior_optab, into_temp1, into_temp2,
1562 				  into_target, unsignedp, next_methods);
1563 	  else
1564 	    inter = 0;
1565 
1566 	  if (inter != 0 && inter != into_target)
1567 	    emit_move_insn (into_target, inter);
1568 
1569 	  outof_temp1 = expand_binop (word_mode, unsigned_shift,
1570 				      into_input, first_shift_count_rtx,
1571 				      NULL_RTX, unsignedp, next_methods);
1572 	  outof_temp2 = expand_binop (word_mode, reverse_unsigned_shift,
1573 				      outof_input, second_shift_count_rtx,
1574 				      NULL_RTX, unsignedp, next_methods);
1575 
1576 	  if (inter != 0 && outof_temp1 != 0 && outof_temp2 != 0)
1577 	    inter = expand_binop (word_mode, ior_optab,
1578 				  outof_temp1, outof_temp2,
1579 				  outof_target, unsignedp, next_methods);
1580 
1581 	  if (inter != 0 && inter != outof_target)
1582 	    emit_move_insn (outof_target, inter);
1583 	}
1584 
1585       insns = get_insns ();
1586       end_sequence ();
1587 
1588       if (inter != 0)
1589 	{
1590 	  emit_insn (insns);
1591 	  return target;
1592 	}
1593     }
1594 
1595   /* These can be done a word at a time by propagating carries.  */
1596   if ((binoptab == add_optab || binoptab == sub_optab)
1597       && is_int_mode (mode, &int_mode)
1598       && GET_MODE_SIZE (int_mode) >= 2 * UNITS_PER_WORD
1599       && optab_handler (binoptab, word_mode) != CODE_FOR_nothing)
1600     {
1601       unsigned int i;
1602       optab otheroptab = binoptab == add_optab ? sub_optab : add_optab;
1603       const unsigned int nwords = GET_MODE_BITSIZE (int_mode) / BITS_PER_WORD;
1604       rtx carry_in = NULL_RTX, carry_out = NULL_RTX;
1605       rtx xop0, xop1, xtarget;
1606 
1607       /* We can handle either a 1 or -1 value for the carry.  If STORE_FLAG
1608 	 value is one of those, use it.  Otherwise, use 1 since it is the
1609 	 one easiest to get.  */
1610 #if STORE_FLAG_VALUE == 1 || STORE_FLAG_VALUE == -1
1611       int normalizep = STORE_FLAG_VALUE;
1612 #else
1613       int normalizep = 1;
1614 #endif
1615 
1616       /* Prepare the operands.  */
1617       xop0 = force_reg (int_mode, op0);
1618       xop1 = force_reg (int_mode, op1);
1619 
1620       xtarget = gen_reg_rtx (int_mode);
1621 
1622       if (target == 0 || !REG_P (target) || !valid_multiword_target_p (target))
1623 	target = xtarget;
1624 
1625       /* Indicate for flow that the entire target reg is being set.  */
1626       if (REG_P (target))
1627 	emit_clobber (xtarget);
1628 
1629       /* Do the actual arithmetic.  */
1630       for (i = 0; i < nwords; i++)
1631 	{
1632 	  int index = (WORDS_BIG_ENDIAN ? nwords - i - 1 : i);
1633 	  rtx target_piece = operand_subword (xtarget, index, 1, int_mode);
1634 	  rtx op0_piece = operand_subword_force (xop0, index, int_mode);
1635 	  rtx op1_piece = operand_subword_force (xop1, index, int_mode);
1636 	  rtx x;
1637 
1638 	  /* Main add/subtract of the input operands.  */
1639 	  x = expand_binop (word_mode, binoptab,
1640 			    op0_piece, op1_piece,
1641 			    target_piece, unsignedp, next_methods);
1642 	  if (x == 0)
1643 	    break;
1644 
1645 	  if (i + 1 < nwords)
1646 	    {
1647 	      /* Store carry from main add/subtract.  */
1648 	      carry_out = gen_reg_rtx (word_mode);
1649 	      carry_out = emit_store_flag_force (carry_out,
1650 						 (binoptab == add_optab
1651 						  ? LT : GT),
1652 						 x, op0_piece,
1653 						 word_mode, 1, normalizep);
1654 	    }
1655 
1656 	  if (i > 0)
1657 	    {
1658 	      rtx newx;
1659 
1660 	      /* Add/subtract previous carry to main result.  */
1661 	      newx = expand_binop (word_mode,
1662 				   normalizep == 1 ? binoptab : otheroptab,
1663 				   x, carry_in,
1664 				   NULL_RTX, 1, next_methods);
1665 
1666 	      if (i + 1 < nwords)
1667 		{
1668 		  /* Get out carry from adding/subtracting carry in.  */
1669 		  rtx carry_tmp = gen_reg_rtx (word_mode);
1670 		  carry_tmp = emit_store_flag_force (carry_tmp,
1671 						     (binoptab == add_optab
1672 						      ? LT : GT),
1673 						     newx, x,
1674 						     word_mode, 1, normalizep);
1675 
1676 		  /* Logical-ior the two poss. carry together.  */
1677 		  carry_out = expand_binop (word_mode, ior_optab,
1678 					    carry_out, carry_tmp,
1679 					    carry_out, 0, next_methods);
1680 		  if (carry_out == 0)
1681 		    break;
1682 		}
1683 	      emit_move_insn (target_piece, newx);
1684 	    }
1685 	  else
1686 	    {
1687 	      if (x != target_piece)
1688 		emit_move_insn (target_piece, x);
1689 	    }
1690 
1691 	  carry_in = carry_out;
1692 	}
1693 
1694       if (i == GET_MODE_BITSIZE (int_mode) / (unsigned) BITS_PER_WORD)
1695 	{
1696 	  if (optab_handler (mov_optab, int_mode) != CODE_FOR_nothing
1697 	      || ! rtx_equal_p (target, xtarget))
1698 	    {
1699 	      rtx_insn *temp = emit_move_insn (target, xtarget);
1700 
1701 	      set_dst_reg_note (temp, REG_EQUAL,
1702 				gen_rtx_fmt_ee (optab_to_code (binoptab),
1703 						int_mode, copy_rtx (xop0),
1704 						copy_rtx (xop1)),
1705 				target);
1706 	    }
1707 	  else
1708 	    target = xtarget;
1709 
1710 	  return target;
1711 	}
1712 
1713       else
1714 	delete_insns_since (last);
1715     }
1716 
1717   /* Attempt to synthesize double word multiplies using a sequence of word
1718      mode multiplications.  We first attempt to generate a sequence using a
1719      more efficient unsigned widening multiply, and if that fails we then
1720      try using a signed widening multiply.  */
1721 
1722   if (binoptab == smul_optab
1723       && is_int_mode (mode, &int_mode)
1724       && GET_MODE_SIZE (int_mode) == 2 * UNITS_PER_WORD
1725       && optab_handler (smul_optab, word_mode) != CODE_FOR_nothing
1726       && optab_handler (add_optab, word_mode) != CODE_FOR_nothing)
1727     {
1728       rtx product = NULL_RTX;
1729       if (convert_optab_handler (umul_widen_optab, int_mode, word_mode)
1730 	  != CODE_FOR_nothing)
1731 	{
1732 	  product = expand_doubleword_mult (int_mode, op0, op1, target,
1733 					    true, methods);
1734 	  if (!product)
1735 	    delete_insns_since (last);
1736 	}
1737 
1738       if (product == NULL_RTX
1739 	  && (convert_optab_handler (smul_widen_optab, int_mode, word_mode)
1740 	      != CODE_FOR_nothing))
1741 	{
1742 	  product = expand_doubleword_mult (int_mode, op0, op1, target,
1743 					    false, methods);
1744 	  if (!product)
1745 	    delete_insns_since (last);
1746 	}
1747 
1748       if (product != NULL_RTX)
1749 	{
1750 	  if (optab_handler (mov_optab, int_mode) != CODE_FOR_nothing)
1751 	    {
1752 	      rtx_insn *move = emit_move_insn (target ? target : product,
1753 					       product);
1754 	      set_dst_reg_note (move,
1755 				REG_EQUAL,
1756 				gen_rtx_fmt_ee (MULT, int_mode,
1757 						copy_rtx (op0),
1758 						copy_rtx (op1)),
1759 				target ? target : product);
1760 	    }
1761 	  return product;
1762 	}
1763     }
1764 
1765   /* It can't be open-coded in this mode.
1766      Use a library call if one is available and caller says that's ok.  */
1767 
1768   libfunc = optab_libfunc (binoptab, mode);
1769   if (libfunc
1770       && (methods == OPTAB_LIB || methods == OPTAB_LIB_WIDEN))
1771     {
1772       rtx_insn *insns;
1773       rtx op1x = op1;
1774       machine_mode op1_mode = mode;
1775       rtx value;
1776 
1777       start_sequence ();
1778 
1779       if (shift_optab_p (binoptab))
1780 	{
1781 	  op1_mode = targetm.libgcc_shift_count_mode ();
1782 	  /* Specify unsigned here,
1783 	     since negative shift counts are meaningless.  */
1784 	  op1x = convert_to_mode (op1_mode, op1, 1);
1785 	}
1786 
1787       if (GET_MODE (op0) != VOIDmode
1788 	  && GET_MODE (op0) != mode)
1789 	op0 = convert_to_mode (mode, op0, unsignedp);
1790 
1791       /* Pass 1 for NO_QUEUE so we don't lose any increments
1792 	 if the libcall is cse'd or moved.  */
1793       value = emit_library_call_value (libfunc,
1794 				       NULL_RTX, LCT_CONST, mode,
1795 				       op0, mode, op1x, op1_mode);
1796 
1797       insns = get_insns ();
1798       end_sequence ();
1799 
1800       bool trapv = trapv_binoptab_p (binoptab);
1801       target = gen_reg_rtx (mode);
1802       emit_libcall_block_1 (insns, target, value,
1803 			    trapv ? NULL_RTX
1804 			    : gen_rtx_fmt_ee (optab_to_code (binoptab),
1805 					      mode, op0, op1), trapv);
1806 
1807       return target;
1808     }
1809 
1810   delete_insns_since (last);
1811 
1812   /* It can't be done in this mode.  Can we do it in a wider mode?  */
1813 
1814   if (! (methods == OPTAB_WIDEN || methods == OPTAB_LIB_WIDEN
1815 	 || methods == OPTAB_MUST_WIDEN))
1816     {
1817       /* Caller says, don't even try.  */
1818       delete_insns_since (entry_last);
1819       return 0;
1820     }
1821 
1822   /* Compute the value of METHODS to pass to recursive calls.
1823      Don't allow widening to be tried recursively.  */
1824 
1825   methods = (methods == OPTAB_LIB_WIDEN ? OPTAB_LIB : OPTAB_DIRECT);
1826 
1827   /* Look for a wider mode of the same class for which it appears we can do
1828      the operation.  */
1829 
1830   if (CLASS_HAS_WIDER_MODES_P (mclass))
1831     {
1832       /* This code doesn't make sense for conversion optabs, since we
1833 	 wouldn't then want to extend the operands to be the same size
1834 	 as the result.  */
1835       gcc_assert (!convert_optab_p (binoptab));
1836       FOR_EACH_WIDER_MODE (wider_mode, mode)
1837 	{
1838 	  if (optab_handler (binoptab, wider_mode)
1839 	      || (methods == OPTAB_LIB
1840 		  && optab_libfunc (binoptab, wider_mode)))
1841 	    {
1842 	      rtx xop0 = op0, xop1 = op1;
1843 	      int no_extend = 0;
1844 
1845 	      /* For certain integer operations, we need not actually extend
1846 		 the narrow operands, as long as we will truncate
1847 		 the results to the same narrowness.  */
1848 
1849 	      if ((binoptab == ior_optab || binoptab == and_optab
1850 		   || binoptab == xor_optab
1851 		   || binoptab == add_optab || binoptab == sub_optab
1852 		   || binoptab == smul_optab || binoptab == ashl_optab)
1853 		  && mclass == MODE_INT)
1854 		no_extend = 1;
1855 
1856 	      xop0 = widen_operand (xop0, wider_mode, mode,
1857 				    unsignedp, no_extend);
1858 
1859 	      /* The second operand of a shift must always be extended.  */
1860 	      xop1 = widen_operand (xop1, wider_mode, mode, unsignedp,
1861 				    no_extend && binoptab != ashl_optab);
1862 
1863 	      temp = expand_binop (wider_mode, binoptab, xop0, xop1, NULL_RTX,
1864 				   unsignedp, methods);
1865 	      if (temp)
1866 		{
1867 		  if (mclass != MODE_INT
1868 		      || !TRULY_NOOP_TRUNCATION_MODES_P (mode, wider_mode))
1869 		    {
1870 		      if (target == 0)
1871 			target = gen_reg_rtx (mode);
1872 		      convert_move (target, temp, 0);
1873 		      return target;
1874 		    }
1875 		  else
1876 		    return gen_lowpart (mode, temp);
1877 		}
1878 	      else
1879 		delete_insns_since (last);
1880 	    }
1881 	}
1882     }
1883 
1884   delete_insns_since (entry_last);
1885   return 0;
1886 }
1887 
1888 /* Expand a binary operator which has both signed and unsigned forms.
1889    UOPTAB is the optab for unsigned operations, and SOPTAB is for
1890    signed operations.
1891 
1892    If we widen unsigned operands, we may use a signed wider operation instead
1893    of an unsigned wider operation, since the result would be the same.  */
1894 
1895 rtx
sign_expand_binop(machine_mode mode,optab uoptab,optab soptab,rtx op0,rtx op1,rtx target,int unsignedp,enum optab_methods methods)1896 sign_expand_binop (machine_mode mode, optab uoptab, optab soptab,
1897 		   rtx op0, rtx op1, rtx target, int unsignedp,
1898 		   enum optab_methods methods)
1899 {
1900   rtx temp;
1901   optab direct_optab = unsignedp ? uoptab : soptab;
1902   bool save_enable;
1903 
1904   /* Do it without widening, if possible.  */
1905   temp = expand_binop (mode, direct_optab, op0, op1, target,
1906 		       unsignedp, OPTAB_DIRECT);
1907   if (temp || methods == OPTAB_DIRECT)
1908     return temp;
1909 
1910   /* Try widening to a signed int.  Disable any direct use of any
1911      signed insn in the current mode.  */
1912   save_enable = swap_optab_enable (soptab, mode, false);
1913 
1914   temp = expand_binop (mode, soptab, op0, op1, target,
1915 		       unsignedp, OPTAB_WIDEN);
1916 
1917   /* For unsigned operands, try widening to an unsigned int.  */
1918   if (!temp && unsignedp)
1919     temp = expand_binop (mode, uoptab, op0, op1, target,
1920 			 unsignedp, OPTAB_WIDEN);
1921   if (temp || methods == OPTAB_WIDEN)
1922     goto egress;
1923 
1924   /* Use the right width libcall if that exists.  */
1925   temp = expand_binop (mode, direct_optab, op0, op1, target,
1926 		       unsignedp, OPTAB_LIB);
1927   if (temp || methods == OPTAB_LIB)
1928     goto egress;
1929 
1930   /* Must widen and use a libcall, use either signed or unsigned.  */
1931   temp = expand_binop (mode, soptab, op0, op1, target,
1932 		       unsignedp, methods);
1933   if (!temp && unsignedp)
1934     temp = expand_binop (mode, uoptab, op0, op1, target,
1935 			 unsignedp, methods);
1936 
1937  egress:
1938   /* Undo the fiddling above.  */
1939   if (save_enable)
1940     swap_optab_enable (soptab, mode, true);
1941   return temp;
1942 }
1943 
1944 /* Generate code to perform an operation specified by UNOPPTAB
1945    on operand OP0, with two results to TARG0 and TARG1.
1946    We assume that the order of the operands for the instruction
1947    is TARG0, TARG1, OP0.
1948 
1949    Either TARG0 or TARG1 may be zero, but what that means is that
1950    the result is not actually wanted.  We will generate it into
1951    a dummy pseudo-reg and discard it.  They may not both be zero.
1952 
1953    Returns 1 if this operation can be performed; 0 if not.  */
1954 
1955 int
expand_twoval_unop(optab unoptab,rtx op0,rtx targ0,rtx targ1,int unsignedp)1956 expand_twoval_unop (optab unoptab, rtx op0, rtx targ0, rtx targ1,
1957 		    int unsignedp)
1958 {
1959   machine_mode mode = GET_MODE (targ0 ? targ0 : targ1);
1960   enum mode_class mclass;
1961   machine_mode wider_mode;
1962   rtx_insn *entry_last = get_last_insn ();
1963   rtx_insn *last;
1964 
1965   mclass = GET_MODE_CLASS (mode);
1966 
1967   if (!targ0)
1968     targ0 = gen_reg_rtx (mode);
1969   if (!targ1)
1970     targ1 = gen_reg_rtx (mode);
1971 
1972   /* Record where to go back to if we fail.  */
1973   last = get_last_insn ();
1974 
1975   if (optab_handler (unoptab, mode) != CODE_FOR_nothing)
1976     {
1977       struct expand_operand ops[3];
1978       enum insn_code icode = optab_handler (unoptab, mode);
1979 
1980       create_fixed_operand (&ops[0], targ0);
1981       create_fixed_operand (&ops[1], targ1);
1982       create_convert_operand_from (&ops[2], op0, mode, unsignedp);
1983       if (maybe_expand_insn (icode, 3, ops))
1984 	return 1;
1985     }
1986 
1987   /* It can't be done in this mode.  Can we do it in a wider mode?  */
1988 
1989   if (CLASS_HAS_WIDER_MODES_P (mclass))
1990     {
1991       FOR_EACH_WIDER_MODE (wider_mode, mode)
1992 	{
1993 	  if (optab_handler (unoptab, wider_mode) != CODE_FOR_nothing)
1994 	    {
1995 	      rtx t0 = gen_reg_rtx (wider_mode);
1996 	      rtx t1 = gen_reg_rtx (wider_mode);
1997 	      rtx cop0 = convert_modes (wider_mode, mode, op0, unsignedp);
1998 
1999 	      if (expand_twoval_unop (unoptab, cop0, t0, t1, unsignedp))
2000 		{
2001 		  convert_move (targ0, t0, unsignedp);
2002 		  convert_move (targ1, t1, unsignedp);
2003 		  return 1;
2004 		}
2005 	      else
2006 		delete_insns_since (last);
2007 	    }
2008 	}
2009     }
2010 
2011   delete_insns_since (entry_last);
2012   return 0;
2013 }
2014 
2015 /* Generate code to perform an operation specified by BINOPTAB
2016    on operands OP0 and OP1, with two results to TARG1 and TARG2.
2017    We assume that the order of the operands for the instruction
2018    is TARG0, OP0, OP1, TARG1, which would fit a pattern like
2019    [(set TARG0 (operate OP0 OP1)) (set TARG1 (operate ...))].
2020 
2021    Either TARG0 or TARG1 may be zero, but what that means is that
2022    the result is not actually wanted.  We will generate it into
2023    a dummy pseudo-reg and discard it.  They may not both be zero.
2024 
2025    Returns 1 if this operation can be performed; 0 if not.  */
2026 
2027 int
expand_twoval_binop(optab binoptab,rtx op0,rtx op1,rtx targ0,rtx targ1,int unsignedp)2028 expand_twoval_binop (optab binoptab, rtx op0, rtx op1, rtx targ0, rtx targ1,
2029 		     int unsignedp)
2030 {
2031   machine_mode mode = GET_MODE (targ0 ? targ0 : targ1);
2032   enum mode_class mclass;
2033   machine_mode wider_mode;
2034   rtx_insn *entry_last = get_last_insn ();
2035   rtx_insn *last;
2036 
2037   mclass = GET_MODE_CLASS (mode);
2038 
2039   if (!targ0)
2040     targ0 = gen_reg_rtx (mode);
2041   if (!targ1)
2042     targ1 = gen_reg_rtx (mode);
2043 
2044   /* Record where to go back to if we fail.  */
2045   last = get_last_insn ();
2046 
2047   if (optab_handler (binoptab, mode) != CODE_FOR_nothing)
2048     {
2049       struct expand_operand ops[4];
2050       enum insn_code icode = optab_handler (binoptab, mode);
2051       machine_mode mode0 = insn_data[icode].operand[1].mode;
2052       machine_mode mode1 = insn_data[icode].operand[2].mode;
2053       rtx xop0 = op0, xop1 = op1;
2054 
2055       /* If we are optimizing, force expensive constants into a register.  */
2056       xop0 = avoid_expensive_constant (mode0, binoptab, 0, xop0, unsignedp);
2057       xop1 = avoid_expensive_constant (mode1, binoptab, 1, xop1, unsignedp);
2058 
2059       create_fixed_operand (&ops[0], targ0);
2060       create_convert_operand_from (&ops[1], op0, mode, unsignedp);
2061       create_convert_operand_from (&ops[2], op1, mode, unsignedp);
2062       create_fixed_operand (&ops[3], targ1);
2063       if (maybe_expand_insn (icode, 4, ops))
2064 	return 1;
2065       delete_insns_since (last);
2066     }
2067 
2068   /* It can't be done in this mode.  Can we do it in a wider mode?  */
2069 
2070   if (CLASS_HAS_WIDER_MODES_P (mclass))
2071     {
2072       FOR_EACH_WIDER_MODE (wider_mode, mode)
2073 	{
2074 	  if (optab_handler (binoptab, wider_mode) != CODE_FOR_nothing)
2075 	    {
2076 	      rtx t0 = gen_reg_rtx (wider_mode);
2077 	      rtx t1 = gen_reg_rtx (wider_mode);
2078 	      rtx cop0 = convert_modes (wider_mode, mode, op0, unsignedp);
2079 	      rtx cop1 = convert_modes (wider_mode, mode, op1, unsignedp);
2080 
2081 	      if (expand_twoval_binop (binoptab, cop0, cop1,
2082 				       t0, t1, unsignedp))
2083 		{
2084 		  convert_move (targ0, t0, unsignedp);
2085 		  convert_move (targ1, t1, unsignedp);
2086 		  return 1;
2087 		}
2088 	      else
2089 		delete_insns_since (last);
2090 	    }
2091 	}
2092     }
2093 
2094   delete_insns_since (entry_last);
2095   return 0;
2096 }
2097 
2098 /* Expand the two-valued library call indicated by BINOPTAB, but
2099    preserve only one of the values.  If TARG0 is non-NULL, the first
2100    value is placed into TARG0; otherwise the second value is placed
2101    into TARG1.  Exactly one of TARG0 and TARG1 must be non-NULL.  The
2102    value stored into TARG0 or TARG1 is equivalent to (CODE OP0 OP1).
2103    This routine assumes that the value returned by the library call is
2104    as if the return value was of an integral mode twice as wide as the
2105    mode of OP0.  Returns 1 if the call was successful.  */
2106 
2107 bool
expand_twoval_binop_libfunc(optab binoptab,rtx op0,rtx op1,rtx targ0,rtx targ1,enum rtx_code code)2108 expand_twoval_binop_libfunc (optab binoptab, rtx op0, rtx op1,
2109 			     rtx targ0, rtx targ1, enum rtx_code code)
2110 {
2111   machine_mode mode;
2112   machine_mode libval_mode;
2113   rtx libval;
2114   rtx_insn *insns;
2115   rtx libfunc;
2116 
2117   /* Exactly one of TARG0 or TARG1 should be non-NULL.  */
2118   gcc_assert (!targ0 != !targ1);
2119 
2120   mode = GET_MODE (op0);
2121   libfunc = optab_libfunc (binoptab, mode);
2122   if (!libfunc)
2123     return false;
2124 
2125   /* The value returned by the library function will have twice as
2126      many bits as the nominal MODE.  */
2127   libval_mode = smallest_int_mode_for_size (2 * GET_MODE_BITSIZE (mode));
2128   start_sequence ();
2129   libval = emit_library_call_value (libfunc, NULL_RTX, LCT_CONST,
2130 				    libval_mode,
2131 				    op0, mode,
2132 				    op1, mode);
2133   /* Get the part of VAL containing the value that we want.  */
2134   libval = simplify_gen_subreg (mode, libval, libval_mode,
2135 				targ0 ? 0 : GET_MODE_SIZE (mode));
2136   insns = get_insns ();
2137   end_sequence ();
2138   /* Move the into the desired location.  */
2139   emit_libcall_block (insns, targ0 ? targ0 : targ1, libval,
2140 		      gen_rtx_fmt_ee (code, mode, op0, op1));
2141 
2142   return true;
2143 }
2144 
2145 
2146 /* Wrapper around expand_unop which takes an rtx code to specify
2147    the operation to perform, not an optab pointer.  All other
2148    arguments are the same.  */
2149 rtx
expand_simple_unop(machine_mode mode,enum rtx_code code,rtx op0,rtx target,int unsignedp)2150 expand_simple_unop (machine_mode mode, enum rtx_code code, rtx op0,
2151 		    rtx target, int unsignedp)
2152 {
2153   optab unop = code_to_optab (code);
2154   gcc_assert (unop);
2155 
2156   return expand_unop (mode, unop, op0, target, unsignedp);
2157 }
2158 
2159 /* Try calculating
2160 	(clz:narrow x)
2161    as
2162 	(clz:wide (zero_extend:wide x)) - ((width wide) - (width narrow)).
2163 
2164    A similar operation can be used for clrsb.  UNOPTAB says which operation
2165    we are trying to expand.  */
2166 static rtx
widen_leading(scalar_int_mode mode,rtx op0,rtx target,optab unoptab)2167 widen_leading (scalar_int_mode mode, rtx op0, rtx target, optab unoptab)
2168 {
2169   opt_scalar_int_mode wider_mode_iter;
2170   FOR_EACH_WIDER_MODE (wider_mode_iter, mode)
2171     {
2172       scalar_int_mode wider_mode = wider_mode_iter.require ();
2173       if (optab_handler (unoptab, wider_mode) != CODE_FOR_nothing)
2174 	{
2175 	  rtx xop0, temp;
2176 	  rtx_insn *last;
2177 
2178 	  last = get_last_insn ();
2179 
2180 	  if (target == 0)
2181 	    target = gen_reg_rtx (mode);
2182 	  xop0 = widen_operand (op0, wider_mode, mode,
2183 				unoptab != clrsb_optab, false);
2184 	  temp = expand_unop (wider_mode, unoptab, xop0, NULL_RTX,
2185 			      unoptab != clrsb_optab);
2186 	  if (temp != 0)
2187 	    temp = expand_binop
2188 	      (wider_mode, sub_optab, temp,
2189 	       gen_int_mode (GET_MODE_PRECISION (wider_mode)
2190 			     - GET_MODE_PRECISION (mode),
2191 			     wider_mode),
2192 	       target, true, OPTAB_DIRECT);
2193 	  if (temp == 0)
2194 	    delete_insns_since (last);
2195 
2196 	  return temp;
2197 	}
2198     }
2199   return 0;
2200 }
2201 
2202 /* Try calculating clz of a double-word quantity as two clz's of word-sized
2203    quantities, choosing which based on whether the high word is nonzero.  */
2204 static rtx
expand_doubleword_clz(scalar_int_mode mode,rtx op0,rtx target)2205 expand_doubleword_clz (scalar_int_mode mode, rtx op0, rtx target)
2206 {
2207   rtx xop0 = force_reg (mode, op0);
2208   rtx subhi = gen_highpart (word_mode, xop0);
2209   rtx sublo = gen_lowpart (word_mode, xop0);
2210   rtx_code_label *hi0_label = gen_label_rtx ();
2211   rtx_code_label *after_label = gen_label_rtx ();
2212   rtx_insn *seq;
2213   rtx temp, result;
2214 
2215   /* If we were not given a target, use a word_mode register, not a
2216      'mode' register.  The result will fit, and nobody is expecting
2217      anything bigger (the return type of __builtin_clz* is int).  */
2218   if (!target)
2219     target = gen_reg_rtx (word_mode);
2220 
2221   /* In any case, write to a word_mode scratch in both branches of the
2222      conditional, so we can ensure there is a single move insn setting
2223      'target' to tag a REG_EQUAL note on.  */
2224   result = gen_reg_rtx (word_mode);
2225 
2226   start_sequence ();
2227 
2228   /* If the high word is not equal to zero,
2229      then clz of the full value is clz of the high word.  */
2230   emit_cmp_and_jump_insns (subhi, CONST0_RTX (word_mode), EQ, 0,
2231 			   word_mode, true, hi0_label);
2232 
2233   temp = expand_unop_direct (word_mode, clz_optab, subhi, result, true);
2234   if (!temp)
2235     goto fail;
2236 
2237   if (temp != result)
2238     convert_move (result, temp, true);
2239 
2240   emit_jump_insn (targetm.gen_jump (after_label));
2241   emit_barrier ();
2242 
2243   /* Else clz of the full value is clz of the low word plus the number
2244      of bits in the high word.  */
2245   emit_label (hi0_label);
2246 
2247   temp = expand_unop_direct (word_mode, clz_optab, sublo, 0, true);
2248   if (!temp)
2249     goto fail;
2250   temp = expand_binop (word_mode, add_optab, temp,
2251 		       gen_int_mode (GET_MODE_BITSIZE (word_mode), word_mode),
2252 		       result, true, OPTAB_DIRECT);
2253   if (!temp)
2254     goto fail;
2255   if (temp != result)
2256     convert_move (result, temp, true);
2257 
2258   emit_label (after_label);
2259   convert_move (target, result, true);
2260 
2261   seq = get_insns ();
2262   end_sequence ();
2263 
2264   add_equal_note (seq, target, CLZ, xop0, NULL_RTX, mode);
2265   emit_insn (seq);
2266   return target;
2267 
2268  fail:
2269   end_sequence ();
2270   return 0;
2271 }
2272 
2273 /* Try calculating popcount of a double-word quantity as two popcount's of
2274    word-sized quantities and summing up the results.  */
2275 static rtx
expand_doubleword_popcount(scalar_int_mode mode,rtx op0,rtx target)2276 expand_doubleword_popcount (scalar_int_mode mode, rtx op0, rtx target)
2277 {
2278   rtx t0, t1, t;
2279   rtx_insn *seq;
2280 
2281   start_sequence ();
2282 
2283   t0 = expand_unop_direct (word_mode, popcount_optab,
2284 			   operand_subword_force (op0, 0, mode), NULL_RTX,
2285 			   true);
2286   t1 = expand_unop_direct (word_mode, popcount_optab,
2287 			   operand_subword_force (op0, 1, mode), NULL_RTX,
2288 			   true);
2289   if (!t0 || !t1)
2290     {
2291       end_sequence ();
2292       return NULL_RTX;
2293     }
2294 
2295   /* If we were not given a target, use a word_mode register, not a
2296      'mode' register.  The result will fit, and nobody is expecting
2297      anything bigger (the return type of __builtin_popcount* is int).  */
2298   if (!target)
2299     target = gen_reg_rtx (word_mode);
2300 
2301   t = expand_binop (word_mode, add_optab, t0, t1, target, 0, OPTAB_DIRECT);
2302 
2303   seq = get_insns ();
2304   end_sequence ();
2305 
2306   add_equal_note (seq, t, POPCOUNT, op0, NULL_RTX, mode);
2307   emit_insn (seq);
2308   return t;
2309 }
2310 
2311 /* Try calculating
2312 	(parity:wide x)
2313    as
2314 	(parity:narrow (low (x) ^ high (x))) */
2315 static rtx
expand_doubleword_parity(scalar_int_mode mode,rtx op0,rtx target)2316 expand_doubleword_parity (scalar_int_mode mode, rtx op0, rtx target)
2317 {
2318   rtx t = expand_binop (word_mode, xor_optab,
2319 			operand_subword_force (op0, 0, mode),
2320 			operand_subword_force (op0, 1, mode),
2321 			NULL_RTX, 0, OPTAB_DIRECT);
2322   return expand_unop (word_mode, parity_optab, t, target, true);
2323 }
2324 
2325 /* Try calculating
2326 	(bswap:narrow x)
2327    as
2328 	(lshiftrt:wide (bswap:wide x) ((width wide) - (width narrow))).  */
2329 static rtx
widen_bswap(scalar_int_mode mode,rtx op0,rtx target)2330 widen_bswap (scalar_int_mode mode, rtx op0, rtx target)
2331 {
2332   rtx x;
2333   rtx_insn *last;
2334   opt_scalar_int_mode wider_mode_iter;
2335 
2336   FOR_EACH_WIDER_MODE (wider_mode_iter, mode)
2337     if (optab_handler (bswap_optab, wider_mode_iter.require ())
2338 	!= CODE_FOR_nothing)
2339       break;
2340 
2341   if (!wider_mode_iter.exists ())
2342     return NULL_RTX;
2343 
2344   scalar_int_mode wider_mode = wider_mode_iter.require ();
2345   last = get_last_insn ();
2346 
2347   x = widen_operand (op0, wider_mode, mode, true, true);
2348   x = expand_unop (wider_mode, bswap_optab, x, NULL_RTX, true);
2349 
2350   gcc_assert (GET_MODE_PRECISION (wider_mode) == GET_MODE_BITSIZE (wider_mode)
2351 	      && GET_MODE_PRECISION (mode) == GET_MODE_BITSIZE (mode));
2352   if (x != 0)
2353     x = expand_shift (RSHIFT_EXPR, wider_mode, x,
2354 		      GET_MODE_BITSIZE (wider_mode)
2355 		      - GET_MODE_BITSIZE (mode),
2356 		      NULL_RTX, true);
2357 
2358   if (x != 0)
2359     {
2360       if (target == 0)
2361 	target = gen_reg_rtx (mode);
2362       emit_move_insn (target, gen_lowpart (mode, x));
2363     }
2364   else
2365     delete_insns_since (last);
2366 
2367   return target;
2368 }
2369 
2370 /* Try calculating bswap as two bswaps of two word-sized operands.  */
2371 
2372 static rtx
expand_doubleword_bswap(machine_mode mode,rtx op,rtx target)2373 expand_doubleword_bswap (machine_mode mode, rtx op, rtx target)
2374 {
2375   rtx t0, t1;
2376 
2377   t1 = expand_unop (word_mode, bswap_optab,
2378 		    operand_subword_force (op, 0, mode), NULL_RTX, true);
2379   t0 = expand_unop (word_mode, bswap_optab,
2380 		    operand_subword_force (op, 1, mode), NULL_RTX, true);
2381 
2382   if (target == 0 || !valid_multiword_target_p (target))
2383     target = gen_reg_rtx (mode);
2384   if (REG_P (target))
2385     emit_clobber (target);
2386   emit_move_insn (operand_subword (target, 0, 1, mode), t0);
2387   emit_move_insn (operand_subword (target, 1, 1, mode), t1);
2388 
2389   return target;
2390 }
2391 
2392 /* Try calculating (parity x) as (and (popcount x) 1), where
2393    popcount can also be done in a wider mode.  */
2394 static rtx
expand_parity(scalar_int_mode mode,rtx op0,rtx target)2395 expand_parity (scalar_int_mode mode, rtx op0, rtx target)
2396 {
2397   enum mode_class mclass = GET_MODE_CLASS (mode);
2398   opt_scalar_int_mode wider_mode_iter;
2399   FOR_EACH_MODE_FROM (wider_mode_iter, mode)
2400     {
2401       scalar_int_mode wider_mode = wider_mode_iter.require ();
2402       if (optab_handler (popcount_optab, wider_mode) != CODE_FOR_nothing)
2403 	{
2404 	  rtx xop0, temp;
2405 	  rtx_insn *last;
2406 
2407 	  last = get_last_insn ();
2408 
2409 	  if (target == 0 || GET_MODE (target) != wider_mode)
2410 	    target = gen_reg_rtx (wider_mode);
2411 
2412 	  xop0 = widen_operand (op0, wider_mode, mode, true, false);
2413 	  temp = expand_unop (wider_mode, popcount_optab, xop0, NULL_RTX,
2414 			      true);
2415 	  if (temp != 0)
2416 	    temp = expand_binop (wider_mode, and_optab, temp, const1_rtx,
2417 				 target, true, OPTAB_DIRECT);
2418 
2419 	  if (temp)
2420 	    {
2421 	      if (mclass != MODE_INT
2422 		  || !TRULY_NOOP_TRUNCATION_MODES_P (mode, wider_mode))
2423 		return convert_to_mode (mode, temp, 0);
2424 	      else
2425 		return gen_lowpart (mode, temp);
2426 	    }
2427 	  else
2428 	    delete_insns_since (last);
2429 	}
2430     }
2431   return 0;
2432 }
2433 
2434 /* Try calculating ctz(x) as K - clz(x & -x) ,
2435    where K is GET_MODE_PRECISION(mode) - 1.
2436 
2437    Both __builtin_ctz and __builtin_clz are undefined at zero, so we
2438    don't have to worry about what the hardware does in that case.  (If
2439    the clz instruction produces the usual value at 0, which is K, the
2440    result of this code sequence will be -1; expand_ffs, below, relies
2441    on this.  It might be nice to have it be K instead, for consistency
2442    with the (very few) processors that provide a ctz with a defined
2443    value, but that would take one more instruction, and it would be
2444    less convenient for expand_ffs anyway.  */
2445 
2446 static rtx
expand_ctz(scalar_int_mode mode,rtx op0,rtx target)2447 expand_ctz (scalar_int_mode mode, rtx op0, rtx target)
2448 {
2449   rtx_insn *seq;
2450   rtx temp;
2451 
2452   if (optab_handler (clz_optab, mode) == CODE_FOR_nothing)
2453     return 0;
2454 
2455   start_sequence ();
2456 
2457   temp = expand_unop_direct (mode, neg_optab, op0, NULL_RTX, true);
2458   if (temp)
2459     temp = expand_binop (mode, and_optab, op0, temp, NULL_RTX,
2460 			 true, OPTAB_DIRECT);
2461   if (temp)
2462     temp = expand_unop_direct (mode, clz_optab, temp, NULL_RTX, true);
2463   if (temp)
2464     temp = expand_binop (mode, sub_optab,
2465 			 gen_int_mode (GET_MODE_PRECISION (mode) - 1, mode),
2466 			 temp, target,
2467 			 true, OPTAB_DIRECT);
2468   if (temp == 0)
2469     {
2470       end_sequence ();
2471       return 0;
2472     }
2473 
2474   seq = get_insns ();
2475   end_sequence ();
2476 
2477   add_equal_note (seq, temp, CTZ, op0, NULL_RTX, mode);
2478   emit_insn (seq);
2479   return temp;
2480 }
2481 
2482 
2483 /* Try calculating ffs(x) using ctz(x) if we have that instruction, or
2484    else with the sequence used by expand_clz.
2485 
2486    The ffs builtin promises to return zero for a zero value and ctz/clz
2487    may have an undefined value in that case.  If they do not give us a
2488    convenient value, we have to generate a test and branch.  */
2489 static rtx
expand_ffs(scalar_int_mode mode,rtx op0,rtx target)2490 expand_ffs (scalar_int_mode mode, rtx op0, rtx target)
2491 {
2492   HOST_WIDE_INT val = 0;
2493   bool defined_at_zero = false;
2494   rtx temp;
2495   rtx_insn *seq;
2496 
2497   if (optab_handler (ctz_optab, mode) != CODE_FOR_nothing)
2498     {
2499       start_sequence ();
2500 
2501       temp = expand_unop_direct (mode, ctz_optab, op0, 0, true);
2502       if (!temp)
2503 	goto fail;
2504 
2505       defined_at_zero = (CTZ_DEFINED_VALUE_AT_ZERO (mode, val) == 2);
2506     }
2507   else if (optab_handler (clz_optab, mode) != CODE_FOR_nothing)
2508     {
2509       start_sequence ();
2510       temp = expand_ctz (mode, op0, 0);
2511       if (!temp)
2512 	goto fail;
2513 
2514       if (CLZ_DEFINED_VALUE_AT_ZERO (mode, val) == 2)
2515 	{
2516 	  defined_at_zero = true;
2517 	  val = (GET_MODE_PRECISION (mode) - 1) - val;
2518 	}
2519     }
2520   else
2521     return 0;
2522 
2523   if (defined_at_zero && val == -1)
2524     /* No correction needed at zero.  */;
2525   else
2526     {
2527       /* We don't try to do anything clever with the situation found
2528 	 on some processors (eg Alpha) where ctz(0:mode) ==
2529 	 bitsize(mode).  If someone can think of a way to send N to -1
2530 	 and leave alone all values in the range 0..N-1 (where N is a
2531 	 power of two), cheaper than this test-and-branch, please add it.
2532 
2533 	 The test-and-branch is done after the operation itself, in case
2534 	 the operation sets condition codes that can be recycled for this.
2535 	 (This is true on i386, for instance.)  */
2536 
2537       rtx_code_label *nonzero_label = gen_label_rtx ();
2538       emit_cmp_and_jump_insns (op0, CONST0_RTX (mode), NE, 0,
2539 			       mode, true, nonzero_label);
2540 
2541       convert_move (temp, GEN_INT (-1), false);
2542       emit_label (nonzero_label);
2543     }
2544 
2545   /* temp now has a value in the range -1..bitsize-1.  ffs is supposed
2546      to produce a value in the range 0..bitsize.  */
2547   temp = expand_binop (mode, add_optab, temp, gen_int_mode (1, mode),
2548 		       target, false, OPTAB_DIRECT);
2549   if (!temp)
2550     goto fail;
2551 
2552   seq = get_insns ();
2553   end_sequence ();
2554 
2555   add_equal_note (seq, temp, FFS, op0, NULL_RTX, mode);
2556   emit_insn (seq);
2557   return temp;
2558 
2559  fail:
2560   end_sequence ();
2561   return 0;
2562 }
2563 
2564 /* Extract the OMODE lowpart from VAL, which has IMODE.  Under certain
2565    conditions, VAL may already be a SUBREG against which we cannot generate
2566    a further SUBREG.  In this case, we expect forcing the value into a
2567    register will work around the situation.  */
2568 
2569 static rtx
lowpart_subreg_maybe_copy(machine_mode omode,rtx val,machine_mode imode)2570 lowpart_subreg_maybe_copy (machine_mode omode, rtx val,
2571 			   machine_mode imode)
2572 {
2573   rtx ret;
2574   ret = lowpart_subreg (omode, val, imode);
2575   if (ret == NULL)
2576     {
2577       val = force_reg (imode, val);
2578       ret = lowpart_subreg (omode, val, imode);
2579       gcc_assert (ret != NULL);
2580     }
2581   return ret;
2582 }
2583 
2584 /* Expand a floating point absolute value or negation operation via a
2585    logical operation on the sign bit.  */
2586 
2587 static rtx
expand_absneg_bit(enum rtx_code code,scalar_float_mode mode,rtx op0,rtx target)2588 expand_absneg_bit (enum rtx_code code, scalar_float_mode mode,
2589 		   rtx op0, rtx target)
2590 {
2591   const struct real_format *fmt;
2592   int bitpos, word, nwords, i;
2593   scalar_int_mode imode;
2594   rtx temp;
2595   rtx_insn *insns;
2596 
2597   /* The format has to have a simple sign bit.  */
2598   fmt = REAL_MODE_FORMAT (mode);
2599   if (fmt == NULL)
2600     return NULL_RTX;
2601 
2602   bitpos = fmt->signbit_rw;
2603   if (bitpos < 0)
2604     return NULL_RTX;
2605 
2606   /* Don't create negative zeros if the format doesn't support them.  */
2607   if (code == NEG && !fmt->has_signed_zero)
2608     return NULL_RTX;
2609 
2610   if (GET_MODE_SIZE (mode) <= UNITS_PER_WORD)
2611     {
2612       if (!int_mode_for_mode (mode).exists (&imode))
2613 	return NULL_RTX;
2614       word = 0;
2615       nwords = 1;
2616     }
2617   else
2618     {
2619       imode = word_mode;
2620 
2621       if (FLOAT_WORDS_BIG_ENDIAN)
2622 	word = (GET_MODE_BITSIZE (mode) - bitpos) / BITS_PER_WORD;
2623       else
2624 	word = bitpos / BITS_PER_WORD;
2625       bitpos = bitpos % BITS_PER_WORD;
2626       nwords = (GET_MODE_BITSIZE (mode) + BITS_PER_WORD - 1) / BITS_PER_WORD;
2627     }
2628 
2629   wide_int mask = wi::set_bit_in_zero (bitpos, GET_MODE_PRECISION (imode));
2630   if (code == ABS)
2631     mask = ~mask;
2632 
2633   if (target == 0
2634       || target == op0
2635       || (nwords > 1 && !valid_multiword_target_p (target)))
2636     target = gen_reg_rtx (mode);
2637 
2638   if (nwords > 1)
2639     {
2640       start_sequence ();
2641 
2642       for (i = 0; i < nwords; ++i)
2643 	{
2644 	  rtx targ_piece = operand_subword (target, i, 1, mode);
2645 	  rtx op0_piece = operand_subword_force (op0, i, mode);
2646 
2647 	  if (i == word)
2648 	    {
2649 	      temp = expand_binop (imode, code == ABS ? and_optab : xor_optab,
2650 				   op0_piece,
2651 				   immed_wide_int_const (mask, imode),
2652 				   targ_piece, 1, OPTAB_LIB_WIDEN);
2653 	      if (temp != targ_piece)
2654 		emit_move_insn (targ_piece, temp);
2655 	    }
2656 	  else
2657 	    emit_move_insn (targ_piece, op0_piece);
2658 	}
2659 
2660       insns = get_insns ();
2661       end_sequence ();
2662 
2663       emit_insn (insns);
2664     }
2665   else
2666     {
2667       temp = expand_binop (imode, code == ABS ? and_optab : xor_optab,
2668 			   gen_lowpart (imode, op0),
2669 			   immed_wide_int_const (mask, imode),
2670 		           gen_lowpart (imode, target), 1, OPTAB_LIB_WIDEN);
2671       target = lowpart_subreg_maybe_copy (mode, temp, imode);
2672 
2673       set_dst_reg_note (get_last_insn (), REG_EQUAL,
2674 			gen_rtx_fmt_e (code, mode, copy_rtx (op0)),
2675 			target);
2676     }
2677 
2678   return target;
2679 }
2680 
2681 /* As expand_unop, but will fail rather than attempt the operation in a
2682    different mode or with a libcall.  */
2683 static rtx
expand_unop_direct(machine_mode mode,optab unoptab,rtx op0,rtx target,int unsignedp)2684 expand_unop_direct (machine_mode mode, optab unoptab, rtx op0, rtx target,
2685 		    int unsignedp)
2686 {
2687   if (optab_handler (unoptab, mode) != CODE_FOR_nothing)
2688     {
2689       struct expand_operand ops[2];
2690       enum insn_code icode = optab_handler (unoptab, mode);
2691       rtx_insn *last = get_last_insn ();
2692       rtx_insn *pat;
2693 
2694       create_output_operand (&ops[0], target, mode);
2695       create_convert_operand_from (&ops[1], op0, mode, unsignedp);
2696       pat = maybe_gen_insn (icode, 2, ops);
2697       if (pat)
2698 	{
2699 	  if (INSN_P (pat) && NEXT_INSN (pat) != NULL_RTX
2700 	      && ! add_equal_note (pat, ops[0].value,
2701 				   optab_to_code (unoptab),
2702 				   ops[1].value, NULL_RTX, mode))
2703 	    {
2704 	      delete_insns_since (last);
2705 	      return expand_unop (mode, unoptab, op0, NULL_RTX, unsignedp);
2706 	    }
2707 
2708 	  emit_insn (pat);
2709 
2710 	  return ops[0].value;
2711 	}
2712     }
2713   return 0;
2714 }
2715 
2716 /* Generate code to perform an operation specified by UNOPTAB
2717    on operand OP0, with result having machine-mode MODE.
2718 
2719    UNSIGNEDP is for the case where we have to widen the operands
2720    to perform the operation.  It says to use zero-extension.
2721 
2722    If TARGET is nonzero, the value
2723    is generated there, if it is convenient to do so.
2724    In all cases an rtx is returned for the locus of the value;
2725    this may or may not be TARGET.  */
2726 
2727 rtx
expand_unop(machine_mode mode,optab unoptab,rtx op0,rtx target,int unsignedp)2728 expand_unop (machine_mode mode, optab unoptab, rtx op0, rtx target,
2729 	     int unsignedp)
2730 {
2731   enum mode_class mclass = GET_MODE_CLASS (mode);
2732   machine_mode wider_mode;
2733   scalar_int_mode int_mode;
2734   scalar_float_mode float_mode;
2735   rtx temp;
2736   rtx libfunc;
2737 
2738   temp = expand_unop_direct (mode, unoptab, op0, target, unsignedp);
2739   if (temp)
2740     return temp;
2741 
2742   /* It can't be done in this mode.  Can we open-code it in a wider mode?  */
2743 
2744   /* Widening (or narrowing) clz needs special treatment.  */
2745   if (unoptab == clz_optab)
2746     {
2747       if (is_a <scalar_int_mode> (mode, &int_mode))
2748 	{
2749 	  temp = widen_leading (int_mode, op0, target, unoptab);
2750 	  if (temp)
2751 	    return temp;
2752 
2753 	  if (GET_MODE_SIZE (int_mode) == 2 * UNITS_PER_WORD
2754 	      && optab_handler (unoptab, word_mode) != CODE_FOR_nothing)
2755 	    {
2756 	      temp = expand_doubleword_clz (int_mode, op0, target);
2757 	      if (temp)
2758 		return temp;
2759 	    }
2760 	}
2761 
2762       goto try_libcall;
2763     }
2764 
2765   if (unoptab == clrsb_optab)
2766     {
2767       if (is_a <scalar_int_mode> (mode, &int_mode))
2768 	{
2769 	  temp = widen_leading (int_mode, op0, target, unoptab);
2770 	  if (temp)
2771 	    return temp;
2772 	}
2773       goto try_libcall;
2774     }
2775 
2776   if (unoptab == popcount_optab
2777       && is_a <scalar_int_mode> (mode, &int_mode)
2778       && GET_MODE_SIZE (int_mode) == 2 * UNITS_PER_WORD
2779       && optab_handler (unoptab, word_mode) != CODE_FOR_nothing
2780       && optimize_insn_for_speed_p ())
2781     {
2782       temp = expand_doubleword_popcount (int_mode, op0, target);
2783       if (temp)
2784 	return temp;
2785     }
2786 
2787   if (unoptab == parity_optab
2788       && is_a <scalar_int_mode> (mode, &int_mode)
2789       && GET_MODE_SIZE (int_mode) == 2 * UNITS_PER_WORD
2790       && (optab_handler (unoptab, word_mode) != CODE_FOR_nothing
2791 	  || optab_handler (popcount_optab, word_mode) != CODE_FOR_nothing)
2792       && optimize_insn_for_speed_p ())
2793     {
2794       temp = expand_doubleword_parity (int_mode, op0, target);
2795       if (temp)
2796 	return temp;
2797     }
2798 
2799   /* Widening (or narrowing) bswap needs special treatment.  */
2800   if (unoptab == bswap_optab)
2801     {
2802       /* HImode is special because in this mode BSWAP is equivalent to ROTATE
2803 	 or ROTATERT.  First try these directly; if this fails, then try the
2804 	 obvious pair of shifts with allowed widening, as this will probably
2805 	 be always more efficient than the other fallback methods.  */
2806       if (mode == HImode)
2807 	{
2808 	  rtx_insn *last;
2809 	  rtx temp1, temp2;
2810 
2811 	  if (optab_handler (rotl_optab, mode) != CODE_FOR_nothing)
2812 	    {
2813 	      temp = expand_binop (mode, rotl_optab, op0,
2814 				   gen_int_shift_amount (mode, 8),
2815 				   target, unsignedp, OPTAB_DIRECT);
2816 	      if (temp)
2817 		return temp;
2818 	     }
2819 
2820 	  if (optab_handler (rotr_optab, mode) != CODE_FOR_nothing)
2821 	    {
2822 	      temp = expand_binop (mode, rotr_optab, op0,
2823 				   gen_int_shift_amount (mode, 8),
2824 				   target, unsignedp, OPTAB_DIRECT);
2825 	      if (temp)
2826 		return temp;
2827 	    }
2828 
2829 	  last = get_last_insn ();
2830 
2831 	  temp1 = expand_binop (mode, ashl_optab, op0,
2832 				gen_int_shift_amount (mode, 8), NULL_RTX,
2833 			        unsignedp, OPTAB_WIDEN);
2834 	  temp2 = expand_binop (mode, lshr_optab, op0,
2835 				gen_int_shift_amount (mode, 8), NULL_RTX,
2836 			        unsignedp, OPTAB_WIDEN);
2837 	  if (temp1 && temp2)
2838 	    {
2839 	      temp = expand_binop (mode, ior_optab, temp1, temp2, target,
2840 				   unsignedp, OPTAB_WIDEN);
2841 	      if (temp)
2842 		return temp;
2843 	    }
2844 
2845 	  delete_insns_since (last);
2846 	}
2847 
2848       if (is_a <scalar_int_mode> (mode, &int_mode))
2849 	{
2850 	  temp = widen_bswap (int_mode, op0, target);
2851 	  if (temp)
2852 	    return temp;
2853 
2854 	  if (GET_MODE_SIZE (int_mode) == 2 * UNITS_PER_WORD
2855 	      && optab_handler (unoptab, word_mode) != CODE_FOR_nothing)
2856 	    {
2857 	      temp = expand_doubleword_bswap (mode, op0, target);
2858 	      if (temp)
2859 		return temp;
2860 	    }
2861 	}
2862 
2863       goto try_libcall;
2864     }
2865 
2866   if (CLASS_HAS_WIDER_MODES_P (mclass))
2867     FOR_EACH_WIDER_MODE (wider_mode, mode)
2868       {
2869 	if (optab_handler (unoptab, wider_mode) != CODE_FOR_nothing)
2870 	  {
2871 	    rtx xop0 = op0;
2872 	    rtx_insn *last = get_last_insn ();
2873 
2874 	    /* For certain operations, we need not actually extend
2875 	       the narrow operand, as long as we will truncate the
2876 	       results to the same narrowness.  */
2877 
2878 	    xop0 = widen_operand (xop0, wider_mode, mode, unsignedp,
2879 				  (unoptab == neg_optab
2880 				   || unoptab == one_cmpl_optab)
2881 				  && mclass == MODE_INT);
2882 
2883 	    temp = expand_unop (wider_mode, unoptab, xop0, NULL_RTX,
2884 				unsignedp);
2885 
2886 	    if (temp)
2887 	      {
2888 		if (mclass != MODE_INT
2889 		    || !TRULY_NOOP_TRUNCATION_MODES_P (mode, wider_mode))
2890 		  {
2891 		    if (target == 0)
2892 		      target = gen_reg_rtx (mode);
2893 		    convert_move (target, temp, 0);
2894 		    return target;
2895 		  }
2896 		else
2897 		  return gen_lowpart (mode, temp);
2898 	      }
2899 	    else
2900 	      delete_insns_since (last);
2901 	  }
2902       }
2903 
2904   /* These can be done a word at a time.  */
2905   if (unoptab == one_cmpl_optab
2906       && is_int_mode (mode, &int_mode)
2907       && GET_MODE_SIZE (int_mode) > UNITS_PER_WORD
2908       && optab_handler (unoptab, word_mode) != CODE_FOR_nothing)
2909     {
2910       int i;
2911       rtx_insn *insns;
2912 
2913       if (target == 0 || target == op0 || !valid_multiword_target_p (target))
2914 	target = gen_reg_rtx (int_mode);
2915 
2916       start_sequence ();
2917 
2918       /* Do the actual arithmetic.  */
2919       for (i = 0; i < GET_MODE_BITSIZE (int_mode) / BITS_PER_WORD; i++)
2920 	{
2921 	  rtx target_piece = operand_subword (target, i, 1, int_mode);
2922 	  rtx x = expand_unop (word_mode, unoptab,
2923 			       operand_subword_force (op0, i, int_mode),
2924 			       target_piece, unsignedp);
2925 
2926 	  if (target_piece != x)
2927 	    emit_move_insn (target_piece, x);
2928 	}
2929 
2930       insns = get_insns ();
2931       end_sequence ();
2932 
2933       emit_insn (insns);
2934       return target;
2935     }
2936 
2937   if (optab_to_code (unoptab) == NEG)
2938     {
2939       /* Try negating floating point values by flipping the sign bit.  */
2940       if (is_a <scalar_float_mode> (mode, &float_mode))
2941 	{
2942 	  temp = expand_absneg_bit (NEG, float_mode, op0, target);
2943 	  if (temp)
2944 	    return temp;
2945 	}
2946 
2947       /* If there is no negation pattern, and we have no negative zero,
2948 	 try subtracting from zero.  */
2949       if (!HONOR_SIGNED_ZEROS (mode))
2950 	{
2951 	  temp = expand_binop (mode, (unoptab == negv_optab
2952 				      ? subv_optab : sub_optab),
2953 			       CONST0_RTX (mode), op0, target,
2954 			       unsignedp, OPTAB_DIRECT);
2955 	  if (temp)
2956 	    return temp;
2957 	}
2958     }
2959 
2960   /* Try calculating parity (x) as popcount (x) % 2.  */
2961   if (unoptab == parity_optab && is_a <scalar_int_mode> (mode, &int_mode))
2962     {
2963       temp = expand_parity (int_mode, op0, target);
2964       if (temp)
2965 	return temp;
2966     }
2967 
2968   /* Try implementing ffs (x) in terms of clz (x).  */
2969   if (unoptab == ffs_optab && is_a <scalar_int_mode> (mode, &int_mode))
2970     {
2971       temp = expand_ffs (int_mode, op0, target);
2972       if (temp)
2973 	return temp;
2974     }
2975 
2976   /* Try implementing ctz (x) in terms of clz (x).  */
2977   if (unoptab == ctz_optab && is_a <scalar_int_mode> (mode, &int_mode))
2978     {
2979       temp = expand_ctz (int_mode, op0, target);
2980       if (temp)
2981 	return temp;
2982     }
2983 
2984  try_libcall:
2985   /* Now try a library call in this mode.  */
2986   libfunc = optab_libfunc (unoptab, mode);
2987   if (libfunc)
2988     {
2989       rtx_insn *insns;
2990       rtx value;
2991       rtx eq_value;
2992       machine_mode outmode = mode;
2993 
2994       /* All of these functions return small values.  Thus we choose to
2995 	 have them return something that isn't a double-word.  */
2996       if (unoptab == ffs_optab || unoptab == clz_optab || unoptab == ctz_optab
2997 	  || unoptab == clrsb_optab || unoptab == popcount_optab
2998 	  || unoptab == parity_optab)
2999 	outmode
3000 	  = GET_MODE (hard_libcall_value (TYPE_MODE (integer_type_node),
3001 					  optab_libfunc (unoptab, mode)));
3002 
3003       start_sequence ();
3004 
3005       /* Pass 1 for NO_QUEUE so we don't lose any increments
3006 	 if the libcall is cse'd or moved.  */
3007       value = emit_library_call_value (libfunc, NULL_RTX, LCT_CONST, outmode,
3008 				       op0, mode);
3009       insns = get_insns ();
3010       end_sequence ();
3011 
3012       target = gen_reg_rtx (outmode);
3013       bool trapv = trapv_unoptab_p (unoptab);
3014       if (trapv)
3015 	eq_value = NULL_RTX;
3016       else
3017 	{
3018 	  eq_value = gen_rtx_fmt_e (optab_to_code (unoptab), mode, op0);
3019 	  if (GET_MODE_UNIT_SIZE (outmode) < GET_MODE_UNIT_SIZE (mode))
3020 	    eq_value = simplify_gen_unary (TRUNCATE, outmode, eq_value, mode);
3021 	  else if (GET_MODE_UNIT_SIZE (outmode) > GET_MODE_UNIT_SIZE (mode))
3022 	    eq_value = simplify_gen_unary (ZERO_EXTEND,
3023 					   outmode, eq_value, mode);
3024 	}
3025       emit_libcall_block_1 (insns, target, value, eq_value, trapv);
3026 
3027       return target;
3028     }
3029 
3030   /* It can't be done in this mode.  Can we do it in a wider mode?  */
3031 
3032   if (CLASS_HAS_WIDER_MODES_P (mclass))
3033     {
3034       FOR_EACH_WIDER_MODE (wider_mode, mode)
3035 	{
3036 	  if (optab_handler (unoptab, wider_mode) != CODE_FOR_nothing
3037 	      || optab_libfunc (unoptab, wider_mode))
3038 	    {
3039 	      rtx xop0 = op0;
3040 	      rtx_insn *last = get_last_insn ();
3041 
3042 	      /* For certain operations, we need not actually extend
3043 		 the narrow operand, as long as we will truncate the
3044 		 results to the same narrowness.  */
3045 	      xop0 = widen_operand (xop0, wider_mode, mode, unsignedp,
3046 				    (unoptab == neg_optab
3047 				     || unoptab == one_cmpl_optab
3048 				     || unoptab == bswap_optab)
3049 				    && mclass == MODE_INT);
3050 
3051 	      temp = expand_unop (wider_mode, unoptab, xop0, NULL_RTX,
3052 				  unsignedp);
3053 
3054 	      /* If we are generating clz using wider mode, adjust the
3055 		 result.  Similarly for clrsb.  */
3056 	      if ((unoptab == clz_optab || unoptab == clrsb_optab)
3057 		  && temp != 0)
3058 		{
3059 		  scalar_int_mode wider_int_mode
3060 		    = as_a <scalar_int_mode> (wider_mode);
3061 		  int_mode = as_a <scalar_int_mode> (mode);
3062 		  temp = expand_binop
3063 		    (wider_mode, sub_optab, temp,
3064 		     gen_int_mode (GET_MODE_PRECISION (wider_int_mode)
3065 				   - GET_MODE_PRECISION (int_mode),
3066 				   wider_int_mode),
3067 		     target, true, OPTAB_DIRECT);
3068 		}
3069 
3070 	      /* Likewise for bswap.  */
3071 	      if (unoptab == bswap_optab && temp != 0)
3072 		{
3073 		  scalar_int_mode wider_int_mode
3074 		    = as_a <scalar_int_mode> (wider_mode);
3075 		  int_mode = as_a <scalar_int_mode> (mode);
3076 		  gcc_assert (GET_MODE_PRECISION (wider_int_mode)
3077 			      == GET_MODE_BITSIZE (wider_int_mode)
3078 			      && GET_MODE_PRECISION (int_mode)
3079 				 == GET_MODE_BITSIZE (int_mode));
3080 
3081 		  temp = expand_shift (RSHIFT_EXPR, wider_int_mode, temp,
3082 				       GET_MODE_BITSIZE (wider_int_mode)
3083 				       - GET_MODE_BITSIZE (int_mode),
3084 				       NULL_RTX, true);
3085 		}
3086 
3087 	      if (temp)
3088 		{
3089 		  if (mclass != MODE_INT)
3090 		    {
3091 		      if (target == 0)
3092 			target = gen_reg_rtx (mode);
3093 		      convert_move (target, temp, 0);
3094 		      return target;
3095 		    }
3096 		  else
3097 		    return gen_lowpart (mode, temp);
3098 		}
3099 	      else
3100 		delete_insns_since (last);
3101 	    }
3102 	}
3103     }
3104 
3105   /* One final attempt at implementing negation via subtraction,
3106      this time allowing widening of the operand.  */
3107   if (optab_to_code (unoptab) == NEG && !HONOR_SIGNED_ZEROS (mode))
3108     {
3109       rtx temp;
3110       temp = expand_binop (mode,
3111                            unoptab == negv_optab ? subv_optab : sub_optab,
3112                            CONST0_RTX (mode), op0,
3113                            target, unsignedp, OPTAB_LIB_WIDEN);
3114       if (temp)
3115         return temp;
3116     }
3117 
3118   return 0;
3119 }
3120 
3121 /* Emit code to compute the absolute value of OP0, with result to
3122    TARGET if convenient.  (TARGET may be 0.)  The return value says
3123    where the result actually is to be found.
3124 
3125    MODE is the mode of the operand; the mode of the result is
3126    different but can be deduced from MODE.
3127 
3128  */
3129 
3130 rtx
expand_abs_nojump(machine_mode mode,rtx op0,rtx target,int result_unsignedp)3131 expand_abs_nojump (machine_mode mode, rtx op0, rtx target,
3132 		   int result_unsignedp)
3133 {
3134   rtx temp;
3135 
3136   if (GET_MODE_CLASS (mode) != MODE_INT
3137       || ! flag_trapv)
3138     result_unsignedp = 1;
3139 
3140   /* First try to do it with a special abs instruction.  */
3141   temp = expand_unop (mode, result_unsignedp ? abs_optab : absv_optab,
3142                       op0, target, 0);
3143   if (temp != 0)
3144     return temp;
3145 
3146   /* For floating point modes, try clearing the sign bit.  */
3147   scalar_float_mode float_mode;
3148   if (is_a <scalar_float_mode> (mode, &float_mode))
3149     {
3150       temp = expand_absneg_bit (ABS, float_mode, op0, target);
3151       if (temp)
3152 	return temp;
3153     }
3154 
3155   /* If we have a MAX insn, we can do this as MAX (x, -x).  */
3156   if (optab_handler (smax_optab, mode) != CODE_FOR_nothing
3157       && !HONOR_SIGNED_ZEROS (mode))
3158     {
3159       rtx_insn *last = get_last_insn ();
3160 
3161       temp = expand_unop (mode, result_unsignedp ? neg_optab : negv_optab,
3162 			  op0, NULL_RTX, 0);
3163       if (temp != 0)
3164 	temp = expand_binop (mode, smax_optab, op0, temp, target, 0,
3165 			     OPTAB_WIDEN);
3166 
3167       if (temp != 0)
3168 	return temp;
3169 
3170       delete_insns_since (last);
3171     }
3172 
3173   /* If this machine has expensive jumps, we can do integer absolute
3174      value of X as (((signed) x >> (W-1)) ^ x) - ((signed) x >> (W-1)),
3175      where W is the width of MODE.  */
3176 
3177   scalar_int_mode int_mode;
3178   if (is_int_mode (mode, &int_mode)
3179       && BRANCH_COST (optimize_insn_for_speed_p (),
3180 	      	      false) >= 2)
3181     {
3182       rtx extended = expand_shift (RSHIFT_EXPR, int_mode, op0,
3183 				   GET_MODE_PRECISION (int_mode) - 1,
3184 				   NULL_RTX, 0);
3185 
3186       temp = expand_binop (int_mode, xor_optab, extended, op0, target, 0,
3187 			   OPTAB_LIB_WIDEN);
3188       if (temp != 0)
3189 	temp = expand_binop (int_mode,
3190 			     result_unsignedp ? sub_optab : subv_optab,
3191                              temp, extended, target, 0, OPTAB_LIB_WIDEN);
3192 
3193       if (temp != 0)
3194 	return temp;
3195     }
3196 
3197   return NULL_RTX;
3198 }
3199 
3200 rtx
expand_abs(machine_mode mode,rtx op0,rtx target,int result_unsignedp,int safe)3201 expand_abs (machine_mode mode, rtx op0, rtx target,
3202 	    int result_unsignedp, int safe)
3203 {
3204   rtx temp;
3205   rtx_code_label *op1;
3206 
3207   if (GET_MODE_CLASS (mode) != MODE_INT
3208       || ! flag_trapv)
3209     result_unsignedp = 1;
3210 
3211   temp = expand_abs_nojump (mode, op0, target, result_unsignedp);
3212   if (temp != 0)
3213     return temp;
3214 
3215   /* If that does not win, use conditional jump and negate.  */
3216 
3217   /* It is safe to use the target if it is the same
3218      as the source if this is also a pseudo register */
3219   if (op0 == target && REG_P (op0)
3220       && REGNO (op0) >= FIRST_PSEUDO_REGISTER)
3221     safe = 1;
3222 
3223   op1 = gen_label_rtx ();
3224   if (target == 0 || ! safe
3225       || GET_MODE (target) != mode
3226       || (MEM_P (target) && MEM_VOLATILE_P (target))
3227       || (REG_P (target)
3228 	  && REGNO (target) < FIRST_PSEUDO_REGISTER))
3229     target = gen_reg_rtx (mode);
3230 
3231   emit_move_insn (target, op0);
3232   NO_DEFER_POP;
3233 
3234   do_compare_rtx_and_jump (target, CONST0_RTX (mode), GE, 0, mode,
3235 			   NULL_RTX, NULL, op1,
3236 			   profile_probability::uninitialized ());
3237 
3238   op0 = expand_unop (mode, result_unsignedp ? neg_optab : negv_optab,
3239                      target, target, 0);
3240   if (op0 != target)
3241     emit_move_insn (target, op0);
3242   emit_label (op1);
3243   OK_DEFER_POP;
3244   return target;
3245 }
3246 
3247 /* Emit code to compute the one's complement absolute value of OP0
3248    (if (OP0 < 0) OP0 = ~OP0), with result to TARGET if convenient.
3249    (TARGET may be NULL_RTX.)  The return value says where the result
3250    actually is to be found.
3251 
3252    MODE is the mode of the operand; the mode of the result is
3253    different but can be deduced from MODE.  */
3254 
3255 rtx
expand_one_cmpl_abs_nojump(machine_mode mode,rtx op0,rtx target)3256 expand_one_cmpl_abs_nojump (machine_mode mode, rtx op0, rtx target)
3257 {
3258   rtx temp;
3259 
3260   /* Not applicable for floating point modes.  */
3261   if (FLOAT_MODE_P (mode))
3262     return NULL_RTX;
3263 
3264   /* If we have a MAX insn, we can do this as MAX (x, ~x).  */
3265   if (optab_handler (smax_optab, mode) != CODE_FOR_nothing)
3266     {
3267       rtx_insn *last = get_last_insn ();
3268 
3269       temp = expand_unop (mode, one_cmpl_optab, op0, NULL_RTX, 0);
3270       if (temp != 0)
3271 	temp = expand_binop (mode, smax_optab, op0, temp, target, 0,
3272 			     OPTAB_WIDEN);
3273 
3274       if (temp != 0)
3275 	return temp;
3276 
3277       delete_insns_since (last);
3278     }
3279 
3280   /* If this machine has expensive jumps, we can do one's complement
3281      absolute value of X as (((signed) x >> (W-1)) ^ x).  */
3282 
3283   scalar_int_mode int_mode;
3284   if (is_int_mode (mode, &int_mode)
3285       && BRANCH_COST (optimize_insn_for_speed_p (),
3286 	             false) >= 2)
3287     {
3288       rtx extended = expand_shift (RSHIFT_EXPR, int_mode, op0,
3289 				   GET_MODE_PRECISION (int_mode) - 1,
3290 				   NULL_RTX, 0);
3291 
3292       temp = expand_binop (int_mode, xor_optab, extended, op0, target, 0,
3293 			   OPTAB_LIB_WIDEN);
3294 
3295       if (temp != 0)
3296 	return temp;
3297     }
3298 
3299   return NULL_RTX;
3300 }
3301 
3302 /* A subroutine of expand_copysign, perform the copysign operation using the
3303    abs and neg primitives advertised to exist on the target.  The assumption
3304    is that we have a split register file, and leaving op0 in fp registers,
3305    and not playing with subregs so much, will help the register allocator.  */
3306 
3307 static rtx
expand_copysign_absneg(scalar_float_mode mode,rtx op0,rtx op1,rtx target,int bitpos,bool op0_is_abs)3308 expand_copysign_absneg (scalar_float_mode mode, rtx op0, rtx op1, rtx target,
3309 		        int bitpos, bool op0_is_abs)
3310 {
3311   scalar_int_mode imode;
3312   enum insn_code icode;
3313   rtx sign;
3314   rtx_code_label *label;
3315 
3316   if (target == op1)
3317     target = NULL_RTX;
3318 
3319   /* Check if the back end provides an insn that handles signbit for the
3320      argument's mode. */
3321   icode = optab_handler (signbit_optab, mode);
3322   if (icode != CODE_FOR_nothing)
3323     {
3324       imode = as_a <scalar_int_mode> (insn_data[(int) icode].operand[0].mode);
3325       sign = gen_reg_rtx (imode);
3326       emit_unop_insn (icode, sign, op1, UNKNOWN);
3327     }
3328   else
3329     {
3330       if (GET_MODE_SIZE (mode) <= UNITS_PER_WORD)
3331 	{
3332 	  if (!int_mode_for_mode (mode).exists (&imode))
3333 	    return NULL_RTX;
3334 	  op1 = gen_lowpart (imode, op1);
3335 	}
3336       else
3337 	{
3338 	  int word;
3339 
3340 	  imode = word_mode;
3341 	  if (FLOAT_WORDS_BIG_ENDIAN)
3342 	    word = (GET_MODE_BITSIZE (mode) - bitpos) / BITS_PER_WORD;
3343 	  else
3344 	    word = bitpos / BITS_PER_WORD;
3345 	  bitpos = bitpos % BITS_PER_WORD;
3346 	  op1 = operand_subword_force (op1, word, mode);
3347 	}
3348 
3349       wide_int mask = wi::set_bit_in_zero (bitpos, GET_MODE_PRECISION (imode));
3350       sign = expand_binop (imode, and_optab, op1,
3351 			   immed_wide_int_const (mask, imode),
3352 			   NULL_RTX, 1, OPTAB_LIB_WIDEN);
3353     }
3354 
3355   if (!op0_is_abs)
3356     {
3357       op0 = expand_unop (mode, abs_optab, op0, target, 0);
3358       if (op0 == NULL)
3359 	return NULL_RTX;
3360       target = op0;
3361     }
3362   else
3363     {
3364       if (target == NULL_RTX)
3365         target = copy_to_reg (op0);
3366       else
3367 	emit_move_insn (target, op0);
3368     }
3369 
3370   label = gen_label_rtx ();
3371   emit_cmp_and_jump_insns (sign, const0_rtx, EQ, NULL_RTX, imode, 1, label);
3372 
3373   if (CONST_DOUBLE_AS_FLOAT_P (op0))
3374     op0 = simplify_unary_operation (NEG, mode, op0, mode);
3375   else
3376     op0 = expand_unop (mode, neg_optab, op0, target, 0);
3377   if (op0 != target)
3378     emit_move_insn (target, op0);
3379 
3380   emit_label (label);
3381 
3382   return target;
3383 }
3384 
3385 
3386 /* A subroutine of expand_copysign, perform the entire copysign operation
3387    with integer bitmasks.  BITPOS is the position of the sign bit; OP0_IS_ABS
3388    is true if op0 is known to have its sign bit clear.  */
3389 
3390 static rtx
expand_copysign_bit(scalar_float_mode mode,rtx op0,rtx op1,rtx target,int bitpos,bool op0_is_abs)3391 expand_copysign_bit (scalar_float_mode mode, rtx op0, rtx op1, rtx target,
3392 		     int bitpos, bool op0_is_abs)
3393 {
3394   scalar_int_mode imode;
3395   int word, nwords, i;
3396   rtx temp;
3397   rtx_insn *insns;
3398 
3399   if (GET_MODE_SIZE (mode) <= UNITS_PER_WORD)
3400     {
3401       if (!int_mode_for_mode (mode).exists (&imode))
3402 	return NULL_RTX;
3403       word = 0;
3404       nwords = 1;
3405     }
3406   else
3407     {
3408       imode = word_mode;
3409 
3410       if (FLOAT_WORDS_BIG_ENDIAN)
3411 	word = (GET_MODE_BITSIZE (mode) - bitpos) / BITS_PER_WORD;
3412       else
3413 	word = bitpos / BITS_PER_WORD;
3414       bitpos = bitpos % BITS_PER_WORD;
3415       nwords = (GET_MODE_BITSIZE (mode) + BITS_PER_WORD - 1) / BITS_PER_WORD;
3416     }
3417 
3418   wide_int mask = wi::set_bit_in_zero (bitpos, GET_MODE_PRECISION (imode));
3419 
3420   if (target == 0
3421       || target == op0
3422       || target == op1
3423       || (nwords > 1 && !valid_multiword_target_p (target)))
3424     target = gen_reg_rtx (mode);
3425 
3426   if (nwords > 1)
3427     {
3428       start_sequence ();
3429 
3430       for (i = 0; i < nwords; ++i)
3431 	{
3432 	  rtx targ_piece = operand_subword (target, i, 1, mode);
3433 	  rtx op0_piece = operand_subword_force (op0, i, mode);
3434 
3435 	  if (i == word)
3436 	    {
3437 	      if (!op0_is_abs)
3438 		op0_piece
3439 		  = expand_binop (imode, and_optab, op0_piece,
3440 				  immed_wide_int_const (~mask, imode),
3441 				  NULL_RTX, 1, OPTAB_LIB_WIDEN);
3442 	      op1 = expand_binop (imode, and_optab,
3443 				  operand_subword_force (op1, i, mode),
3444 				  immed_wide_int_const (mask, imode),
3445 				  NULL_RTX, 1, OPTAB_LIB_WIDEN);
3446 
3447 	      temp = expand_binop (imode, ior_optab, op0_piece, op1,
3448 				   targ_piece, 1, OPTAB_LIB_WIDEN);
3449 	      if (temp != targ_piece)
3450 		emit_move_insn (targ_piece, temp);
3451 	    }
3452 	  else
3453 	    emit_move_insn (targ_piece, op0_piece);
3454 	}
3455 
3456       insns = get_insns ();
3457       end_sequence ();
3458 
3459       emit_insn (insns);
3460     }
3461   else
3462     {
3463       op1 = expand_binop (imode, and_optab, gen_lowpart (imode, op1),
3464 		          immed_wide_int_const (mask, imode),
3465 		          NULL_RTX, 1, OPTAB_LIB_WIDEN);
3466 
3467       op0 = gen_lowpart (imode, op0);
3468       if (!op0_is_abs)
3469 	op0 = expand_binop (imode, and_optab, op0,
3470 			    immed_wide_int_const (~mask, imode),
3471 			    NULL_RTX, 1, OPTAB_LIB_WIDEN);
3472 
3473       temp = expand_binop (imode, ior_optab, op0, op1,
3474 			   gen_lowpart (imode, target), 1, OPTAB_LIB_WIDEN);
3475       target = lowpart_subreg_maybe_copy (mode, temp, imode);
3476     }
3477 
3478   return target;
3479 }
3480 
3481 /* Expand the C99 copysign operation.  OP0 and OP1 must be the same
3482    scalar floating point mode.  Return NULL if we do not know how to
3483    expand the operation inline.  */
3484 
3485 rtx
expand_copysign(rtx op0,rtx op1,rtx target)3486 expand_copysign (rtx op0, rtx op1, rtx target)
3487 {
3488   scalar_float_mode mode;
3489   const struct real_format *fmt;
3490   bool op0_is_abs;
3491   rtx temp;
3492 
3493   mode = as_a <scalar_float_mode> (GET_MODE (op0));
3494   gcc_assert (GET_MODE (op1) == mode);
3495 
3496   /* First try to do it with a special instruction.  */
3497   temp = expand_binop (mode, copysign_optab, op0, op1,
3498 		       target, 0, OPTAB_DIRECT);
3499   if (temp)
3500     return temp;
3501 
3502   fmt = REAL_MODE_FORMAT (mode);
3503   if (fmt == NULL || !fmt->has_signed_zero)
3504     return NULL_RTX;
3505 
3506   op0_is_abs = false;
3507   if (CONST_DOUBLE_AS_FLOAT_P (op0))
3508     {
3509       if (real_isneg (CONST_DOUBLE_REAL_VALUE (op0)))
3510 	op0 = simplify_unary_operation (ABS, mode, op0, mode);
3511       op0_is_abs = true;
3512     }
3513 
3514   if (fmt->signbit_ro >= 0
3515       && (CONST_DOUBLE_AS_FLOAT_P (op0)
3516 	  || (optab_handler (neg_optab, mode) != CODE_FOR_nothing
3517 	      && optab_handler (abs_optab, mode) != CODE_FOR_nothing)))
3518     {
3519       temp = expand_copysign_absneg (mode, op0, op1, target,
3520 				     fmt->signbit_ro, op0_is_abs);
3521       if (temp)
3522 	return temp;
3523     }
3524 
3525   if (fmt->signbit_rw < 0)
3526     return NULL_RTX;
3527   return expand_copysign_bit (mode, op0, op1, target,
3528 			      fmt->signbit_rw, op0_is_abs);
3529 }
3530 
3531 /* Generate an instruction whose insn-code is INSN_CODE,
3532    with two operands: an output TARGET and an input OP0.
3533    TARGET *must* be nonzero, and the output is always stored there.
3534    CODE is an rtx code such that (CODE OP0) is an rtx that describes
3535    the value that is stored into TARGET.
3536 
3537    Return false if expansion failed.  */
3538 
3539 bool
maybe_emit_unop_insn(enum insn_code icode,rtx target,rtx op0,enum rtx_code code)3540 maybe_emit_unop_insn (enum insn_code icode, rtx target, rtx op0,
3541 		      enum rtx_code code)
3542 {
3543   struct expand_operand ops[2];
3544   rtx_insn *pat;
3545 
3546   create_output_operand (&ops[0], target, GET_MODE (target));
3547   create_input_operand (&ops[1], op0, GET_MODE (op0));
3548   pat = maybe_gen_insn (icode, 2, ops);
3549   if (!pat)
3550     return false;
3551 
3552   if (INSN_P (pat) && NEXT_INSN (pat) != NULL_RTX
3553       && code != UNKNOWN)
3554     add_equal_note (pat, ops[0].value, code, ops[1].value, NULL_RTX,
3555 		    GET_MODE (op0));
3556 
3557   emit_insn (pat);
3558 
3559   if (ops[0].value != target)
3560     emit_move_insn (target, ops[0].value);
3561   return true;
3562 }
3563 /* Generate an instruction whose insn-code is INSN_CODE,
3564    with two operands: an output TARGET and an input OP0.
3565    TARGET *must* be nonzero, and the output is always stored there.
3566    CODE is an rtx code such that (CODE OP0) is an rtx that describes
3567    the value that is stored into TARGET.  */
3568 
3569 void
emit_unop_insn(enum insn_code icode,rtx target,rtx op0,enum rtx_code code)3570 emit_unop_insn (enum insn_code icode, rtx target, rtx op0, enum rtx_code code)
3571 {
3572   bool ok = maybe_emit_unop_insn (icode, target, op0, code);
3573   gcc_assert (ok);
3574 }
3575 
3576 struct no_conflict_data
3577 {
3578   rtx target;
3579   rtx_insn *first, *insn;
3580   bool must_stay;
3581 };
3582 
3583 /* Called via note_stores by emit_libcall_block.  Set P->must_stay if
3584    the currently examined clobber / store has to stay in the list of
3585    insns that constitute the actual libcall block.  */
3586 static void
no_conflict_move_test(rtx dest,const_rtx set,void * p0)3587 no_conflict_move_test (rtx dest, const_rtx set, void *p0)
3588 {
3589   struct no_conflict_data *p= (struct no_conflict_data *) p0;
3590 
3591   /* If this inns directly contributes to setting the target, it must stay.  */
3592   if (reg_overlap_mentioned_p (p->target, dest))
3593     p->must_stay = true;
3594   /* If we haven't committed to keeping any other insns in the list yet,
3595      there is nothing more to check.  */
3596   else if (p->insn == p->first)
3597     return;
3598   /* If this insn sets / clobbers a register that feeds one of the insns
3599      already in the list, this insn has to stay too.  */
3600   else if (reg_overlap_mentioned_p (dest, PATTERN (p->first))
3601 	   || (CALL_P (p->first) && (find_reg_fusage (p->first, USE, dest)))
3602 	   || reg_used_between_p (dest, p->first, p->insn)
3603 	   /* Likewise if this insn depends on a register set by a previous
3604 	      insn in the list, or if it sets a result (presumably a hard
3605 	      register) that is set or clobbered by a previous insn.
3606 	      N.B. the modified_*_p (SET_DEST...) tests applied to a MEM
3607 	      SET_DEST perform the former check on the address, and the latter
3608 	      check on the MEM.  */
3609 	   || (GET_CODE (set) == SET
3610 	       && (modified_in_p (SET_SRC (set), p->first)
3611 		   || modified_in_p (SET_DEST (set), p->first)
3612 		   || modified_between_p (SET_SRC (set), p->first, p->insn)
3613 		   || modified_between_p (SET_DEST (set), p->first, p->insn))))
3614     p->must_stay = true;
3615 }
3616 
3617 
3618 /* Emit code to make a call to a constant function or a library call.
3619 
3620    INSNS is a list containing all insns emitted in the call.
3621    These insns leave the result in RESULT.  Our block is to copy RESULT
3622    to TARGET, which is logically equivalent to EQUIV.
3623 
3624    We first emit any insns that set a pseudo on the assumption that these are
3625    loading constants into registers; doing so allows them to be safely cse'ed
3626    between blocks.  Then we emit all the other insns in the block, followed by
3627    an insn to move RESULT to TARGET.  This last insn will have a REQ_EQUAL
3628    note with an operand of EQUIV.  */
3629 
3630 static void
emit_libcall_block_1(rtx_insn * insns,rtx target,rtx result,rtx equiv,bool equiv_may_trap)3631 emit_libcall_block_1 (rtx_insn *insns, rtx target, rtx result, rtx equiv,
3632 		      bool equiv_may_trap)
3633 {
3634   rtx final_dest = target;
3635   rtx_insn *next, *last, *insn;
3636 
3637   /* If this is a reg with REG_USERVAR_P set, then it could possibly turn
3638      into a MEM later.  Protect the libcall block from this change.  */
3639   if (! REG_P (target) || REG_USERVAR_P (target))
3640     target = gen_reg_rtx (GET_MODE (target));
3641 
3642   /* If we're using non-call exceptions, a libcall corresponding to an
3643      operation that may trap may also trap.  */
3644   /* ??? See the comment in front of make_reg_eh_region_note.  */
3645   if (cfun->can_throw_non_call_exceptions
3646       && (equiv_may_trap || may_trap_p (equiv)))
3647     {
3648       for (insn = insns; insn; insn = NEXT_INSN (insn))
3649 	if (CALL_P (insn))
3650 	  {
3651 	    rtx note = find_reg_note (insn, REG_EH_REGION, NULL_RTX);
3652 	    if (note)
3653 	      {
3654 		int lp_nr = INTVAL (XEXP (note, 0));
3655 		if (lp_nr == 0 || lp_nr == INT_MIN)
3656 		  remove_note (insn, note);
3657 	      }
3658 	  }
3659     }
3660   else
3661     {
3662       /* Look for any CALL_INSNs in this sequence, and attach a REG_EH_REGION
3663 	 reg note to indicate that this call cannot throw or execute a nonlocal
3664 	 goto (unless there is already a REG_EH_REGION note, in which case
3665 	 we update it).  */
3666       for (insn = insns; insn; insn = NEXT_INSN (insn))
3667 	if (CALL_P (insn))
3668 	  make_reg_eh_region_note_nothrow_nononlocal (insn);
3669     }
3670 
3671   /* First emit all insns that set pseudos.  Remove them from the list as
3672      we go.  Avoid insns that set pseudos which were referenced in previous
3673      insns.  These can be generated by move_by_pieces, for example,
3674      to update an address.  Similarly, avoid insns that reference things
3675      set in previous insns.  */
3676 
3677   for (insn = insns; insn; insn = next)
3678     {
3679       rtx set = single_set (insn);
3680 
3681       next = NEXT_INSN (insn);
3682 
3683       if (set != 0 && REG_P (SET_DEST (set))
3684 	  && REGNO (SET_DEST (set)) >= FIRST_PSEUDO_REGISTER)
3685 	{
3686 	  struct no_conflict_data data;
3687 
3688 	  data.target = const0_rtx;
3689 	  data.first = insns;
3690 	  data.insn = insn;
3691 	  data.must_stay = 0;
3692 	  note_stores (PATTERN (insn), no_conflict_move_test, &data);
3693 	  if (! data.must_stay)
3694 	    {
3695 	      if (PREV_INSN (insn))
3696 		SET_NEXT_INSN (PREV_INSN (insn)) = next;
3697 	      else
3698 		insns = next;
3699 
3700 	      if (next)
3701 		SET_PREV_INSN (next) = PREV_INSN (insn);
3702 
3703 	      add_insn (insn);
3704 	    }
3705 	}
3706 
3707       /* Some ports use a loop to copy large arguments onto the stack.
3708 	 Don't move anything outside such a loop.  */
3709       if (LABEL_P (insn))
3710 	break;
3711     }
3712 
3713   /* Write the remaining insns followed by the final copy.  */
3714   for (insn = insns; insn; insn = next)
3715     {
3716       next = NEXT_INSN (insn);
3717 
3718       add_insn (insn);
3719     }
3720 
3721   last = emit_move_insn (target, result);
3722   if (equiv)
3723     set_dst_reg_note (last, REG_EQUAL, copy_rtx (equiv), target);
3724 
3725   if (final_dest != target)
3726     emit_move_insn (final_dest, target);
3727 }
3728 
3729 void
emit_libcall_block(rtx_insn * insns,rtx target,rtx result,rtx equiv)3730 emit_libcall_block (rtx_insn *insns, rtx target, rtx result, rtx equiv)
3731 {
3732   emit_libcall_block_1 (insns, target, result, equiv, false);
3733 }
3734 
3735 /* Nonzero if we can perform a comparison of mode MODE straightforwardly.
3736    PURPOSE describes how this comparison will be used.  CODE is the rtx
3737    comparison code we will be using.
3738 
3739    ??? Actually, CODE is slightly weaker than that.  A target is still
3740    required to implement all of the normal bcc operations, but not
3741    required to implement all (or any) of the unordered bcc operations.  */
3742 
3743 int
can_compare_p(enum rtx_code code,machine_mode mode,enum can_compare_purpose purpose)3744 can_compare_p (enum rtx_code code, machine_mode mode,
3745 	       enum can_compare_purpose purpose)
3746 {
3747   rtx test;
3748   test = gen_rtx_fmt_ee (code, mode, const0_rtx, const0_rtx);
3749   do
3750     {
3751       enum insn_code icode;
3752 
3753       if (purpose == ccp_jump
3754           && (icode = optab_handler (cbranch_optab, mode)) != CODE_FOR_nothing
3755           && insn_operand_matches (icode, 0, test))
3756         return 1;
3757       if (purpose == ccp_store_flag
3758           && (icode = optab_handler (cstore_optab, mode)) != CODE_FOR_nothing
3759           && insn_operand_matches (icode, 1, test))
3760         return 1;
3761       if (purpose == ccp_cmov
3762 	  && optab_handler (cmov_optab, mode) != CODE_FOR_nothing)
3763 	return 1;
3764 
3765       mode = GET_MODE_WIDER_MODE (mode).else_void ();
3766       PUT_MODE (test, mode);
3767     }
3768   while (mode != VOIDmode);
3769 
3770   return 0;
3771 }
3772 
3773 /* This function is called when we are going to emit a compare instruction that
3774    compares the values found in X and Y, using the rtl operator COMPARISON.
3775 
3776    If they have mode BLKmode, then SIZE specifies the size of both operands.
3777 
3778    UNSIGNEDP nonzero says that the operands are unsigned;
3779    this matters if they need to be widened (as given by METHODS).
3780 
3781    *PTEST is where the resulting comparison RTX is returned or NULL_RTX
3782    if we failed to produce one.
3783 
3784    *PMODE is the mode of the inputs (in case they are const_int).
3785 
3786    This function performs all the setup necessary so that the caller only has
3787    to emit a single comparison insn.  This setup can involve doing a BLKmode
3788    comparison or emitting a library call to perform the comparison if no insn
3789    is available to handle it.
3790    The values which are passed in through pointers can be modified; the caller
3791    should perform the comparison on the modified values.  Constant
3792    comparisons must have already been folded.  */
3793 
3794 static void
prepare_cmp_insn(rtx x,rtx y,enum rtx_code comparison,rtx size,int unsignedp,enum optab_methods methods,rtx * ptest,machine_mode * pmode)3795 prepare_cmp_insn (rtx x, rtx y, enum rtx_code comparison, rtx size,
3796 		  int unsignedp, enum optab_methods methods,
3797 		  rtx *ptest, machine_mode *pmode)
3798 {
3799   machine_mode mode = *pmode;
3800   rtx libfunc, test;
3801   machine_mode cmp_mode;
3802   enum mode_class mclass;
3803 
3804   /* The other methods are not needed.  */
3805   gcc_assert (methods == OPTAB_DIRECT || methods == OPTAB_WIDEN
3806 	      || methods == OPTAB_LIB_WIDEN);
3807 
3808   /* If we are optimizing, force expensive constants into a register.  */
3809   if (CONSTANT_P (x) && optimize
3810       && (rtx_cost (x, mode, COMPARE, 0, optimize_insn_for_speed_p ())
3811           > COSTS_N_INSNS (1)))
3812     x = force_reg (mode, x);
3813 
3814   if (CONSTANT_P (y) && optimize
3815       && (rtx_cost (y, mode, COMPARE, 1, optimize_insn_for_speed_p ())
3816           > COSTS_N_INSNS (1)))
3817     y = force_reg (mode, y);
3818 
3819 #if HAVE_cc0
3820   /* Make sure if we have a canonical comparison.  The RTL
3821      documentation states that canonical comparisons are required only
3822      for targets which have cc0.  */
3823   gcc_assert (!CONSTANT_P (x) || CONSTANT_P (y));
3824 #endif
3825 
3826   /* Don't let both operands fail to indicate the mode.  */
3827   if (GET_MODE (x) == VOIDmode && GET_MODE (y) == VOIDmode)
3828     x = force_reg (mode, x);
3829   if (mode == VOIDmode)
3830     mode = GET_MODE (x) != VOIDmode ? GET_MODE (x) : GET_MODE (y);
3831 
3832   /* Handle all BLKmode compares.  */
3833 
3834   if (mode == BLKmode)
3835     {
3836       machine_mode result_mode;
3837       enum insn_code cmp_code;
3838       rtx result;
3839       rtx opalign
3840 	= GEN_INT (MIN (MEM_ALIGN (x), MEM_ALIGN (y)) / BITS_PER_UNIT);
3841 
3842       gcc_assert (size);
3843 
3844       /* Try to use a memory block compare insn - either cmpstr
3845 	 or cmpmem will do.  */
3846       opt_scalar_int_mode cmp_mode_iter;
3847       FOR_EACH_MODE_IN_CLASS (cmp_mode_iter, MODE_INT)
3848 	{
3849 	  scalar_int_mode cmp_mode = cmp_mode_iter.require ();
3850 	  cmp_code = direct_optab_handler (cmpmem_optab, cmp_mode);
3851 	  if (cmp_code == CODE_FOR_nothing)
3852 	    cmp_code = direct_optab_handler (cmpstr_optab, cmp_mode);
3853 	  if (cmp_code == CODE_FOR_nothing)
3854 	    cmp_code = direct_optab_handler (cmpstrn_optab, cmp_mode);
3855 	  if (cmp_code == CODE_FOR_nothing)
3856 	    continue;
3857 
3858 	  /* Must make sure the size fits the insn's mode.  */
3859 	  if (CONST_INT_P (size)
3860 	      ? UINTVAL (size) > GET_MODE_MASK (cmp_mode)
3861 	      : (GET_MODE_BITSIZE (as_a <scalar_int_mode> (GET_MODE (size)))
3862 		 > GET_MODE_BITSIZE (cmp_mode)))
3863 	    continue;
3864 
3865 	  result_mode = insn_data[cmp_code].operand[0].mode;
3866 	  result = gen_reg_rtx (result_mode);
3867 	  size = convert_to_mode (cmp_mode, size, 1);
3868 	  emit_insn (GEN_FCN (cmp_code) (result, x, y, size, opalign));
3869 
3870           *ptest = gen_rtx_fmt_ee (comparison, VOIDmode, result, const0_rtx);
3871           *pmode = result_mode;
3872 	  return;
3873 	}
3874 
3875       if (methods != OPTAB_LIB && methods != OPTAB_LIB_WIDEN)
3876 	goto fail;
3877 
3878       /* Otherwise call a library function.  */
3879       result = emit_block_comp_via_libcall (x, y, size);
3880 
3881       x = result;
3882       y = const0_rtx;
3883       mode = TYPE_MODE (integer_type_node);
3884       methods = OPTAB_LIB_WIDEN;
3885       unsignedp = false;
3886     }
3887 
3888   /* Don't allow operands to the compare to trap, as that can put the
3889      compare and branch in different basic blocks.  */
3890   if (cfun->can_throw_non_call_exceptions)
3891     {
3892       if (may_trap_p (x))
3893 	x = copy_to_reg (x);
3894       if (may_trap_p (y))
3895 	y = copy_to_reg (y);
3896     }
3897 
3898   if (GET_MODE_CLASS (mode) == MODE_CC)
3899     {
3900       enum insn_code icode = optab_handler (cbranch_optab, CCmode);
3901       test = gen_rtx_fmt_ee (comparison, VOIDmode, x, y);
3902       gcc_assert (icode != CODE_FOR_nothing
3903                   && insn_operand_matches (icode, 0, test));
3904       *ptest = test;
3905       return;
3906     }
3907 
3908   mclass = GET_MODE_CLASS (mode);
3909   test = gen_rtx_fmt_ee (comparison, VOIDmode, x, y);
3910   FOR_EACH_MODE_FROM (cmp_mode, mode)
3911     {
3912       enum insn_code icode;
3913       icode = optab_handler (cbranch_optab, cmp_mode);
3914       if (icode != CODE_FOR_nothing
3915 	  && insn_operand_matches (icode, 0, test))
3916 	{
3917 	  rtx_insn *last = get_last_insn ();
3918 	  rtx op0 = prepare_operand (icode, x, 1, mode, cmp_mode, unsignedp);
3919 	  rtx op1 = prepare_operand (icode, y, 2, mode, cmp_mode, unsignedp);
3920 	  if (op0 && op1
3921 	      && insn_operand_matches (icode, 1, op0)
3922 	      && insn_operand_matches (icode, 2, op1))
3923 	    {
3924 	      XEXP (test, 0) = op0;
3925 	      XEXP (test, 1) = op1;
3926 	      *ptest = test;
3927 	      *pmode = cmp_mode;
3928 	      return;
3929 	    }
3930 	  delete_insns_since (last);
3931 	}
3932 
3933       if (methods == OPTAB_DIRECT || !CLASS_HAS_WIDER_MODES_P (mclass))
3934 	break;
3935     }
3936 
3937   if (methods != OPTAB_LIB_WIDEN)
3938     goto fail;
3939 
3940   if (SCALAR_FLOAT_MODE_P (mode))
3941     {
3942       /* Small trick if UNORDERED isn't implemented by the hardware.  */
3943       if (comparison == UNORDERED && rtx_equal_p (x, y))
3944 	{
3945 	  prepare_cmp_insn (x, y, UNLT, NULL_RTX, unsignedp, OPTAB_WIDEN,
3946 			    ptest, pmode);
3947 	  if (*ptest)
3948 	    return;
3949 	}
3950 
3951       prepare_float_lib_cmp (x, y, comparison, ptest, pmode);
3952     }
3953   else
3954     {
3955       rtx result;
3956       machine_mode ret_mode;
3957 
3958       /* Handle a libcall just for the mode we are using.  */
3959       libfunc = optab_libfunc (cmp_optab, mode);
3960       gcc_assert (libfunc);
3961 
3962       /* If we want unsigned, and this mode has a distinct unsigned
3963 	 comparison routine, use that.  */
3964       if (unsignedp)
3965 	{
3966 	  rtx ulibfunc = optab_libfunc (ucmp_optab, mode);
3967 	  if (ulibfunc)
3968 	    libfunc = ulibfunc;
3969 	}
3970 
3971       ret_mode = targetm.libgcc_cmp_return_mode ();
3972       result = emit_library_call_value (libfunc, NULL_RTX, LCT_CONST,
3973 					ret_mode, x, mode, y, mode);
3974 
3975       /* There are two kinds of comparison routines. Biased routines
3976 	 return 0/1/2, and unbiased routines return -1/0/1. Other parts
3977 	 of gcc expect that the comparison operation is equivalent
3978 	 to the modified comparison. For signed comparisons compare the
3979 	 result against 1 in the biased case, and zero in the unbiased
3980 	 case. For unsigned comparisons always compare against 1 after
3981 	 biasing the unbiased result by adding 1. This gives us a way to
3982 	 represent LTU.
3983 	 The comparisons in the fixed-point helper library are always
3984 	 biased.  */
3985       x = result;
3986       y = const1_rtx;
3987 
3988       if (!TARGET_LIB_INT_CMP_BIASED && !ALL_FIXED_POINT_MODE_P (mode))
3989 	{
3990 	  if (unsignedp)
3991 	    x = plus_constant (ret_mode, result, 1);
3992 	  else
3993 	    y = const0_rtx;
3994 	}
3995 
3996       *pmode = ret_mode;
3997       prepare_cmp_insn (x, y, comparison, NULL_RTX, unsignedp, methods,
3998 			ptest, pmode);
3999     }
4000 
4001   return;
4002 
4003  fail:
4004   *ptest = NULL_RTX;
4005 }
4006 
4007 /* Before emitting an insn with code ICODE, make sure that X, which is going
4008    to be used for operand OPNUM of the insn, is converted from mode MODE to
4009    WIDER_MODE (UNSIGNEDP determines whether it is an unsigned conversion), and
4010    that it is accepted by the operand predicate.  Return the new value.  */
4011 
4012 rtx
prepare_operand(enum insn_code icode,rtx x,int opnum,machine_mode mode,machine_mode wider_mode,int unsignedp)4013 prepare_operand (enum insn_code icode, rtx x, int opnum, machine_mode mode,
4014 		 machine_mode wider_mode, int unsignedp)
4015 {
4016   if (mode != wider_mode)
4017     x = convert_modes (wider_mode, mode, x, unsignedp);
4018 
4019   if (!insn_operand_matches (icode, opnum, x))
4020     {
4021       machine_mode op_mode = insn_data[(int) icode].operand[opnum].mode;
4022       if (reload_completed)
4023 	return NULL_RTX;
4024       if (GET_MODE (x) != op_mode && GET_MODE (x) != VOIDmode)
4025 	return NULL_RTX;
4026       x = copy_to_mode_reg (op_mode, x);
4027     }
4028 
4029   return x;
4030 }
4031 
4032 /* Subroutine of emit_cmp_and_jump_insns; this function is called when we know
4033    we can do the branch.  */
4034 
4035 static void
emit_cmp_and_jump_insn_1(rtx test,machine_mode mode,rtx label,profile_probability prob)4036 emit_cmp_and_jump_insn_1 (rtx test, machine_mode mode, rtx label,
4037 			  profile_probability prob)
4038 {
4039   machine_mode optab_mode;
4040   enum mode_class mclass;
4041   enum insn_code icode;
4042   rtx_insn *insn;
4043 
4044   mclass = GET_MODE_CLASS (mode);
4045   optab_mode = (mclass == MODE_CC) ? CCmode : mode;
4046   icode = optab_handler (cbranch_optab, optab_mode);
4047 
4048   gcc_assert (icode != CODE_FOR_nothing);
4049   gcc_assert (insn_operand_matches (icode, 0, test));
4050   insn = emit_jump_insn (GEN_FCN (icode) (test, XEXP (test, 0),
4051                                           XEXP (test, 1), label));
4052   if (prob.initialized_p ()
4053       && profile_status_for_fn (cfun) != PROFILE_ABSENT
4054       && insn
4055       && JUMP_P (insn)
4056       && any_condjump_p (insn)
4057       && !find_reg_note (insn, REG_BR_PROB, 0))
4058     add_reg_br_prob_note (insn, prob);
4059 }
4060 
4061 /* Generate code to compare X with Y so that the condition codes are
4062    set and to jump to LABEL if the condition is true.  If X is a
4063    constant and Y is not a constant, then the comparison is swapped to
4064    ensure that the comparison RTL has the canonical form.
4065 
4066    UNSIGNEDP nonzero says that X and Y are unsigned; this matters if they
4067    need to be widened.  UNSIGNEDP is also used to select the proper
4068    branch condition code.
4069 
4070    If X and Y have mode BLKmode, then SIZE specifies the size of both X and Y.
4071 
4072    MODE is the mode of the inputs (in case they are const_int).
4073 
4074    COMPARISON is the rtl operator to compare with (EQ, NE, GT, etc.).
4075    It will be potentially converted into an unsigned variant based on
4076    UNSIGNEDP to select a proper jump instruction.
4077 
4078    PROB is the probability of jumping to LABEL.  */
4079 
4080 void
emit_cmp_and_jump_insns(rtx x,rtx y,enum rtx_code comparison,rtx size,machine_mode mode,int unsignedp,rtx label,profile_probability prob)4081 emit_cmp_and_jump_insns (rtx x, rtx y, enum rtx_code comparison, rtx size,
4082 			 machine_mode mode, int unsignedp, rtx label,
4083                          profile_probability prob)
4084 {
4085   rtx op0 = x, op1 = y;
4086   rtx test;
4087 
4088   /* Swap operands and condition to ensure canonical RTL.  */
4089   if (swap_commutative_operands_p (x, y)
4090       && can_compare_p (swap_condition (comparison), mode, ccp_jump))
4091     {
4092       op0 = y, op1 = x;
4093       comparison = swap_condition (comparison);
4094     }
4095 
4096   /* If OP0 is still a constant, then both X and Y must be constants
4097      or the opposite comparison is not supported.  Force X into a register
4098      to create canonical RTL.  */
4099   if (CONSTANT_P (op0))
4100     op0 = force_reg (mode, op0);
4101 
4102   if (unsignedp)
4103     comparison = unsigned_condition (comparison);
4104 
4105   prepare_cmp_insn (op0, op1, comparison, size, unsignedp, OPTAB_LIB_WIDEN,
4106 		    &test, &mode);
4107   emit_cmp_and_jump_insn_1 (test, mode, label, prob);
4108 }
4109 
4110 
4111 /* Emit a library call comparison between floating point X and Y.
4112    COMPARISON is the rtl operator to compare with (EQ, NE, GT, etc.).  */
4113 
4114 static void
prepare_float_lib_cmp(rtx x,rtx y,enum rtx_code comparison,rtx * ptest,machine_mode * pmode)4115 prepare_float_lib_cmp (rtx x, rtx y, enum rtx_code comparison,
4116 		       rtx *ptest, machine_mode *pmode)
4117 {
4118   enum rtx_code swapped = swap_condition (comparison);
4119   enum rtx_code reversed = reverse_condition_maybe_unordered (comparison);
4120   machine_mode orig_mode = GET_MODE (x);
4121   machine_mode mode;
4122   rtx true_rtx, false_rtx;
4123   rtx value, target, equiv;
4124   rtx_insn *insns;
4125   rtx libfunc = 0;
4126   bool reversed_p = false;
4127   scalar_int_mode cmp_mode = targetm.libgcc_cmp_return_mode ();
4128 
4129   FOR_EACH_MODE_FROM (mode, orig_mode)
4130     {
4131       if (code_to_optab (comparison)
4132 	  && (libfunc = optab_libfunc (code_to_optab (comparison), mode)))
4133 	break;
4134 
4135       if (code_to_optab (swapped)
4136 	  && (libfunc = optab_libfunc (code_to_optab (swapped), mode)))
4137 	{
4138 	  std::swap (x, y);
4139 	  comparison = swapped;
4140 	  break;
4141 	}
4142 
4143       if (code_to_optab (reversed)
4144 	  && (libfunc = optab_libfunc (code_to_optab (reversed), mode)))
4145 	{
4146 	  comparison = reversed;
4147 	  reversed_p = true;
4148 	  break;
4149 	}
4150     }
4151 
4152   gcc_assert (mode != VOIDmode);
4153 
4154   if (mode != orig_mode)
4155     {
4156       x = convert_to_mode (mode, x, 0);
4157       y = convert_to_mode (mode, y, 0);
4158     }
4159 
4160   /* Attach a REG_EQUAL note describing the semantics of the libcall to
4161      the RTL.  The allows the RTL optimizers to delete the libcall if the
4162      condition can be determined at compile-time.  */
4163   if (comparison == UNORDERED
4164       || FLOAT_LIB_COMPARE_RETURNS_BOOL (mode, comparison))
4165     {
4166       true_rtx = const_true_rtx;
4167       false_rtx = const0_rtx;
4168     }
4169   else
4170     {
4171       switch (comparison)
4172         {
4173         case EQ:
4174           true_rtx = const0_rtx;
4175           false_rtx = const_true_rtx;
4176           break;
4177 
4178         case NE:
4179           true_rtx = const_true_rtx;
4180           false_rtx = const0_rtx;
4181           break;
4182 
4183         case GT:
4184           true_rtx = const1_rtx;
4185           false_rtx = const0_rtx;
4186           break;
4187 
4188         case GE:
4189           true_rtx = const0_rtx;
4190           false_rtx = constm1_rtx;
4191           break;
4192 
4193         case LT:
4194           true_rtx = constm1_rtx;
4195           false_rtx = const0_rtx;
4196           break;
4197 
4198         case LE:
4199           true_rtx = const0_rtx;
4200           false_rtx = const1_rtx;
4201           break;
4202 
4203         default:
4204           gcc_unreachable ();
4205         }
4206     }
4207 
4208   if (comparison == UNORDERED)
4209     {
4210       rtx temp = simplify_gen_relational (NE, cmp_mode, mode, x, x);
4211       equiv = simplify_gen_relational (NE, cmp_mode, mode, y, y);
4212       equiv = simplify_gen_ternary (IF_THEN_ELSE, cmp_mode, cmp_mode,
4213 				    temp, const_true_rtx, equiv);
4214     }
4215   else
4216     {
4217       equiv = simplify_gen_relational (comparison, cmp_mode, mode, x, y);
4218       if (! FLOAT_LIB_COMPARE_RETURNS_BOOL (mode, comparison))
4219         equiv = simplify_gen_ternary (IF_THEN_ELSE, cmp_mode, cmp_mode,
4220                                       equiv, true_rtx, false_rtx);
4221     }
4222 
4223   start_sequence ();
4224   value = emit_library_call_value (libfunc, NULL_RTX, LCT_CONST,
4225 				   cmp_mode, x, mode, y, mode);
4226   insns = get_insns ();
4227   end_sequence ();
4228 
4229   target = gen_reg_rtx (cmp_mode);
4230   emit_libcall_block (insns, target, value, equiv);
4231 
4232   if (comparison == UNORDERED
4233       || FLOAT_LIB_COMPARE_RETURNS_BOOL (mode, comparison)
4234       || reversed_p)
4235     *ptest = gen_rtx_fmt_ee (reversed_p ? EQ : NE, VOIDmode, target, false_rtx);
4236   else
4237     *ptest = gen_rtx_fmt_ee (comparison, VOIDmode, target, const0_rtx);
4238 
4239   *pmode = cmp_mode;
4240 }
4241 
4242 /* Generate code to indirectly jump to a location given in the rtx LOC.  */
4243 
4244 void
emit_indirect_jump(rtx loc)4245 emit_indirect_jump (rtx loc)
4246 {
4247   if (!targetm.have_indirect_jump ())
4248     sorry ("indirect jumps are not available on this target");
4249   else
4250     {
4251       struct expand_operand ops[1];
4252       create_address_operand (&ops[0], loc);
4253       expand_jump_insn (targetm.code_for_indirect_jump, 1, ops);
4254       emit_barrier ();
4255     }
4256 }
4257 
4258 
4259 /* Emit a conditional move instruction if the machine supports one for that
4260    condition and machine mode.
4261 
4262    OP0 and OP1 are the operands that should be compared using CODE.  CMODE is
4263    the mode to use should they be constants.  If it is VOIDmode, they cannot
4264    both be constants.
4265 
4266    OP2 should be stored in TARGET if the comparison is true, otherwise OP3
4267    should be stored there.  MODE is the mode to use should they be constants.
4268    If it is VOIDmode, they cannot both be constants.
4269 
4270    The result is either TARGET (perhaps modified) or NULL_RTX if the operation
4271    is not supported.  */
4272 
4273 rtx
emit_conditional_move(rtx target,enum rtx_code code,rtx op0,rtx op1,machine_mode cmode,rtx op2,rtx op3,machine_mode mode,int unsignedp)4274 emit_conditional_move (rtx target, enum rtx_code code, rtx op0, rtx op1,
4275 		       machine_mode cmode, rtx op2, rtx op3,
4276 		       machine_mode mode, int unsignedp)
4277 {
4278   rtx comparison;
4279   rtx_insn *last;
4280   enum insn_code icode;
4281   enum rtx_code reversed;
4282 
4283   /* If the two source operands are identical, that's just a move.  */
4284 
4285   if (rtx_equal_p (op2, op3))
4286     {
4287       if (!target)
4288 	target = gen_reg_rtx (mode);
4289 
4290       emit_move_insn (target, op3);
4291       return target;
4292     }
4293 
4294   /* If one operand is constant, make it the second one.  Only do this
4295      if the other operand is not constant as well.  */
4296 
4297   if (swap_commutative_operands_p (op0, op1))
4298     {
4299       std::swap (op0, op1);
4300       code = swap_condition (code);
4301     }
4302 
4303   /* get_condition will prefer to generate LT and GT even if the old
4304      comparison was against zero, so undo that canonicalization here since
4305      comparisons against zero are cheaper.  */
4306   if (code == LT && op1 == const1_rtx)
4307     code = LE, op1 = const0_rtx;
4308   else if (code == GT && op1 == constm1_rtx)
4309     code = GE, op1 = const0_rtx;
4310 
4311   if (cmode == VOIDmode)
4312     cmode = GET_MODE (op0);
4313 
4314   enum rtx_code orig_code = code;
4315   bool swapped = false;
4316   if (swap_commutative_operands_p (op2, op3)
4317       && ((reversed = reversed_comparison_code_parts (code, op0, op1, NULL))
4318           != UNKNOWN))
4319     {
4320       std::swap (op2, op3);
4321       code = reversed;
4322       swapped = true;
4323     }
4324 
4325   if (mode == VOIDmode)
4326     mode = GET_MODE (op2);
4327 
4328   icode = direct_optab_handler (movcc_optab, mode);
4329 
4330   if (icode == CODE_FOR_nothing)
4331     return NULL_RTX;
4332 
4333   if (!target)
4334     target = gen_reg_rtx (mode);
4335 
4336   for (int pass = 0; ; pass++)
4337     {
4338       code = unsignedp ? unsigned_condition (code) : code;
4339       comparison = simplify_gen_relational (code, VOIDmode, cmode, op0, op1);
4340 
4341       /* We can get const0_rtx or const_true_rtx in some circumstances.  Just
4342 	 punt and let the caller figure out how best to deal with this
4343 	 situation.  */
4344       if (COMPARISON_P (comparison))
4345 	{
4346 	  saved_pending_stack_adjust save;
4347 	  save_pending_stack_adjust (&save);
4348 	  last = get_last_insn ();
4349 	  do_pending_stack_adjust ();
4350 	  machine_mode cmpmode = cmode;
4351 	  prepare_cmp_insn (XEXP (comparison, 0), XEXP (comparison, 1),
4352 			    GET_CODE (comparison), NULL_RTX, unsignedp,
4353 			    OPTAB_WIDEN, &comparison, &cmpmode);
4354 	  if (comparison)
4355 	    {
4356 	      struct expand_operand ops[4];
4357 
4358 	      create_output_operand (&ops[0], target, mode);
4359 	      create_fixed_operand (&ops[1], comparison);
4360 	      create_input_operand (&ops[2], op2, mode);
4361 	      create_input_operand (&ops[3], op3, mode);
4362 	      if (maybe_expand_insn (icode, 4, ops))
4363 		{
4364 		  if (ops[0].value != target)
4365 		    convert_move (target, ops[0].value, false);
4366 		  return target;
4367 		}
4368 	    }
4369 	  delete_insns_since (last);
4370 	  restore_pending_stack_adjust (&save);
4371 	}
4372 
4373       if (pass == 1)
4374 	return NULL_RTX;
4375 
4376       /* If the preferred op2/op3 order is not usable, retry with other
4377 	 operand order, perhaps it will expand successfully.  */
4378       if (swapped)
4379 	code = orig_code;
4380       else if ((reversed = reversed_comparison_code_parts (orig_code, op0, op1,
4381 							   NULL))
4382 	       != UNKNOWN)
4383 	code = reversed;
4384       else
4385 	return NULL_RTX;
4386       std::swap (op2, op3);
4387     }
4388 }
4389 
4390 
4391 /* Emit a conditional negate or bitwise complement using the
4392    negcc or notcc optabs if available.  Return NULL_RTX if such operations
4393    are not available.  Otherwise return the RTX holding the result.
4394    TARGET is the desired destination of the result.  COMP is the comparison
4395    on which to negate.  If COND is true move into TARGET the negation
4396    or bitwise complement of OP1.  Otherwise move OP2 into TARGET.
4397    CODE is either NEG or NOT.  MODE is the machine mode in which the
4398    operation is performed.  */
4399 
4400 rtx
emit_conditional_neg_or_complement(rtx target,rtx_code code,machine_mode mode,rtx cond,rtx op1,rtx op2)4401 emit_conditional_neg_or_complement (rtx target, rtx_code code,
4402 				     machine_mode mode, rtx cond, rtx op1,
4403 				     rtx op2)
4404 {
4405   optab op = unknown_optab;
4406   if (code == NEG)
4407     op = negcc_optab;
4408   else if (code == NOT)
4409     op = notcc_optab;
4410   else
4411     gcc_unreachable ();
4412 
4413   insn_code icode = direct_optab_handler (op, mode);
4414 
4415   if (icode == CODE_FOR_nothing)
4416     return NULL_RTX;
4417 
4418   if (!target)
4419     target = gen_reg_rtx (mode);
4420 
4421   rtx_insn *last = get_last_insn ();
4422   struct expand_operand ops[4];
4423 
4424   create_output_operand (&ops[0], target, mode);
4425   create_fixed_operand (&ops[1], cond);
4426   create_input_operand (&ops[2], op1, mode);
4427   create_input_operand (&ops[3], op2, mode);
4428 
4429   if (maybe_expand_insn (icode, 4, ops))
4430     {
4431       if (ops[0].value != target)
4432 	convert_move (target, ops[0].value, false);
4433 
4434       return target;
4435     }
4436   delete_insns_since (last);
4437   return NULL_RTX;
4438 }
4439 
4440 /* Emit a conditional addition instruction if the machine supports one for that
4441    condition and machine mode.
4442 
4443    OP0 and OP1 are the operands that should be compared using CODE.  CMODE is
4444    the mode to use should they be constants.  If it is VOIDmode, they cannot
4445    both be constants.
4446 
4447    OP2 should be stored in TARGET if the comparison is false, otherwise OP2+OP3
4448    should be stored there.  MODE is the mode to use should they be constants.
4449    If it is VOIDmode, they cannot both be constants.
4450 
4451    The result is either TARGET (perhaps modified) or NULL_RTX if the operation
4452    is not supported.  */
4453 
4454 rtx
emit_conditional_add(rtx target,enum rtx_code code,rtx op0,rtx op1,machine_mode cmode,rtx op2,rtx op3,machine_mode mode,int unsignedp)4455 emit_conditional_add (rtx target, enum rtx_code code, rtx op0, rtx op1,
4456 		      machine_mode cmode, rtx op2, rtx op3,
4457 		      machine_mode mode, int unsignedp)
4458 {
4459   rtx comparison;
4460   rtx_insn *last;
4461   enum insn_code icode;
4462 
4463   /* If one operand is constant, make it the second one.  Only do this
4464      if the other operand is not constant as well.  */
4465 
4466   if (swap_commutative_operands_p (op0, op1))
4467     {
4468       std::swap (op0, op1);
4469       code = swap_condition (code);
4470     }
4471 
4472   /* get_condition will prefer to generate LT and GT even if the old
4473      comparison was against zero, so undo that canonicalization here since
4474      comparisons against zero are cheaper.  */
4475   if (code == LT && op1 == const1_rtx)
4476     code = LE, op1 = const0_rtx;
4477   else if (code == GT && op1 == constm1_rtx)
4478     code = GE, op1 = const0_rtx;
4479 
4480   if (cmode == VOIDmode)
4481     cmode = GET_MODE (op0);
4482 
4483   if (mode == VOIDmode)
4484     mode = GET_MODE (op2);
4485 
4486   icode = optab_handler (addcc_optab, mode);
4487 
4488   if (icode == CODE_FOR_nothing)
4489     return 0;
4490 
4491   if (!target)
4492     target = gen_reg_rtx (mode);
4493 
4494   code = unsignedp ? unsigned_condition (code) : code;
4495   comparison = simplify_gen_relational (code, VOIDmode, cmode, op0, op1);
4496 
4497   /* We can get const0_rtx or const_true_rtx in some circumstances.  Just
4498      return NULL and let the caller figure out how best to deal with this
4499      situation.  */
4500   if (!COMPARISON_P (comparison))
4501     return NULL_RTX;
4502 
4503   do_pending_stack_adjust ();
4504   last = get_last_insn ();
4505   prepare_cmp_insn (XEXP (comparison, 0), XEXP (comparison, 1),
4506                     GET_CODE (comparison), NULL_RTX, unsignedp, OPTAB_WIDEN,
4507                     &comparison, &cmode);
4508   if (comparison)
4509     {
4510       struct expand_operand ops[4];
4511 
4512       create_output_operand (&ops[0], target, mode);
4513       create_fixed_operand (&ops[1], comparison);
4514       create_input_operand (&ops[2], op2, mode);
4515       create_input_operand (&ops[3], op3, mode);
4516       if (maybe_expand_insn (icode, 4, ops))
4517 	{
4518 	  if (ops[0].value != target)
4519 	    convert_move (target, ops[0].value, false);
4520 	  return target;
4521 	}
4522     }
4523   delete_insns_since (last);
4524   return NULL_RTX;
4525 }
4526 
4527 /* These functions attempt to generate an insn body, rather than
4528    emitting the insn, but if the gen function already emits them, we
4529    make no attempt to turn them back into naked patterns.  */
4530 
4531 /* Generate and return an insn body to add Y to X.  */
4532 
4533 rtx_insn *
gen_add2_insn(rtx x,rtx y)4534 gen_add2_insn (rtx x, rtx y)
4535 {
4536   enum insn_code icode = optab_handler (add_optab, GET_MODE (x));
4537 
4538   gcc_assert (insn_operand_matches (icode, 0, x));
4539   gcc_assert (insn_operand_matches (icode, 1, x));
4540   gcc_assert (insn_operand_matches (icode, 2, y));
4541 
4542   return GEN_FCN (icode) (x, x, y);
4543 }
4544 
4545 /* Generate and return an insn body to add r1 and c,
4546    storing the result in r0.  */
4547 
4548 rtx_insn *
gen_add3_insn(rtx r0,rtx r1,rtx c)4549 gen_add3_insn (rtx r0, rtx r1, rtx c)
4550 {
4551   enum insn_code icode = optab_handler (add_optab, GET_MODE (r0));
4552 
4553   if (icode == CODE_FOR_nothing
4554       || !insn_operand_matches (icode, 0, r0)
4555       || !insn_operand_matches (icode, 1, r1)
4556       || !insn_operand_matches (icode, 2, c))
4557     return NULL;
4558 
4559   return GEN_FCN (icode) (r0, r1, c);
4560 }
4561 
4562 int
have_add2_insn(rtx x,rtx y)4563 have_add2_insn (rtx x, rtx y)
4564 {
4565   enum insn_code icode;
4566 
4567   gcc_assert (GET_MODE (x) != VOIDmode);
4568 
4569   icode = optab_handler (add_optab, GET_MODE (x));
4570 
4571   if (icode == CODE_FOR_nothing)
4572     return 0;
4573 
4574   if (!insn_operand_matches (icode, 0, x)
4575       || !insn_operand_matches (icode, 1, x)
4576       || !insn_operand_matches (icode, 2, y))
4577     return 0;
4578 
4579   return 1;
4580 }
4581 
4582 /* Generate and return an insn body to add Y to X.  */
4583 
4584 rtx_insn *
gen_addptr3_insn(rtx x,rtx y,rtx z)4585 gen_addptr3_insn (rtx x, rtx y, rtx z)
4586 {
4587   enum insn_code icode = optab_handler (addptr3_optab, GET_MODE (x));
4588 
4589   gcc_assert (insn_operand_matches (icode, 0, x));
4590   gcc_assert (insn_operand_matches (icode, 1, y));
4591   gcc_assert (insn_operand_matches (icode, 2, z));
4592 
4593   return GEN_FCN (icode) (x, y, z);
4594 }
4595 
4596 /* Return true if the target implements an addptr pattern and X, Y,
4597    and Z are valid for the pattern predicates.  */
4598 
4599 int
have_addptr3_insn(rtx x,rtx y,rtx z)4600 have_addptr3_insn (rtx x, rtx y, rtx z)
4601 {
4602   enum insn_code icode;
4603 
4604   gcc_assert (GET_MODE (x) != VOIDmode);
4605 
4606   icode = optab_handler (addptr3_optab, GET_MODE (x));
4607 
4608   if (icode == CODE_FOR_nothing)
4609     return 0;
4610 
4611   if (!insn_operand_matches (icode, 0, x)
4612       || !insn_operand_matches (icode, 1, y)
4613       || !insn_operand_matches (icode, 2, z))
4614     return 0;
4615 
4616   return 1;
4617 }
4618 
4619 /* Generate and return an insn body to subtract Y from X.  */
4620 
4621 rtx_insn *
gen_sub2_insn(rtx x,rtx y)4622 gen_sub2_insn (rtx x, rtx y)
4623 {
4624   enum insn_code icode = optab_handler (sub_optab, GET_MODE (x));
4625 
4626   gcc_assert (insn_operand_matches (icode, 0, x));
4627   gcc_assert (insn_operand_matches (icode, 1, x));
4628   gcc_assert (insn_operand_matches (icode, 2, y));
4629 
4630   return GEN_FCN (icode) (x, x, y);
4631 }
4632 
4633 /* Generate and return an insn body to subtract r1 and c,
4634    storing the result in r0.  */
4635 
4636 rtx_insn *
gen_sub3_insn(rtx r0,rtx r1,rtx c)4637 gen_sub3_insn (rtx r0, rtx r1, rtx c)
4638 {
4639   enum insn_code icode = optab_handler (sub_optab, GET_MODE (r0));
4640 
4641   if (icode == CODE_FOR_nothing
4642       || !insn_operand_matches (icode, 0, r0)
4643       || !insn_operand_matches (icode, 1, r1)
4644       || !insn_operand_matches (icode, 2, c))
4645     return NULL;
4646 
4647   return GEN_FCN (icode) (r0, r1, c);
4648 }
4649 
4650 int
have_sub2_insn(rtx x,rtx y)4651 have_sub2_insn (rtx x, rtx y)
4652 {
4653   enum insn_code icode;
4654 
4655   gcc_assert (GET_MODE (x) != VOIDmode);
4656 
4657   icode = optab_handler (sub_optab, GET_MODE (x));
4658 
4659   if (icode == CODE_FOR_nothing)
4660     return 0;
4661 
4662   if (!insn_operand_matches (icode, 0, x)
4663       || !insn_operand_matches (icode, 1, x)
4664       || !insn_operand_matches (icode, 2, y))
4665     return 0;
4666 
4667   return 1;
4668 }
4669 
4670 /* Generate the body of an insn to extend Y (with mode MFROM)
4671    into X (with mode MTO).  Do zero-extension if UNSIGNEDP is nonzero.  */
4672 
4673 rtx_insn *
gen_extend_insn(rtx x,rtx y,machine_mode mto,machine_mode mfrom,int unsignedp)4674 gen_extend_insn (rtx x, rtx y, machine_mode mto,
4675 		 machine_mode mfrom, int unsignedp)
4676 {
4677   enum insn_code icode = can_extend_p (mto, mfrom, unsignedp);
4678   return GEN_FCN (icode) (x, y);
4679 }
4680 
4681 /* Generate code to convert FROM to floating point
4682    and store in TO.  FROM must be fixed point and not VOIDmode.
4683    UNSIGNEDP nonzero means regard FROM as unsigned.
4684    Normally this is done by correcting the final value
4685    if it is negative.  */
4686 
4687 void
expand_float(rtx to,rtx from,int unsignedp)4688 expand_float (rtx to, rtx from, int unsignedp)
4689 {
4690   enum insn_code icode;
4691   rtx target = to;
4692   scalar_mode from_mode, to_mode;
4693   machine_mode fmode, imode;
4694   bool can_do_signed = false;
4695 
4696   /* Crash now, because we won't be able to decide which mode to use.  */
4697   gcc_assert (GET_MODE (from) != VOIDmode);
4698 
4699   /* Look for an insn to do the conversion.  Do it in the specified
4700      modes if possible; otherwise convert either input, output or both to
4701      wider mode.  If the integer mode is wider than the mode of FROM,
4702      we can do the conversion signed even if the input is unsigned.  */
4703 
4704   FOR_EACH_MODE_FROM (fmode, GET_MODE (to))
4705     FOR_EACH_MODE_FROM (imode, GET_MODE (from))
4706       {
4707 	int doing_unsigned = unsignedp;
4708 
4709 	if (fmode != GET_MODE (to)
4710 	    && (significand_size (fmode)
4711 		< GET_MODE_UNIT_PRECISION (GET_MODE (from))))
4712 	  continue;
4713 
4714 	icode = can_float_p (fmode, imode, unsignedp);
4715 	if (icode == CODE_FOR_nothing && unsignedp)
4716 	  {
4717 	    enum insn_code scode = can_float_p (fmode, imode, 0);
4718 	    if (scode != CODE_FOR_nothing)
4719 	      can_do_signed = true;
4720 	    if (imode != GET_MODE (from))
4721 	      icode = scode, doing_unsigned = 0;
4722 	  }
4723 
4724 	if (icode != CODE_FOR_nothing)
4725 	  {
4726 	    if (imode != GET_MODE (from))
4727 	      from = convert_to_mode (imode, from, unsignedp);
4728 
4729 	    if (fmode != GET_MODE (to))
4730 	      target = gen_reg_rtx (fmode);
4731 
4732 	    emit_unop_insn (icode, target, from,
4733 			    doing_unsigned ? UNSIGNED_FLOAT : FLOAT);
4734 
4735 	    if (target != to)
4736 	      convert_move (to, target, 0);
4737 	    return;
4738 	  }
4739       }
4740 
4741   /* Unsigned integer, and no way to convert directly.  Convert as signed,
4742      then unconditionally adjust the result.  */
4743   if (unsignedp
4744       && can_do_signed
4745       && is_a <scalar_mode> (GET_MODE (to), &to_mode)
4746       && is_a <scalar_mode> (GET_MODE (from), &from_mode))
4747     {
4748       opt_scalar_mode fmode_iter;
4749       rtx_code_label *label = gen_label_rtx ();
4750       rtx temp;
4751       REAL_VALUE_TYPE offset;
4752 
4753       /* Look for a usable floating mode FMODE wider than the source and at
4754 	 least as wide as the target.  Using FMODE will avoid rounding woes
4755 	 with unsigned values greater than the signed maximum value.  */
4756 
4757       FOR_EACH_MODE_FROM (fmode_iter, to_mode)
4758 	{
4759 	  scalar_mode fmode = fmode_iter.require ();
4760 	  if (GET_MODE_PRECISION (from_mode) < GET_MODE_BITSIZE (fmode)
4761 	      && can_float_p (fmode, from_mode, 0) != CODE_FOR_nothing)
4762 	    break;
4763 	}
4764 
4765       if (!fmode_iter.exists (&fmode))
4766 	{
4767 	  /* There is no such mode.  Pretend the target is wide enough.  */
4768 	  fmode = to_mode;
4769 
4770 	  /* Avoid double-rounding when TO is narrower than FROM.  */
4771 	  if ((significand_size (fmode) + 1)
4772 	      < GET_MODE_PRECISION (from_mode))
4773 	    {
4774 	      rtx temp1;
4775 	      rtx_code_label *neglabel = gen_label_rtx ();
4776 
4777 	      /* Don't use TARGET if it isn't a register, is a hard register,
4778 		 or is the wrong mode.  */
4779 	      if (!REG_P (target)
4780 		  || REGNO (target) < FIRST_PSEUDO_REGISTER
4781 		  || GET_MODE (target) != fmode)
4782 		target = gen_reg_rtx (fmode);
4783 
4784 	      imode = from_mode;
4785 	      do_pending_stack_adjust ();
4786 
4787 	      /* Test whether the sign bit is set.  */
4788 	      emit_cmp_and_jump_insns (from, const0_rtx, LT, NULL_RTX, imode,
4789 				       0, neglabel);
4790 
4791 	      /* The sign bit is not set.  Convert as signed.  */
4792 	      expand_float (target, from, 0);
4793 	      emit_jump_insn (targetm.gen_jump (label));
4794 	      emit_barrier ();
4795 
4796 	      /* The sign bit is set.
4797 		 Convert to a usable (positive signed) value by shifting right
4798 		 one bit, while remembering if a nonzero bit was shifted
4799 		 out; i.e., compute  (from & 1) | (from >> 1).  */
4800 
4801 	      emit_label (neglabel);
4802 	      temp = expand_binop (imode, and_optab, from, const1_rtx,
4803 				   NULL_RTX, 1, OPTAB_LIB_WIDEN);
4804 	      temp1 = expand_shift (RSHIFT_EXPR, imode, from, 1, NULL_RTX, 1);
4805 	      temp = expand_binop (imode, ior_optab, temp, temp1, temp, 1,
4806 				   OPTAB_LIB_WIDEN);
4807 	      expand_float (target, temp, 0);
4808 
4809 	      /* Multiply by 2 to undo the shift above.  */
4810 	      temp = expand_binop (fmode, add_optab, target, target,
4811 				   target, 0, OPTAB_LIB_WIDEN);
4812 	      if (temp != target)
4813 		emit_move_insn (target, temp);
4814 
4815 	      do_pending_stack_adjust ();
4816 	      emit_label (label);
4817 	      goto done;
4818 	    }
4819 	}
4820 
4821       /* If we are about to do some arithmetic to correct for an
4822 	 unsigned operand, do it in a pseudo-register.  */
4823 
4824       if (to_mode != fmode
4825 	  || !REG_P (to) || REGNO (to) < FIRST_PSEUDO_REGISTER)
4826 	target = gen_reg_rtx (fmode);
4827 
4828       /* Convert as signed integer to floating.  */
4829       expand_float (target, from, 0);
4830 
4831       /* If FROM is negative (and therefore TO is negative),
4832 	 correct its value by 2**bitwidth.  */
4833 
4834       do_pending_stack_adjust ();
4835       emit_cmp_and_jump_insns (from, const0_rtx, GE, NULL_RTX, from_mode,
4836 			       0, label);
4837 
4838 
4839       real_2expN (&offset, GET_MODE_PRECISION (from_mode), fmode);
4840       temp = expand_binop (fmode, add_optab, target,
4841 			   const_double_from_real_value (offset, fmode),
4842 			   target, 0, OPTAB_LIB_WIDEN);
4843       if (temp != target)
4844 	emit_move_insn (target, temp);
4845 
4846       do_pending_stack_adjust ();
4847       emit_label (label);
4848       goto done;
4849     }
4850 
4851   /* No hardware instruction available; call a library routine.  */
4852     {
4853       rtx libfunc;
4854       rtx_insn *insns;
4855       rtx value;
4856       convert_optab tab = unsignedp ? ufloat_optab : sfloat_optab;
4857 
4858       if (is_narrower_int_mode (GET_MODE (from), SImode))
4859 	from = convert_to_mode (SImode, from, unsignedp);
4860 
4861       libfunc = convert_optab_libfunc (tab, GET_MODE (to), GET_MODE (from));
4862       gcc_assert (libfunc);
4863 
4864       start_sequence ();
4865 
4866       value = emit_library_call_value (libfunc, NULL_RTX, LCT_CONST,
4867 				       GET_MODE (to), from, GET_MODE (from));
4868       insns = get_insns ();
4869       end_sequence ();
4870 
4871       emit_libcall_block (insns, target, value,
4872 			  gen_rtx_fmt_e (unsignedp ? UNSIGNED_FLOAT : FLOAT,
4873 					 GET_MODE (to), from));
4874     }
4875 
4876  done:
4877 
4878   /* Copy result to requested destination
4879      if we have been computing in a temp location.  */
4880 
4881   if (target != to)
4882     {
4883       if (GET_MODE (target) == GET_MODE (to))
4884 	emit_move_insn (to, target);
4885       else
4886 	convert_move (to, target, 0);
4887     }
4888 }
4889 
4890 /* Generate code to convert FROM to fixed point and store in TO.  FROM
4891    must be floating point.  */
4892 
4893 void
expand_fix(rtx to,rtx from,int unsignedp)4894 expand_fix (rtx to, rtx from, int unsignedp)
4895 {
4896   enum insn_code icode;
4897   rtx target = to;
4898   machine_mode fmode, imode;
4899   opt_scalar_mode fmode_iter;
4900   bool must_trunc = false;
4901 
4902   /* We first try to find a pair of modes, one real and one integer, at
4903      least as wide as FROM and TO, respectively, in which we can open-code
4904      this conversion.  If the integer mode is wider than the mode of TO,
4905      we can do the conversion either signed or unsigned.  */
4906 
4907   FOR_EACH_MODE_FROM (fmode, GET_MODE (from))
4908     FOR_EACH_MODE_FROM (imode, GET_MODE (to))
4909       {
4910 	int doing_unsigned = unsignedp;
4911 
4912 	icode = can_fix_p (imode, fmode, unsignedp, &must_trunc);
4913 	if (icode == CODE_FOR_nothing && imode != GET_MODE (to) && unsignedp)
4914 	  icode = can_fix_p (imode, fmode, 0, &must_trunc), doing_unsigned = 0;
4915 
4916 	if (icode != CODE_FOR_nothing)
4917 	  {
4918 	    rtx_insn *last = get_last_insn ();
4919 	    if (fmode != GET_MODE (from))
4920 	      from = convert_to_mode (fmode, from, 0);
4921 
4922 	    if (must_trunc)
4923 	      {
4924 		rtx temp = gen_reg_rtx (GET_MODE (from));
4925 		from = expand_unop (GET_MODE (from), ftrunc_optab, from,
4926 				    temp, 0);
4927 	      }
4928 
4929 	    if (imode != GET_MODE (to))
4930 	      target = gen_reg_rtx (imode);
4931 
4932 	    if (maybe_emit_unop_insn (icode, target, from,
4933 				      doing_unsigned ? UNSIGNED_FIX : FIX))
4934 	      {
4935 		if (target != to)
4936 		  convert_move (to, target, unsignedp);
4937 		return;
4938 	      }
4939 	    delete_insns_since (last);
4940 	  }
4941       }
4942 
4943   /* For an unsigned conversion, there is one more way to do it.
4944      If we have a signed conversion, we generate code that compares
4945      the real value to the largest representable positive number.  If if
4946      is smaller, the conversion is done normally.  Otherwise, subtract
4947      one plus the highest signed number, convert, and add it back.
4948 
4949      We only need to check all real modes, since we know we didn't find
4950      anything with a wider integer mode.
4951 
4952      This code used to extend FP value into mode wider than the destination.
4953      This is needed for decimal float modes which cannot accurately
4954      represent one plus the highest signed number of the same size, but
4955      not for binary modes.  Consider, for instance conversion from SFmode
4956      into DImode.
4957 
4958      The hot path through the code is dealing with inputs smaller than 2^63
4959      and doing just the conversion, so there is no bits to lose.
4960 
4961      In the other path we know the value is positive in the range 2^63..2^64-1
4962      inclusive.  (as for other input overflow happens and result is undefined)
4963      So we know that the most important bit set in mantissa corresponds to
4964      2^63.  The subtraction of 2^63 should not generate any rounding as it
4965      simply clears out that bit.  The rest is trivial.  */
4966 
4967   scalar_int_mode to_mode;
4968   if (unsignedp
4969       && is_a <scalar_int_mode> (GET_MODE (to), &to_mode)
4970       && HWI_COMPUTABLE_MODE_P (to_mode))
4971     FOR_EACH_MODE_FROM (fmode_iter, as_a <scalar_mode> (GET_MODE (from)))
4972       {
4973 	scalar_mode fmode = fmode_iter.require ();
4974 	if (CODE_FOR_nothing != can_fix_p (to_mode, fmode,
4975 					   0, &must_trunc)
4976 	    && (!DECIMAL_FLOAT_MODE_P (fmode)
4977 		|| (GET_MODE_BITSIZE (fmode) > GET_MODE_PRECISION (to_mode))))
4978 	  {
4979 	    int bitsize;
4980 	    REAL_VALUE_TYPE offset;
4981 	    rtx limit;
4982 	    rtx_code_label *lab1, *lab2;
4983 	    rtx_insn *insn;
4984 
4985 	    bitsize = GET_MODE_PRECISION (to_mode);
4986 	    real_2expN (&offset, bitsize - 1, fmode);
4987 	    limit = const_double_from_real_value (offset, fmode);
4988 	    lab1 = gen_label_rtx ();
4989 	    lab2 = gen_label_rtx ();
4990 
4991 	    if (fmode != GET_MODE (from))
4992 	      from = convert_to_mode (fmode, from, 0);
4993 
4994 	    /* See if we need to do the subtraction.  */
4995 	    do_pending_stack_adjust ();
4996 	    emit_cmp_and_jump_insns (from, limit, GE, NULL_RTX,
4997 				     GET_MODE (from), 0, lab1);
4998 
4999 	    /* If not, do the signed "fix" and branch around fixup code.  */
5000 	    expand_fix (to, from, 0);
5001 	    emit_jump_insn (targetm.gen_jump (lab2));
5002 	    emit_barrier ();
5003 
5004 	    /* Otherwise, subtract 2**(N-1), convert to signed number,
5005 	       then add 2**(N-1).  Do the addition using XOR since this
5006 	       will often generate better code.  */
5007 	    emit_label (lab1);
5008 	    target = expand_binop (GET_MODE (from), sub_optab, from, limit,
5009 				   NULL_RTX, 0, OPTAB_LIB_WIDEN);
5010 	    expand_fix (to, target, 0);
5011 	    target = expand_binop (to_mode, xor_optab, to,
5012 				   gen_int_mode
5013 				   (HOST_WIDE_INT_1 << (bitsize - 1),
5014 				    to_mode),
5015 				   to, 1, OPTAB_LIB_WIDEN);
5016 
5017 	    if (target != to)
5018 	      emit_move_insn (to, target);
5019 
5020 	    emit_label (lab2);
5021 
5022 	    if (optab_handler (mov_optab, to_mode) != CODE_FOR_nothing)
5023 	      {
5024 		/* Make a place for a REG_NOTE and add it.  */
5025 		insn = emit_move_insn (to, to);
5026 		set_dst_reg_note (insn, REG_EQUAL,
5027 				  gen_rtx_fmt_e (UNSIGNED_FIX, to_mode,
5028 						 copy_rtx (from)),
5029 				  to);
5030 	      }
5031 
5032 	    return;
5033 	  }
5034       }
5035 
5036   /* We can't do it with an insn, so use a library call.  But first ensure
5037      that the mode of TO is at least as wide as SImode, since those are the
5038      only library calls we know about.  */
5039 
5040   if (is_narrower_int_mode (GET_MODE (to), SImode))
5041     {
5042       target = gen_reg_rtx (SImode);
5043 
5044       expand_fix (target, from, unsignedp);
5045     }
5046   else
5047     {
5048       rtx_insn *insns;
5049       rtx value;
5050       rtx libfunc;
5051 
5052       convert_optab tab = unsignedp ? ufix_optab : sfix_optab;
5053       libfunc = convert_optab_libfunc (tab, GET_MODE (to), GET_MODE (from));
5054       gcc_assert (libfunc);
5055 
5056       start_sequence ();
5057 
5058       value = emit_library_call_value (libfunc, NULL_RTX, LCT_CONST,
5059 				       GET_MODE (to), from, GET_MODE (from));
5060       insns = get_insns ();
5061       end_sequence ();
5062 
5063       emit_libcall_block (insns, target, value,
5064 			  gen_rtx_fmt_e (unsignedp ? UNSIGNED_FIX : FIX,
5065 					 GET_MODE (to), from));
5066     }
5067 
5068   if (target != to)
5069     {
5070       if (GET_MODE (to) == GET_MODE (target))
5071         emit_move_insn (to, target);
5072       else
5073         convert_move (to, target, 0);
5074     }
5075 }
5076 
5077 
5078 /* Promote integer arguments for a libcall if necessary.
5079    emit_library_call_value cannot do the promotion because it does not
5080    know if it should do a signed or unsigned promotion.  This is because
5081    there are no tree types defined for libcalls.  */
5082 
5083 static rtx
prepare_libcall_arg(rtx arg,int uintp)5084 prepare_libcall_arg (rtx arg, int uintp)
5085 {
5086   scalar_int_mode mode;
5087   machine_mode arg_mode;
5088   if (is_a <scalar_int_mode> (GET_MODE (arg), &mode))
5089     {
5090       /*  If we need to promote the integer function argument we need to do
5091 	  it here instead of inside emit_library_call_value because in
5092 	  emit_library_call_value we don't know if we should do a signed or
5093 	  unsigned promotion.  */
5094 
5095       int unsigned_p = 0;
5096       arg_mode = promote_function_mode (NULL_TREE, mode,
5097 					&unsigned_p, NULL_TREE, 0);
5098       if (arg_mode != mode)
5099 	return convert_to_mode (arg_mode, arg, uintp);
5100     }
5101     return arg;
5102 }
5103 
5104 /* Generate code to convert FROM or TO a fixed-point.
5105    If UINTP is true, either TO or FROM is an unsigned integer.
5106    If SATP is true, we need to saturate the result.  */
5107 
5108 void
expand_fixed_convert(rtx to,rtx from,int uintp,int satp)5109 expand_fixed_convert (rtx to, rtx from, int uintp, int satp)
5110 {
5111   machine_mode to_mode = GET_MODE (to);
5112   machine_mode from_mode = GET_MODE (from);
5113   convert_optab tab;
5114   enum rtx_code this_code;
5115   enum insn_code code;
5116   rtx_insn *insns;
5117   rtx value;
5118   rtx libfunc;
5119 
5120   if (to_mode == from_mode)
5121     {
5122       emit_move_insn (to, from);
5123       return;
5124     }
5125 
5126   if (uintp)
5127     {
5128       tab = satp ? satfractuns_optab : fractuns_optab;
5129       this_code = satp ? UNSIGNED_SAT_FRACT : UNSIGNED_FRACT_CONVERT;
5130     }
5131   else
5132     {
5133       tab = satp ? satfract_optab : fract_optab;
5134       this_code = satp ? SAT_FRACT : FRACT_CONVERT;
5135     }
5136   code = convert_optab_handler (tab, to_mode, from_mode);
5137   if (code != CODE_FOR_nothing)
5138     {
5139       emit_unop_insn (code, to, from, this_code);
5140       return;
5141     }
5142 
5143   libfunc = convert_optab_libfunc (tab, to_mode, from_mode);
5144   gcc_assert (libfunc);
5145 
5146   from = prepare_libcall_arg (from, uintp);
5147   from_mode = GET_MODE (from);
5148 
5149   start_sequence ();
5150   value = emit_library_call_value (libfunc, NULL_RTX, LCT_CONST, to_mode,
5151 				   from, from_mode);
5152   insns = get_insns ();
5153   end_sequence ();
5154 
5155   emit_libcall_block (insns, to, value,
5156 		      gen_rtx_fmt_e (optab_to_code (tab), to_mode, from));
5157 }
5158 
5159 /* Generate code to convert FROM to fixed point and store in TO.  FROM
5160    must be floating point, TO must be signed.  Use the conversion optab
5161    TAB to do the conversion.  */
5162 
5163 bool
expand_sfix_optab(rtx to,rtx from,convert_optab tab)5164 expand_sfix_optab (rtx to, rtx from, convert_optab tab)
5165 {
5166   enum insn_code icode;
5167   rtx target = to;
5168   machine_mode fmode, imode;
5169 
5170   /* We first try to find a pair of modes, one real and one integer, at
5171      least as wide as FROM and TO, respectively, in which we can open-code
5172      this conversion.  If the integer mode is wider than the mode of TO,
5173      we can do the conversion either signed or unsigned.  */
5174 
5175   FOR_EACH_MODE_FROM (fmode, GET_MODE (from))
5176     FOR_EACH_MODE_FROM (imode, GET_MODE (to))
5177       {
5178 	icode = convert_optab_handler (tab, imode, fmode);
5179 	if (icode != CODE_FOR_nothing)
5180 	  {
5181 	    rtx_insn *last = get_last_insn ();
5182 	    if (fmode != GET_MODE (from))
5183 	      from = convert_to_mode (fmode, from, 0);
5184 
5185 	    if (imode != GET_MODE (to))
5186 	      target = gen_reg_rtx (imode);
5187 
5188 	    if (!maybe_emit_unop_insn (icode, target, from, UNKNOWN))
5189 	      {
5190 	        delete_insns_since (last);
5191 		continue;
5192 	      }
5193 	    if (target != to)
5194 	      convert_move (to, target, 0);
5195 	    return true;
5196 	  }
5197       }
5198 
5199   return false;
5200 }
5201 
5202 /* Report whether we have an instruction to perform the operation
5203    specified by CODE on operands of mode MODE.  */
5204 int
have_insn_for(enum rtx_code code,machine_mode mode)5205 have_insn_for (enum rtx_code code, machine_mode mode)
5206 {
5207   return (code_to_optab (code)
5208 	  && (optab_handler (code_to_optab (code), mode)
5209 	      != CODE_FOR_nothing));
5210 }
5211 
5212 /* Print information about the current contents of the optabs on
5213    STDERR.  */
5214 
5215 DEBUG_FUNCTION void
debug_optab_libfuncs(void)5216 debug_optab_libfuncs (void)
5217 {
5218   int i, j, k;
5219 
5220   /* Dump the arithmetic optabs.  */
5221   for (i = FIRST_NORM_OPTAB; i <= LAST_NORMLIB_OPTAB; ++i)
5222     for (j = 0; j < NUM_MACHINE_MODES; ++j)
5223       {
5224 	rtx l = optab_libfunc ((optab) i, (machine_mode) j);
5225 	if (l)
5226 	  {
5227 	    gcc_assert (GET_CODE (l) == SYMBOL_REF);
5228 	    fprintf (stderr, "%s\t%s:\t%s\n",
5229 		     GET_RTX_NAME (optab_to_code ((optab) i)),
5230 		     GET_MODE_NAME (j),
5231 		     XSTR (l, 0));
5232 	  }
5233       }
5234 
5235   /* Dump the conversion optabs.  */
5236   for (i = FIRST_CONV_OPTAB; i <= LAST_CONVLIB_OPTAB; ++i)
5237     for (j = 0; j < NUM_MACHINE_MODES; ++j)
5238       for (k = 0; k < NUM_MACHINE_MODES; ++k)
5239 	{
5240 	  rtx l = convert_optab_libfunc ((optab) i, (machine_mode) j,
5241 					 (machine_mode) k);
5242 	  if (l)
5243 	    {
5244 	      gcc_assert (GET_CODE (l) == SYMBOL_REF);
5245 	      fprintf (stderr, "%s\t%s\t%s:\t%s\n",
5246 		       GET_RTX_NAME (optab_to_code ((optab) i)),
5247 		       GET_MODE_NAME (j),
5248 		       GET_MODE_NAME (k),
5249 		       XSTR (l, 0));
5250 	    }
5251 	}
5252 }
5253 
5254 /* Generate insns to trap with code TCODE if OP1 and OP2 satisfy condition
5255    CODE.  Return 0 on failure.  */
5256 
5257 rtx_insn *
gen_cond_trap(enum rtx_code code,rtx op1,rtx op2,rtx tcode)5258 gen_cond_trap (enum rtx_code code, rtx op1, rtx op2, rtx tcode)
5259 {
5260   machine_mode mode = GET_MODE (op1);
5261   enum insn_code icode;
5262   rtx_insn *insn;
5263   rtx trap_rtx;
5264 
5265   if (mode == VOIDmode)
5266     return 0;
5267 
5268   icode = optab_handler (ctrap_optab, mode);
5269   if (icode == CODE_FOR_nothing)
5270     return 0;
5271 
5272   /* Some targets only accept a zero trap code.  */
5273   if (!insn_operand_matches (icode, 3, tcode))
5274     return 0;
5275 
5276   do_pending_stack_adjust ();
5277   start_sequence ();
5278   prepare_cmp_insn (op1, op2, code, NULL_RTX, false, OPTAB_DIRECT,
5279 		    &trap_rtx, &mode);
5280   if (!trap_rtx)
5281     insn = NULL;
5282   else
5283     insn = GEN_FCN (icode) (trap_rtx, XEXP (trap_rtx, 0), XEXP (trap_rtx, 1),
5284 			    tcode);
5285 
5286   /* If that failed, then give up.  */
5287   if (insn == 0)
5288     {
5289       end_sequence ();
5290       return 0;
5291     }
5292 
5293   emit_insn (insn);
5294   insn = get_insns ();
5295   end_sequence ();
5296   return insn;
5297 }
5298 
5299 /* Return rtx code for TCODE. Use UNSIGNEDP to select signed
5300    or unsigned operation code.  */
5301 
5302 enum rtx_code
get_rtx_code(enum tree_code tcode,bool unsignedp)5303 get_rtx_code (enum tree_code tcode, bool unsignedp)
5304 {
5305   enum rtx_code code;
5306   switch (tcode)
5307     {
5308     case EQ_EXPR:
5309       code = EQ;
5310       break;
5311     case NE_EXPR:
5312       code = NE;
5313       break;
5314     case LT_EXPR:
5315       code = unsignedp ? LTU : LT;
5316       break;
5317     case LE_EXPR:
5318       code = unsignedp ? LEU : LE;
5319       break;
5320     case GT_EXPR:
5321       code = unsignedp ? GTU : GT;
5322       break;
5323     case GE_EXPR:
5324       code = unsignedp ? GEU : GE;
5325       break;
5326 
5327     case UNORDERED_EXPR:
5328       code = UNORDERED;
5329       break;
5330     case ORDERED_EXPR:
5331       code = ORDERED;
5332       break;
5333     case UNLT_EXPR:
5334       code = UNLT;
5335       break;
5336     case UNLE_EXPR:
5337       code = UNLE;
5338       break;
5339     case UNGT_EXPR:
5340       code = UNGT;
5341       break;
5342     case UNGE_EXPR:
5343       code = UNGE;
5344       break;
5345     case UNEQ_EXPR:
5346       code = UNEQ;
5347       break;
5348     case LTGT_EXPR:
5349       code = LTGT;
5350       break;
5351 
5352     case BIT_AND_EXPR:
5353       code = AND;
5354       break;
5355 
5356     case BIT_IOR_EXPR:
5357       code = IOR;
5358       break;
5359 
5360     default:
5361       gcc_unreachable ();
5362     }
5363   return code;
5364 }
5365 
5366 /* Return a comparison rtx of mode CMP_MODE for COND.  Use UNSIGNEDP to
5367    select signed or unsigned operators.  OPNO holds the index of the
5368    first comparison operand for insn ICODE.  Do not generate the
5369    compare instruction itself.  */
5370 
5371 static rtx
vector_compare_rtx(machine_mode cmp_mode,enum tree_code tcode,tree t_op0,tree t_op1,bool unsignedp,enum insn_code icode,unsigned int opno)5372 vector_compare_rtx (machine_mode cmp_mode, enum tree_code tcode,
5373 		    tree t_op0, tree t_op1, bool unsignedp,
5374 		    enum insn_code icode, unsigned int opno)
5375 {
5376   struct expand_operand ops[2];
5377   rtx rtx_op0, rtx_op1;
5378   machine_mode m0, m1;
5379   enum rtx_code rcode = get_rtx_code (tcode, unsignedp);
5380 
5381   gcc_assert (TREE_CODE_CLASS (tcode) == tcc_comparison);
5382 
5383   /* Expand operands.  For vector types with scalar modes, e.g. where int64x1_t
5384      has mode DImode, this can produce a constant RTX of mode VOIDmode; in such
5385      cases, use the original mode.  */
5386   rtx_op0 = expand_expr (t_op0, NULL_RTX, TYPE_MODE (TREE_TYPE (t_op0)),
5387 			 EXPAND_STACK_PARM);
5388   m0 = GET_MODE (rtx_op0);
5389   if (m0 == VOIDmode)
5390     m0 = TYPE_MODE (TREE_TYPE (t_op0));
5391 
5392   rtx_op1 = expand_expr (t_op1, NULL_RTX, TYPE_MODE (TREE_TYPE (t_op1)),
5393 			 EXPAND_STACK_PARM);
5394   m1 = GET_MODE (rtx_op1);
5395   if (m1 == VOIDmode)
5396     m1 = TYPE_MODE (TREE_TYPE (t_op1));
5397 
5398   create_input_operand (&ops[0], rtx_op0, m0);
5399   create_input_operand (&ops[1], rtx_op1, m1);
5400   if (!maybe_legitimize_operands (icode, opno, 2, ops))
5401     gcc_unreachable ();
5402   return gen_rtx_fmt_ee (rcode, cmp_mode, ops[0].value, ops[1].value);
5403 }
5404 
5405 /* Check if vec_perm mask SEL is a constant equivalent to a shift of
5406    the first vec_perm operand, assuming the second operand is a constant
5407    vector of zeros.  Return the shift distance in bits if so, or NULL_RTX
5408    if the vec_perm is not a shift.  MODE is the mode of the value being
5409    shifted.  */
5410 static rtx
shift_amt_for_vec_perm_mask(machine_mode mode,const vec_perm_indices & sel)5411 shift_amt_for_vec_perm_mask (machine_mode mode, const vec_perm_indices &sel)
5412 {
5413   unsigned int bitsize = GET_MODE_UNIT_BITSIZE (mode);
5414   poly_int64 first = sel[0];
5415   if (maybe_ge (sel[0], GET_MODE_NUNITS (mode)))
5416     return NULL_RTX;
5417 
5418   if (!sel.series_p (0, 1, first, 1))
5419     {
5420       unsigned int nelt;
5421       if (!GET_MODE_NUNITS (mode).is_constant (&nelt))
5422 	return NULL_RTX;
5423       for (unsigned int i = 1; i < nelt; i++)
5424 	{
5425 	  poly_int64 expected = i + first;
5426 	  /* Indices into the second vector are all equivalent.  */
5427 	  if (maybe_lt (sel[i], nelt)
5428 	      ? maybe_ne (sel[i], expected)
5429 	      : maybe_lt (expected, nelt))
5430 	    return NULL_RTX;
5431 	}
5432     }
5433 
5434   return gen_int_shift_amount (mode, first * bitsize);
5435 }
5436 
5437 /* A subroutine of expand_vec_perm_var for expanding one vec_perm insn.  */
5438 
5439 static rtx
expand_vec_perm_1(enum insn_code icode,rtx target,rtx v0,rtx v1,rtx sel)5440 expand_vec_perm_1 (enum insn_code icode, rtx target,
5441 		   rtx v0, rtx v1, rtx sel)
5442 {
5443   machine_mode tmode = GET_MODE (target);
5444   machine_mode smode = GET_MODE (sel);
5445   struct expand_operand ops[4];
5446 
5447   gcc_assert (GET_MODE_CLASS (smode) == MODE_VECTOR_INT
5448 	      || mode_for_int_vector (tmode).require () == smode);
5449   create_output_operand (&ops[0], target, tmode);
5450   create_input_operand (&ops[3], sel, smode);
5451 
5452   /* Make an effort to preserve v0 == v1.  The target expander is able to
5453      rely on this to determine if we're permuting a single input operand.  */
5454   if (rtx_equal_p (v0, v1))
5455     {
5456       if (!insn_operand_matches (icode, 1, v0))
5457         v0 = force_reg (tmode, v0);
5458       gcc_checking_assert (insn_operand_matches (icode, 1, v0));
5459       gcc_checking_assert (insn_operand_matches (icode, 2, v0));
5460 
5461       create_fixed_operand (&ops[1], v0);
5462       create_fixed_operand (&ops[2], v0);
5463     }
5464   else
5465     {
5466       create_input_operand (&ops[1], v0, tmode);
5467       create_input_operand (&ops[2], v1, tmode);
5468     }
5469 
5470   if (maybe_expand_insn (icode, 4, ops))
5471     return ops[0].value;
5472   return NULL_RTX;
5473 }
5474 
5475 /* Implement a permutation of vectors v0 and v1 using the permutation
5476    vector in SEL and return the result.  Use TARGET to hold the result
5477    if nonnull and convenient.
5478 
5479    MODE is the mode of the vectors being permuted (V0 and V1).  SEL_MODE
5480    is the TYPE_MODE associated with SEL, or BLKmode if SEL isn't known
5481    to have a particular mode.  */
5482 
5483 rtx
expand_vec_perm_const(machine_mode mode,rtx v0,rtx v1,const vec_perm_builder & sel,machine_mode sel_mode,rtx target)5484 expand_vec_perm_const (machine_mode mode, rtx v0, rtx v1,
5485 		       const vec_perm_builder &sel, machine_mode sel_mode,
5486 		       rtx target)
5487 {
5488   if (!target || !register_operand (target, mode))
5489     target = gen_reg_rtx (mode);
5490 
5491   /* Set QIMODE to a different vector mode with byte elements.
5492      If no such mode, or if MODE already has byte elements, use VOIDmode.  */
5493   machine_mode qimode;
5494   if (!qimode_for_vec_perm (mode).exists (&qimode))
5495     qimode = VOIDmode;
5496 
5497   rtx_insn *last = get_last_insn ();
5498 
5499   bool single_arg_p = rtx_equal_p (v0, v1);
5500   /* Always specify two input vectors here and leave the target to handle
5501      cases in which the inputs are equal.  Not all backends can cope with
5502      the single-input representation when testing for a double-input
5503      target instruction.  */
5504   vec_perm_indices indices (sel, 2, GET_MODE_NUNITS (mode));
5505 
5506   /* See if this can be handled with a vec_shr.  We only do this if the
5507      second vector is all zeroes.  */
5508   insn_code shift_code = optab_handler (vec_shr_optab, mode);
5509   insn_code shift_code_qi = ((qimode != VOIDmode && qimode != mode)
5510 			     ? optab_handler (vec_shr_optab, qimode)
5511 			     : CODE_FOR_nothing);
5512 
5513   if (v1 == CONST0_RTX (GET_MODE (v1))
5514       && (shift_code != CODE_FOR_nothing
5515 	  || shift_code_qi != CODE_FOR_nothing))
5516     {
5517       rtx shift_amt = shift_amt_for_vec_perm_mask (mode, indices);
5518       if (shift_amt)
5519 	{
5520 	  struct expand_operand ops[3];
5521 	  if (shift_code != CODE_FOR_nothing)
5522 	    {
5523 	      create_output_operand (&ops[0], target, mode);
5524 	      create_input_operand (&ops[1], v0, mode);
5525 	      create_convert_operand_from_type (&ops[2], shift_amt, sizetype);
5526 	      if (maybe_expand_insn (shift_code, 3, ops))
5527 		return ops[0].value;
5528 	    }
5529 	  if (shift_code_qi != CODE_FOR_nothing)
5530 	    {
5531 	      rtx tmp = gen_reg_rtx (qimode);
5532 	      create_output_operand (&ops[0], tmp, qimode);
5533 	      create_input_operand (&ops[1], gen_lowpart (qimode, v0), qimode);
5534 	      create_convert_operand_from_type (&ops[2], shift_amt, sizetype);
5535 	      if (maybe_expand_insn (shift_code_qi, 3, ops))
5536 		return gen_lowpart (mode, ops[0].value);
5537 	    }
5538 	}
5539     }
5540 
5541   if (targetm.vectorize.vec_perm_const != NULL)
5542     {
5543       v0 = force_reg (mode, v0);
5544       if (single_arg_p)
5545 	v1 = v0;
5546       else
5547 	v1 = force_reg (mode, v1);
5548 
5549       if (targetm.vectorize.vec_perm_const (mode, target, v0, v1, indices))
5550 	return target;
5551     }
5552 
5553   /* Fall back to a constant byte-based permutation.  */
5554   vec_perm_indices qimode_indices;
5555   rtx target_qi = NULL_RTX, v0_qi = NULL_RTX, v1_qi = NULL_RTX;
5556   if (qimode != VOIDmode)
5557     {
5558       qimode_indices.new_expanded_vector (indices, GET_MODE_UNIT_SIZE (mode));
5559       target_qi = gen_reg_rtx (qimode);
5560       v0_qi = gen_lowpart (qimode, v0);
5561       v1_qi = gen_lowpart (qimode, v1);
5562       if (targetm.vectorize.vec_perm_const != NULL
5563 	  && targetm.vectorize.vec_perm_const (qimode, target_qi, v0_qi,
5564 					       v1_qi, qimode_indices))
5565 	return gen_lowpart (mode, target_qi);
5566     }
5567 
5568   /* Otherwise expand as a fully variable permuation.  */
5569 
5570   /* The optabs are only defined for selectors with the same width
5571      as the values being permuted.  */
5572   machine_mode required_sel_mode;
5573   if (!mode_for_int_vector (mode).exists (&required_sel_mode)
5574       || !VECTOR_MODE_P (required_sel_mode))
5575     {
5576       delete_insns_since (last);
5577       return NULL_RTX;
5578     }
5579 
5580   /* We know that it is semantically valid to treat SEL as having SEL_MODE.
5581      If that isn't the mode we want then we need to prove that using
5582      REQUIRED_SEL_MODE is OK.  */
5583   if (sel_mode != required_sel_mode)
5584     {
5585       if (!selector_fits_mode_p (required_sel_mode, indices))
5586 	{
5587 	  delete_insns_since (last);
5588 	  return NULL_RTX;
5589 	}
5590       sel_mode = required_sel_mode;
5591     }
5592 
5593   insn_code icode = direct_optab_handler (vec_perm_optab, mode);
5594   if (icode != CODE_FOR_nothing)
5595     {
5596       rtx sel_rtx = vec_perm_indices_to_rtx (sel_mode, indices);
5597       rtx tmp = expand_vec_perm_1 (icode, target, v0, v1, sel_rtx);
5598       if (tmp)
5599 	return tmp;
5600     }
5601 
5602   if (qimode != VOIDmode
5603       && selector_fits_mode_p (qimode, qimode_indices))
5604     {
5605       icode = direct_optab_handler (vec_perm_optab, qimode);
5606       if (icode != CODE_FOR_nothing)
5607 	{
5608 	  rtx sel_qi = vec_perm_indices_to_rtx (qimode, qimode_indices);
5609 	  rtx tmp = expand_vec_perm_1 (icode, target_qi, v0_qi, v1_qi, sel_qi);
5610 	  if (tmp)
5611 	    return gen_lowpart (mode, tmp);
5612 	}
5613     }
5614 
5615   delete_insns_since (last);
5616   return NULL_RTX;
5617 }
5618 
5619 /* Implement a permutation of vectors v0 and v1 using the permutation
5620    vector in SEL and return the result.  Use TARGET to hold the result
5621    if nonnull and convenient.
5622 
5623    MODE is the mode of the vectors being permuted (V0 and V1).
5624    SEL must have the integer equivalent of MODE and is known to be
5625    unsuitable for permutes with a constant permutation vector.  */
5626 
5627 rtx
expand_vec_perm_var(machine_mode mode,rtx v0,rtx v1,rtx sel,rtx target)5628 expand_vec_perm_var (machine_mode mode, rtx v0, rtx v1, rtx sel, rtx target)
5629 {
5630   enum insn_code icode;
5631   unsigned int i, u;
5632   rtx tmp, sel_qi;
5633 
5634   u = GET_MODE_UNIT_SIZE (mode);
5635 
5636   if (!target || GET_MODE (target) != mode)
5637     target = gen_reg_rtx (mode);
5638 
5639   icode = direct_optab_handler (vec_perm_optab, mode);
5640   if (icode != CODE_FOR_nothing)
5641     {
5642       tmp = expand_vec_perm_1 (icode, target, v0, v1, sel);
5643       if (tmp)
5644 	return tmp;
5645     }
5646 
5647   /* As a special case to aid several targets, lower the element-based
5648      permutation to a byte-based permutation and try again.  */
5649   machine_mode qimode;
5650   if (!qimode_for_vec_perm (mode).exists (&qimode)
5651       || maybe_gt (GET_MODE_NUNITS (qimode), GET_MODE_MASK (QImode) + 1))
5652     return NULL_RTX;
5653   icode = direct_optab_handler (vec_perm_optab, qimode);
5654   if (icode == CODE_FOR_nothing)
5655     return NULL_RTX;
5656 
5657   /* Multiply each element by its byte size.  */
5658   machine_mode selmode = GET_MODE (sel);
5659   if (u == 2)
5660     sel = expand_simple_binop (selmode, PLUS, sel, sel,
5661 			       NULL, 0, OPTAB_DIRECT);
5662   else
5663     sel = expand_simple_binop (selmode, ASHIFT, sel,
5664 			       gen_int_shift_amount (selmode, exact_log2 (u)),
5665 			       NULL, 0, OPTAB_DIRECT);
5666   gcc_assert (sel != NULL);
5667 
5668   /* Broadcast the low byte each element into each of its bytes.
5669      The encoding has U interleaved stepped patterns, one for each
5670      byte of an element.  */
5671   vec_perm_builder const_sel (GET_MODE_SIZE (mode), u, 3);
5672   unsigned int low_byte_in_u = BYTES_BIG_ENDIAN ? u - 1 : 0;
5673   for (i = 0; i < 3; ++i)
5674     for (unsigned int j = 0; j < u; ++j)
5675       const_sel.quick_push (i * u + low_byte_in_u);
5676   sel = gen_lowpart (qimode, sel);
5677   sel = expand_vec_perm_const (qimode, sel, sel, const_sel, qimode, NULL);
5678   gcc_assert (sel != NULL);
5679 
5680   /* Add the byte offset to each byte element.  */
5681   /* Note that the definition of the indicies here is memory ordering,
5682      so there should be no difference between big and little endian.  */
5683   rtx_vector_builder byte_indices (qimode, u, 1);
5684   for (i = 0; i < u; ++i)
5685     byte_indices.quick_push (GEN_INT (i));
5686   tmp = byte_indices.build ();
5687   sel_qi = expand_simple_binop (qimode, PLUS, sel, tmp,
5688 				sel, 0, OPTAB_DIRECT);
5689   gcc_assert (sel_qi != NULL);
5690 
5691   tmp = mode != qimode ? gen_reg_rtx (qimode) : target;
5692   tmp = expand_vec_perm_1 (icode, tmp, gen_lowpart (qimode, v0),
5693 			   gen_lowpart (qimode, v1), sel_qi);
5694   if (tmp)
5695     tmp = gen_lowpart (mode, tmp);
5696   return tmp;
5697 }
5698 
5699 /* Generate insns for a VEC_COND_EXPR with mask, given its TYPE and its
5700    three operands.  */
5701 
5702 rtx
expand_vec_cond_mask_expr(tree vec_cond_type,tree op0,tree op1,tree op2,rtx target)5703 expand_vec_cond_mask_expr (tree vec_cond_type, tree op0, tree op1, tree op2,
5704 			   rtx target)
5705 {
5706   struct expand_operand ops[4];
5707   machine_mode mode = TYPE_MODE (vec_cond_type);
5708   machine_mode mask_mode = TYPE_MODE (TREE_TYPE (op0));
5709   enum insn_code icode = get_vcond_mask_icode (mode, mask_mode);
5710   rtx mask, rtx_op1, rtx_op2;
5711 
5712   if (icode == CODE_FOR_nothing)
5713     return 0;
5714 
5715   mask = expand_normal (op0);
5716   rtx_op1 = expand_normal (op1);
5717   rtx_op2 = expand_normal (op2);
5718 
5719   mask = force_reg (mask_mode, mask);
5720   rtx_op1 = force_reg (GET_MODE (rtx_op1), rtx_op1);
5721 
5722   create_output_operand (&ops[0], target, mode);
5723   create_input_operand (&ops[1], rtx_op1, mode);
5724   create_input_operand (&ops[2], rtx_op2, mode);
5725   create_input_operand (&ops[3], mask, mask_mode);
5726   expand_insn (icode, 4, ops);
5727 
5728   return ops[0].value;
5729 }
5730 
5731 /* Generate insns for a VEC_COND_EXPR, given its TYPE and its
5732    three operands.  */
5733 
5734 rtx
expand_vec_cond_expr(tree vec_cond_type,tree op0,tree op1,tree op2,rtx target)5735 expand_vec_cond_expr (tree vec_cond_type, tree op0, tree op1, tree op2,
5736 		      rtx target)
5737 {
5738   struct expand_operand ops[6];
5739   enum insn_code icode;
5740   rtx comparison, rtx_op1, rtx_op2;
5741   machine_mode mode = TYPE_MODE (vec_cond_type);
5742   machine_mode cmp_op_mode;
5743   bool unsignedp;
5744   tree op0a, op0b;
5745   enum tree_code tcode;
5746 
5747   if (COMPARISON_CLASS_P (op0))
5748     {
5749       op0a = TREE_OPERAND (op0, 0);
5750       op0b = TREE_OPERAND (op0, 1);
5751       tcode = TREE_CODE (op0);
5752     }
5753   else
5754     {
5755       gcc_assert (VECTOR_BOOLEAN_TYPE_P (TREE_TYPE (op0)));
5756       if (get_vcond_mask_icode (mode, TYPE_MODE (TREE_TYPE (op0)))
5757 	  != CODE_FOR_nothing)
5758 	return expand_vec_cond_mask_expr (vec_cond_type, op0, op1,
5759 					  op2, target);
5760       /* Fake op0 < 0.  */
5761       else
5762 	{
5763 	  gcc_assert (GET_MODE_CLASS (TYPE_MODE (TREE_TYPE (op0)))
5764 		      == MODE_VECTOR_INT);
5765 	  op0a = op0;
5766 	  op0b = build_zero_cst (TREE_TYPE (op0));
5767 	  tcode = LT_EXPR;
5768 	}
5769     }
5770   cmp_op_mode = TYPE_MODE (TREE_TYPE (op0a));
5771   unsignedp = TYPE_UNSIGNED (TREE_TYPE (op0a));
5772 
5773 
5774   gcc_assert (known_eq (GET_MODE_SIZE (mode), GET_MODE_SIZE (cmp_op_mode))
5775 	      && known_eq (GET_MODE_NUNITS (mode),
5776 			   GET_MODE_NUNITS (cmp_op_mode)));
5777 
5778   icode = get_vcond_icode (mode, cmp_op_mode, unsignedp);
5779   if (icode == CODE_FOR_nothing)
5780     {
5781       if (tcode == EQ_EXPR || tcode == NE_EXPR)
5782 	icode = get_vcond_eq_icode (mode, cmp_op_mode);
5783       if (icode == CODE_FOR_nothing)
5784 	return 0;
5785     }
5786 
5787   comparison = vector_compare_rtx (VOIDmode, tcode, op0a, op0b, unsignedp,
5788 				   icode, 4);
5789   rtx_op1 = expand_normal (op1);
5790   rtx_op2 = expand_normal (op2);
5791 
5792   create_output_operand (&ops[0], target, mode);
5793   create_input_operand (&ops[1], rtx_op1, mode);
5794   create_input_operand (&ops[2], rtx_op2, mode);
5795   create_fixed_operand (&ops[3], comparison);
5796   create_fixed_operand (&ops[4], XEXP (comparison, 0));
5797   create_fixed_operand (&ops[5], XEXP (comparison, 1));
5798   expand_insn (icode, 6, ops);
5799   return ops[0].value;
5800 }
5801 
5802 /* Generate VEC_SERIES_EXPR <OP0, OP1>, returning a value of mode VMODE.
5803    Use TARGET for the result if nonnull and convenient.  */
5804 
5805 rtx
expand_vec_series_expr(machine_mode vmode,rtx op0,rtx op1,rtx target)5806 expand_vec_series_expr (machine_mode vmode, rtx op0, rtx op1, rtx target)
5807 {
5808   struct expand_operand ops[3];
5809   enum insn_code icode;
5810   machine_mode emode = GET_MODE_INNER (vmode);
5811 
5812   icode = direct_optab_handler (vec_series_optab, vmode);
5813   gcc_assert (icode != CODE_FOR_nothing);
5814 
5815   create_output_operand (&ops[0], target, vmode);
5816   create_input_operand (&ops[1], op0, emode);
5817   create_input_operand (&ops[2], op1, emode);
5818 
5819   expand_insn (icode, 3, ops);
5820   return ops[0].value;
5821 }
5822 
5823 /* Generate insns for a vector comparison into a mask.  */
5824 
5825 rtx
expand_vec_cmp_expr(tree type,tree exp,rtx target)5826 expand_vec_cmp_expr (tree type, tree exp, rtx target)
5827 {
5828   struct expand_operand ops[4];
5829   enum insn_code icode;
5830   rtx comparison;
5831   machine_mode mask_mode = TYPE_MODE (type);
5832   machine_mode vmode;
5833   bool unsignedp;
5834   tree op0a, op0b;
5835   enum tree_code tcode;
5836 
5837   op0a = TREE_OPERAND (exp, 0);
5838   op0b = TREE_OPERAND (exp, 1);
5839   tcode = TREE_CODE (exp);
5840 
5841   unsignedp = TYPE_UNSIGNED (TREE_TYPE (op0a));
5842   vmode = TYPE_MODE (TREE_TYPE (op0a));
5843 
5844   icode = get_vec_cmp_icode (vmode, mask_mode, unsignedp);
5845   if (icode == CODE_FOR_nothing)
5846     {
5847       if (tcode == EQ_EXPR || tcode == NE_EXPR)
5848 	icode = get_vec_cmp_eq_icode (vmode, mask_mode);
5849       if (icode == CODE_FOR_nothing)
5850 	return 0;
5851     }
5852 
5853   comparison = vector_compare_rtx (mask_mode, tcode, op0a, op0b,
5854 				   unsignedp, icode, 2);
5855   create_output_operand (&ops[0], target, mask_mode);
5856   create_fixed_operand (&ops[1], comparison);
5857   create_fixed_operand (&ops[2], XEXP (comparison, 0));
5858   create_fixed_operand (&ops[3], XEXP (comparison, 1));
5859   expand_insn (icode, 4, ops);
5860   return ops[0].value;
5861 }
5862 
5863 /* Expand a highpart multiply.  */
5864 
5865 rtx
expand_mult_highpart(machine_mode mode,rtx op0,rtx op1,rtx target,bool uns_p)5866 expand_mult_highpart (machine_mode mode, rtx op0, rtx op1,
5867 		      rtx target, bool uns_p)
5868 {
5869   struct expand_operand eops[3];
5870   enum insn_code icode;
5871   int method, i;
5872   machine_mode wmode;
5873   rtx m1, m2;
5874   optab tab1, tab2;
5875 
5876   method = can_mult_highpart_p (mode, uns_p);
5877   switch (method)
5878     {
5879     case 0:
5880       return NULL_RTX;
5881     case 1:
5882       tab1 = uns_p ? umul_highpart_optab : smul_highpart_optab;
5883       return expand_binop (mode, tab1, op0, op1, target, uns_p,
5884 			   OPTAB_LIB_WIDEN);
5885     case 2:
5886       tab1 = uns_p ? vec_widen_umult_even_optab : vec_widen_smult_even_optab;
5887       tab2 = uns_p ? vec_widen_umult_odd_optab : vec_widen_smult_odd_optab;
5888       break;
5889     case 3:
5890       tab1 = uns_p ? vec_widen_umult_lo_optab : vec_widen_smult_lo_optab;
5891       tab2 = uns_p ? vec_widen_umult_hi_optab : vec_widen_smult_hi_optab;
5892       if (BYTES_BIG_ENDIAN)
5893 	std::swap (tab1, tab2);
5894       break;
5895     default:
5896       gcc_unreachable ();
5897     }
5898 
5899   icode = optab_handler (tab1, mode);
5900   wmode = insn_data[icode].operand[0].mode;
5901   gcc_checking_assert (known_eq (2 * GET_MODE_NUNITS (wmode),
5902 				 GET_MODE_NUNITS (mode)));
5903   gcc_checking_assert (known_eq (GET_MODE_SIZE (wmode), GET_MODE_SIZE (mode)));
5904 
5905   create_output_operand (&eops[0], gen_reg_rtx (wmode), wmode);
5906   create_input_operand (&eops[1], op0, mode);
5907   create_input_operand (&eops[2], op1, mode);
5908   expand_insn (icode, 3, eops);
5909   m1 = gen_lowpart (mode, eops[0].value);
5910 
5911   create_output_operand (&eops[0], gen_reg_rtx (wmode), wmode);
5912   create_input_operand (&eops[1], op0, mode);
5913   create_input_operand (&eops[2], op1, mode);
5914   expand_insn (optab_handler (tab2, mode), 3, eops);
5915   m2 = gen_lowpart (mode, eops[0].value);
5916 
5917   vec_perm_builder sel;
5918   if (method == 2)
5919     {
5920       /* The encoding has 2 interleaved stepped patterns.  */
5921       sel.new_vector (GET_MODE_NUNITS (mode), 2, 3);
5922       for (i = 0; i < 6; ++i)
5923 	sel.quick_push (!BYTES_BIG_ENDIAN + (i & ~1)
5924 			+ ((i & 1) ? GET_MODE_NUNITS (mode) : 0));
5925     }
5926   else
5927     {
5928       /* The encoding has a single interleaved stepped pattern.  */
5929       sel.new_vector (GET_MODE_NUNITS (mode), 1, 3);
5930       for (i = 0; i < 3; ++i)
5931 	sel.quick_push (2 * i + (BYTES_BIG_ENDIAN ? 0 : 1));
5932     }
5933 
5934   return expand_vec_perm_const (mode, m1, m2, sel, BLKmode, target);
5935 }
5936 
5937 /* Helper function to find the MODE_CC set in a sync_compare_and_swap
5938    pattern.  */
5939 
5940 static void
find_cc_set(rtx x,const_rtx pat,void * data)5941 find_cc_set (rtx x, const_rtx pat, void *data)
5942 {
5943   if (REG_P (x) && GET_MODE_CLASS (GET_MODE (x)) == MODE_CC
5944       && GET_CODE (pat) == SET)
5945     {
5946       rtx *p_cc_reg = (rtx *) data;
5947       gcc_assert (!*p_cc_reg);
5948       *p_cc_reg = x;
5949     }
5950 }
5951 
5952 /* This is a helper function for the other atomic operations.  This function
5953    emits a loop that contains SEQ that iterates until a compare-and-swap
5954    operation at the end succeeds.  MEM is the memory to be modified.  SEQ is
5955    a set of instructions that takes a value from OLD_REG as an input and
5956    produces a value in NEW_REG as an output.  Before SEQ, OLD_REG will be
5957    set to the current contents of MEM.  After SEQ, a compare-and-swap will
5958    attempt to update MEM with NEW_REG.  The function returns true when the
5959    loop was generated successfully.  */
5960 
5961 static bool
expand_compare_and_swap_loop(rtx mem,rtx old_reg,rtx new_reg,rtx seq)5962 expand_compare_and_swap_loop (rtx mem, rtx old_reg, rtx new_reg, rtx seq)
5963 {
5964   machine_mode mode = GET_MODE (mem);
5965   rtx_code_label *label;
5966   rtx cmp_reg, success, oldval;
5967 
5968   /* The loop we want to generate looks like
5969 
5970 	cmp_reg = mem;
5971       label:
5972         old_reg = cmp_reg;
5973 	seq;
5974 	(success, cmp_reg) = compare-and-swap(mem, old_reg, new_reg)
5975 	if (success)
5976 	  goto label;
5977 
5978      Note that we only do the plain load from memory once.  Subsequent
5979      iterations use the value loaded by the compare-and-swap pattern.  */
5980 
5981   label = gen_label_rtx ();
5982   cmp_reg = gen_reg_rtx (mode);
5983 
5984   emit_move_insn (cmp_reg, mem);
5985   emit_label (label);
5986   emit_move_insn (old_reg, cmp_reg);
5987   if (seq)
5988     emit_insn (seq);
5989 
5990   success = NULL_RTX;
5991   oldval = cmp_reg;
5992   if (!expand_atomic_compare_and_swap (&success, &oldval, mem, old_reg,
5993 				       new_reg, false, MEMMODEL_SYNC_SEQ_CST,
5994 				       MEMMODEL_RELAXED))
5995     return false;
5996 
5997   if (oldval != cmp_reg)
5998     emit_move_insn (cmp_reg, oldval);
5999 
6000   /* Mark this jump predicted not taken.  */
6001   emit_cmp_and_jump_insns (success, const0_rtx, EQ, const0_rtx,
6002 			   GET_MODE (success), 1, label,
6003 			   profile_probability::guessed_never ());
6004   return true;
6005 }
6006 
6007 
6008 /* This function tries to emit an atomic_exchange intruction.  VAL is written
6009    to *MEM using memory model MODEL. The previous contents of *MEM are returned,
6010    using TARGET if possible.  */
6011 
6012 static rtx
maybe_emit_atomic_exchange(rtx target,rtx mem,rtx val,enum memmodel model)6013 maybe_emit_atomic_exchange (rtx target, rtx mem, rtx val, enum memmodel model)
6014 {
6015   machine_mode mode = GET_MODE (mem);
6016   enum insn_code icode;
6017 
6018   /* If the target supports the exchange directly, great.  */
6019   icode = direct_optab_handler (atomic_exchange_optab, mode);
6020   if (icode != CODE_FOR_nothing)
6021     {
6022       struct expand_operand ops[4];
6023 
6024       create_output_operand (&ops[0], target, mode);
6025       create_fixed_operand (&ops[1], mem);
6026       create_input_operand (&ops[2], val, mode);
6027       create_integer_operand (&ops[3], model);
6028       if (maybe_expand_insn (icode, 4, ops))
6029 	return ops[0].value;
6030     }
6031 
6032   return NULL_RTX;
6033 }
6034 
6035 /* This function tries to implement an atomic exchange operation using
6036    __sync_lock_test_and_set. VAL is written to *MEM using memory model MODEL.
6037    The previous contents of *MEM are returned, using TARGET if possible.
6038    Since this instructionn is an acquire barrier only, stronger memory
6039    models may require additional barriers to be emitted.  */
6040 
6041 static rtx
maybe_emit_sync_lock_test_and_set(rtx target,rtx mem,rtx val,enum memmodel model)6042 maybe_emit_sync_lock_test_and_set (rtx target, rtx mem, rtx val,
6043 				   enum memmodel model)
6044 {
6045   machine_mode mode = GET_MODE (mem);
6046   enum insn_code icode;
6047   rtx_insn *last_insn = get_last_insn ();
6048 
6049   icode = optab_handler (sync_lock_test_and_set_optab, mode);
6050 
6051   /* Legacy sync_lock_test_and_set is an acquire barrier.  If the pattern
6052      exists, and the memory model is stronger than acquire, add a release
6053      barrier before the instruction.  */
6054 
6055   if (is_mm_seq_cst (model) || is_mm_release (model) || is_mm_acq_rel (model))
6056     expand_mem_thread_fence (model);
6057 
6058   if (icode != CODE_FOR_nothing)
6059     {
6060       struct expand_operand ops[3];
6061       create_output_operand (&ops[0], target, mode);
6062       create_fixed_operand (&ops[1], mem);
6063       create_input_operand (&ops[2], val, mode);
6064       if (maybe_expand_insn (icode, 3, ops))
6065 	return ops[0].value;
6066     }
6067 
6068   /* If an external test-and-set libcall is provided, use that instead of
6069      any external compare-and-swap that we might get from the compare-and-
6070      swap-loop expansion later.  */
6071   if (!can_compare_and_swap_p (mode, false))
6072     {
6073       rtx libfunc = optab_libfunc (sync_lock_test_and_set_optab, mode);
6074       if (libfunc != NULL)
6075 	{
6076 	  rtx addr;
6077 
6078 	  addr = convert_memory_address (ptr_mode, XEXP (mem, 0));
6079 	  return emit_library_call_value (libfunc, NULL_RTX, LCT_NORMAL,
6080 					  mode, addr, ptr_mode,
6081 					  val, mode);
6082 	}
6083     }
6084 
6085   /* If the test_and_set can't be emitted, eliminate any barrier that might
6086      have been emitted.  */
6087   delete_insns_since (last_insn);
6088   return NULL_RTX;
6089 }
6090 
6091 /* This function tries to implement an atomic exchange operation using a
6092    compare_and_swap loop. VAL is written to *MEM.  The previous contents of
6093    *MEM are returned, using TARGET if possible.  No memory model is required
6094    since a compare_and_swap loop is seq-cst.  */
6095 
6096 static rtx
maybe_emit_compare_and_swap_exchange_loop(rtx target,rtx mem,rtx val)6097 maybe_emit_compare_and_swap_exchange_loop (rtx target, rtx mem, rtx val)
6098 {
6099   machine_mode mode = GET_MODE (mem);
6100 
6101   if (can_compare_and_swap_p (mode, true))
6102     {
6103       if (!target || !register_operand (target, mode))
6104 	target = gen_reg_rtx (mode);
6105       if (expand_compare_and_swap_loop (mem, target, val, NULL_RTX))
6106 	return target;
6107     }
6108 
6109   return NULL_RTX;
6110 }
6111 
6112 /* This function tries to implement an atomic test-and-set operation
6113    using the atomic_test_and_set instruction pattern.  A boolean value
6114    is returned from the operation, using TARGET if possible.  */
6115 
6116 static rtx
maybe_emit_atomic_test_and_set(rtx target,rtx mem,enum memmodel model)6117 maybe_emit_atomic_test_and_set (rtx target, rtx mem, enum memmodel model)
6118 {
6119   machine_mode pat_bool_mode;
6120   struct expand_operand ops[3];
6121 
6122   if (!targetm.have_atomic_test_and_set ())
6123     return NULL_RTX;
6124 
6125   /* While we always get QImode from __atomic_test_and_set, we get
6126      other memory modes from __sync_lock_test_and_set.  Note that we
6127      use no endian adjustment here.  This matches the 4.6 behavior
6128      in the Sparc backend.  */
6129   enum insn_code icode = targetm.code_for_atomic_test_and_set;
6130   gcc_checking_assert (insn_data[icode].operand[1].mode == QImode);
6131   if (GET_MODE (mem) != QImode)
6132     mem = adjust_address_nv (mem, QImode, 0);
6133 
6134   pat_bool_mode = insn_data[icode].operand[0].mode;
6135   create_output_operand (&ops[0], target, pat_bool_mode);
6136   create_fixed_operand (&ops[1], mem);
6137   create_integer_operand (&ops[2], model);
6138 
6139   if (maybe_expand_insn (icode, 3, ops))
6140     return ops[0].value;
6141   return NULL_RTX;
6142 }
6143 
6144 /* This function expands the legacy _sync_lock test_and_set operation which is
6145    generally an atomic exchange.  Some limited targets only allow the
6146    constant 1 to be stored.  This is an ACQUIRE operation.
6147 
6148    TARGET is an optional place to stick the return value.
6149    MEM is where VAL is stored.  */
6150 
6151 rtx
expand_sync_lock_test_and_set(rtx target,rtx mem,rtx val)6152 expand_sync_lock_test_and_set (rtx target, rtx mem, rtx val)
6153 {
6154   rtx ret;
6155 
6156   /* Try an atomic_exchange first.  */
6157   ret = maybe_emit_atomic_exchange (target, mem, val, MEMMODEL_SYNC_ACQUIRE);
6158   if (ret)
6159     return ret;
6160 
6161   ret = maybe_emit_sync_lock_test_and_set (target, mem, val,
6162 					   MEMMODEL_SYNC_ACQUIRE);
6163   if (ret)
6164     return ret;
6165 
6166   ret = maybe_emit_compare_and_swap_exchange_loop (target, mem, val);
6167   if (ret)
6168     return ret;
6169 
6170   /* If there are no other options, try atomic_test_and_set if the value
6171      being stored is 1.  */
6172   if (val == const1_rtx)
6173     ret = maybe_emit_atomic_test_and_set (target, mem, MEMMODEL_SYNC_ACQUIRE);
6174 
6175   return ret;
6176 }
6177 
6178 /* This function expands the atomic test_and_set operation:
6179    atomically store a boolean TRUE into MEM and return the previous value.
6180 
6181    MEMMODEL is the memory model variant to use.
6182    TARGET is an optional place to stick the return value.  */
6183 
6184 rtx
expand_atomic_test_and_set(rtx target,rtx mem,enum memmodel model)6185 expand_atomic_test_and_set (rtx target, rtx mem, enum memmodel model)
6186 {
6187   machine_mode mode = GET_MODE (mem);
6188   rtx ret, trueval, subtarget;
6189 
6190   ret = maybe_emit_atomic_test_and_set (target, mem, model);
6191   if (ret)
6192     return ret;
6193 
6194   /* Be binary compatible with non-default settings of trueval, and different
6195      cpu revisions.  E.g. one revision may have atomic-test-and-set, but
6196      another only has atomic-exchange.  */
6197   if (targetm.atomic_test_and_set_trueval == 1)
6198     {
6199       trueval = const1_rtx;
6200       subtarget = target ? target : gen_reg_rtx (mode);
6201     }
6202   else
6203     {
6204       trueval = gen_int_mode (targetm.atomic_test_and_set_trueval, mode);
6205       subtarget = gen_reg_rtx (mode);
6206     }
6207 
6208   /* Try the atomic-exchange optab...  */
6209   ret = maybe_emit_atomic_exchange (subtarget, mem, trueval, model);
6210 
6211   /* ... then an atomic-compare-and-swap loop ... */
6212   if (!ret)
6213     ret = maybe_emit_compare_and_swap_exchange_loop (subtarget, mem, trueval);
6214 
6215   /* ... before trying the vaguely defined legacy lock_test_and_set. */
6216   if (!ret)
6217     ret = maybe_emit_sync_lock_test_and_set (subtarget, mem, trueval, model);
6218 
6219   /* Recall that the legacy lock_test_and_set optab was allowed to do magic
6220      things with the value 1.  Thus we try again without trueval.  */
6221   if (!ret && targetm.atomic_test_and_set_trueval != 1)
6222     ret = maybe_emit_sync_lock_test_and_set (subtarget, mem, const1_rtx, model);
6223 
6224   /* Failing all else, assume a single threaded environment and simply
6225      perform the operation.  */
6226   if (!ret)
6227     {
6228       /* If the result is ignored skip the move to target.  */
6229       if (subtarget != const0_rtx)
6230         emit_move_insn (subtarget, mem);
6231 
6232       emit_move_insn (mem, trueval);
6233       ret = subtarget;
6234     }
6235 
6236   /* Recall that have to return a boolean value; rectify if trueval
6237      is not exactly one.  */
6238   if (targetm.atomic_test_and_set_trueval != 1)
6239     ret = emit_store_flag_force (target, NE, ret, const0_rtx, mode, 0, 1);
6240 
6241   return ret;
6242 }
6243 
6244 /* This function expands the atomic exchange operation:
6245    atomically store VAL in MEM and return the previous value in MEM.
6246 
6247    MEMMODEL is the memory model variant to use.
6248    TARGET is an optional place to stick the return value.  */
6249 
6250 rtx
expand_atomic_exchange(rtx target,rtx mem,rtx val,enum memmodel model)6251 expand_atomic_exchange (rtx target, rtx mem, rtx val, enum memmodel model)
6252 {
6253   machine_mode mode = GET_MODE (mem);
6254   rtx ret;
6255 
6256   /* If loads are not atomic for the required size and we are not called to
6257      provide a __sync builtin, do not do anything so that we stay consistent
6258      with atomic loads of the same size.  */
6259   if (!can_atomic_load_p (mode) && !is_mm_sync (model))
6260     return NULL_RTX;
6261 
6262   ret = maybe_emit_atomic_exchange (target, mem, val, model);
6263 
6264   /* Next try a compare-and-swap loop for the exchange.  */
6265   if (!ret)
6266     ret = maybe_emit_compare_and_swap_exchange_loop (target, mem, val);
6267 
6268   return ret;
6269 }
6270 
6271 /* This function expands the atomic compare exchange operation:
6272 
6273    *PTARGET_BOOL is an optional place to store the boolean success/failure.
6274    *PTARGET_OVAL is an optional place to store the old value from memory.
6275    Both target parameters may be NULL or const0_rtx to indicate that we do
6276    not care about that return value.  Both target parameters are updated on
6277    success to the actual location of the corresponding result.
6278 
6279    MEMMODEL is the memory model variant to use.
6280 
6281    The return value of the function is true for success.  */
6282 
6283 bool
expand_atomic_compare_and_swap(rtx * ptarget_bool,rtx * ptarget_oval,rtx mem,rtx expected,rtx desired,bool is_weak,enum memmodel succ_model,enum memmodel fail_model)6284 expand_atomic_compare_and_swap (rtx *ptarget_bool, rtx *ptarget_oval,
6285 				rtx mem, rtx expected, rtx desired,
6286 				bool is_weak, enum memmodel succ_model,
6287 				enum memmodel fail_model)
6288 {
6289   machine_mode mode = GET_MODE (mem);
6290   struct expand_operand ops[8];
6291   enum insn_code icode;
6292   rtx target_oval, target_bool = NULL_RTX;
6293   rtx libfunc;
6294 
6295   /* If loads are not atomic for the required size and we are not called to
6296      provide a __sync builtin, do not do anything so that we stay consistent
6297      with atomic loads of the same size.  */
6298   if (!can_atomic_load_p (mode) && !is_mm_sync (succ_model))
6299     return false;
6300 
6301   /* Load expected into a register for the compare and swap.  */
6302   if (MEM_P (expected))
6303     expected = copy_to_reg (expected);
6304 
6305   /* Make sure we always have some place to put the return oldval.
6306      Further, make sure that place is distinct from the input expected,
6307      just in case we need that path down below.  */
6308   if (ptarget_oval && *ptarget_oval == const0_rtx)
6309     ptarget_oval = NULL;
6310 
6311   if (ptarget_oval == NULL
6312       || (target_oval = *ptarget_oval) == NULL
6313       || reg_overlap_mentioned_p (expected, target_oval))
6314     target_oval = gen_reg_rtx (mode);
6315 
6316   icode = direct_optab_handler (atomic_compare_and_swap_optab, mode);
6317   if (icode != CODE_FOR_nothing)
6318     {
6319       machine_mode bool_mode = insn_data[icode].operand[0].mode;
6320 
6321       if (ptarget_bool && *ptarget_bool == const0_rtx)
6322 	ptarget_bool = NULL;
6323 
6324       /* Make sure we always have a place for the bool operand.  */
6325       if (ptarget_bool == NULL
6326 	  || (target_bool = *ptarget_bool) == NULL
6327 	  || GET_MODE (target_bool) != bool_mode)
6328 	target_bool = gen_reg_rtx (bool_mode);
6329 
6330       /* Emit the compare_and_swap.  */
6331       create_output_operand (&ops[0], target_bool, bool_mode);
6332       create_output_operand (&ops[1], target_oval, mode);
6333       create_fixed_operand (&ops[2], mem);
6334       create_input_operand (&ops[3], expected, mode);
6335       create_input_operand (&ops[4], desired, mode);
6336       create_integer_operand (&ops[5], is_weak);
6337       create_integer_operand (&ops[6], succ_model);
6338       create_integer_operand (&ops[7], fail_model);
6339       if (maybe_expand_insn (icode, 8, ops))
6340 	{
6341 	  /* Return success/failure.  */
6342 	  target_bool = ops[0].value;
6343 	  target_oval = ops[1].value;
6344 	  goto success;
6345 	}
6346     }
6347 
6348   /* Otherwise fall back to the original __sync_val_compare_and_swap
6349      which is always seq-cst.  */
6350   icode = optab_handler (sync_compare_and_swap_optab, mode);
6351   if (icode != CODE_FOR_nothing)
6352     {
6353       rtx cc_reg;
6354 
6355       create_output_operand (&ops[0], target_oval, mode);
6356       create_fixed_operand (&ops[1], mem);
6357       create_input_operand (&ops[2], expected, mode);
6358       create_input_operand (&ops[3], desired, mode);
6359       if (!maybe_expand_insn (icode, 4, ops))
6360 	return false;
6361 
6362       target_oval = ops[0].value;
6363 
6364       /* If the caller isn't interested in the boolean return value,
6365 	 skip the computation of it.  */
6366       if (ptarget_bool == NULL)
6367 	goto success;
6368 
6369       /* Otherwise, work out if the compare-and-swap succeeded.  */
6370       cc_reg = NULL_RTX;
6371       if (have_insn_for (COMPARE, CCmode))
6372 	note_stores (PATTERN (get_last_insn ()), find_cc_set, &cc_reg);
6373       if (cc_reg)
6374 	{
6375 	  target_bool = emit_store_flag_force (target_bool, EQ, cc_reg,
6376 					       const0_rtx, VOIDmode, 0, 1);
6377 	  goto success;
6378 	}
6379       goto success_bool_from_val;
6380     }
6381 
6382   /* Also check for library support for __sync_val_compare_and_swap.  */
6383   libfunc = optab_libfunc (sync_compare_and_swap_optab, mode);
6384   if (libfunc != NULL)
6385     {
6386       rtx addr = convert_memory_address (ptr_mode, XEXP (mem, 0));
6387       rtx target = emit_library_call_value (libfunc, NULL_RTX, LCT_NORMAL,
6388 					    mode, addr, ptr_mode,
6389 					    expected, mode, desired, mode);
6390       emit_move_insn (target_oval, target);
6391 
6392       /* Compute the boolean return value only if requested.  */
6393       if (ptarget_bool)
6394 	goto success_bool_from_val;
6395       else
6396 	goto success;
6397     }
6398 
6399   /* Failure.  */
6400   return false;
6401 
6402  success_bool_from_val:
6403    target_bool = emit_store_flag_force (target_bool, EQ, target_oval,
6404 					expected, VOIDmode, 1, 1);
6405  success:
6406   /* Make sure that the oval output winds up where the caller asked.  */
6407   if (ptarget_oval)
6408     *ptarget_oval = target_oval;
6409   if (ptarget_bool)
6410     *ptarget_bool = target_bool;
6411   return true;
6412 }
6413 
6414 /* Generate asm volatile("" : : : "memory") as the memory blockage.  */
6415 
6416 static void
expand_asm_memory_blockage(void)6417 expand_asm_memory_blockage (void)
6418 {
6419   rtx asm_op, clob;
6420 
6421   asm_op = gen_rtx_ASM_OPERANDS (VOIDmode, "", "", 0,
6422 				 rtvec_alloc (0), rtvec_alloc (0),
6423 				 rtvec_alloc (0), UNKNOWN_LOCATION);
6424   MEM_VOLATILE_P (asm_op) = 1;
6425 
6426   clob = gen_rtx_SCRATCH (VOIDmode);
6427   clob = gen_rtx_MEM (BLKmode, clob);
6428   clob = gen_rtx_CLOBBER (VOIDmode, clob);
6429 
6430   emit_insn (gen_rtx_PARALLEL (VOIDmode, gen_rtvec (2, asm_op, clob)));
6431 }
6432 
6433 /* Do not propagate memory accesses across this point.  */
6434 
6435 static void
expand_memory_blockage(void)6436 expand_memory_blockage (void)
6437 {
6438   if (targetm.have_memory_blockage ())
6439     emit_insn (targetm.gen_memory_blockage ());
6440   else
6441     expand_asm_memory_blockage ();
6442 }
6443 
6444 /* This routine will either emit the mem_thread_fence pattern or issue a
6445    sync_synchronize to generate a fence for memory model MEMMODEL.  */
6446 
6447 void
expand_mem_thread_fence(enum memmodel model)6448 expand_mem_thread_fence (enum memmodel model)
6449 {
6450   if (is_mm_relaxed (model))
6451     return;
6452   if (targetm.have_mem_thread_fence ())
6453     {
6454       emit_insn (targetm.gen_mem_thread_fence (GEN_INT (model)));
6455       expand_memory_blockage ();
6456     }
6457   else if (targetm.have_memory_barrier ())
6458     emit_insn (targetm.gen_memory_barrier ());
6459   else if (synchronize_libfunc != NULL_RTX)
6460     emit_library_call (synchronize_libfunc, LCT_NORMAL, VOIDmode);
6461   else
6462     expand_memory_blockage ();
6463 }
6464 
6465 /* Emit a signal fence with given memory model.  */
6466 
6467 void
expand_mem_signal_fence(enum memmodel model)6468 expand_mem_signal_fence (enum memmodel model)
6469 {
6470   /* No machine barrier is required to implement a signal fence, but
6471      a compiler memory barrier must be issued, except for relaxed MM.  */
6472   if (!is_mm_relaxed (model))
6473     expand_memory_blockage ();
6474 }
6475 
6476 /* This function expands the atomic load operation:
6477    return the atomically loaded value in MEM.
6478 
6479    MEMMODEL is the memory model variant to use.
6480    TARGET is an option place to stick the return value.  */
6481 
6482 rtx
expand_atomic_load(rtx target,rtx mem,enum memmodel model)6483 expand_atomic_load (rtx target, rtx mem, enum memmodel model)
6484 {
6485   machine_mode mode = GET_MODE (mem);
6486   enum insn_code icode;
6487 
6488   /* If the target supports the load directly, great.  */
6489   icode = direct_optab_handler (atomic_load_optab, mode);
6490   if (icode != CODE_FOR_nothing)
6491     {
6492       struct expand_operand ops[3];
6493       rtx_insn *last = get_last_insn ();
6494       if (is_mm_seq_cst (model))
6495 	expand_memory_blockage ();
6496 
6497       create_output_operand (&ops[0], target, mode);
6498       create_fixed_operand (&ops[1], mem);
6499       create_integer_operand (&ops[2], model);
6500       if (maybe_expand_insn (icode, 3, ops))
6501 	{
6502 	  if (!is_mm_relaxed (model))
6503 	    expand_memory_blockage ();
6504 	  return ops[0].value;
6505 	}
6506       delete_insns_since (last);
6507     }
6508 
6509   /* If the size of the object is greater than word size on this target,
6510      then we assume that a load will not be atomic.  We could try to
6511      emulate a load with a compare-and-swap operation, but the store that
6512      doing this could result in would be incorrect if this is a volatile
6513      atomic load or targetting read-only-mapped memory.  */
6514   if (maybe_gt (GET_MODE_PRECISION (mode), BITS_PER_WORD))
6515     /* If there is no atomic load, leave the library call.  */
6516     return NULL_RTX;
6517 
6518   /* Otherwise assume loads are atomic, and emit the proper barriers.  */
6519   if (!target || target == const0_rtx)
6520     target = gen_reg_rtx (mode);
6521 
6522   /* For SEQ_CST, emit a barrier before the load.  */
6523   if (is_mm_seq_cst (model))
6524     expand_mem_thread_fence (model);
6525 
6526   emit_move_insn (target, mem);
6527 
6528   /* Emit the appropriate barrier after the load.  */
6529   expand_mem_thread_fence (model);
6530 
6531   return target;
6532 }
6533 
6534 /* This function expands the atomic store operation:
6535    Atomically store VAL in MEM.
6536    MEMMODEL is the memory model variant to use.
6537    USE_RELEASE is true if __sync_lock_release can be used as a fall back.
6538    function returns const0_rtx if a pattern was emitted.  */
6539 
6540 rtx
expand_atomic_store(rtx mem,rtx val,enum memmodel model,bool use_release)6541 expand_atomic_store (rtx mem, rtx val, enum memmodel model, bool use_release)
6542 {
6543   machine_mode mode = GET_MODE (mem);
6544   enum insn_code icode;
6545   struct expand_operand ops[3];
6546 
6547   /* If the target supports the store directly, great.  */
6548   icode = direct_optab_handler (atomic_store_optab, mode);
6549   if (icode != CODE_FOR_nothing)
6550     {
6551       rtx_insn *last = get_last_insn ();
6552       if (!is_mm_relaxed (model))
6553 	expand_memory_blockage ();
6554       create_fixed_operand (&ops[0], mem);
6555       create_input_operand (&ops[1], val, mode);
6556       create_integer_operand (&ops[2], model);
6557       if (maybe_expand_insn (icode, 3, ops))
6558 	{
6559 	  if (is_mm_seq_cst (model))
6560 	    expand_memory_blockage ();
6561 	  return const0_rtx;
6562 	}
6563       delete_insns_since (last);
6564     }
6565 
6566   /* If using __sync_lock_release is a viable alternative, try it.
6567      Note that this will not be set to true if we are expanding a generic
6568      __atomic_store_n.  */
6569   if (use_release)
6570     {
6571       icode = direct_optab_handler (sync_lock_release_optab, mode);
6572       if (icode != CODE_FOR_nothing)
6573 	{
6574 	  create_fixed_operand (&ops[0], mem);
6575 	  create_input_operand (&ops[1], const0_rtx, mode);
6576 	  if (maybe_expand_insn (icode, 2, ops))
6577 	    {
6578 	      /* lock_release is only a release barrier.  */
6579 	      if (is_mm_seq_cst (model))
6580 		expand_mem_thread_fence (model);
6581 	      return const0_rtx;
6582 	    }
6583 	}
6584     }
6585 
6586   /* If the size of the object is greater than word size on this target,
6587      a default store will not be atomic.  */
6588   if (maybe_gt (GET_MODE_PRECISION (mode), BITS_PER_WORD))
6589     {
6590       /* If loads are atomic or we are called to provide a __sync builtin,
6591 	 we can try a atomic_exchange and throw away the result.  Otherwise,
6592 	 don't do anything so that we do not create an inconsistency between
6593 	 loads and stores.  */
6594       if (can_atomic_load_p (mode) || is_mm_sync (model))
6595 	{
6596 	  rtx target = maybe_emit_atomic_exchange (NULL_RTX, mem, val, model);
6597 	  if (!target)
6598 	    target = maybe_emit_compare_and_swap_exchange_loop (NULL_RTX, mem,
6599 								val);
6600 	  if (target)
6601 	    return const0_rtx;
6602 	}
6603         return NULL_RTX;
6604     }
6605 
6606   /* Otherwise assume stores are atomic, and emit the proper barriers.  */
6607   expand_mem_thread_fence (model);
6608 
6609   emit_move_insn (mem, val);
6610 
6611   /* For SEQ_CST, also emit a barrier after the store.  */
6612   if (is_mm_seq_cst (model))
6613     expand_mem_thread_fence (model);
6614 
6615   return const0_rtx;
6616 }
6617 
6618 
6619 /* Structure containing the pointers and values required to process the
6620    various forms of the atomic_fetch_op and atomic_op_fetch builtins.  */
6621 
6622 struct atomic_op_functions
6623 {
6624   direct_optab mem_fetch_before;
6625   direct_optab mem_fetch_after;
6626   direct_optab mem_no_result;
6627   optab fetch_before;
6628   optab fetch_after;
6629   direct_optab no_result;
6630   enum rtx_code reverse_code;
6631 };
6632 
6633 
6634 /* Fill in structure pointed to by OP with the various optab entries for an
6635    operation of type CODE.  */
6636 
6637 static void
get_atomic_op_for_code(struct atomic_op_functions * op,enum rtx_code code)6638 get_atomic_op_for_code (struct atomic_op_functions *op, enum rtx_code code)
6639 {
6640   gcc_assert (op!= NULL);
6641 
6642   /* If SWITCHABLE_TARGET is defined, then subtargets can be switched
6643      in the source code during compilation, and the optab entries are not
6644      computable until runtime.  Fill in the values at runtime.  */
6645   switch (code)
6646     {
6647     case PLUS:
6648       op->mem_fetch_before = atomic_fetch_add_optab;
6649       op->mem_fetch_after = atomic_add_fetch_optab;
6650       op->mem_no_result = atomic_add_optab;
6651       op->fetch_before = sync_old_add_optab;
6652       op->fetch_after = sync_new_add_optab;
6653       op->no_result = sync_add_optab;
6654       op->reverse_code = MINUS;
6655       break;
6656     case MINUS:
6657       op->mem_fetch_before = atomic_fetch_sub_optab;
6658       op->mem_fetch_after = atomic_sub_fetch_optab;
6659       op->mem_no_result = atomic_sub_optab;
6660       op->fetch_before = sync_old_sub_optab;
6661       op->fetch_after = sync_new_sub_optab;
6662       op->no_result = sync_sub_optab;
6663       op->reverse_code = PLUS;
6664       break;
6665     case XOR:
6666       op->mem_fetch_before = atomic_fetch_xor_optab;
6667       op->mem_fetch_after = atomic_xor_fetch_optab;
6668       op->mem_no_result = atomic_xor_optab;
6669       op->fetch_before = sync_old_xor_optab;
6670       op->fetch_after = sync_new_xor_optab;
6671       op->no_result = sync_xor_optab;
6672       op->reverse_code = XOR;
6673       break;
6674     case AND:
6675       op->mem_fetch_before = atomic_fetch_and_optab;
6676       op->mem_fetch_after = atomic_and_fetch_optab;
6677       op->mem_no_result = atomic_and_optab;
6678       op->fetch_before = sync_old_and_optab;
6679       op->fetch_after = sync_new_and_optab;
6680       op->no_result = sync_and_optab;
6681       op->reverse_code = UNKNOWN;
6682       break;
6683     case IOR:
6684       op->mem_fetch_before = atomic_fetch_or_optab;
6685       op->mem_fetch_after = atomic_or_fetch_optab;
6686       op->mem_no_result = atomic_or_optab;
6687       op->fetch_before = sync_old_ior_optab;
6688       op->fetch_after = sync_new_ior_optab;
6689       op->no_result = sync_ior_optab;
6690       op->reverse_code = UNKNOWN;
6691       break;
6692     case NOT:
6693       op->mem_fetch_before = atomic_fetch_nand_optab;
6694       op->mem_fetch_after = atomic_nand_fetch_optab;
6695       op->mem_no_result = atomic_nand_optab;
6696       op->fetch_before = sync_old_nand_optab;
6697       op->fetch_after = sync_new_nand_optab;
6698       op->no_result = sync_nand_optab;
6699       op->reverse_code = UNKNOWN;
6700       break;
6701     default:
6702       gcc_unreachable ();
6703     }
6704 }
6705 
6706 /* See if there is a more optimal way to implement the operation "*MEM CODE VAL"
6707    using memory order MODEL.  If AFTER is true the operation needs to return
6708    the value of *MEM after the operation, otherwise the previous value.
6709    TARGET is an optional place to place the result.  The result is unused if
6710    it is const0_rtx.
6711    Return the result if there is a better sequence, otherwise NULL_RTX.  */
6712 
6713 static rtx
maybe_optimize_fetch_op(rtx target,rtx mem,rtx val,enum rtx_code code,enum memmodel model,bool after)6714 maybe_optimize_fetch_op (rtx target, rtx mem, rtx val, enum rtx_code code,
6715 			 enum memmodel model, bool after)
6716 {
6717   /* If the value is prefetched, or not used, it may be possible to replace
6718      the sequence with a native exchange operation.  */
6719   if (!after || target == const0_rtx)
6720     {
6721       /* fetch_and (&x, 0, m) can be replaced with exchange (&x, 0, m).  */
6722       if (code == AND && val == const0_rtx)
6723         {
6724 	  if (target == const0_rtx)
6725 	    target = gen_reg_rtx (GET_MODE (mem));
6726 	  return maybe_emit_atomic_exchange (target, mem, val, model);
6727 	}
6728 
6729       /* fetch_or (&x, -1, m) can be replaced with exchange (&x, -1, m).  */
6730       if (code == IOR && val == constm1_rtx)
6731         {
6732 	  if (target == const0_rtx)
6733 	    target = gen_reg_rtx (GET_MODE (mem));
6734 	  return maybe_emit_atomic_exchange (target, mem, val, model);
6735 	}
6736     }
6737 
6738   return NULL_RTX;
6739 }
6740 
6741 /* Try to emit an instruction for a specific operation varaition.
6742    OPTAB contains the OP functions.
6743    TARGET is an optional place to return the result. const0_rtx means unused.
6744    MEM is the memory location to operate on.
6745    VAL is the value to use in the operation.
6746    USE_MEMMODEL is TRUE if the variation with a memory model should be tried.
6747    MODEL is the memory model, if used.
6748    AFTER is true if the returned result is the value after the operation.  */
6749 
6750 static rtx
maybe_emit_op(const struct atomic_op_functions * optab,rtx target,rtx mem,rtx val,bool use_memmodel,enum memmodel model,bool after)6751 maybe_emit_op (const struct atomic_op_functions *optab, rtx target, rtx mem,
6752 	       rtx val, bool use_memmodel, enum memmodel model, bool after)
6753 {
6754   machine_mode mode = GET_MODE (mem);
6755   struct expand_operand ops[4];
6756   enum insn_code icode;
6757   int op_counter = 0;
6758   int num_ops;
6759 
6760   /* Check to see if there is a result returned.  */
6761   if (target == const0_rtx)
6762     {
6763       if (use_memmodel)
6764         {
6765 	  icode = direct_optab_handler (optab->mem_no_result, mode);
6766 	  create_integer_operand (&ops[2], model);
6767 	  num_ops = 3;
6768 	}
6769       else
6770         {
6771 	  icode = direct_optab_handler (optab->no_result, mode);
6772 	  num_ops = 2;
6773 	}
6774     }
6775   /* Otherwise, we need to generate a result.  */
6776   else
6777     {
6778       if (use_memmodel)
6779         {
6780 	  icode = direct_optab_handler (after ? optab->mem_fetch_after
6781 					: optab->mem_fetch_before, mode);
6782 	  create_integer_operand (&ops[3], model);
6783 	  num_ops = 4;
6784 	}
6785       else
6786 	{
6787 	  icode = optab_handler (after ? optab->fetch_after
6788 				 : optab->fetch_before, mode);
6789 	  num_ops = 3;
6790 	}
6791       create_output_operand (&ops[op_counter++], target, mode);
6792     }
6793   if (icode == CODE_FOR_nothing)
6794     return NULL_RTX;
6795 
6796   create_fixed_operand (&ops[op_counter++], mem);
6797   /* VAL may have been promoted to a wider mode.  Shrink it if so.  */
6798   create_convert_operand_to (&ops[op_counter++], val, mode, true);
6799 
6800   if (maybe_expand_insn (icode, num_ops, ops))
6801     return (target == const0_rtx ? const0_rtx : ops[0].value);
6802 
6803   return NULL_RTX;
6804 }
6805 
6806 
6807 /* This function expands an atomic fetch_OP or OP_fetch operation:
6808    TARGET is an option place to stick the return value.  const0_rtx indicates
6809    the result is unused.
6810    atomically fetch MEM, perform the operation with VAL and return it to MEM.
6811    CODE is the operation being performed (OP)
6812    MEMMODEL is the memory model variant to use.
6813    AFTER is true to return the result of the operation (OP_fetch).
6814    AFTER is false to return the value before the operation (fetch_OP).
6815 
6816    This function will *only* generate instructions if there is a direct
6817    optab. No compare and swap loops or libcalls will be generated. */
6818 
6819 static rtx
expand_atomic_fetch_op_no_fallback(rtx target,rtx mem,rtx val,enum rtx_code code,enum memmodel model,bool after)6820 expand_atomic_fetch_op_no_fallback (rtx target, rtx mem, rtx val,
6821 				    enum rtx_code code, enum memmodel model,
6822 				    bool after)
6823 {
6824   machine_mode mode = GET_MODE (mem);
6825   struct atomic_op_functions optab;
6826   rtx result;
6827   bool unused_result = (target == const0_rtx);
6828 
6829   get_atomic_op_for_code (&optab, code);
6830 
6831   /* Check to see if there are any better instructions.  */
6832   result = maybe_optimize_fetch_op (target, mem, val, code, model, after);
6833   if (result)
6834     return result;
6835 
6836   /* Check for the case where the result isn't used and try those patterns.  */
6837   if (unused_result)
6838     {
6839       /* Try the memory model variant first.  */
6840       result = maybe_emit_op (&optab, target, mem, val, true, model, true);
6841       if (result)
6842         return result;
6843 
6844       /* Next try the old style withuot a memory model.  */
6845       result = maybe_emit_op (&optab, target, mem, val, false, model, true);
6846       if (result)
6847         return result;
6848 
6849       /* There is no no-result pattern, so try patterns with a result.  */
6850       target = NULL_RTX;
6851     }
6852 
6853   /* Try the __atomic version.  */
6854   result = maybe_emit_op (&optab, target, mem, val, true, model, after);
6855   if (result)
6856     return result;
6857 
6858   /* Try the older __sync version.  */
6859   result = maybe_emit_op (&optab, target, mem, val, false, model, after);
6860   if (result)
6861     return result;
6862 
6863   /* If the fetch value can be calculated from the other variation of fetch,
6864      try that operation.  */
6865   if (after || unused_result || optab.reverse_code != UNKNOWN)
6866     {
6867       /* Try the __atomic version, then the older __sync version.  */
6868       result = maybe_emit_op (&optab, target, mem, val, true, model, !after);
6869       if (!result)
6870 	result = maybe_emit_op (&optab, target, mem, val, false, model, !after);
6871 
6872       if (result)
6873 	{
6874 	  /* If the result isn't used, no need to do compensation code.  */
6875 	  if (unused_result)
6876 	    return result;
6877 
6878 	  /* Issue compensation code.  Fetch_after  == fetch_before OP val.
6879 	     Fetch_before == after REVERSE_OP val.  */
6880 	  if (!after)
6881 	    code = optab.reverse_code;
6882 	  if (code == NOT)
6883 	    {
6884 	      result = expand_simple_binop (mode, AND, result, val, NULL_RTX,
6885 					    true, OPTAB_LIB_WIDEN);
6886 	      result = expand_simple_unop (mode, NOT, result, target, true);
6887 	    }
6888 	  else
6889 	    result = expand_simple_binop (mode, code, result, val, target,
6890 					  true, OPTAB_LIB_WIDEN);
6891 	  return result;
6892 	}
6893     }
6894 
6895   /* No direct opcode can be generated.  */
6896   return NULL_RTX;
6897 }
6898 
6899 
6900 
6901 /* This function expands an atomic fetch_OP or OP_fetch operation:
6902    TARGET is an option place to stick the return value.  const0_rtx indicates
6903    the result is unused.
6904    atomically fetch MEM, perform the operation with VAL and return it to MEM.
6905    CODE is the operation being performed (OP)
6906    MEMMODEL is the memory model variant to use.
6907    AFTER is true to return the result of the operation (OP_fetch).
6908    AFTER is false to return the value before the operation (fetch_OP).  */
6909 rtx
expand_atomic_fetch_op(rtx target,rtx mem,rtx val,enum rtx_code code,enum memmodel model,bool after)6910 expand_atomic_fetch_op (rtx target, rtx mem, rtx val, enum rtx_code code,
6911 			enum memmodel model, bool after)
6912 {
6913   machine_mode mode = GET_MODE (mem);
6914   rtx result;
6915   bool unused_result = (target == const0_rtx);
6916 
6917   /* If loads are not atomic for the required size and we are not called to
6918      provide a __sync builtin, do not do anything so that we stay consistent
6919      with atomic loads of the same size.  */
6920   if (!can_atomic_load_p (mode) && !is_mm_sync (model))
6921     return NULL_RTX;
6922 
6923   result = expand_atomic_fetch_op_no_fallback (target, mem, val, code, model,
6924 					       after);
6925 
6926   if (result)
6927     return result;
6928 
6929   /* Add/sub can be implemented by doing the reverse operation with -(val).  */
6930   if (code == PLUS || code == MINUS)
6931     {
6932       rtx tmp;
6933       enum rtx_code reverse = (code == PLUS ? MINUS : PLUS);
6934 
6935       start_sequence ();
6936       tmp = expand_simple_unop (mode, NEG, val, NULL_RTX, true);
6937       result = expand_atomic_fetch_op_no_fallback (target, mem, tmp, reverse,
6938 						   model, after);
6939       if (result)
6940 	{
6941 	  /* PLUS worked so emit the insns and return.  */
6942 	  tmp = get_insns ();
6943 	  end_sequence ();
6944 	  emit_insn (tmp);
6945           return result;
6946 	}
6947 
6948       /* PLUS did not work, so throw away the negation code and continue.  */
6949       end_sequence ();
6950     }
6951 
6952   /* Try the __sync libcalls only if we can't do compare-and-swap inline.  */
6953   if (!can_compare_and_swap_p (mode, false))
6954     {
6955       rtx libfunc;
6956       bool fixup = false;
6957       enum rtx_code orig_code = code;
6958       struct atomic_op_functions optab;
6959 
6960       get_atomic_op_for_code (&optab, code);
6961       libfunc = optab_libfunc (after ? optab.fetch_after
6962 			       : optab.fetch_before, mode);
6963       if (libfunc == NULL
6964 	  && (after || unused_result || optab.reverse_code != UNKNOWN))
6965 	{
6966 	  fixup = true;
6967 	  if (!after)
6968 	    code = optab.reverse_code;
6969 	  libfunc = optab_libfunc (after ? optab.fetch_before
6970 				   : optab.fetch_after, mode);
6971 	}
6972       if (libfunc != NULL)
6973 	{
6974 	  rtx addr = convert_memory_address (ptr_mode, XEXP (mem, 0));
6975 	  result = emit_library_call_value (libfunc, NULL, LCT_NORMAL, mode,
6976 					    addr, ptr_mode, val, mode);
6977 
6978 	  if (!unused_result && fixup)
6979 	    result = expand_simple_binop (mode, code, result, val, target,
6980 					  true, OPTAB_LIB_WIDEN);
6981 	  return result;
6982 	}
6983 
6984       /* We need the original code for any further attempts.  */
6985       code = orig_code;
6986     }
6987 
6988   /* If nothing else has succeeded, default to a compare and swap loop.  */
6989   if (can_compare_and_swap_p (mode, true))
6990     {
6991       rtx_insn *insn;
6992       rtx t0 = gen_reg_rtx (mode), t1;
6993 
6994       start_sequence ();
6995 
6996       /* If the result is used, get a register for it.  */
6997       if (!unused_result)
6998         {
6999 	  if (!target || !register_operand (target, mode))
7000 	    target = gen_reg_rtx (mode);
7001 	  /* If fetch_before, copy the value now.  */
7002 	  if (!after)
7003 	    emit_move_insn (target, t0);
7004 	}
7005       else
7006         target = const0_rtx;
7007 
7008       t1 = t0;
7009       if (code == NOT)
7010         {
7011 	  t1 = expand_simple_binop (mode, AND, t1, val, NULL_RTX,
7012 				    true, OPTAB_LIB_WIDEN);
7013 	  t1 = expand_simple_unop (mode, code, t1, NULL_RTX, true);
7014 	}
7015       else
7016 	t1 = expand_simple_binop (mode, code, t1, val, NULL_RTX, true,
7017 				  OPTAB_LIB_WIDEN);
7018 
7019       /* For after, copy the value now.  */
7020       if (!unused_result && after)
7021         emit_move_insn (target, t1);
7022       insn = get_insns ();
7023       end_sequence ();
7024 
7025       if (t1 != NULL && expand_compare_and_swap_loop (mem, t0, t1, insn))
7026         return target;
7027     }
7028 
7029   return NULL_RTX;
7030 }
7031 
7032 /* Return true if OPERAND is suitable for operand number OPNO of
7033    instruction ICODE.  */
7034 
7035 bool
insn_operand_matches(enum insn_code icode,unsigned int opno,rtx operand)7036 insn_operand_matches (enum insn_code icode, unsigned int opno, rtx operand)
7037 {
7038   return (!insn_data[(int) icode].operand[opno].predicate
7039 	  || (insn_data[(int) icode].operand[opno].predicate
7040 	      (operand, insn_data[(int) icode].operand[opno].mode)));
7041 }
7042 
7043 /* TARGET is a target of a multiword operation that we are going to
7044    implement as a series of word-mode operations.  Return true if
7045    TARGET is suitable for this purpose.  */
7046 
7047 bool
valid_multiword_target_p(rtx target)7048 valid_multiword_target_p (rtx target)
7049 {
7050   machine_mode mode;
7051   int i, size;
7052 
7053   mode = GET_MODE (target);
7054   if (!GET_MODE_SIZE (mode).is_constant (&size))
7055     return false;
7056   for (i = 0; i < size; i += UNITS_PER_WORD)
7057     if (!validate_subreg (word_mode, mode, target, i))
7058       return false;
7059   return true;
7060 }
7061 
7062 /* Make OP describe an input operand that has value INTVAL and that has
7063    no inherent mode.  This function should only be used for operands that
7064    are always expand-time constants.  The backend may request that INTVAL
7065    be copied into a different kind of rtx, but it must specify the mode
7066    of that rtx if so.  */
7067 
7068 void
create_integer_operand(struct expand_operand * op,poly_int64 intval)7069 create_integer_operand (struct expand_operand *op, poly_int64 intval)
7070 {
7071   create_expand_operand (op, EXPAND_INTEGER,
7072 			 gen_int_mode (intval, MAX_MODE_INT),
7073 			 VOIDmode, false, intval);
7074 }
7075 
7076 /* Like maybe_legitimize_operand, but do not change the code of the
7077    current rtx value.  */
7078 
7079 static bool
maybe_legitimize_operand_same_code(enum insn_code icode,unsigned int opno,struct expand_operand * op)7080 maybe_legitimize_operand_same_code (enum insn_code icode, unsigned int opno,
7081 				    struct expand_operand *op)
7082 {
7083   /* See if the operand matches in its current form.  */
7084   if (insn_operand_matches (icode, opno, op->value))
7085     return true;
7086 
7087   /* If the operand is a memory whose address has no side effects,
7088      try forcing the address into a non-virtual pseudo register.
7089      The check for side effects is important because copy_to_mode_reg
7090      cannot handle things like auto-modified addresses.  */
7091   if (insn_data[(int) icode].operand[opno].allows_mem && MEM_P (op->value))
7092     {
7093       rtx addr, mem;
7094 
7095       mem = op->value;
7096       addr = XEXP (mem, 0);
7097       if (!(REG_P (addr) && REGNO (addr) > LAST_VIRTUAL_REGISTER)
7098 	  && !side_effects_p (addr))
7099 	{
7100 	  rtx_insn *last;
7101 	  machine_mode mode;
7102 
7103 	  last = get_last_insn ();
7104 	  mode = get_address_mode (mem);
7105 	  mem = replace_equiv_address (mem, copy_to_mode_reg (mode, addr));
7106 	  if (insn_operand_matches (icode, opno, mem))
7107 	    {
7108 	      op->value = mem;
7109 	      return true;
7110 	    }
7111 	  delete_insns_since (last);
7112 	}
7113     }
7114 
7115   return false;
7116 }
7117 
7118 /* Try to make OP match operand OPNO of instruction ICODE.  Return true
7119    on success, storing the new operand value back in OP.  */
7120 
7121 static bool
maybe_legitimize_operand(enum insn_code icode,unsigned int opno,struct expand_operand * op)7122 maybe_legitimize_operand (enum insn_code icode, unsigned int opno,
7123 			  struct expand_operand *op)
7124 {
7125   machine_mode mode, imode;
7126   bool old_volatile_ok, result;
7127 
7128   mode = op->mode;
7129   switch (op->type)
7130     {
7131     case EXPAND_FIXED:
7132       old_volatile_ok = volatile_ok;
7133       volatile_ok = true;
7134       result = maybe_legitimize_operand_same_code (icode, opno, op);
7135       volatile_ok = old_volatile_ok;
7136       return result;
7137 
7138     case EXPAND_OUTPUT:
7139       gcc_assert (mode != VOIDmode);
7140       if (op->value
7141 	  && op->value != const0_rtx
7142 	  && GET_MODE (op->value) == mode
7143 	  && maybe_legitimize_operand_same_code (icode, opno, op))
7144 	return true;
7145 
7146       op->value = gen_reg_rtx (mode);
7147       op->target = 0;
7148       break;
7149 
7150     case EXPAND_INPUT:
7151     input:
7152       gcc_assert (mode != VOIDmode);
7153       gcc_assert (GET_MODE (op->value) == VOIDmode
7154 		  || GET_MODE (op->value) == mode);
7155       if (maybe_legitimize_operand_same_code (icode, opno, op))
7156 	return true;
7157 
7158       op->value = copy_to_mode_reg (mode, op->value);
7159       break;
7160 
7161     case EXPAND_CONVERT_TO:
7162       gcc_assert (mode != VOIDmode);
7163       op->value = convert_to_mode (mode, op->value, op->unsigned_p);
7164       goto input;
7165 
7166     case EXPAND_CONVERT_FROM:
7167       if (GET_MODE (op->value) != VOIDmode)
7168 	mode = GET_MODE (op->value);
7169       else
7170 	/* The caller must tell us what mode this value has.  */
7171 	gcc_assert (mode != VOIDmode);
7172 
7173       imode = insn_data[(int) icode].operand[opno].mode;
7174       if (imode != VOIDmode && imode != mode)
7175 	{
7176 	  op->value = convert_modes (imode, mode, op->value, op->unsigned_p);
7177 	  mode = imode;
7178 	}
7179       goto input;
7180 
7181     case EXPAND_ADDRESS:
7182       op->value = convert_memory_address (as_a <scalar_int_mode> (mode),
7183 					  op->value);
7184       goto input;
7185 
7186     case EXPAND_INTEGER:
7187       mode = insn_data[(int) icode].operand[opno].mode;
7188       if (mode != VOIDmode
7189 	  && known_eq (trunc_int_for_mode (op->int_value, mode),
7190 		       op->int_value))
7191 	{
7192 	  op->value = gen_int_mode (op->int_value, mode);
7193 	  goto input;
7194 	}
7195       break;
7196     }
7197   return insn_operand_matches (icode, opno, op->value);
7198 }
7199 
7200 /* Make OP describe an input operand that should have the same value
7201    as VALUE, after any mode conversion that the target might request.
7202    TYPE is the type of VALUE.  */
7203 
7204 void
create_convert_operand_from_type(struct expand_operand * op,rtx value,tree type)7205 create_convert_operand_from_type (struct expand_operand *op,
7206 				  rtx value, tree type)
7207 {
7208   create_convert_operand_from (op, value, TYPE_MODE (type),
7209 			       TYPE_UNSIGNED (type));
7210 }
7211 
7212 /* Try to make operands [OPS, OPS + NOPS) match operands [OPNO, OPNO + NOPS)
7213    of instruction ICODE.  Return true on success, leaving the new operand
7214    values in the OPS themselves.  Emit no code on failure.  */
7215 
7216 bool
maybe_legitimize_operands(enum insn_code icode,unsigned int opno,unsigned int nops,struct expand_operand * ops)7217 maybe_legitimize_operands (enum insn_code icode, unsigned int opno,
7218 			   unsigned int nops, struct expand_operand *ops)
7219 {
7220   rtx_insn *last;
7221   unsigned int i;
7222 
7223   last = get_last_insn ();
7224   for (i = 0; i < nops; i++)
7225     if (!maybe_legitimize_operand (icode, opno + i, &ops[i]))
7226       {
7227 	delete_insns_since (last);
7228 	return false;
7229       }
7230   return true;
7231 }
7232 
7233 /* Try to generate instruction ICODE, using operands [OPS, OPS + NOPS)
7234    as its operands.  Return the instruction pattern on success,
7235    and emit any necessary set-up code.  Return null and emit no
7236    code on failure.  */
7237 
7238 rtx_insn *
maybe_gen_insn(enum insn_code icode,unsigned int nops,struct expand_operand * ops)7239 maybe_gen_insn (enum insn_code icode, unsigned int nops,
7240 		struct expand_operand *ops)
7241 {
7242   gcc_assert (nops == (unsigned int) insn_data[(int) icode].n_generator_args);
7243   if (!maybe_legitimize_operands (icode, 0, nops, ops))
7244     return NULL;
7245 
7246   switch (nops)
7247     {
7248     case 1:
7249       return GEN_FCN (icode) (ops[0].value);
7250     case 2:
7251       return GEN_FCN (icode) (ops[0].value, ops[1].value);
7252     case 3:
7253       return GEN_FCN (icode) (ops[0].value, ops[1].value, ops[2].value);
7254     case 4:
7255       return GEN_FCN (icode) (ops[0].value, ops[1].value, ops[2].value,
7256 			      ops[3].value);
7257     case 5:
7258       return GEN_FCN (icode) (ops[0].value, ops[1].value, ops[2].value,
7259 			      ops[3].value, ops[4].value);
7260     case 6:
7261       return GEN_FCN (icode) (ops[0].value, ops[1].value, ops[2].value,
7262 			      ops[3].value, ops[4].value, ops[5].value);
7263     case 7:
7264       return GEN_FCN (icode) (ops[0].value, ops[1].value, ops[2].value,
7265 			      ops[3].value, ops[4].value, ops[5].value,
7266 			      ops[6].value);
7267     case 8:
7268       return GEN_FCN (icode) (ops[0].value, ops[1].value, ops[2].value,
7269 			      ops[3].value, ops[4].value, ops[5].value,
7270 			      ops[6].value, ops[7].value);
7271     case 9:
7272       return GEN_FCN (icode) (ops[0].value, ops[1].value, ops[2].value,
7273 			      ops[3].value, ops[4].value, ops[5].value,
7274 			      ops[6].value, ops[7].value, ops[8].value);
7275     }
7276   gcc_unreachable ();
7277 }
7278 
7279 /* Try to emit instruction ICODE, using operands [OPS, OPS + NOPS)
7280    as its operands.  Return true on success and emit no code on failure.  */
7281 
7282 bool
maybe_expand_insn(enum insn_code icode,unsigned int nops,struct expand_operand * ops)7283 maybe_expand_insn (enum insn_code icode, unsigned int nops,
7284 		   struct expand_operand *ops)
7285 {
7286   rtx_insn *pat = maybe_gen_insn (icode, nops, ops);
7287   if (pat)
7288     {
7289       emit_insn (pat);
7290       return true;
7291     }
7292   return false;
7293 }
7294 
7295 /* Like maybe_expand_insn, but for jumps.  */
7296 
7297 bool
maybe_expand_jump_insn(enum insn_code icode,unsigned int nops,struct expand_operand * ops)7298 maybe_expand_jump_insn (enum insn_code icode, unsigned int nops,
7299 			struct expand_operand *ops)
7300 {
7301   rtx_insn *pat = maybe_gen_insn (icode, nops, ops);
7302   if (pat)
7303     {
7304       emit_jump_insn (pat);
7305       return true;
7306     }
7307   return false;
7308 }
7309 
7310 /* Emit instruction ICODE, using operands [OPS, OPS + NOPS)
7311    as its operands.  */
7312 
7313 void
expand_insn(enum insn_code icode,unsigned int nops,struct expand_operand * ops)7314 expand_insn (enum insn_code icode, unsigned int nops,
7315 	     struct expand_operand *ops)
7316 {
7317   if (!maybe_expand_insn (icode, nops, ops))
7318     gcc_unreachable ();
7319 }
7320 
7321 /* Like expand_insn, but for jumps.  */
7322 
7323 void
expand_jump_insn(enum insn_code icode,unsigned int nops,struct expand_operand * ops)7324 expand_jump_insn (enum insn_code icode, unsigned int nops,
7325 		  struct expand_operand *ops)
7326 {
7327   if (!maybe_expand_jump_insn (icode, nops, ops))
7328     gcc_unreachable ();
7329 }
7330