1 /* Internal functions.
2 Copyright (C) 2011-2016 Free Software Foundation, Inc.
3
4 This file is part of GCC.
5
6 GCC is free software; you can redistribute it and/or modify it under
7 the terms of the GNU General Public License as published by the Free
8 Software Foundation; either version 3, or (at your option) any later
9 version.
10
11 GCC is distributed in the hope that it will be useful, but WITHOUT ANY
12 WARRANTY; without even the implied warranty of MERCHANTABILITY or
13 FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
14 for more details.
15
16 You should have received a copy of the GNU General Public License
17 along with GCC; see the file COPYING3. If not see
18 <http://www.gnu.org/licenses/>. */
19
20 #include "config.h"
21 #include "system.h"
22 #include "coretypes.h"
23 #include "backend.h"
24 #include "target.h"
25 #include "rtl.h"
26 #include "tree.h"
27 #include "gimple.h"
28 #include "predict.h"
29 #include "stringpool.h"
30 #include "tree-ssanames.h"
31 #include "expmed.h"
32 #include "optabs.h"
33 #include "emit-rtl.h"
34 #include "diagnostic-core.h"
35 #include "fold-const.h"
36 #include "internal-fn.h"
37 #include "stor-layout.h"
38 #include "dojump.h"
39 #include "expr.h"
40 #include "ubsan.h"
41 #include "recog.h"
42
43 /* The names of each internal function, indexed by function number. */
44 const char *const internal_fn_name_array[] = {
45 #define DEF_INTERNAL_FN(CODE, FLAGS, FNSPEC) #CODE,
46 #include "internal-fn.def"
47 "<invalid-fn>"
48 };
49
50 /* The ECF_* flags of each internal function, indexed by function number. */
51 const int internal_fn_flags_array[] = {
52 #define DEF_INTERNAL_FN(CODE, FLAGS, FNSPEC) FLAGS,
53 #include "internal-fn.def"
54 0
55 };
56
57 /* Fnspec of each internal function, indexed by function number. */
58 const_tree internal_fn_fnspec_array[IFN_LAST + 1];
59
60 void
init_internal_fns()61 init_internal_fns ()
62 {
63 #define DEF_INTERNAL_FN(CODE, FLAGS, FNSPEC) \
64 if (FNSPEC) internal_fn_fnspec_array[IFN_##CODE] = \
65 build_string ((int) sizeof (FNSPEC), FNSPEC ? FNSPEC : "");
66 #include "internal-fn.def"
67 internal_fn_fnspec_array[IFN_LAST] = 0;
68 }
69
70 /* Create static initializers for the information returned by
71 direct_internal_fn. */
72 #define not_direct { -2, -2, false }
73 #define mask_load_direct { -1, 2, false }
74 #define load_lanes_direct { -1, -1, false }
75 #define mask_store_direct { 3, 2, false }
76 #define store_lanes_direct { 0, 0, false }
77 #define unary_direct { 0, 0, true }
78 #define binary_direct { 0, 0, true }
79
80 const direct_internal_fn_info direct_internal_fn_array[IFN_LAST + 1] = {
81 #define DEF_INTERNAL_FN(CODE, FLAGS, FNSPEC) not_direct,
82 #define DEF_INTERNAL_OPTAB_FN(CODE, FLAGS, OPTAB, TYPE) TYPE##_direct,
83 #include "internal-fn.def"
84 not_direct
85 };
86
87 /* ARRAY_TYPE is an array of vector modes. Return the associated insn
88 for load-lanes-style optab OPTAB, or CODE_FOR_nothing if none. */
89
90 static enum insn_code
get_multi_vector_move(tree array_type,convert_optab optab)91 get_multi_vector_move (tree array_type, convert_optab optab)
92 {
93 machine_mode imode;
94 machine_mode vmode;
95
96 gcc_assert (TREE_CODE (array_type) == ARRAY_TYPE);
97 imode = TYPE_MODE (array_type);
98 vmode = TYPE_MODE (TREE_TYPE (array_type));
99
100 return convert_optab_handler (optab, imode, vmode);
101 }
102
103 /* Expand LOAD_LANES call STMT using optab OPTAB. */
104
105 static void
expand_load_lanes_optab_fn(internal_fn,gcall * stmt,convert_optab optab)106 expand_load_lanes_optab_fn (internal_fn, gcall *stmt, convert_optab optab)
107 {
108 struct expand_operand ops[2];
109 tree type, lhs, rhs;
110 rtx target, mem;
111
112 lhs = gimple_call_lhs (stmt);
113 rhs = gimple_call_arg (stmt, 0);
114 type = TREE_TYPE (lhs);
115
116 target = expand_expr (lhs, NULL_RTX, VOIDmode, EXPAND_WRITE);
117 mem = expand_normal (rhs);
118
119 gcc_assert (MEM_P (mem));
120 PUT_MODE (mem, TYPE_MODE (type));
121
122 create_output_operand (&ops[0], target, TYPE_MODE (type));
123 create_fixed_operand (&ops[1], mem);
124 expand_insn (get_multi_vector_move (type, optab), 2, ops);
125 }
126
127 /* Expand STORE_LANES call STMT using optab OPTAB. */
128
129 static void
expand_store_lanes_optab_fn(internal_fn,gcall * stmt,convert_optab optab)130 expand_store_lanes_optab_fn (internal_fn, gcall *stmt, convert_optab optab)
131 {
132 struct expand_operand ops[2];
133 tree type, lhs, rhs;
134 rtx target, reg;
135
136 lhs = gimple_call_lhs (stmt);
137 rhs = gimple_call_arg (stmt, 0);
138 type = TREE_TYPE (rhs);
139
140 target = expand_expr (lhs, NULL_RTX, VOIDmode, EXPAND_WRITE);
141 reg = expand_normal (rhs);
142
143 gcc_assert (MEM_P (target));
144 PUT_MODE (target, TYPE_MODE (type));
145
146 create_fixed_operand (&ops[0], target);
147 create_input_operand (&ops[1], reg, TYPE_MODE (type));
148 expand_insn (get_multi_vector_move (type, optab), 2, ops);
149 }
150
151 static void
expand_ANNOTATE(internal_fn,gcall *)152 expand_ANNOTATE (internal_fn, gcall *)
153 {
154 gcc_unreachable ();
155 }
156
157 /* This should get expanded in adjust_simduid_builtins. */
158
159 static void
expand_GOMP_SIMD_LANE(internal_fn,gcall *)160 expand_GOMP_SIMD_LANE (internal_fn, gcall *)
161 {
162 gcc_unreachable ();
163 }
164
165 /* This should get expanded in adjust_simduid_builtins. */
166
167 static void
expand_GOMP_SIMD_VF(internal_fn,gcall *)168 expand_GOMP_SIMD_VF (internal_fn, gcall *)
169 {
170 gcc_unreachable ();
171 }
172
173 /* This should get expanded in adjust_simduid_builtins. */
174
175 static void
expand_GOMP_SIMD_LAST_LANE(internal_fn,gcall *)176 expand_GOMP_SIMD_LAST_LANE (internal_fn, gcall *)
177 {
178 gcc_unreachable ();
179 }
180
181 /* This should get expanded in adjust_simduid_builtins. */
182
183 static void
expand_GOMP_SIMD_ORDERED_START(internal_fn,gcall *)184 expand_GOMP_SIMD_ORDERED_START (internal_fn, gcall *)
185 {
186 gcc_unreachable ();
187 }
188
189 /* This should get expanded in adjust_simduid_builtins. */
190
191 static void
expand_GOMP_SIMD_ORDERED_END(internal_fn,gcall *)192 expand_GOMP_SIMD_ORDERED_END (internal_fn, gcall *)
193 {
194 gcc_unreachable ();
195 }
196
197 /* This should get expanded in the sanopt pass. */
198
199 static void
expand_UBSAN_NULL(internal_fn,gcall *)200 expand_UBSAN_NULL (internal_fn, gcall *)
201 {
202 gcc_unreachable ();
203 }
204
205 /* This should get expanded in the sanopt pass. */
206
207 static void
expand_UBSAN_BOUNDS(internal_fn,gcall *)208 expand_UBSAN_BOUNDS (internal_fn, gcall *)
209 {
210 gcc_unreachable ();
211 }
212
213 /* This should get expanded in the sanopt pass. */
214
215 static void
expand_UBSAN_VPTR(internal_fn,gcall *)216 expand_UBSAN_VPTR (internal_fn, gcall *)
217 {
218 gcc_unreachable ();
219 }
220
221 /* This should get expanded in the sanopt pass. */
222
223 static void
expand_UBSAN_OBJECT_SIZE(internal_fn,gcall *)224 expand_UBSAN_OBJECT_SIZE (internal_fn, gcall *)
225 {
226 gcc_unreachable ();
227 }
228
229 /* This should get expanded in the sanopt pass. */
230
231 static void
expand_ASAN_CHECK(internal_fn,gcall *)232 expand_ASAN_CHECK (internal_fn, gcall *)
233 {
234 gcc_unreachable ();
235 }
236
237 /* This should get expanded in the tsan pass. */
238
239 static void
expand_TSAN_FUNC_EXIT(internal_fn,gcall *)240 expand_TSAN_FUNC_EXIT (internal_fn, gcall *)
241 {
242 gcc_unreachable ();
243 }
244
245 /* Helper function for expand_addsub_overflow. Return 1
246 if ARG interpreted as signed in its precision is known to be always
247 positive or 2 if ARG is known to be always negative, or 3 if ARG may
248 be positive or negative. */
249
250 static int
get_range_pos_neg(tree arg)251 get_range_pos_neg (tree arg)
252 {
253 if (arg == error_mark_node)
254 return 3;
255
256 int prec = TYPE_PRECISION (TREE_TYPE (arg));
257 int cnt = 0;
258 if (TREE_CODE (arg) == INTEGER_CST)
259 {
260 wide_int w = wi::sext (arg, prec);
261 if (wi::neg_p (w))
262 return 2;
263 else
264 return 1;
265 }
266 while (CONVERT_EXPR_P (arg)
267 && INTEGRAL_TYPE_P (TREE_TYPE (TREE_OPERAND (arg, 0)))
268 && TYPE_PRECISION (TREE_TYPE (TREE_OPERAND (arg, 0))) <= prec)
269 {
270 arg = TREE_OPERAND (arg, 0);
271 /* Narrower value zero extended into wider type
272 will always result in positive values. */
273 if (TYPE_UNSIGNED (TREE_TYPE (arg))
274 && TYPE_PRECISION (TREE_TYPE (arg)) < prec)
275 return 1;
276 prec = TYPE_PRECISION (TREE_TYPE (arg));
277 if (++cnt > 30)
278 return 3;
279 }
280
281 if (TREE_CODE (arg) != SSA_NAME)
282 return 3;
283 wide_int arg_min, arg_max;
284 while (get_range_info (arg, &arg_min, &arg_max) != VR_RANGE)
285 {
286 gimple *g = SSA_NAME_DEF_STMT (arg);
287 if (is_gimple_assign (g)
288 && CONVERT_EXPR_CODE_P (gimple_assign_rhs_code (g)))
289 {
290 tree t = gimple_assign_rhs1 (g);
291 if (INTEGRAL_TYPE_P (TREE_TYPE (t))
292 && TYPE_PRECISION (TREE_TYPE (t)) <= prec)
293 {
294 if (TYPE_UNSIGNED (TREE_TYPE (t))
295 && TYPE_PRECISION (TREE_TYPE (t)) < prec)
296 return 1;
297 prec = TYPE_PRECISION (TREE_TYPE (t));
298 arg = t;
299 if (++cnt > 30)
300 return 3;
301 continue;
302 }
303 }
304 return 3;
305 }
306 if (TYPE_UNSIGNED (TREE_TYPE (arg)))
307 {
308 /* For unsigned values, the "positive" range comes
309 below the "negative" range. */
310 if (!wi::neg_p (wi::sext (arg_max, prec), SIGNED))
311 return 1;
312 if (wi::neg_p (wi::sext (arg_min, prec), SIGNED))
313 return 2;
314 }
315 else
316 {
317 if (!wi::neg_p (wi::sext (arg_min, prec), SIGNED))
318 return 1;
319 if (wi::neg_p (wi::sext (arg_max, prec), SIGNED))
320 return 2;
321 }
322 return 3;
323 }
324
325 /* Return minimum precision needed to represent all values
326 of ARG in SIGNed integral type. */
327
328 static int
get_min_precision(tree arg,signop sign)329 get_min_precision (tree arg, signop sign)
330 {
331 int prec = TYPE_PRECISION (TREE_TYPE (arg));
332 int cnt = 0;
333 signop orig_sign = sign;
334 if (TREE_CODE (arg) == INTEGER_CST)
335 {
336 int p;
337 if (TYPE_SIGN (TREE_TYPE (arg)) != sign)
338 {
339 widest_int w = wi::to_widest (arg);
340 w = wi::ext (w, prec, sign);
341 p = wi::min_precision (w, sign);
342 }
343 else
344 p = wi::min_precision (arg, sign);
345 return MIN (p, prec);
346 }
347 while (CONVERT_EXPR_P (arg)
348 && INTEGRAL_TYPE_P (TREE_TYPE (TREE_OPERAND (arg, 0)))
349 && TYPE_PRECISION (TREE_TYPE (TREE_OPERAND (arg, 0))) <= prec)
350 {
351 arg = TREE_OPERAND (arg, 0);
352 if (TYPE_PRECISION (TREE_TYPE (arg)) < prec)
353 {
354 if (TYPE_UNSIGNED (TREE_TYPE (arg)))
355 sign = UNSIGNED;
356 else if (sign == UNSIGNED && get_range_pos_neg (arg) != 1)
357 return prec + (orig_sign != sign);
358 prec = TYPE_PRECISION (TREE_TYPE (arg));
359 }
360 if (++cnt > 30)
361 return prec + (orig_sign != sign);
362 }
363 if (TREE_CODE (arg) != SSA_NAME)
364 return prec + (orig_sign != sign);
365 wide_int arg_min, arg_max;
366 while (get_range_info (arg, &arg_min, &arg_max) != VR_RANGE)
367 {
368 gimple *g = SSA_NAME_DEF_STMT (arg);
369 if (is_gimple_assign (g)
370 && CONVERT_EXPR_CODE_P (gimple_assign_rhs_code (g)))
371 {
372 tree t = gimple_assign_rhs1 (g);
373 if (INTEGRAL_TYPE_P (TREE_TYPE (t))
374 && TYPE_PRECISION (TREE_TYPE (t)) <= prec)
375 {
376 arg = t;
377 if (TYPE_PRECISION (TREE_TYPE (arg)) < prec)
378 {
379 if (TYPE_UNSIGNED (TREE_TYPE (arg)))
380 sign = UNSIGNED;
381 else if (sign == UNSIGNED && get_range_pos_neg (arg) != 1)
382 return prec + (orig_sign != sign);
383 prec = TYPE_PRECISION (TREE_TYPE (arg));
384 }
385 if (++cnt > 30)
386 return prec + (orig_sign != sign);
387 continue;
388 }
389 }
390 return prec + (orig_sign != sign);
391 }
392 if (sign == TYPE_SIGN (TREE_TYPE (arg)))
393 {
394 int p1 = wi::min_precision (arg_min, sign);
395 int p2 = wi::min_precision (arg_max, sign);
396 p1 = MAX (p1, p2);
397 prec = MIN (prec, p1);
398 }
399 else if (sign == UNSIGNED && !wi::neg_p (arg_min, SIGNED))
400 {
401 int p = wi::min_precision (arg_max, UNSIGNED);
402 prec = MIN (prec, p);
403 }
404 return prec + (orig_sign != sign);
405 }
406
407 /* Helper for expand_*_overflow. Store RES into the __real__ part
408 of TARGET. If RES has larger MODE than __real__ part of TARGET,
409 set the __imag__ part to 1 if RES doesn't fit into it. */
410
411 static void
expand_arith_overflow_result_store(tree lhs,rtx target,machine_mode mode,rtx res)412 expand_arith_overflow_result_store (tree lhs, rtx target,
413 machine_mode mode, rtx res)
414 {
415 machine_mode tgtmode = GET_MODE_INNER (GET_MODE (target));
416 rtx lres = res;
417 if (tgtmode != mode)
418 {
419 rtx_code_label *done_label = gen_label_rtx ();
420 int uns = TYPE_UNSIGNED (TREE_TYPE (TREE_TYPE (lhs)));
421 lres = convert_modes (tgtmode, mode, res, uns);
422 gcc_assert (GET_MODE_PRECISION (tgtmode) < GET_MODE_PRECISION (mode));
423 do_compare_rtx_and_jump (res, convert_modes (mode, tgtmode, lres, uns),
424 EQ, true, mode, NULL_RTX, NULL, done_label,
425 PROB_VERY_LIKELY);
426 write_complex_part (target, const1_rtx, true);
427 emit_label (done_label);
428 }
429 write_complex_part (target, lres, false);
430 }
431
432 /* Helper for expand_*_overflow. Store RES into TARGET. */
433
434 static void
expand_ubsan_result_store(rtx target,rtx res)435 expand_ubsan_result_store (rtx target, rtx res)
436 {
437 if (GET_CODE (target) == SUBREG && SUBREG_PROMOTED_VAR_P (target))
438 /* If this is a scalar in a register that is stored in a wider mode
439 than the declared mode, compute the result into its declared mode
440 and then convert to the wider mode. Our value is the computed
441 expression. */
442 convert_move (SUBREG_REG (target), res, SUBREG_PROMOTED_SIGN (target));
443 else
444 emit_move_insn (target, res);
445 }
446
447 /* Add sub/add overflow checking to the statement STMT.
448 CODE says whether the operation is +, or -. */
449
450 static void
expand_addsub_overflow(location_t loc,tree_code code,tree lhs,tree arg0,tree arg1,bool unsr_p,bool uns0_p,bool uns1_p,bool is_ubsan)451 expand_addsub_overflow (location_t loc, tree_code code, tree lhs,
452 tree arg0, tree arg1, bool unsr_p, bool uns0_p,
453 bool uns1_p, bool is_ubsan)
454 {
455 rtx res, target = NULL_RTX;
456 tree fn;
457 rtx_code_label *done_label = gen_label_rtx ();
458 rtx_code_label *do_error = gen_label_rtx ();
459 do_pending_stack_adjust ();
460 rtx op0 = expand_normal (arg0);
461 rtx op1 = expand_normal (arg1);
462 machine_mode mode = TYPE_MODE (TREE_TYPE (arg0));
463 int prec = GET_MODE_PRECISION (mode);
464 rtx sgn = immed_wide_int_const (wi::min_value (prec, SIGNED), mode);
465 bool do_xor = false;
466
467 if (is_ubsan)
468 gcc_assert (!unsr_p && !uns0_p && !uns1_p);
469
470 if (lhs)
471 {
472 target = expand_expr (lhs, NULL_RTX, VOIDmode, EXPAND_WRITE);
473 if (!is_ubsan)
474 write_complex_part (target, const0_rtx, true);
475 }
476
477 /* We assume both operands and result have the same precision
478 here (GET_MODE_BITSIZE (mode)), S stands for signed type
479 with that precision, U for unsigned type with that precision,
480 sgn for unsigned most significant bit in that precision.
481 s1 is signed first operand, u1 is unsigned first operand,
482 s2 is signed second operand, u2 is unsigned second operand,
483 sr is signed result, ur is unsigned result and the following
484 rules say how to compute result (which is always result of
485 the operands as if both were unsigned, cast to the right
486 signedness) and how to compute whether operation overflowed.
487
488 s1 + s2 -> sr
489 res = (S) ((U) s1 + (U) s2)
490 ovf = s2 < 0 ? res > s1 : res < s1 (or jump on overflow)
491 s1 - s2 -> sr
492 res = (S) ((U) s1 - (U) s2)
493 ovf = s2 < 0 ? res < s1 : res > s2 (or jump on overflow)
494 u1 + u2 -> ur
495 res = u1 + u2
496 ovf = res < u1 (or jump on carry, but RTL opts will handle it)
497 u1 - u2 -> ur
498 res = u1 - u2
499 ovf = res > u1 (or jump on carry, but RTL opts will handle it)
500 s1 + u2 -> sr
501 res = (S) ((U) s1 + u2)
502 ovf = ((U) res ^ sgn) < u2
503 s1 + u2 -> ur
504 t1 = (S) (u2 ^ sgn)
505 t2 = s1 + t1
506 res = (U) t2 ^ sgn
507 ovf = t1 < 0 ? t2 > s1 : t2 < s1 (or jump on overflow)
508 s1 - u2 -> sr
509 res = (S) ((U) s1 - u2)
510 ovf = u2 > ((U) s1 ^ sgn)
511 s1 - u2 -> ur
512 res = (U) s1 - u2
513 ovf = s1 < 0 || u2 > (U) s1
514 u1 - s2 -> sr
515 res = u1 - (U) s2
516 ovf = u1 >= ((U) s2 ^ sgn)
517 u1 - s2 -> ur
518 t1 = u1 ^ sgn
519 t2 = t1 - (U) s2
520 res = t2 ^ sgn
521 ovf = s2 < 0 ? (S) t2 < (S) t1 : (S) t2 > (S) t1 (or jump on overflow)
522 s1 + s2 -> ur
523 res = (U) s1 + (U) s2
524 ovf = s2 < 0 ? (s1 | (S) res) < 0) : (s1 & (S) res) < 0)
525 u1 + u2 -> sr
526 res = (S) (u1 + u2)
527 ovf = (U) res < u2 || res < 0
528 u1 - u2 -> sr
529 res = (S) (u1 - u2)
530 ovf = u1 >= u2 ? res < 0 : res >= 0
531 s1 - s2 -> ur
532 res = (U) s1 - (U) s2
533 ovf = s2 >= 0 ? ((s1 | (S) res) < 0) : ((s1 & (S) res) < 0) */
534
535 if (code == PLUS_EXPR && uns0_p && !uns1_p)
536 {
537 /* PLUS_EXPR is commutative, if operand signedness differs,
538 canonicalize to the first operand being signed and second
539 unsigned to simplify following code. */
540 std::swap (op0, op1);
541 std::swap (arg0, arg1);
542 uns0_p = false;
543 uns1_p = true;
544 }
545
546 /* u1 +- u2 -> ur */
547 if (uns0_p && uns1_p && unsr_p)
548 {
549 insn_code icode = optab_handler (code == PLUS_EXPR ? uaddv4_optab
550 : usubv4_optab, mode);
551 if (icode != CODE_FOR_nothing)
552 {
553 struct expand_operand ops[4];
554 rtx_insn *last = get_last_insn ();
555
556 res = gen_reg_rtx (mode);
557 create_output_operand (&ops[0], res, mode);
558 create_input_operand (&ops[1], op0, mode);
559 create_input_operand (&ops[2], op1, mode);
560 create_fixed_operand (&ops[3], do_error);
561 if (maybe_expand_insn (icode, 4, ops))
562 {
563 last = get_last_insn ();
564 if (profile_status_for_fn (cfun) != PROFILE_ABSENT
565 && JUMP_P (last)
566 && any_condjump_p (last)
567 && !find_reg_note (last, REG_BR_PROB, 0))
568 add_int_reg_note (last, REG_BR_PROB, PROB_VERY_UNLIKELY);
569 emit_jump (done_label);
570 goto do_error_label;
571 }
572
573 delete_insns_since (last);
574 }
575
576 /* Compute the operation. On RTL level, the addition is always
577 unsigned. */
578 res = expand_binop (mode, code == PLUS_EXPR ? add_optab : sub_optab,
579 op0, op1, NULL_RTX, false, OPTAB_LIB_WIDEN);
580 rtx tem = op0;
581 /* For PLUS_EXPR, the operation is commutative, so we can pick
582 operand to compare against. For prec <= BITS_PER_WORD, I think
583 preferring REG operand is better over CONST_INT, because
584 the CONST_INT might enlarge the instruction or CSE would need
585 to figure out we'd already loaded it into a register before.
586 For prec > BITS_PER_WORD, I think CONST_INT might be more beneficial,
587 as then the multi-word comparison can be perhaps simplified. */
588 if (code == PLUS_EXPR
589 && (prec <= BITS_PER_WORD
590 ? (CONST_SCALAR_INT_P (op0) && REG_P (op1))
591 : CONST_SCALAR_INT_P (op1)))
592 tem = op1;
593 do_compare_rtx_and_jump (res, tem, code == PLUS_EXPR ? GEU : LEU,
594 true, mode, NULL_RTX, NULL, done_label,
595 PROB_VERY_LIKELY);
596 goto do_error_label;
597 }
598
599 /* s1 +- u2 -> sr */
600 if (!uns0_p && uns1_p && !unsr_p)
601 {
602 /* Compute the operation. On RTL level, the addition is always
603 unsigned. */
604 res = expand_binop (mode, code == PLUS_EXPR ? add_optab : sub_optab,
605 op0, op1, NULL_RTX, false, OPTAB_LIB_WIDEN);
606 rtx tem = expand_binop (mode, add_optab,
607 code == PLUS_EXPR ? res : op0, sgn,
608 NULL_RTX, false, OPTAB_LIB_WIDEN);
609 do_compare_rtx_and_jump (tem, op1, GEU, true, mode, NULL_RTX, NULL,
610 done_label, PROB_VERY_LIKELY);
611 goto do_error_label;
612 }
613
614 /* s1 + u2 -> ur */
615 if (code == PLUS_EXPR && !uns0_p && uns1_p && unsr_p)
616 {
617 op1 = expand_binop (mode, add_optab, op1, sgn, NULL_RTX, false,
618 OPTAB_LIB_WIDEN);
619 /* As we've changed op1, we have to avoid using the value range
620 for the original argument. */
621 arg1 = error_mark_node;
622 do_xor = true;
623 goto do_signed;
624 }
625
626 /* u1 - s2 -> ur */
627 if (code == MINUS_EXPR && uns0_p && !uns1_p && unsr_p)
628 {
629 op0 = expand_binop (mode, add_optab, op0, sgn, NULL_RTX, false,
630 OPTAB_LIB_WIDEN);
631 /* As we've changed op0, we have to avoid using the value range
632 for the original argument. */
633 arg0 = error_mark_node;
634 do_xor = true;
635 goto do_signed;
636 }
637
638 /* s1 - u2 -> ur */
639 if (code == MINUS_EXPR && !uns0_p && uns1_p && unsr_p)
640 {
641 /* Compute the operation. On RTL level, the addition is always
642 unsigned. */
643 res = expand_binop (mode, sub_optab, op0, op1, NULL_RTX, false,
644 OPTAB_LIB_WIDEN);
645 int pos_neg = get_range_pos_neg (arg0);
646 if (pos_neg == 2)
647 /* If ARG0 is known to be always negative, this is always overflow. */
648 emit_jump (do_error);
649 else if (pos_neg == 3)
650 /* If ARG0 is not known to be always positive, check at runtime. */
651 do_compare_rtx_and_jump (op0, const0_rtx, LT, false, mode, NULL_RTX,
652 NULL, do_error, PROB_VERY_UNLIKELY);
653 do_compare_rtx_and_jump (op1, op0, LEU, true, mode, NULL_RTX, NULL,
654 done_label, PROB_VERY_LIKELY);
655 goto do_error_label;
656 }
657
658 /* u1 - s2 -> sr */
659 if (code == MINUS_EXPR && uns0_p && !uns1_p && !unsr_p)
660 {
661 /* Compute the operation. On RTL level, the addition is always
662 unsigned. */
663 res = expand_binop (mode, sub_optab, op0, op1, NULL_RTX, false,
664 OPTAB_LIB_WIDEN);
665 rtx tem = expand_binop (mode, add_optab, op1, sgn, NULL_RTX, false,
666 OPTAB_LIB_WIDEN);
667 do_compare_rtx_and_jump (op0, tem, LTU, true, mode, NULL_RTX, NULL,
668 done_label, PROB_VERY_LIKELY);
669 goto do_error_label;
670 }
671
672 /* u1 + u2 -> sr */
673 if (code == PLUS_EXPR && uns0_p && uns1_p && !unsr_p)
674 {
675 /* Compute the operation. On RTL level, the addition is always
676 unsigned. */
677 res = expand_binop (mode, add_optab, op0, op1, NULL_RTX, false,
678 OPTAB_LIB_WIDEN);
679 do_compare_rtx_and_jump (res, const0_rtx, LT, false, mode, NULL_RTX,
680 NULL, do_error, PROB_VERY_UNLIKELY);
681 rtx tem = op1;
682 /* The operation is commutative, so we can pick operand to compare
683 against. For prec <= BITS_PER_WORD, I think preferring REG operand
684 is better over CONST_INT, because the CONST_INT might enlarge the
685 instruction or CSE would need to figure out we'd already loaded it
686 into a register before. For prec > BITS_PER_WORD, I think CONST_INT
687 might be more beneficial, as then the multi-word comparison can be
688 perhaps simplified. */
689 if (prec <= BITS_PER_WORD
690 ? (CONST_SCALAR_INT_P (op1) && REG_P (op0))
691 : CONST_SCALAR_INT_P (op0))
692 tem = op0;
693 do_compare_rtx_and_jump (res, tem, GEU, true, mode, NULL_RTX, NULL,
694 done_label, PROB_VERY_LIKELY);
695 goto do_error_label;
696 }
697
698 /* s1 +- s2 -> ur */
699 if (!uns0_p && !uns1_p && unsr_p)
700 {
701 /* Compute the operation. On RTL level, the addition is always
702 unsigned. */
703 res = expand_binop (mode, code == PLUS_EXPR ? add_optab : sub_optab,
704 op0, op1, NULL_RTX, false, OPTAB_LIB_WIDEN);
705 int pos_neg = get_range_pos_neg (arg1);
706 if (code == PLUS_EXPR)
707 {
708 int pos_neg0 = get_range_pos_neg (arg0);
709 if (pos_neg0 != 3 && pos_neg == 3)
710 {
711 std::swap (op0, op1);
712 pos_neg = pos_neg0;
713 }
714 }
715 rtx tem;
716 if (pos_neg != 3)
717 {
718 tem = expand_binop (mode, ((pos_neg == 1) ^ (code == MINUS_EXPR))
719 ? and_optab : ior_optab,
720 op0, res, NULL_RTX, false, OPTAB_LIB_WIDEN);
721 do_compare_rtx_and_jump (tem, const0_rtx, GE, false, mode, NULL,
722 NULL, done_label, PROB_VERY_LIKELY);
723 }
724 else
725 {
726 rtx_code_label *do_ior_label = gen_label_rtx ();
727 do_compare_rtx_and_jump (op1, const0_rtx,
728 code == MINUS_EXPR ? GE : LT, false, mode,
729 NULL_RTX, NULL, do_ior_label,
730 PROB_EVEN);
731 tem = expand_binop (mode, and_optab, op0, res, NULL_RTX, false,
732 OPTAB_LIB_WIDEN);
733 do_compare_rtx_and_jump (tem, const0_rtx, GE, false, mode, NULL_RTX,
734 NULL, done_label, PROB_VERY_LIKELY);
735 emit_jump (do_error);
736 emit_label (do_ior_label);
737 tem = expand_binop (mode, ior_optab, op0, res, NULL_RTX, false,
738 OPTAB_LIB_WIDEN);
739 do_compare_rtx_and_jump (tem, const0_rtx, GE, false, mode, NULL_RTX,
740 NULL, done_label, PROB_VERY_LIKELY);
741 }
742 goto do_error_label;
743 }
744
745 /* u1 - u2 -> sr */
746 if (code == MINUS_EXPR && uns0_p && uns1_p && !unsr_p)
747 {
748 /* Compute the operation. On RTL level, the addition is always
749 unsigned. */
750 res = expand_binop (mode, sub_optab, op0, op1, NULL_RTX, false,
751 OPTAB_LIB_WIDEN);
752 rtx_code_label *op0_geu_op1 = gen_label_rtx ();
753 do_compare_rtx_and_jump (op0, op1, GEU, true, mode, NULL_RTX, NULL,
754 op0_geu_op1, PROB_EVEN);
755 do_compare_rtx_and_jump (res, const0_rtx, LT, false, mode, NULL_RTX,
756 NULL, done_label, PROB_VERY_LIKELY);
757 emit_jump (do_error);
758 emit_label (op0_geu_op1);
759 do_compare_rtx_and_jump (res, const0_rtx, GE, false, mode, NULL_RTX,
760 NULL, done_label, PROB_VERY_LIKELY);
761 goto do_error_label;
762 }
763
764 gcc_assert (!uns0_p && !uns1_p && !unsr_p);
765
766 /* s1 +- s2 -> sr */
767 do_signed:
768 {
769 insn_code icode = optab_handler (code == PLUS_EXPR ? addv4_optab
770 : subv4_optab, mode);
771 if (icode != CODE_FOR_nothing)
772 {
773 struct expand_operand ops[4];
774 rtx_insn *last = get_last_insn ();
775
776 res = gen_reg_rtx (mode);
777 create_output_operand (&ops[0], res, mode);
778 create_input_operand (&ops[1], op0, mode);
779 create_input_operand (&ops[2], op1, mode);
780 create_fixed_operand (&ops[3], do_error);
781 if (maybe_expand_insn (icode, 4, ops))
782 {
783 last = get_last_insn ();
784 if (profile_status_for_fn (cfun) != PROFILE_ABSENT
785 && JUMP_P (last)
786 && any_condjump_p (last)
787 && !find_reg_note (last, REG_BR_PROB, 0))
788 add_int_reg_note (last, REG_BR_PROB, PROB_VERY_UNLIKELY);
789 emit_jump (done_label);
790 goto do_error_label;
791 }
792
793 delete_insns_since (last);
794 }
795
796 rtx_code_label *sub_check = gen_label_rtx ();
797 int pos_neg = 3;
798
799 /* Compute the operation. On RTL level, the addition is always
800 unsigned. */
801 res = expand_binop (mode, code == PLUS_EXPR ? add_optab : sub_optab,
802 op0, op1, NULL_RTX, false, OPTAB_LIB_WIDEN);
803
804 /* If we can prove one of the arguments (for MINUS_EXPR only
805 the second operand, as subtraction is not commutative) is always
806 non-negative or always negative, we can do just one comparison
807 and conditional jump instead of 2 at runtime, 3 present in the
808 emitted code. If one of the arguments is CONST_INT, all we
809 need is to make sure it is op1, then the first
810 do_compare_rtx_and_jump will be just folded. Otherwise try
811 to use range info if available. */
812 if (code == PLUS_EXPR && CONST_INT_P (op0))
813 std::swap (op0, op1);
814 else if (CONST_INT_P (op1))
815 ;
816 else if (code == PLUS_EXPR && TREE_CODE (arg0) == SSA_NAME)
817 {
818 pos_neg = get_range_pos_neg (arg0);
819 if (pos_neg != 3)
820 std::swap (op0, op1);
821 }
822 if (pos_neg == 3 && !CONST_INT_P (op1) && TREE_CODE (arg1) == SSA_NAME)
823 pos_neg = get_range_pos_neg (arg1);
824
825 /* If the op1 is negative, we have to use a different check. */
826 if (pos_neg == 3)
827 do_compare_rtx_and_jump (op1, const0_rtx, LT, false, mode, NULL_RTX,
828 NULL, sub_check, PROB_EVEN);
829
830 /* Compare the result of the operation with one of the operands. */
831 if (pos_neg & 1)
832 do_compare_rtx_and_jump (res, op0, code == PLUS_EXPR ? GE : LE,
833 false, mode, NULL_RTX, NULL, done_label,
834 PROB_VERY_LIKELY);
835
836 /* If we get here, we have to print the error. */
837 if (pos_neg == 3)
838 {
839 emit_jump (do_error);
840 emit_label (sub_check);
841 }
842
843 /* We have k = a + b for b < 0 here. k <= a must hold. */
844 if (pos_neg & 2)
845 do_compare_rtx_and_jump (res, op0, code == PLUS_EXPR ? LE : GE,
846 false, mode, NULL_RTX, NULL, done_label,
847 PROB_VERY_LIKELY);
848 }
849
850 do_error_label:
851 emit_label (do_error);
852 if (is_ubsan)
853 {
854 /* Expand the ubsan builtin call. */
855 push_temp_slots ();
856 fn = ubsan_build_overflow_builtin (code, loc, TREE_TYPE (arg0),
857 arg0, arg1);
858 expand_normal (fn);
859 pop_temp_slots ();
860 do_pending_stack_adjust ();
861 }
862 else if (lhs)
863 write_complex_part (target, const1_rtx, true);
864
865 /* We're done. */
866 emit_label (done_label);
867
868 if (lhs)
869 {
870 if (is_ubsan)
871 expand_ubsan_result_store (target, res);
872 else
873 {
874 if (do_xor)
875 res = expand_binop (mode, add_optab, res, sgn, NULL_RTX, false,
876 OPTAB_LIB_WIDEN);
877
878 expand_arith_overflow_result_store (lhs, target, mode, res);
879 }
880 }
881 }
882
883 /* Add negate overflow checking to the statement STMT. */
884
885 static void
expand_neg_overflow(location_t loc,tree lhs,tree arg1,bool is_ubsan)886 expand_neg_overflow (location_t loc, tree lhs, tree arg1, bool is_ubsan)
887 {
888 rtx res, op1;
889 tree fn;
890 rtx_code_label *done_label, *do_error;
891 rtx target = NULL_RTX;
892
893 done_label = gen_label_rtx ();
894 do_error = gen_label_rtx ();
895
896 do_pending_stack_adjust ();
897 op1 = expand_normal (arg1);
898
899 machine_mode mode = TYPE_MODE (TREE_TYPE (arg1));
900 if (lhs)
901 {
902 target = expand_expr (lhs, NULL_RTX, VOIDmode, EXPAND_WRITE);
903 if (!is_ubsan)
904 write_complex_part (target, const0_rtx, true);
905 }
906
907 enum insn_code icode = optab_handler (negv3_optab, mode);
908 if (icode != CODE_FOR_nothing)
909 {
910 struct expand_operand ops[3];
911 rtx_insn *last = get_last_insn ();
912
913 res = gen_reg_rtx (mode);
914 create_output_operand (&ops[0], res, mode);
915 create_input_operand (&ops[1], op1, mode);
916 create_fixed_operand (&ops[2], do_error);
917 if (maybe_expand_insn (icode, 3, ops))
918 {
919 last = get_last_insn ();
920 if (profile_status_for_fn (cfun) != PROFILE_ABSENT
921 && JUMP_P (last)
922 && any_condjump_p (last)
923 && !find_reg_note (last, REG_BR_PROB, 0))
924 add_int_reg_note (last, REG_BR_PROB, PROB_VERY_UNLIKELY);
925 emit_jump (done_label);
926 }
927 else
928 {
929 delete_insns_since (last);
930 icode = CODE_FOR_nothing;
931 }
932 }
933
934 if (icode == CODE_FOR_nothing)
935 {
936 /* Compute the operation. On RTL level, the addition is always
937 unsigned. */
938 res = expand_unop (mode, neg_optab, op1, NULL_RTX, false);
939
940 /* Compare the operand with the most negative value. */
941 rtx minv = expand_normal (TYPE_MIN_VALUE (TREE_TYPE (arg1)));
942 do_compare_rtx_and_jump (op1, minv, NE, true, mode, NULL_RTX, NULL,
943 done_label, PROB_VERY_LIKELY);
944 }
945
946 emit_label (do_error);
947 if (is_ubsan)
948 {
949 /* Expand the ubsan builtin call. */
950 push_temp_slots ();
951 fn = ubsan_build_overflow_builtin (NEGATE_EXPR, loc, TREE_TYPE (arg1),
952 arg1, NULL_TREE);
953 expand_normal (fn);
954 pop_temp_slots ();
955 do_pending_stack_adjust ();
956 }
957 else if (lhs)
958 write_complex_part (target, const1_rtx, true);
959
960 /* We're done. */
961 emit_label (done_label);
962
963 if (lhs)
964 {
965 if (is_ubsan)
966 expand_ubsan_result_store (target, res);
967 else
968 expand_arith_overflow_result_store (lhs, target, mode, res);
969 }
970 }
971
972 /* Add mul overflow checking to the statement STMT. */
973
974 static void
expand_mul_overflow(location_t loc,tree lhs,tree arg0,tree arg1,bool unsr_p,bool uns0_p,bool uns1_p,bool is_ubsan)975 expand_mul_overflow (location_t loc, tree lhs, tree arg0, tree arg1,
976 bool unsr_p, bool uns0_p, bool uns1_p, bool is_ubsan)
977 {
978 rtx res, op0, op1;
979 tree fn, type;
980 rtx_code_label *done_label, *do_error;
981 rtx target = NULL_RTX;
982 signop sign;
983 enum insn_code icode;
984
985 done_label = gen_label_rtx ();
986 do_error = gen_label_rtx ();
987
988 do_pending_stack_adjust ();
989 op0 = expand_normal (arg0);
990 op1 = expand_normal (arg1);
991
992 machine_mode mode = TYPE_MODE (TREE_TYPE (arg0));
993 bool uns = unsr_p;
994 if (lhs)
995 {
996 target = expand_expr (lhs, NULL_RTX, VOIDmode, EXPAND_WRITE);
997 if (!is_ubsan)
998 write_complex_part (target, const0_rtx, true);
999 }
1000
1001 if (is_ubsan)
1002 gcc_assert (!unsr_p && !uns0_p && !uns1_p);
1003
1004 /* We assume both operands and result have the same precision
1005 here (GET_MODE_BITSIZE (mode)), S stands for signed type
1006 with that precision, U for unsigned type with that precision,
1007 sgn for unsigned most significant bit in that precision.
1008 s1 is signed first operand, u1 is unsigned first operand,
1009 s2 is signed second operand, u2 is unsigned second operand,
1010 sr is signed result, ur is unsigned result and the following
1011 rules say how to compute result (which is always result of
1012 the operands as if both were unsigned, cast to the right
1013 signedness) and how to compute whether operation overflowed.
1014 main_ovf (false) stands for jump on signed multiplication
1015 overflow or the main algorithm with uns == false.
1016 main_ovf (true) stands for jump on unsigned multiplication
1017 overflow or the main algorithm with uns == true.
1018
1019 s1 * s2 -> sr
1020 res = (S) ((U) s1 * (U) s2)
1021 ovf = main_ovf (false)
1022 u1 * u2 -> ur
1023 res = u1 * u2
1024 ovf = main_ovf (true)
1025 s1 * u2 -> ur
1026 res = (U) s1 * u2
1027 ovf = (s1 < 0 && u2) || main_ovf (true)
1028 u1 * u2 -> sr
1029 res = (S) (u1 * u2)
1030 ovf = res < 0 || main_ovf (true)
1031 s1 * u2 -> sr
1032 res = (S) ((U) s1 * u2)
1033 ovf = (S) u2 >= 0 ? main_ovf (false)
1034 : (s1 != 0 && (s1 != -1 || u2 != (U) res))
1035 s1 * s2 -> ur
1036 t1 = (s1 & s2) < 0 ? (-(U) s1) : ((U) s1)
1037 t2 = (s1 & s2) < 0 ? (-(U) s2) : ((U) s2)
1038 res = t1 * t2
1039 ovf = (s1 ^ s2) < 0 ? (s1 && s2) : main_ovf (true) */
1040
1041 if (uns0_p && !uns1_p)
1042 {
1043 /* Multiplication is commutative, if operand signedness differs,
1044 canonicalize to the first operand being signed and second
1045 unsigned to simplify following code. */
1046 std::swap (op0, op1);
1047 std::swap (arg0, arg1);
1048 uns0_p = false;
1049 uns1_p = true;
1050 }
1051
1052 int pos_neg0 = get_range_pos_neg (arg0);
1053 int pos_neg1 = get_range_pos_neg (arg1);
1054
1055 /* s1 * u2 -> ur */
1056 if (!uns0_p && uns1_p && unsr_p)
1057 {
1058 switch (pos_neg0)
1059 {
1060 case 1:
1061 /* If s1 is non-negative, just perform normal u1 * u2 -> ur. */
1062 goto do_main;
1063 case 2:
1064 /* If s1 is negative, avoid the main code, just multiply and
1065 signal overflow if op1 is not 0. */
1066 struct separate_ops ops;
1067 ops.code = MULT_EXPR;
1068 ops.type = TREE_TYPE (arg1);
1069 ops.op0 = make_tree (ops.type, op0);
1070 ops.op1 = make_tree (ops.type, op1);
1071 ops.op2 = NULL_TREE;
1072 ops.location = loc;
1073 res = expand_expr_real_2 (&ops, NULL_RTX, mode, EXPAND_NORMAL);
1074 do_compare_rtx_and_jump (op1, const0_rtx, EQ, true, mode, NULL_RTX,
1075 NULL, done_label, PROB_VERY_LIKELY);
1076 goto do_error_label;
1077 case 3:
1078 rtx_code_label *do_main_label;
1079 do_main_label = gen_label_rtx ();
1080 do_compare_rtx_and_jump (op0, const0_rtx, GE, false, mode, NULL_RTX,
1081 NULL, do_main_label, PROB_VERY_LIKELY);
1082 do_compare_rtx_and_jump (op1, const0_rtx, EQ, true, mode, NULL_RTX,
1083 NULL, do_main_label, PROB_VERY_LIKELY);
1084 write_complex_part (target, const1_rtx, true);
1085 emit_label (do_main_label);
1086 goto do_main;
1087 default:
1088 gcc_unreachable ();
1089 }
1090 }
1091
1092 /* u1 * u2 -> sr */
1093 if (uns0_p && uns1_p && !unsr_p)
1094 {
1095 uns = true;
1096 /* Rest of handling of this case after res is computed. */
1097 goto do_main;
1098 }
1099
1100 /* s1 * u2 -> sr */
1101 if (!uns0_p && uns1_p && !unsr_p)
1102 {
1103 switch (pos_neg1)
1104 {
1105 case 1:
1106 goto do_main;
1107 case 2:
1108 /* If (S) u2 is negative (i.e. u2 is larger than maximum of S,
1109 avoid the main code, just multiply and signal overflow
1110 unless 0 * u2 or -1 * ((U) Smin). */
1111 struct separate_ops ops;
1112 ops.code = MULT_EXPR;
1113 ops.type = TREE_TYPE (arg1);
1114 ops.op0 = make_tree (ops.type, op0);
1115 ops.op1 = make_tree (ops.type, op1);
1116 ops.op2 = NULL_TREE;
1117 ops.location = loc;
1118 res = expand_expr_real_2 (&ops, NULL_RTX, mode, EXPAND_NORMAL);
1119 do_compare_rtx_and_jump (op0, const0_rtx, EQ, true, mode, NULL_RTX,
1120 NULL, done_label, PROB_VERY_LIKELY);
1121 do_compare_rtx_and_jump (op0, constm1_rtx, NE, true, mode, NULL_RTX,
1122 NULL, do_error, PROB_VERY_UNLIKELY);
1123 int prec;
1124 prec = GET_MODE_PRECISION (mode);
1125 rtx sgn;
1126 sgn = immed_wide_int_const (wi::min_value (prec, SIGNED), mode);
1127 do_compare_rtx_and_jump (op1, sgn, EQ, true, mode, NULL_RTX,
1128 NULL, done_label, PROB_VERY_LIKELY);
1129 goto do_error_label;
1130 case 3:
1131 /* Rest of handling of this case after res is computed. */
1132 goto do_main;
1133 default:
1134 gcc_unreachable ();
1135 }
1136 }
1137
1138 /* s1 * s2 -> ur */
1139 if (!uns0_p && !uns1_p && unsr_p)
1140 {
1141 rtx tem, tem2;
1142 switch (pos_neg0 | pos_neg1)
1143 {
1144 case 1: /* Both operands known to be non-negative. */
1145 goto do_main;
1146 case 2: /* Both operands known to be negative. */
1147 op0 = expand_unop (mode, neg_optab, op0, NULL_RTX, false);
1148 op1 = expand_unop (mode, neg_optab, op1, NULL_RTX, false);
1149 /* Avoid looking at arg0/arg1 ranges, as we've changed
1150 the arguments. */
1151 arg0 = error_mark_node;
1152 arg1 = error_mark_node;
1153 goto do_main;
1154 case 3:
1155 if ((pos_neg0 ^ pos_neg1) == 3)
1156 {
1157 /* If one operand is known to be negative and the other
1158 non-negative, this overflows always, unless the non-negative
1159 one is 0. Just do normal multiply and set overflow
1160 unless one of the operands is 0. */
1161 struct separate_ops ops;
1162 ops.code = MULT_EXPR;
1163 ops.type
1164 = build_nonstandard_integer_type (GET_MODE_PRECISION (mode),
1165 1);
1166 ops.op0 = make_tree (ops.type, op0);
1167 ops.op1 = make_tree (ops.type, op1);
1168 ops.op2 = NULL_TREE;
1169 ops.location = loc;
1170 res = expand_expr_real_2 (&ops, NULL_RTX, mode, EXPAND_NORMAL);
1171 tem = expand_binop (mode, and_optab, op0, op1, NULL_RTX, false,
1172 OPTAB_LIB_WIDEN);
1173 do_compare_rtx_and_jump (tem, const0_rtx, EQ, true, mode,
1174 NULL_RTX, NULL, done_label,
1175 PROB_VERY_LIKELY);
1176 goto do_error_label;
1177 }
1178 /* The general case, do all the needed comparisons at runtime. */
1179 rtx_code_label *do_main_label, *after_negate_label;
1180 rtx rop0, rop1;
1181 rop0 = gen_reg_rtx (mode);
1182 rop1 = gen_reg_rtx (mode);
1183 emit_move_insn (rop0, op0);
1184 emit_move_insn (rop1, op1);
1185 op0 = rop0;
1186 op1 = rop1;
1187 do_main_label = gen_label_rtx ();
1188 after_negate_label = gen_label_rtx ();
1189 tem = expand_binop (mode, and_optab, op0, op1, NULL_RTX, false,
1190 OPTAB_LIB_WIDEN);
1191 do_compare_rtx_and_jump (tem, const0_rtx, GE, false, mode, NULL_RTX,
1192 NULL, after_negate_label, PROB_VERY_LIKELY);
1193 /* Both arguments negative here, negate them and continue with
1194 normal unsigned overflow checking multiplication. */
1195 emit_move_insn (op0, expand_unop (mode, neg_optab, op0,
1196 NULL_RTX, false));
1197 emit_move_insn (op1, expand_unop (mode, neg_optab, op1,
1198 NULL_RTX, false));
1199 /* Avoid looking at arg0/arg1 ranges, as we might have changed
1200 the arguments. */
1201 arg0 = error_mark_node;
1202 arg1 = error_mark_node;
1203 emit_jump (do_main_label);
1204 emit_label (after_negate_label);
1205 tem2 = expand_binop (mode, xor_optab, op0, op1, NULL_RTX, false,
1206 OPTAB_LIB_WIDEN);
1207 do_compare_rtx_and_jump (tem2, const0_rtx, GE, false, mode, NULL_RTX,
1208 NULL, do_main_label, PROB_VERY_LIKELY);
1209 /* One argument is negative here, the other positive. This
1210 overflows always, unless one of the arguments is 0. But
1211 if e.g. s2 is 0, (U) s1 * 0 doesn't overflow, whatever s1
1212 is, thus we can keep do_main code oring in overflow as is. */
1213 do_compare_rtx_and_jump (tem, const0_rtx, EQ, true, mode, NULL_RTX,
1214 NULL, do_main_label, PROB_VERY_LIKELY);
1215 write_complex_part (target, const1_rtx, true);
1216 emit_label (do_main_label);
1217 goto do_main;
1218 default:
1219 gcc_unreachable ();
1220 }
1221 }
1222
1223 do_main:
1224 type = build_nonstandard_integer_type (GET_MODE_PRECISION (mode), uns);
1225 sign = uns ? UNSIGNED : SIGNED;
1226 icode = optab_handler (uns ? umulv4_optab : mulv4_optab, mode);
1227 if (icode != CODE_FOR_nothing)
1228 {
1229 struct expand_operand ops[4];
1230 rtx_insn *last = get_last_insn ();
1231
1232 res = gen_reg_rtx (mode);
1233 create_output_operand (&ops[0], res, mode);
1234 create_input_operand (&ops[1], op0, mode);
1235 create_input_operand (&ops[2], op1, mode);
1236 create_fixed_operand (&ops[3], do_error);
1237 if (maybe_expand_insn (icode, 4, ops))
1238 {
1239 last = get_last_insn ();
1240 if (profile_status_for_fn (cfun) != PROFILE_ABSENT
1241 && JUMP_P (last)
1242 && any_condjump_p (last)
1243 && !find_reg_note (last, REG_BR_PROB, 0))
1244 add_int_reg_note (last, REG_BR_PROB, PROB_VERY_UNLIKELY);
1245 emit_jump (done_label);
1246 }
1247 else
1248 {
1249 delete_insns_since (last);
1250 icode = CODE_FOR_nothing;
1251 }
1252 }
1253
1254 if (icode == CODE_FOR_nothing)
1255 {
1256 struct separate_ops ops;
1257 int prec = GET_MODE_PRECISION (mode);
1258 machine_mode hmode = mode_for_size (prec / 2, MODE_INT, 1);
1259 ops.op0 = make_tree (type, op0);
1260 ops.op1 = make_tree (type, op1);
1261 ops.op2 = NULL_TREE;
1262 ops.location = loc;
1263 if (GET_MODE_2XWIDER_MODE (mode) != VOIDmode
1264 && targetm.scalar_mode_supported_p (GET_MODE_2XWIDER_MODE (mode)))
1265 {
1266 machine_mode wmode = GET_MODE_2XWIDER_MODE (mode);
1267 ops.code = WIDEN_MULT_EXPR;
1268 ops.type
1269 = build_nonstandard_integer_type (GET_MODE_PRECISION (wmode), uns);
1270
1271 res = expand_expr_real_2 (&ops, NULL_RTX, wmode, EXPAND_NORMAL);
1272 rtx hipart = expand_shift (RSHIFT_EXPR, wmode, res, prec,
1273 NULL_RTX, uns);
1274 hipart = convert_modes (mode, wmode, hipart, uns);
1275 res = convert_modes (mode, wmode, res, uns);
1276 if (uns)
1277 /* For the unsigned multiplication, there was overflow if
1278 HIPART is non-zero. */
1279 do_compare_rtx_and_jump (hipart, const0_rtx, EQ, true, mode,
1280 NULL_RTX, NULL, done_label,
1281 PROB_VERY_LIKELY);
1282 else
1283 {
1284 rtx signbit = expand_shift (RSHIFT_EXPR, mode, res, prec - 1,
1285 NULL_RTX, 0);
1286 /* RES is low half of the double width result, HIPART
1287 the high half. There was overflow if
1288 HIPART is different from RES < 0 ? -1 : 0. */
1289 do_compare_rtx_and_jump (signbit, hipart, EQ, true, mode,
1290 NULL_RTX, NULL, done_label,
1291 PROB_VERY_LIKELY);
1292 }
1293 }
1294 else if (hmode != BLKmode && 2 * GET_MODE_PRECISION (hmode) == prec)
1295 {
1296 rtx_code_label *large_op0 = gen_label_rtx ();
1297 rtx_code_label *small_op0_large_op1 = gen_label_rtx ();
1298 rtx_code_label *one_small_one_large = gen_label_rtx ();
1299 rtx_code_label *both_ops_large = gen_label_rtx ();
1300 rtx_code_label *after_hipart_neg = uns ? NULL : gen_label_rtx ();
1301 rtx_code_label *after_lopart_neg = uns ? NULL : gen_label_rtx ();
1302 rtx_code_label *do_overflow = gen_label_rtx ();
1303 rtx_code_label *hipart_different = uns ? NULL : gen_label_rtx ();
1304
1305 unsigned int hprec = GET_MODE_PRECISION (hmode);
1306 rtx hipart0 = expand_shift (RSHIFT_EXPR, mode, op0, hprec,
1307 NULL_RTX, uns);
1308 hipart0 = convert_modes (hmode, mode, hipart0, uns);
1309 rtx lopart0 = convert_modes (hmode, mode, op0, uns);
1310 rtx signbit0 = const0_rtx;
1311 if (!uns)
1312 signbit0 = expand_shift (RSHIFT_EXPR, hmode, lopart0, hprec - 1,
1313 NULL_RTX, 0);
1314 rtx hipart1 = expand_shift (RSHIFT_EXPR, mode, op1, hprec,
1315 NULL_RTX, uns);
1316 hipart1 = convert_modes (hmode, mode, hipart1, uns);
1317 rtx lopart1 = convert_modes (hmode, mode, op1, uns);
1318 rtx signbit1 = const0_rtx;
1319 if (!uns)
1320 signbit1 = expand_shift (RSHIFT_EXPR, hmode, lopart1, hprec - 1,
1321 NULL_RTX, 0);
1322
1323 res = gen_reg_rtx (mode);
1324
1325 /* True if op0 resp. op1 are known to be in the range of
1326 halfstype. */
1327 bool op0_small_p = false;
1328 bool op1_small_p = false;
1329 /* True if op0 resp. op1 are known to have all zeros or all ones
1330 in the upper half of bits, but are not known to be
1331 op{0,1}_small_p. */
1332 bool op0_medium_p = false;
1333 bool op1_medium_p = false;
1334 /* -1 if op{0,1} is known to be negative, 0 if it is known to be
1335 nonnegative, 1 if unknown. */
1336 int op0_sign = 1;
1337 int op1_sign = 1;
1338
1339 if (pos_neg0 == 1)
1340 op0_sign = 0;
1341 else if (pos_neg0 == 2)
1342 op0_sign = -1;
1343 if (pos_neg1 == 1)
1344 op1_sign = 0;
1345 else if (pos_neg1 == 2)
1346 op1_sign = -1;
1347
1348 unsigned int mprec0 = prec;
1349 if (arg0 != error_mark_node)
1350 mprec0 = get_min_precision (arg0, sign);
1351 if (mprec0 <= hprec)
1352 op0_small_p = true;
1353 else if (!uns && mprec0 <= hprec + 1)
1354 op0_medium_p = true;
1355 unsigned int mprec1 = prec;
1356 if (arg1 != error_mark_node)
1357 mprec1 = get_min_precision (arg1, sign);
1358 if (mprec1 <= hprec)
1359 op1_small_p = true;
1360 else if (!uns && mprec1 <= hprec + 1)
1361 op1_medium_p = true;
1362
1363 int smaller_sign = 1;
1364 int larger_sign = 1;
1365 if (op0_small_p)
1366 {
1367 smaller_sign = op0_sign;
1368 larger_sign = op1_sign;
1369 }
1370 else if (op1_small_p)
1371 {
1372 smaller_sign = op1_sign;
1373 larger_sign = op0_sign;
1374 }
1375 else if (op0_sign == op1_sign)
1376 {
1377 smaller_sign = op0_sign;
1378 larger_sign = op0_sign;
1379 }
1380
1381 if (!op0_small_p)
1382 do_compare_rtx_and_jump (signbit0, hipart0, NE, true, hmode,
1383 NULL_RTX, NULL, large_op0,
1384 PROB_UNLIKELY);
1385
1386 if (!op1_small_p)
1387 do_compare_rtx_and_jump (signbit1, hipart1, NE, true, hmode,
1388 NULL_RTX, NULL, small_op0_large_op1,
1389 PROB_UNLIKELY);
1390
1391 /* If both op0 and op1 are sign (!uns) or zero (uns) extended from
1392 hmode to mode, the multiplication will never overflow. We can
1393 do just one hmode x hmode => mode widening multiplication. */
1394 rtx lopart0s = lopart0, lopart1s = lopart1;
1395 if (GET_CODE (lopart0) == SUBREG)
1396 {
1397 lopart0s = shallow_copy_rtx (lopart0);
1398 SUBREG_PROMOTED_VAR_P (lopart0s) = 1;
1399 SUBREG_PROMOTED_SET (lopart0s, uns ? SRP_UNSIGNED : SRP_SIGNED);
1400 }
1401 if (GET_CODE (lopart1) == SUBREG)
1402 {
1403 lopart1s = shallow_copy_rtx (lopart1);
1404 SUBREG_PROMOTED_VAR_P (lopart1s) = 1;
1405 SUBREG_PROMOTED_SET (lopart1s, uns ? SRP_UNSIGNED : SRP_SIGNED);
1406 }
1407 tree halfstype = build_nonstandard_integer_type (hprec, uns);
1408 ops.op0 = make_tree (halfstype, lopart0s);
1409 ops.op1 = make_tree (halfstype, lopart1s);
1410 ops.code = WIDEN_MULT_EXPR;
1411 ops.type = type;
1412 rtx thisres
1413 = expand_expr_real_2 (&ops, NULL_RTX, mode, EXPAND_NORMAL);
1414 emit_move_insn (res, thisres);
1415 emit_jump (done_label);
1416
1417 emit_label (small_op0_large_op1);
1418
1419 /* If op0 is sign (!uns) or zero (uns) extended from hmode to mode,
1420 but op1 is not, just swap the arguments and handle it as op1
1421 sign/zero extended, op0 not. */
1422 rtx larger = gen_reg_rtx (mode);
1423 rtx hipart = gen_reg_rtx (hmode);
1424 rtx lopart = gen_reg_rtx (hmode);
1425 emit_move_insn (larger, op1);
1426 emit_move_insn (hipart, hipart1);
1427 emit_move_insn (lopart, lopart0);
1428 emit_jump (one_small_one_large);
1429
1430 emit_label (large_op0);
1431
1432 if (!op1_small_p)
1433 do_compare_rtx_and_jump (signbit1, hipart1, NE, true, hmode,
1434 NULL_RTX, NULL, both_ops_large,
1435 PROB_UNLIKELY);
1436
1437 /* If op1 is sign (!uns) or zero (uns) extended from hmode to mode,
1438 but op0 is not, prepare larger, hipart and lopart pseudos and
1439 handle it together with small_op0_large_op1. */
1440 emit_move_insn (larger, op0);
1441 emit_move_insn (hipart, hipart0);
1442 emit_move_insn (lopart, lopart1);
1443
1444 emit_label (one_small_one_large);
1445
1446 /* lopart is the low part of the operand that is sign extended
1447 to mode, larger is the other operand, hipart is the
1448 high part of larger and lopart0 and lopart1 are the low parts
1449 of both operands.
1450 We perform lopart0 * lopart1 and lopart * hipart widening
1451 multiplications. */
1452 tree halfutype = build_nonstandard_integer_type (hprec, 1);
1453 ops.op0 = make_tree (halfutype, lopart0);
1454 ops.op1 = make_tree (halfutype, lopart1);
1455 rtx lo0xlo1
1456 = expand_expr_real_2 (&ops, NULL_RTX, mode, EXPAND_NORMAL);
1457
1458 ops.op0 = make_tree (halfutype, lopart);
1459 ops.op1 = make_tree (halfutype, hipart);
1460 rtx loxhi = gen_reg_rtx (mode);
1461 rtx tem = expand_expr_real_2 (&ops, NULL_RTX, mode, EXPAND_NORMAL);
1462 emit_move_insn (loxhi, tem);
1463
1464 if (!uns)
1465 {
1466 /* if (hipart < 0) loxhi -= lopart << (bitsize / 2); */
1467 if (larger_sign == 0)
1468 emit_jump (after_hipart_neg);
1469 else if (larger_sign != -1)
1470 do_compare_rtx_and_jump (hipart, const0_rtx, GE, false, hmode,
1471 NULL_RTX, NULL, after_hipart_neg,
1472 PROB_EVEN);
1473
1474 tem = convert_modes (mode, hmode, lopart, 1);
1475 tem = expand_shift (LSHIFT_EXPR, mode, tem, hprec, NULL_RTX, 1);
1476 tem = expand_simple_binop (mode, MINUS, loxhi, tem, NULL_RTX,
1477 1, OPTAB_DIRECT);
1478 emit_move_insn (loxhi, tem);
1479
1480 emit_label (after_hipart_neg);
1481
1482 /* if (lopart < 0) loxhi -= larger; */
1483 if (smaller_sign == 0)
1484 emit_jump (after_lopart_neg);
1485 else if (smaller_sign != -1)
1486 do_compare_rtx_and_jump (lopart, const0_rtx, GE, false, hmode,
1487 NULL_RTX, NULL, after_lopart_neg,
1488 PROB_EVEN);
1489
1490 tem = expand_simple_binop (mode, MINUS, loxhi, larger, NULL_RTX,
1491 1, OPTAB_DIRECT);
1492 emit_move_insn (loxhi, tem);
1493
1494 emit_label (after_lopart_neg);
1495 }
1496
1497 /* loxhi += (uns) lo0xlo1 >> (bitsize / 2); */
1498 tem = expand_shift (RSHIFT_EXPR, mode, lo0xlo1, hprec, NULL_RTX, 1);
1499 tem = expand_simple_binop (mode, PLUS, loxhi, tem, NULL_RTX,
1500 1, OPTAB_DIRECT);
1501 emit_move_insn (loxhi, tem);
1502
1503 /* if (loxhi >> (bitsize / 2)
1504 == (hmode) loxhi >> (bitsize / 2 - 1)) (if !uns)
1505 if (loxhi >> (bitsize / 2) == 0 (if uns). */
1506 rtx hipartloxhi = expand_shift (RSHIFT_EXPR, mode, loxhi, hprec,
1507 NULL_RTX, 0);
1508 hipartloxhi = convert_modes (hmode, mode, hipartloxhi, 0);
1509 rtx signbitloxhi = const0_rtx;
1510 if (!uns)
1511 signbitloxhi = expand_shift (RSHIFT_EXPR, hmode,
1512 convert_modes (hmode, mode,
1513 loxhi, 0),
1514 hprec - 1, NULL_RTX, 0);
1515
1516 do_compare_rtx_and_jump (signbitloxhi, hipartloxhi, NE, true, hmode,
1517 NULL_RTX, NULL, do_overflow,
1518 PROB_VERY_UNLIKELY);
1519
1520 /* res = (loxhi << (bitsize / 2)) | (hmode) lo0xlo1; */
1521 rtx loxhishifted = expand_shift (LSHIFT_EXPR, mode, loxhi, hprec,
1522 NULL_RTX, 1);
1523 tem = convert_modes (mode, hmode,
1524 convert_modes (hmode, mode, lo0xlo1, 1), 1);
1525
1526 tem = expand_simple_binop (mode, IOR, loxhishifted, tem, res,
1527 1, OPTAB_DIRECT);
1528 if (tem != res)
1529 emit_move_insn (res, tem);
1530 emit_jump (done_label);
1531
1532 emit_label (both_ops_large);
1533
1534 /* If both operands are large (not sign (!uns) or zero (uns)
1535 extended from hmode), then perform the full multiplication
1536 which will be the result of the operation.
1537 The only cases which don't overflow are for signed multiplication
1538 some cases where both hipart0 and highpart1 are 0 or -1.
1539 For unsigned multiplication when high parts are both non-zero
1540 this overflows always. */
1541 ops.code = MULT_EXPR;
1542 ops.op0 = make_tree (type, op0);
1543 ops.op1 = make_tree (type, op1);
1544 tem = expand_expr_real_2 (&ops, NULL_RTX, mode, EXPAND_NORMAL);
1545 emit_move_insn (res, tem);
1546
1547 if (!uns)
1548 {
1549 if (!op0_medium_p)
1550 {
1551 tem = expand_simple_binop (hmode, PLUS, hipart0, const1_rtx,
1552 NULL_RTX, 1, OPTAB_DIRECT);
1553 do_compare_rtx_and_jump (tem, const1_rtx, GTU, true, hmode,
1554 NULL_RTX, NULL, do_error,
1555 PROB_VERY_UNLIKELY);
1556 }
1557
1558 if (!op1_medium_p)
1559 {
1560 tem = expand_simple_binop (hmode, PLUS, hipart1, const1_rtx,
1561 NULL_RTX, 1, OPTAB_DIRECT);
1562 do_compare_rtx_and_jump (tem, const1_rtx, GTU, true, hmode,
1563 NULL_RTX, NULL, do_error,
1564 PROB_VERY_UNLIKELY);
1565 }
1566
1567 /* At this point hipart{0,1} are both in [-1, 0]. If they are
1568 the same, overflow happened if res is non-positive, if they
1569 are different, overflow happened if res is positive. */
1570 if (op0_sign != 1 && op1_sign != 1 && op0_sign != op1_sign)
1571 emit_jump (hipart_different);
1572 else if (op0_sign == 1 || op1_sign == 1)
1573 do_compare_rtx_and_jump (hipart0, hipart1, NE, true, hmode,
1574 NULL_RTX, NULL, hipart_different,
1575 PROB_EVEN);
1576
1577 do_compare_rtx_and_jump (res, const0_rtx, LE, false, mode,
1578 NULL_RTX, NULL, do_error,
1579 PROB_VERY_UNLIKELY);
1580 emit_jump (done_label);
1581
1582 emit_label (hipart_different);
1583
1584 do_compare_rtx_and_jump (res, const0_rtx, GE, false, mode,
1585 NULL_RTX, NULL, do_error,
1586 PROB_VERY_UNLIKELY);
1587 emit_jump (done_label);
1588 }
1589
1590 emit_label (do_overflow);
1591
1592 /* Overflow, do full multiplication and fallthru into do_error. */
1593 ops.op0 = make_tree (type, op0);
1594 ops.op1 = make_tree (type, op1);
1595 tem = expand_expr_real_2 (&ops, NULL_RTX, mode, EXPAND_NORMAL);
1596 emit_move_insn (res, tem);
1597 }
1598 else
1599 {
1600 gcc_assert (!is_ubsan);
1601 ops.code = MULT_EXPR;
1602 ops.type = type;
1603 res = expand_expr_real_2 (&ops, NULL_RTX, mode, EXPAND_NORMAL);
1604 emit_jump (done_label);
1605 }
1606 }
1607
1608 do_error_label:
1609 emit_label (do_error);
1610 if (is_ubsan)
1611 {
1612 /* Expand the ubsan builtin call. */
1613 push_temp_slots ();
1614 fn = ubsan_build_overflow_builtin (MULT_EXPR, loc, TREE_TYPE (arg0),
1615 arg0, arg1);
1616 expand_normal (fn);
1617 pop_temp_slots ();
1618 do_pending_stack_adjust ();
1619 }
1620 else if (lhs)
1621 write_complex_part (target, const1_rtx, true);
1622
1623 /* We're done. */
1624 emit_label (done_label);
1625
1626 /* u1 * u2 -> sr */
1627 if (uns0_p && uns1_p && !unsr_p)
1628 {
1629 rtx_code_label *all_done_label = gen_label_rtx ();
1630 do_compare_rtx_and_jump (res, const0_rtx, GE, false, mode, NULL_RTX,
1631 NULL, all_done_label, PROB_VERY_LIKELY);
1632 write_complex_part (target, const1_rtx, true);
1633 emit_label (all_done_label);
1634 }
1635
1636 /* s1 * u2 -> sr */
1637 if (!uns0_p && uns1_p && !unsr_p && pos_neg1 == 3)
1638 {
1639 rtx_code_label *all_done_label = gen_label_rtx ();
1640 rtx_code_label *set_noovf = gen_label_rtx ();
1641 do_compare_rtx_and_jump (op1, const0_rtx, GE, false, mode, NULL_RTX,
1642 NULL, all_done_label, PROB_VERY_LIKELY);
1643 write_complex_part (target, const1_rtx, true);
1644 do_compare_rtx_and_jump (op0, const0_rtx, EQ, true, mode, NULL_RTX,
1645 NULL, set_noovf, PROB_VERY_LIKELY);
1646 do_compare_rtx_and_jump (op0, constm1_rtx, NE, true, mode, NULL_RTX,
1647 NULL, all_done_label, PROB_VERY_UNLIKELY);
1648 do_compare_rtx_and_jump (op1, res, NE, true, mode, NULL_RTX, NULL,
1649 all_done_label, PROB_VERY_UNLIKELY);
1650 emit_label (set_noovf);
1651 write_complex_part (target, const0_rtx, true);
1652 emit_label (all_done_label);
1653 }
1654
1655 if (lhs)
1656 {
1657 if (is_ubsan)
1658 expand_ubsan_result_store (target, res);
1659 else
1660 expand_arith_overflow_result_store (lhs, target, mode, res);
1661 }
1662 }
1663
1664 /* Expand UBSAN_CHECK_ADD call STMT. */
1665
1666 static void
expand_UBSAN_CHECK_ADD(internal_fn,gcall * stmt)1667 expand_UBSAN_CHECK_ADD (internal_fn, gcall *stmt)
1668 {
1669 location_t loc = gimple_location (stmt);
1670 tree lhs = gimple_call_lhs (stmt);
1671 tree arg0 = gimple_call_arg (stmt, 0);
1672 tree arg1 = gimple_call_arg (stmt, 1);
1673 expand_addsub_overflow (loc, PLUS_EXPR, lhs, arg0, arg1,
1674 false, false, false, true);
1675 }
1676
1677 /* Expand UBSAN_CHECK_SUB call STMT. */
1678
1679 static void
expand_UBSAN_CHECK_SUB(internal_fn,gcall * stmt)1680 expand_UBSAN_CHECK_SUB (internal_fn, gcall *stmt)
1681 {
1682 location_t loc = gimple_location (stmt);
1683 tree lhs = gimple_call_lhs (stmt);
1684 tree arg0 = gimple_call_arg (stmt, 0);
1685 tree arg1 = gimple_call_arg (stmt, 1);
1686 if (integer_zerop (arg0))
1687 expand_neg_overflow (loc, lhs, arg1, true);
1688 else
1689 expand_addsub_overflow (loc, MINUS_EXPR, lhs, arg0, arg1,
1690 false, false, false, true);
1691 }
1692
1693 /* Expand UBSAN_CHECK_MUL call STMT. */
1694
1695 static void
expand_UBSAN_CHECK_MUL(internal_fn,gcall * stmt)1696 expand_UBSAN_CHECK_MUL (internal_fn, gcall *stmt)
1697 {
1698 location_t loc = gimple_location (stmt);
1699 tree lhs = gimple_call_lhs (stmt);
1700 tree arg0 = gimple_call_arg (stmt, 0);
1701 tree arg1 = gimple_call_arg (stmt, 1);
1702 expand_mul_overflow (loc, lhs, arg0, arg1, false, false, false, true);
1703 }
1704
1705 /* Helper function for {ADD,SUB,MUL}_OVERFLOW call stmt expansion. */
1706
1707 static void
expand_arith_overflow(enum tree_code code,gimple * stmt)1708 expand_arith_overflow (enum tree_code code, gimple *stmt)
1709 {
1710 tree lhs = gimple_call_lhs (stmt);
1711 if (lhs == NULL_TREE)
1712 return;
1713 tree arg0 = gimple_call_arg (stmt, 0);
1714 tree arg1 = gimple_call_arg (stmt, 1);
1715 tree type = TREE_TYPE (TREE_TYPE (lhs));
1716 int uns0_p = TYPE_UNSIGNED (TREE_TYPE (arg0));
1717 int uns1_p = TYPE_UNSIGNED (TREE_TYPE (arg1));
1718 int unsr_p = TYPE_UNSIGNED (type);
1719 int prec0 = TYPE_PRECISION (TREE_TYPE (arg0));
1720 int prec1 = TYPE_PRECISION (TREE_TYPE (arg1));
1721 int precres = TYPE_PRECISION (type);
1722 location_t loc = gimple_location (stmt);
1723 if (!uns0_p && get_range_pos_neg (arg0) == 1)
1724 uns0_p = true;
1725 if (!uns1_p && get_range_pos_neg (arg1) == 1)
1726 uns1_p = true;
1727 int pr = get_min_precision (arg0, uns0_p ? UNSIGNED : SIGNED);
1728 prec0 = MIN (prec0, pr);
1729 pr = get_min_precision (arg1, uns1_p ? UNSIGNED : SIGNED);
1730 prec1 = MIN (prec1, pr);
1731
1732 /* If uns0_p && uns1_p, precop is minimum needed precision
1733 of unsigned type to hold the exact result, otherwise
1734 precop is minimum needed precision of signed type to
1735 hold the exact result. */
1736 int precop;
1737 if (code == MULT_EXPR)
1738 precop = prec0 + prec1 + (uns0_p != uns1_p);
1739 else
1740 {
1741 if (uns0_p == uns1_p)
1742 precop = MAX (prec0, prec1) + 1;
1743 else if (uns0_p)
1744 precop = MAX (prec0 + 1, prec1) + 1;
1745 else
1746 precop = MAX (prec0, prec1 + 1) + 1;
1747 }
1748 int orig_precres = precres;
1749
1750 do
1751 {
1752 if ((uns0_p && uns1_p)
1753 ? ((precop + !unsr_p) <= precres
1754 /* u1 - u2 -> ur can overflow, no matter what precision
1755 the result has. */
1756 && (code != MINUS_EXPR || !unsr_p))
1757 : (!unsr_p && precop <= precres))
1758 {
1759 /* The infinity precision result will always fit into result. */
1760 rtx target = expand_expr (lhs, NULL_RTX, VOIDmode, EXPAND_WRITE);
1761 write_complex_part (target, const0_rtx, true);
1762 enum machine_mode mode = TYPE_MODE (type);
1763 struct separate_ops ops;
1764 ops.code = code;
1765 ops.type = type;
1766 ops.op0 = fold_convert_loc (loc, type, arg0);
1767 ops.op1 = fold_convert_loc (loc, type, arg1);
1768 ops.op2 = NULL_TREE;
1769 ops.location = loc;
1770 rtx tem = expand_expr_real_2 (&ops, NULL_RTX, mode, EXPAND_NORMAL);
1771 expand_arith_overflow_result_store (lhs, target, mode, tem);
1772 return;
1773 }
1774
1775 /* For sub-word operations, if target doesn't have them, start
1776 with precres widening right away, otherwise do it only
1777 if the most simple cases can't be used. */
1778 if (WORD_REGISTER_OPERATIONS
1779 && orig_precres == precres
1780 && precres < BITS_PER_WORD)
1781 ;
1782 else if ((uns0_p && uns1_p && unsr_p && prec0 <= precres
1783 && prec1 <= precres)
1784 || ((!uns0_p || !uns1_p) && !unsr_p
1785 && prec0 + uns0_p <= precres
1786 && prec1 + uns1_p <= precres))
1787 {
1788 arg0 = fold_convert_loc (loc, type, arg0);
1789 arg1 = fold_convert_loc (loc, type, arg1);
1790 switch (code)
1791 {
1792 case MINUS_EXPR:
1793 if (integer_zerop (arg0) && !unsr_p)
1794 {
1795 expand_neg_overflow (loc, lhs, arg1, false);
1796 return;
1797 }
1798 /* FALLTHRU */
1799 case PLUS_EXPR:
1800 expand_addsub_overflow (loc, code, lhs, arg0, arg1,
1801 unsr_p, unsr_p, unsr_p, false);
1802 return;
1803 case MULT_EXPR:
1804 expand_mul_overflow (loc, lhs, arg0, arg1,
1805 unsr_p, unsr_p, unsr_p, false);
1806 return;
1807 default:
1808 gcc_unreachable ();
1809 }
1810 }
1811
1812 /* For sub-word operations, retry with a wider type first. */
1813 if (orig_precres == precres && precop <= BITS_PER_WORD)
1814 {
1815 #if WORD_REGISTER_OPERATIONS
1816 int p = BITS_PER_WORD;
1817 #else
1818 int p = precop;
1819 #endif
1820 enum machine_mode m = smallest_mode_for_size (p, MODE_INT);
1821 tree optype = build_nonstandard_integer_type (GET_MODE_PRECISION (m),
1822 uns0_p && uns1_p
1823 && unsr_p);
1824 p = TYPE_PRECISION (optype);
1825 if (p > precres)
1826 {
1827 precres = p;
1828 unsr_p = TYPE_UNSIGNED (optype);
1829 type = optype;
1830 continue;
1831 }
1832 }
1833
1834 if (prec0 <= precres && prec1 <= precres)
1835 {
1836 tree types[2];
1837 if (unsr_p)
1838 {
1839 types[0] = build_nonstandard_integer_type (precres, 0);
1840 types[1] = type;
1841 }
1842 else
1843 {
1844 types[0] = type;
1845 types[1] = build_nonstandard_integer_type (precres, 1);
1846 }
1847 arg0 = fold_convert_loc (loc, types[uns0_p], arg0);
1848 arg1 = fold_convert_loc (loc, types[uns1_p], arg1);
1849 if (code != MULT_EXPR)
1850 expand_addsub_overflow (loc, code, lhs, arg0, arg1, unsr_p,
1851 uns0_p, uns1_p, false);
1852 else
1853 expand_mul_overflow (loc, lhs, arg0, arg1, unsr_p,
1854 uns0_p, uns1_p, false);
1855 return;
1856 }
1857
1858 /* Retry with a wider type. */
1859 if (orig_precres == precres)
1860 {
1861 int p = MAX (prec0, prec1);
1862 enum machine_mode m = smallest_mode_for_size (p, MODE_INT);
1863 tree optype = build_nonstandard_integer_type (GET_MODE_PRECISION (m),
1864 uns0_p && uns1_p
1865 && unsr_p);
1866 p = TYPE_PRECISION (optype);
1867 if (p > precres)
1868 {
1869 precres = p;
1870 unsr_p = TYPE_UNSIGNED (optype);
1871 type = optype;
1872 continue;
1873 }
1874 }
1875
1876 gcc_unreachable ();
1877 }
1878 while (1);
1879 }
1880
1881 /* Expand ADD_OVERFLOW STMT. */
1882
1883 static void
expand_ADD_OVERFLOW(internal_fn,gcall * stmt)1884 expand_ADD_OVERFLOW (internal_fn, gcall *stmt)
1885 {
1886 expand_arith_overflow (PLUS_EXPR, stmt);
1887 }
1888
1889 /* Expand SUB_OVERFLOW STMT. */
1890
1891 static void
expand_SUB_OVERFLOW(internal_fn,gcall * stmt)1892 expand_SUB_OVERFLOW (internal_fn, gcall *stmt)
1893 {
1894 expand_arith_overflow (MINUS_EXPR, stmt);
1895 }
1896
1897 /* Expand MUL_OVERFLOW STMT. */
1898
1899 static void
expand_MUL_OVERFLOW(internal_fn,gcall * stmt)1900 expand_MUL_OVERFLOW (internal_fn, gcall *stmt)
1901 {
1902 expand_arith_overflow (MULT_EXPR, stmt);
1903 }
1904
1905 /* This should get folded in tree-vectorizer.c. */
1906
1907 static void
expand_LOOP_VECTORIZED(internal_fn,gcall *)1908 expand_LOOP_VECTORIZED (internal_fn, gcall *)
1909 {
1910 gcc_unreachable ();
1911 }
1912
1913 /* Expand MASK_LOAD call STMT using optab OPTAB. */
1914
1915 static void
expand_mask_load_optab_fn(internal_fn,gcall * stmt,convert_optab optab)1916 expand_mask_load_optab_fn (internal_fn, gcall *stmt, convert_optab optab)
1917 {
1918 struct expand_operand ops[3];
1919 tree type, lhs, rhs, maskt, ptr;
1920 rtx mem, target, mask;
1921 unsigned align;
1922
1923 maskt = gimple_call_arg (stmt, 2);
1924 lhs = gimple_call_lhs (stmt);
1925 if (lhs == NULL_TREE)
1926 return;
1927 type = TREE_TYPE (lhs);
1928 ptr = build_int_cst (TREE_TYPE (gimple_call_arg (stmt, 1)), 0);
1929 align = tree_to_shwi (gimple_call_arg (stmt, 1));
1930 if (TYPE_ALIGN (type) != align)
1931 type = build_aligned_type (type, align);
1932 rhs = fold_build2 (MEM_REF, type, gimple_call_arg (stmt, 0), ptr);
1933
1934 mem = expand_expr (rhs, NULL_RTX, VOIDmode, EXPAND_WRITE);
1935 gcc_assert (MEM_P (mem));
1936 mask = expand_normal (maskt);
1937 target = expand_expr (lhs, NULL_RTX, VOIDmode, EXPAND_WRITE);
1938 create_output_operand (&ops[0], target, TYPE_MODE (type));
1939 create_fixed_operand (&ops[1], mem);
1940 create_input_operand (&ops[2], mask, TYPE_MODE (TREE_TYPE (maskt)));
1941 expand_insn (convert_optab_handler (optab, TYPE_MODE (type),
1942 TYPE_MODE (TREE_TYPE (maskt))),
1943 3, ops);
1944 }
1945
1946 /* Expand MASK_STORE call STMT using optab OPTAB. */
1947
1948 static void
expand_mask_store_optab_fn(internal_fn,gcall * stmt,convert_optab optab)1949 expand_mask_store_optab_fn (internal_fn, gcall *stmt, convert_optab optab)
1950 {
1951 struct expand_operand ops[3];
1952 tree type, lhs, rhs, maskt, ptr;
1953 rtx mem, reg, mask;
1954 unsigned align;
1955
1956 maskt = gimple_call_arg (stmt, 2);
1957 rhs = gimple_call_arg (stmt, 3);
1958 type = TREE_TYPE (rhs);
1959 ptr = build_int_cst (TREE_TYPE (gimple_call_arg (stmt, 1)), 0);
1960 align = tree_to_shwi (gimple_call_arg (stmt, 1));
1961 if (TYPE_ALIGN (type) != align)
1962 type = build_aligned_type (type, align);
1963 lhs = fold_build2 (MEM_REF, type, gimple_call_arg (stmt, 0), ptr);
1964
1965 mem = expand_expr (lhs, NULL_RTX, VOIDmode, EXPAND_WRITE);
1966 gcc_assert (MEM_P (mem));
1967 mask = expand_normal (maskt);
1968 reg = expand_normal (rhs);
1969 create_fixed_operand (&ops[0], mem);
1970 create_input_operand (&ops[1], reg, TYPE_MODE (type));
1971 create_input_operand (&ops[2], mask, TYPE_MODE (TREE_TYPE (maskt)));
1972 expand_insn (convert_optab_handler (optab, TYPE_MODE (type),
1973 TYPE_MODE (TREE_TYPE (maskt))),
1974 3, ops);
1975 }
1976
1977 static void
expand_ABNORMAL_DISPATCHER(internal_fn,gcall *)1978 expand_ABNORMAL_DISPATCHER (internal_fn, gcall *)
1979 {
1980 }
1981
1982 static void
expand_BUILTIN_EXPECT(internal_fn,gcall * stmt)1983 expand_BUILTIN_EXPECT (internal_fn, gcall *stmt)
1984 {
1985 /* When guessing was done, the hints should be already stripped away. */
1986 gcc_assert (!flag_guess_branch_prob || optimize == 0 || seen_error ());
1987
1988 rtx target;
1989 tree lhs = gimple_call_lhs (stmt);
1990 if (lhs)
1991 target = expand_expr (lhs, NULL_RTX, VOIDmode, EXPAND_WRITE);
1992 else
1993 target = const0_rtx;
1994 rtx val = expand_expr (gimple_call_arg (stmt, 0), target, VOIDmode, EXPAND_NORMAL);
1995 if (lhs && val != target)
1996 emit_move_insn (target, val);
1997 }
1998
1999 /* IFN_VA_ARG is supposed to be expanded at pass_stdarg. So this dummy function
2000 should never be called. */
2001
2002 static void
expand_VA_ARG(internal_fn,gcall *)2003 expand_VA_ARG (internal_fn, gcall *)
2004 {
2005 gcc_unreachable ();
2006 }
2007
2008 /* Expand the IFN_UNIQUE function according to its first argument. */
2009
2010 static void
expand_UNIQUE(internal_fn,gcall * stmt)2011 expand_UNIQUE (internal_fn, gcall *stmt)
2012 {
2013 rtx pattern = NULL_RTX;
2014 enum ifn_unique_kind kind
2015 = (enum ifn_unique_kind) TREE_INT_CST_LOW (gimple_call_arg (stmt, 0));
2016
2017 switch (kind)
2018 {
2019 default:
2020 gcc_unreachable ();
2021
2022 case IFN_UNIQUE_UNSPEC:
2023 if (targetm.have_unique ())
2024 pattern = targetm.gen_unique ();
2025 break;
2026
2027 case IFN_UNIQUE_OACC_FORK:
2028 case IFN_UNIQUE_OACC_JOIN:
2029 if (targetm.have_oacc_fork () && targetm.have_oacc_join ())
2030 {
2031 tree lhs = gimple_call_lhs (stmt);
2032 rtx target = const0_rtx;
2033
2034 if (lhs)
2035 target = expand_expr (lhs, NULL_RTX, VOIDmode, EXPAND_WRITE);
2036
2037 rtx data_dep = expand_normal (gimple_call_arg (stmt, 1));
2038 rtx axis = expand_normal (gimple_call_arg (stmt, 2));
2039
2040 if (kind == IFN_UNIQUE_OACC_FORK)
2041 pattern = targetm.gen_oacc_fork (target, data_dep, axis);
2042 else
2043 pattern = targetm.gen_oacc_join (target, data_dep, axis);
2044 }
2045 else
2046 gcc_unreachable ();
2047 break;
2048 }
2049
2050 if (pattern)
2051 emit_insn (pattern);
2052 }
2053
2054 /* The size of an OpenACC compute dimension. */
2055
2056 static void
expand_GOACC_DIM_SIZE(internal_fn,gcall * stmt)2057 expand_GOACC_DIM_SIZE (internal_fn, gcall *stmt)
2058 {
2059 tree lhs = gimple_call_lhs (stmt);
2060
2061 if (!lhs)
2062 return;
2063
2064 rtx target = expand_expr (lhs, NULL_RTX, VOIDmode, EXPAND_WRITE);
2065 if (targetm.have_oacc_dim_size ())
2066 {
2067 rtx dim = expand_expr (gimple_call_arg (stmt, 0), NULL_RTX,
2068 VOIDmode, EXPAND_NORMAL);
2069 emit_insn (targetm.gen_oacc_dim_size (target, dim));
2070 }
2071 else
2072 emit_move_insn (target, GEN_INT (1));
2073 }
2074
2075 /* The position of an OpenACC execution engine along one compute axis. */
2076
2077 static void
expand_GOACC_DIM_POS(internal_fn,gcall * stmt)2078 expand_GOACC_DIM_POS (internal_fn, gcall *stmt)
2079 {
2080 tree lhs = gimple_call_lhs (stmt);
2081
2082 if (!lhs)
2083 return;
2084
2085 rtx target = expand_expr (lhs, NULL_RTX, VOIDmode, EXPAND_WRITE);
2086 if (targetm.have_oacc_dim_pos ())
2087 {
2088 rtx dim = expand_expr (gimple_call_arg (stmt, 0), NULL_RTX,
2089 VOIDmode, EXPAND_NORMAL);
2090 emit_insn (targetm.gen_oacc_dim_pos (target, dim));
2091 }
2092 else
2093 emit_move_insn (target, const0_rtx);
2094 }
2095
2096 /* This is expanded by oacc_device_lower pass. */
2097
2098 static void
expand_GOACC_LOOP(internal_fn,gcall *)2099 expand_GOACC_LOOP (internal_fn, gcall *)
2100 {
2101 gcc_unreachable ();
2102 }
2103
2104 /* This is expanded by oacc_device_lower pass. */
2105
2106 static void
expand_GOACC_REDUCTION(internal_fn,gcall *)2107 expand_GOACC_REDUCTION (internal_fn, gcall *)
2108 {
2109 gcc_unreachable ();
2110 }
2111
2112 /* Set errno to EDOM. */
2113
2114 static void
expand_SET_EDOM(internal_fn,gcall *)2115 expand_SET_EDOM (internal_fn, gcall *)
2116 {
2117 #ifdef TARGET_EDOM
2118 #ifdef GEN_ERRNO_RTX
2119 rtx errno_rtx = GEN_ERRNO_RTX;
2120 #else
2121 rtx errno_rtx = gen_rtx_MEM (word_mode, gen_rtx_SYMBOL_REF (Pmode, "errno"));
2122 #endif
2123 emit_move_insn (errno_rtx,
2124 gen_int_mode (TARGET_EDOM, GET_MODE (errno_rtx)));
2125 #else
2126 gcc_unreachable ();
2127 #endif
2128 }
2129
2130 /* Expand a call to FN using the operands in STMT. FN has a single
2131 output operand and NARGS input operands. */
2132
2133 static void
expand_direct_optab_fn(internal_fn fn,gcall * stmt,direct_optab optab,unsigned int nargs)2134 expand_direct_optab_fn (internal_fn fn, gcall *stmt, direct_optab optab,
2135 unsigned int nargs)
2136 {
2137 expand_operand *ops = XALLOCAVEC (expand_operand, nargs + 1);
2138
2139 tree_pair types = direct_internal_fn_types (fn, stmt);
2140 insn_code icode = direct_optab_handler (optab, TYPE_MODE (types.first));
2141
2142 tree lhs = gimple_call_lhs (stmt);
2143 tree lhs_type = TREE_TYPE (lhs);
2144 rtx lhs_rtx = expand_expr (lhs, NULL_RTX, VOIDmode, EXPAND_WRITE);
2145 create_output_operand (&ops[0], lhs_rtx, insn_data[icode].operand[0].mode);
2146
2147 for (unsigned int i = 0; i < nargs; ++i)
2148 {
2149 tree rhs = gimple_call_arg (stmt, i);
2150 tree rhs_type = TREE_TYPE (rhs);
2151 rtx rhs_rtx = expand_normal (rhs);
2152 if (INTEGRAL_TYPE_P (rhs_type))
2153 create_convert_operand_from (&ops[i + 1], rhs_rtx,
2154 TYPE_MODE (rhs_type),
2155 TYPE_UNSIGNED (rhs_type));
2156 else
2157 create_input_operand (&ops[i + 1], rhs_rtx, TYPE_MODE (rhs_type));
2158 }
2159
2160 expand_insn (icode, nargs + 1, ops);
2161 if (!rtx_equal_p (lhs_rtx, ops[0].value))
2162 {
2163 /* If the return value has an integral type, convert the instruction
2164 result to that type. This is useful for things that return an
2165 int regardless of the size of the input. If the instruction result
2166 is smaller than required, assume that it is signed.
2167
2168 If the return value has a nonintegral type, its mode must match
2169 the instruction result. */
2170 if (GET_CODE (lhs_rtx) == SUBREG && SUBREG_PROMOTED_VAR_P (lhs_rtx))
2171 {
2172 /* If this is a scalar in a register that is stored in a wider
2173 mode than the declared mode, compute the result into its
2174 declared mode and then convert to the wider mode. */
2175 gcc_checking_assert (INTEGRAL_TYPE_P (lhs_type));
2176 rtx tmp = convert_to_mode (GET_MODE (lhs_rtx), ops[0].value, 0);
2177 convert_move (SUBREG_REG (lhs_rtx), tmp,
2178 SUBREG_PROMOTED_SIGN (lhs_rtx));
2179 }
2180 else if (GET_MODE (lhs_rtx) == GET_MODE (ops[0].value))
2181 emit_move_insn (lhs_rtx, ops[0].value);
2182 else
2183 {
2184 gcc_checking_assert (INTEGRAL_TYPE_P (lhs_type));
2185 convert_move (lhs_rtx, ops[0].value, 0);
2186 }
2187 }
2188 }
2189
2190 /* Expanders for optabs that can use expand_direct_optab_fn. */
2191
2192 #define expand_unary_optab_fn(FN, STMT, OPTAB) \
2193 expand_direct_optab_fn (FN, STMT, OPTAB, 1)
2194
2195 #define expand_binary_optab_fn(FN, STMT, OPTAB) \
2196 expand_direct_optab_fn (FN, STMT, OPTAB, 2)
2197
2198 /* RETURN_TYPE and ARGS are a return type and argument list that are
2199 in principle compatible with FN (which satisfies direct_internal_fn_p).
2200 Return the types that should be used to determine whether the
2201 target supports FN. */
2202
2203 tree_pair
direct_internal_fn_types(internal_fn fn,tree return_type,tree * args)2204 direct_internal_fn_types (internal_fn fn, tree return_type, tree *args)
2205 {
2206 const direct_internal_fn_info &info = direct_internal_fn (fn);
2207 tree type0 = (info.type0 < 0 ? return_type : TREE_TYPE (args[info.type0]));
2208 tree type1 = (info.type1 < 0 ? return_type : TREE_TYPE (args[info.type1]));
2209 return tree_pair (type0, type1);
2210 }
2211
2212 /* CALL is a call whose return type and arguments are in principle
2213 compatible with FN (which satisfies direct_internal_fn_p). Return the
2214 types that should be used to determine whether the target supports FN. */
2215
2216 tree_pair
direct_internal_fn_types(internal_fn fn,gcall * call)2217 direct_internal_fn_types (internal_fn fn, gcall *call)
2218 {
2219 const direct_internal_fn_info &info = direct_internal_fn (fn);
2220 tree op0 = (info.type0 < 0
2221 ? gimple_call_lhs (call)
2222 : gimple_call_arg (call, info.type0));
2223 tree op1 = (info.type1 < 0
2224 ? gimple_call_lhs (call)
2225 : gimple_call_arg (call, info.type1));
2226 return tree_pair (TREE_TYPE (op0), TREE_TYPE (op1));
2227 }
2228
2229 /* Return true if OPTAB is supported for TYPES (whose modes should be
2230 the same) when the optimization type is OPT_TYPE. Used for simple
2231 direct optabs. */
2232
2233 static bool
direct_optab_supported_p(direct_optab optab,tree_pair types,optimization_type opt_type)2234 direct_optab_supported_p (direct_optab optab, tree_pair types,
2235 optimization_type opt_type)
2236 {
2237 machine_mode mode = TYPE_MODE (types.first);
2238 gcc_checking_assert (mode == TYPE_MODE (types.second));
2239 return direct_optab_handler (optab, mode, opt_type) != CODE_FOR_nothing;
2240 }
2241
2242 /* Return true if load/store lanes optab OPTAB is supported for
2243 array type TYPES.first when the optimization type is OPT_TYPE. */
2244
2245 static bool
multi_vector_optab_supported_p(convert_optab optab,tree_pair types,optimization_type opt_type)2246 multi_vector_optab_supported_p (convert_optab optab, tree_pair types,
2247 optimization_type opt_type)
2248 {
2249 gcc_assert (TREE_CODE (types.first) == ARRAY_TYPE);
2250 machine_mode imode = TYPE_MODE (types.first);
2251 machine_mode vmode = TYPE_MODE (TREE_TYPE (types.first));
2252 return (convert_optab_handler (optab, imode, vmode, opt_type)
2253 != CODE_FOR_nothing);
2254 }
2255
2256 #define direct_unary_optab_supported_p direct_optab_supported_p
2257 #define direct_binary_optab_supported_p direct_optab_supported_p
2258 #define direct_mask_load_optab_supported_p direct_optab_supported_p
2259 #define direct_load_lanes_optab_supported_p multi_vector_optab_supported_p
2260 #define direct_mask_store_optab_supported_p direct_optab_supported_p
2261 #define direct_store_lanes_optab_supported_p multi_vector_optab_supported_p
2262
2263 /* Return true if FN is supported for the types in TYPES when the
2264 optimization type is OPT_TYPE. The types are those associated with
2265 the "type0" and "type1" fields of FN's direct_internal_fn_info
2266 structure. */
2267
2268 bool
direct_internal_fn_supported_p(internal_fn fn,tree_pair types,optimization_type opt_type)2269 direct_internal_fn_supported_p (internal_fn fn, tree_pair types,
2270 optimization_type opt_type)
2271 {
2272 switch (fn)
2273 {
2274 #define DEF_INTERNAL_FN(CODE, FLAGS, FNSPEC) \
2275 case IFN_##CODE: break;
2276 #define DEF_INTERNAL_OPTAB_FN(CODE, FLAGS, OPTAB, TYPE) \
2277 case IFN_##CODE: \
2278 return direct_##TYPE##_optab_supported_p (OPTAB##_optab, types, \
2279 opt_type);
2280 #include "internal-fn.def"
2281
2282 case IFN_LAST:
2283 break;
2284 }
2285 gcc_unreachable ();
2286 }
2287
2288 /* Return true if FN is supported for type TYPE when the optimization
2289 type is OPT_TYPE. The caller knows that the "type0" and "type1"
2290 fields of FN's direct_internal_fn_info structure are the same. */
2291
2292 bool
direct_internal_fn_supported_p(internal_fn fn,tree type,optimization_type opt_type)2293 direct_internal_fn_supported_p (internal_fn fn, tree type,
2294 optimization_type opt_type)
2295 {
2296 const direct_internal_fn_info &info = direct_internal_fn (fn);
2297 gcc_checking_assert (info.type0 == info.type1);
2298 return direct_internal_fn_supported_p (fn, tree_pair (type, type), opt_type);
2299 }
2300
2301 /* Return true if IFN_SET_EDOM is supported. */
2302
2303 bool
set_edom_supported_p(void)2304 set_edom_supported_p (void)
2305 {
2306 #ifdef TARGET_EDOM
2307 return true;
2308 #else
2309 return false;
2310 #endif
2311 }
2312
2313 #define DEF_INTERNAL_OPTAB_FN(CODE, FLAGS, OPTAB, TYPE) \
2314 static void \
2315 expand_##CODE (internal_fn fn, gcall *stmt) \
2316 { \
2317 expand_##TYPE##_optab_fn (fn, stmt, OPTAB##_optab); \
2318 }
2319 #include "internal-fn.def"
2320
2321 /* Routines to expand each internal function, indexed by function number.
2322 Each routine has the prototype:
2323
2324 expand_<NAME> (gcall *stmt)
2325
2326 where STMT is the statement that performs the call. */
2327 static void (*const internal_fn_expanders[]) (internal_fn, gcall *) = {
2328 #define DEF_INTERNAL_FN(CODE, FLAGS, FNSPEC) expand_##CODE,
2329 #include "internal-fn.def"
2330 0
2331 };
2332
2333 /* Expand STMT as though it were a call to internal function FN. */
2334
2335 void
expand_internal_call(internal_fn fn,gcall * stmt)2336 expand_internal_call (internal_fn fn, gcall *stmt)
2337 {
2338 internal_fn_expanders[fn] (fn, stmt);
2339 }
2340
2341 /* Expand STMT, which is a call to internal function FN. */
2342
2343 void
expand_internal_call(gcall * stmt)2344 expand_internal_call (gcall *stmt)
2345 {
2346 expand_internal_call (gimple_call_internal_fn (stmt), stmt);
2347 }
2348