1 /* Internal functions.
2 Copyright (C) 2011-2019 Free Software Foundation, Inc.
3
4 This file is part of GCC.
5
6 GCC is free software; you can redistribute it and/or modify it under
7 the terms of the GNU General Public License as published by the Free
8 Software Foundation; either version 3, or (at your option) any later
9 version.
10
11 GCC is distributed in the hope that it will be useful, but WITHOUT ANY
12 WARRANTY; without even the implied warranty of MERCHANTABILITY or
13 FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
14 for more details.
15
16 You should have received a copy of the GNU General Public License
17 along with GCC; see the file COPYING3. If not see
18 <http://www.gnu.org/licenses/>. */
19
20 #include "config.h"
21 #include "system.h"
22 #include "coretypes.h"
23 #include "backend.h"
24 #include "target.h"
25 #include "rtl.h"
26 #include "tree.h"
27 #include "gimple.h"
28 #include "predict.h"
29 #include "stringpool.h"
30 #include "tree-vrp.h"
31 #include "tree-ssanames.h"
32 #include "expmed.h"
33 #include "memmodel.h"
34 #include "optabs.h"
35 #include "emit-rtl.h"
36 #include "diagnostic-core.h"
37 #include "fold-const.h"
38 #include "internal-fn.h"
39 #include "stor-layout.h"
40 #include "dojump.h"
41 #include "expr.h"
42 #include "stringpool.h"
43 #include "attribs.h"
44 #include "asan.h"
45 #include "ubsan.h"
46 #include "recog.h"
47 #include "builtins.h"
48 #include "optabs-tree.h"
49 #include "gimple-ssa.h"
50 #include "tree-phinodes.h"
51 #include "ssa-iterators.h"
52
53 /* The names of each internal function, indexed by function number. */
54 const char *const internal_fn_name_array[] = {
55 #define DEF_INTERNAL_FN(CODE, FLAGS, FNSPEC) #CODE,
56 #include "internal-fn.def"
57 "<invalid-fn>"
58 };
59
60 /* The ECF_* flags of each internal function, indexed by function number. */
61 const int internal_fn_flags_array[] = {
62 #define DEF_INTERNAL_FN(CODE, FLAGS, FNSPEC) FLAGS,
63 #include "internal-fn.def"
64 0
65 };
66
67 /* Return the internal function called NAME, or IFN_LAST if there's
68 no such function. */
69
70 internal_fn
lookup_internal_fn(const char * name)71 lookup_internal_fn (const char *name)
72 {
73 typedef hash_map<nofree_string_hash, internal_fn> name_to_fn_map_type;
74 static name_to_fn_map_type *name_to_fn_map;
75
76 if (!name_to_fn_map)
77 {
78 name_to_fn_map = new name_to_fn_map_type (IFN_LAST);
79 for (unsigned int i = 0; i < IFN_LAST; ++i)
80 name_to_fn_map->put (internal_fn_name (internal_fn (i)),
81 internal_fn (i));
82 }
83 internal_fn *entry = name_to_fn_map->get (name);
84 return entry ? *entry : IFN_LAST;
85 }
86
87 /* Fnspec of each internal function, indexed by function number. */
88 const_tree internal_fn_fnspec_array[IFN_LAST + 1];
89
90 void
init_internal_fns()91 init_internal_fns ()
92 {
93 #define DEF_INTERNAL_FN(CODE, FLAGS, FNSPEC) \
94 if (FNSPEC) internal_fn_fnspec_array[IFN_##CODE] = \
95 build_string ((int) sizeof (FNSPEC), FNSPEC ? FNSPEC : "");
96 #include "internal-fn.def"
97 internal_fn_fnspec_array[IFN_LAST] = 0;
98 }
99
100 /* Create static initializers for the information returned by
101 direct_internal_fn. */
102 #define not_direct { -2, -2, false }
103 #define mask_load_direct { -1, 2, false }
104 #define load_lanes_direct { -1, -1, false }
105 #define mask_load_lanes_direct { -1, -1, false }
106 #define gather_load_direct { -1, -1, false }
107 #define mask_store_direct { 3, 2, false }
108 #define store_lanes_direct { 0, 0, false }
109 #define mask_store_lanes_direct { 0, 0, false }
110 #define scatter_store_direct { 3, 3, false }
111 #define unary_direct { 0, 0, true }
112 #define binary_direct { 0, 0, true }
113 #define ternary_direct { 0, 0, true }
114 #define cond_unary_direct { 1, 1, true }
115 #define cond_binary_direct { 1, 1, true }
116 #define cond_ternary_direct { 1, 1, true }
117 #define while_direct { 0, 2, false }
118 #define fold_extract_direct { 2, 2, false }
119 #define fold_left_direct { 1, 1, false }
120
121 const direct_internal_fn_info direct_internal_fn_array[IFN_LAST + 1] = {
122 #define DEF_INTERNAL_FN(CODE, FLAGS, FNSPEC) not_direct,
123 #define DEF_INTERNAL_OPTAB_FN(CODE, FLAGS, OPTAB, TYPE) TYPE##_direct,
124 #define DEF_INTERNAL_SIGNED_OPTAB_FN(CODE, FLAGS, SELECTOR, SIGNED_OPTAB, \
125 UNSIGNED_OPTAB, TYPE) TYPE##_direct,
126 #include "internal-fn.def"
127 not_direct
128 };
129
130 /* ARRAY_TYPE is an array of vector modes. Return the associated insn
131 for load-lanes-style optab OPTAB, or CODE_FOR_nothing if none. */
132
133 static enum insn_code
get_multi_vector_move(tree array_type,convert_optab optab)134 get_multi_vector_move (tree array_type, convert_optab optab)
135 {
136 machine_mode imode;
137 machine_mode vmode;
138
139 gcc_assert (TREE_CODE (array_type) == ARRAY_TYPE);
140 imode = TYPE_MODE (array_type);
141 vmode = TYPE_MODE (TREE_TYPE (array_type));
142
143 return convert_optab_handler (optab, imode, vmode);
144 }
145
146 /* Expand LOAD_LANES call STMT using optab OPTAB. */
147
148 static void
expand_load_lanes_optab_fn(internal_fn,gcall * stmt,convert_optab optab)149 expand_load_lanes_optab_fn (internal_fn, gcall *stmt, convert_optab optab)
150 {
151 struct expand_operand ops[2];
152 tree type, lhs, rhs;
153 rtx target, mem;
154
155 lhs = gimple_call_lhs (stmt);
156 rhs = gimple_call_arg (stmt, 0);
157 type = TREE_TYPE (lhs);
158
159 target = expand_expr (lhs, NULL_RTX, VOIDmode, EXPAND_WRITE);
160 mem = expand_normal (rhs);
161
162 gcc_assert (MEM_P (mem));
163 PUT_MODE (mem, TYPE_MODE (type));
164
165 create_output_operand (&ops[0], target, TYPE_MODE (type));
166 create_fixed_operand (&ops[1], mem);
167 expand_insn (get_multi_vector_move (type, optab), 2, ops);
168 }
169
170 /* Expand STORE_LANES call STMT using optab OPTAB. */
171
172 static void
expand_store_lanes_optab_fn(internal_fn,gcall * stmt,convert_optab optab)173 expand_store_lanes_optab_fn (internal_fn, gcall *stmt, convert_optab optab)
174 {
175 struct expand_operand ops[2];
176 tree type, lhs, rhs;
177 rtx target, reg;
178
179 lhs = gimple_call_lhs (stmt);
180 rhs = gimple_call_arg (stmt, 0);
181 type = TREE_TYPE (rhs);
182
183 target = expand_expr (lhs, NULL_RTX, VOIDmode, EXPAND_WRITE);
184 reg = expand_normal (rhs);
185
186 gcc_assert (MEM_P (target));
187 PUT_MODE (target, TYPE_MODE (type));
188
189 create_fixed_operand (&ops[0], target);
190 create_input_operand (&ops[1], reg, TYPE_MODE (type));
191 expand_insn (get_multi_vector_move (type, optab), 2, ops);
192 }
193
194 static void
expand_ANNOTATE(internal_fn,gcall *)195 expand_ANNOTATE (internal_fn, gcall *)
196 {
197 gcc_unreachable ();
198 }
199
200 /* This should get expanded in omp_device_lower pass. */
201
202 static void
expand_GOMP_USE_SIMT(internal_fn,gcall *)203 expand_GOMP_USE_SIMT (internal_fn, gcall *)
204 {
205 gcc_unreachable ();
206 }
207
208 /* This should get expanded in omp_device_lower pass. */
209
210 static void
expand_GOMP_SIMT_ENTER(internal_fn,gcall *)211 expand_GOMP_SIMT_ENTER (internal_fn, gcall *)
212 {
213 gcc_unreachable ();
214 }
215
216 /* Allocate per-lane storage and begin non-uniform execution region. */
217
218 static void
expand_GOMP_SIMT_ENTER_ALLOC(internal_fn,gcall * stmt)219 expand_GOMP_SIMT_ENTER_ALLOC (internal_fn, gcall *stmt)
220 {
221 rtx target;
222 tree lhs = gimple_call_lhs (stmt);
223 if (lhs)
224 target = expand_expr (lhs, NULL_RTX, VOIDmode, EXPAND_WRITE);
225 else
226 target = gen_reg_rtx (Pmode);
227 rtx size = expand_normal (gimple_call_arg (stmt, 0));
228 rtx align = expand_normal (gimple_call_arg (stmt, 1));
229 struct expand_operand ops[3];
230 create_output_operand (&ops[0], target, Pmode);
231 create_input_operand (&ops[1], size, Pmode);
232 create_input_operand (&ops[2], align, Pmode);
233 gcc_assert (targetm.have_omp_simt_enter ());
234 expand_insn (targetm.code_for_omp_simt_enter, 3, ops);
235 }
236
237 /* Deallocate per-lane storage and leave non-uniform execution region. */
238
239 static void
expand_GOMP_SIMT_EXIT(internal_fn,gcall * stmt)240 expand_GOMP_SIMT_EXIT (internal_fn, gcall *stmt)
241 {
242 gcc_checking_assert (!gimple_call_lhs (stmt));
243 rtx arg = expand_normal (gimple_call_arg (stmt, 0));
244 struct expand_operand ops[1];
245 create_input_operand (&ops[0], arg, Pmode);
246 gcc_assert (targetm.have_omp_simt_exit ());
247 expand_insn (targetm.code_for_omp_simt_exit, 1, ops);
248 }
249
250 /* Lane index on SIMT targets: thread index in the warp on NVPTX. On targets
251 without SIMT execution this should be expanded in omp_device_lower pass. */
252
253 static void
expand_GOMP_SIMT_LANE(internal_fn,gcall * stmt)254 expand_GOMP_SIMT_LANE (internal_fn, gcall *stmt)
255 {
256 tree lhs = gimple_call_lhs (stmt);
257 if (!lhs)
258 return;
259
260 rtx target = expand_expr (lhs, NULL_RTX, VOIDmode, EXPAND_WRITE);
261 gcc_assert (targetm.have_omp_simt_lane ());
262 emit_insn (targetm.gen_omp_simt_lane (target));
263 }
264
265 /* This should get expanded in omp_device_lower pass. */
266
267 static void
expand_GOMP_SIMT_VF(internal_fn,gcall *)268 expand_GOMP_SIMT_VF (internal_fn, gcall *)
269 {
270 gcc_unreachable ();
271 }
272
273 /* Lane index of the first SIMT lane that supplies a non-zero argument.
274 This is a SIMT counterpart to GOMP_SIMD_LAST_LANE, used to represent the
275 lane that executed the last iteration for handling OpenMP lastprivate. */
276
277 static void
expand_GOMP_SIMT_LAST_LANE(internal_fn,gcall * stmt)278 expand_GOMP_SIMT_LAST_LANE (internal_fn, gcall *stmt)
279 {
280 tree lhs = gimple_call_lhs (stmt);
281 if (!lhs)
282 return;
283
284 rtx target = expand_expr (lhs, NULL_RTX, VOIDmode, EXPAND_WRITE);
285 rtx cond = expand_normal (gimple_call_arg (stmt, 0));
286 machine_mode mode = TYPE_MODE (TREE_TYPE (lhs));
287 struct expand_operand ops[2];
288 create_output_operand (&ops[0], target, mode);
289 create_input_operand (&ops[1], cond, mode);
290 gcc_assert (targetm.have_omp_simt_last_lane ());
291 expand_insn (targetm.code_for_omp_simt_last_lane, 2, ops);
292 }
293
294 /* Non-transparent predicate used in SIMT lowering of OpenMP "ordered". */
295
296 static void
expand_GOMP_SIMT_ORDERED_PRED(internal_fn,gcall * stmt)297 expand_GOMP_SIMT_ORDERED_PRED (internal_fn, gcall *stmt)
298 {
299 tree lhs = gimple_call_lhs (stmt);
300 if (!lhs)
301 return;
302
303 rtx target = expand_expr (lhs, NULL_RTX, VOIDmode, EXPAND_WRITE);
304 rtx ctr = expand_normal (gimple_call_arg (stmt, 0));
305 machine_mode mode = TYPE_MODE (TREE_TYPE (lhs));
306 struct expand_operand ops[2];
307 create_output_operand (&ops[0], target, mode);
308 create_input_operand (&ops[1], ctr, mode);
309 gcc_assert (targetm.have_omp_simt_ordered ());
310 expand_insn (targetm.code_for_omp_simt_ordered, 2, ops);
311 }
312
313 /* "Or" boolean reduction across SIMT lanes: return non-zero in all lanes if
314 any lane supplies a non-zero argument. */
315
316 static void
expand_GOMP_SIMT_VOTE_ANY(internal_fn,gcall * stmt)317 expand_GOMP_SIMT_VOTE_ANY (internal_fn, gcall *stmt)
318 {
319 tree lhs = gimple_call_lhs (stmt);
320 if (!lhs)
321 return;
322
323 rtx target = expand_expr (lhs, NULL_RTX, VOIDmode, EXPAND_WRITE);
324 rtx cond = expand_normal (gimple_call_arg (stmt, 0));
325 machine_mode mode = TYPE_MODE (TREE_TYPE (lhs));
326 struct expand_operand ops[2];
327 create_output_operand (&ops[0], target, mode);
328 create_input_operand (&ops[1], cond, mode);
329 gcc_assert (targetm.have_omp_simt_vote_any ());
330 expand_insn (targetm.code_for_omp_simt_vote_any, 2, ops);
331 }
332
333 /* Exchange between SIMT lanes with a "butterfly" pattern: source lane index
334 is destination lane index XOR given offset. */
335
336 static void
expand_GOMP_SIMT_XCHG_BFLY(internal_fn,gcall * stmt)337 expand_GOMP_SIMT_XCHG_BFLY (internal_fn, gcall *stmt)
338 {
339 tree lhs = gimple_call_lhs (stmt);
340 if (!lhs)
341 return;
342
343 rtx target = expand_expr (lhs, NULL_RTX, VOIDmode, EXPAND_WRITE);
344 rtx src = expand_normal (gimple_call_arg (stmt, 0));
345 rtx idx = expand_normal (gimple_call_arg (stmt, 1));
346 machine_mode mode = TYPE_MODE (TREE_TYPE (lhs));
347 struct expand_operand ops[3];
348 create_output_operand (&ops[0], target, mode);
349 create_input_operand (&ops[1], src, mode);
350 create_input_operand (&ops[2], idx, SImode);
351 gcc_assert (targetm.have_omp_simt_xchg_bfly ());
352 expand_insn (targetm.code_for_omp_simt_xchg_bfly, 3, ops);
353 }
354
355 /* Exchange between SIMT lanes according to given source lane index. */
356
357 static void
expand_GOMP_SIMT_XCHG_IDX(internal_fn,gcall * stmt)358 expand_GOMP_SIMT_XCHG_IDX (internal_fn, gcall *stmt)
359 {
360 tree lhs = gimple_call_lhs (stmt);
361 if (!lhs)
362 return;
363
364 rtx target = expand_expr (lhs, NULL_RTX, VOIDmode, EXPAND_WRITE);
365 rtx src = expand_normal (gimple_call_arg (stmt, 0));
366 rtx idx = expand_normal (gimple_call_arg (stmt, 1));
367 machine_mode mode = TYPE_MODE (TREE_TYPE (lhs));
368 struct expand_operand ops[3];
369 create_output_operand (&ops[0], target, mode);
370 create_input_operand (&ops[1], src, mode);
371 create_input_operand (&ops[2], idx, SImode);
372 gcc_assert (targetm.have_omp_simt_xchg_idx ());
373 expand_insn (targetm.code_for_omp_simt_xchg_idx, 3, ops);
374 }
375
376 /* This should get expanded in adjust_simduid_builtins. */
377
378 static void
expand_GOMP_SIMD_LANE(internal_fn,gcall *)379 expand_GOMP_SIMD_LANE (internal_fn, gcall *)
380 {
381 gcc_unreachable ();
382 }
383
384 /* This should get expanded in adjust_simduid_builtins. */
385
386 static void
expand_GOMP_SIMD_VF(internal_fn,gcall *)387 expand_GOMP_SIMD_VF (internal_fn, gcall *)
388 {
389 gcc_unreachable ();
390 }
391
392 /* This should get expanded in adjust_simduid_builtins. */
393
394 static void
expand_GOMP_SIMD_LAST_LANE(internal_fn,gcall *)395 expand_GOMP_SIMD_LAST_LANE (internal_fn, gcall *)
396 {
397 gcc_unreachable ();
398 }
399
400 /* This should get expanded in adjust_simduid_builtins. */
401
402 static void
expand_GOMP_SIMD_ORDERED_START(internal_fn,gcall *)403 expand_GOMP_SIMD_ORDERED_START (internal_fn, gcall *)
404 {
405 gcc_unreachable ();
406 }
407
408 /* This should get expanded in adjust_simduid_builtins. */
409
410 static void
expand_GOMP_SIMD_ORDERED_END(internal_fn,gcall *)411 expand_GOMP_SIMD_ORDERED_END (internal_fn, gcall *)
412 {
413 gcc_unreachable ();
414 }
415
416 /* This should get expanded in the sanopt pass. */
417
418 static void
expand_UBSAN_NULL(internal_fn,gcall *)419 expand_UBSAN_NULL (internal_fn, gcall *)
420 {
421 gcc_unreachable ();
422 }
423
424 /* This should get expanded in the sanopt pass. */
425
426 static void
expand_UBSAN_BOUNDS(internal_fn,gcall *)427 expand_UBSAN_BOUNDS (internal_fn, gcall *)
428 {
429 gcc_unreachable ();
430 }
431
432 /* This should get expanded in the sanopt pass. */
433
434 static void
expand_UBSAN_VPTR(internal_fn,gcall *)435 expand_UBSAN_VPTR (internal_fn, gcall *)
436 {
437 gcc_unreachable ();
438 }
439
440 /* This should get expanded in the sanopt pass. */
441
442 static void
expand_UBSAN_PTR(internal_fn,gcall *)443 expand_UBSAN_PTR (internal_fn, gcall *)
444 {
445 gcc_unreachable ();
446 }
447
448 /* This should get expanded in the sanopt pass. */
449
450 static void
expand_UBSAN_OBJECT_SIZE(internal_fn,gcall *)451 expand_UBSAN_OBJECT_SIZE (internal_fn, gcall *)
452 {
453 gcc_unreachable ();
454 }
455
456 /* This should get expanded in the sanopt pass. */
457
458 static void
expand_ASAN_CHECK(internal_fn,gcall *)459 expand_ASAN_CHECK (internal_fn, gcall *)
460 {
461 gcc_unreachable ();
462 }
463
464 /* This should get expanded in the sanopt pass. */
465
466 static void
expand_ASAN_MARK(internal_fn,gcall *)467 expand_ASAN_MARK (internal_fn, gcall *)
468 {
469 gcc_unreachable ();
470 }
471
472 /* This should get expanded in the sanopt pass. */
473
474 static void
expand_ASAN_POISON(internal_fn,gcall *)475 expand_ASAN_POISON (internal_fn, gcall *)
476 {
477 gcc_unreachable ();
478 }
479
480 /* This should get expanded in the sanopt pass. */
481
482 static void
expand_ASAN_POISON_USE(internal_fn,gcall *)483 expand_ASAN_POISON_USE (internal_fn, gcall *)
484 {
485 gcc_unreachable ();
486 }
487
488 /* This should get expanded in the tsan pass. */
489
490 static void
expand_TSAN_FUNC_EXIT(internal_fn,gcall *)491 expand_TSAN_FUNC_EXIT (internal_fn, gcall *)
492 {
493 gcc_unreachable ();
494 }
495
496 /* This should get expanded in the lower pass. */
497
498 static void
expand_FALLTHROUGH(internal_fn,gcall * call)499 expand_FALLTHROUGH (internal_fn, gcall *call)
500 {
501 error_at (gimple_location (call),
502 "invalid use of attribute %<fallthrough%>");
503 }
504
505 /* Return minimum precision needed to represent all values
506 of ARG in SIGNed integral type. */
507
508 static int
get_min_precision(tree arg,signop sign)509 get_min_precision (tree arg, signop sign)
510 {
511 int prec = TYPE_PRECISION (TREE_TYPE (arg));
512 int cnt = 0;
513 signop orig_sign = sign;
514 if (TREE_CODE (arg) == INTEGER_CST)
515 {
516 int p;
517 if (TYPE_SIGN (TREE_TYPE (arg)) != sign)
518 {
519 widest_int w = wi::to_widest (arg);
520 w = wi::ext (w, prec, sign);
521 p = wi::min_precision (w, sign);
522 }
523 else
524 p = wi::min_precision (wi::to_wide (arg), sign);
525 return MIN (p, prec);
526 }
527 while (CONVERT_EXPR_P (arg)
528 && INTEGRAL_TYPE_P (TREE_TYPE (TREE_OPERAND (arg, 0)))
529 && TYPE_PRECISION (TREE_TYPE (TREE_OPERAND (arg, 0))) <= prec)
530 {
531 arg = TREE_OPERAND (arg, 0);
532 if (TYPE_PRECISION (TREE_TYPE (arg)) < prec)
533 {
534 if (TYPE_UNSIGNED (TREE_TYPE (arg)))
535 sign = UNSIGNED;
536 else if (sign == UNSIGNED && get_range_pos_neg (arg) != 1)
537 return prec + (orig_sign != sign);
538 prec = TYPE_PRECISION (TREE_TYPE (arg));
539 }
540 if (++cnt > 30)
541 return prec + (orig_sign != sign);
542 }
543 if (TREE_CODE (arg) != SSA_NAME)
544 return prec + (orig_sign != sign);
545 wide_int arg_min, arg_max;
546 while (get_range_info (arg, &arg_min, &arg_max) != VR_RANGE)
547 {
548 gimple *g = SSA_NAME_DEF_STMT (arg);
549 if (is_gimple_assign (g)
550 && CONVERT_EXPR_CODE_P (gimple_assign_rhs_code (g)))
551 {
552 tree t = gimple_assign_rhs1 (g);
553 if (INTEGRAL_TYPE_P (TREE_TYPE (t))
554 && TYPE_PRECISION (TREE_TYPE (t)) <= prec)
555 {
556 arg = t;
557 if (TYPE_PRECISION (TREE_TYPE (arg)) < prec)
558 {
559 if (TYPE_UNSIGNED (TREE_TYPE (arg)))
560 sign = UNSIGNED;
561 else if (sign == UNSIGNED && get_range_pos_neg (arg) != 1)
562 return prec + (orig_sign != sign);
563 prec = TYPE_PRECISION (TREE_TYPE (arg));
564 }
565 if (++cnt > 30)
566 return prec + (orig_sign != sign);
567 continue;
568 }
569 }
570 return prec + (orig_sign != sign);
571 }
572 if (sign == TYPE_SIGN (TREE_TYPE (arg)))
573 {
574 int p1 = wi::min_precision (arg_min, sign);
575 int p2 = wi::min_precision (arg_max, sign);
576 p1 = MAX (p1, p2);
577 prec = MIN (prec, p1);
578 }
579 else if (sign == UNSIGNED && !wi::neg_p (arg_min, SIGNED))
580 {
581 int p = wi::min_precision (arg_max, UNSIGNED);
582 prec = MIN (prec, p);
583 }
584 return prec + (orig_sign != sign);
585 }
586
587 /* Helper for expand_*_overflow. Set the __imag__ part to true
588 (1 except for signed:1 type, in which case store -1). */
589
590 static void
expand_arith_set_overflow(tree lhs,rtx target)591 expand_arith_set_overflow (tree lhs, rtx target)
592 {
593 if (TYPE_PRECISION (TREE_TYPE (TREE_TYPE (lhs))) == 1
594 && !TYPE_UNSIGNED (TREE_TYPE (TREE_TYPE (lhs))))
595 write_complex_part (target, constm1_rtx, true);
596 else
597 write_complex_part (target, const1_rtx, true);
598 }
599
600 /* Helper for expand_*_overflow. Store RES into the __real__ part
601 of TARGET. If RES has larger MODE than __real__ part of TARGET,
602 set the __imag__ part to 1 if RES doesn't fit into it. Similarly
603 if LHS has smaller precision than its mode. */
604
605 static void
expand_arith_overflow_result_store(tree lhs,rtx target,scalar_int_mode mode,rtx res)606 expand_arith_overflow_result_store (tree lhs, rtx target,
607 scalar_int_mode mode, rtx res)
608 {
609 scalar_int_mode tgtmode
610 = as_a <scalar_int_mode> (GET_MODE_INNER (GET_MODE (target)));
611 rtx lres = res;
612 if (tgtmode != mode)
613 {
614 rtx_code_label *done_label = gen_label_rtx ();
615 int uns = TYPE_UNSIGNED (TREE_TYPE (TREE_TYPE (lhs)));
616 lres = convert_modes (tgtmode, mode, res, uns);
617 gcc_assert (GET_MODE_PRECISION (tgtmode) < GET_MODE_PRECISION (mode));
618 do_compare_rtx_and_jump (res, convert_modes (mode, tgtmode, lres, uns),
619 EQ, true, mode, NULL_RTX, NULL, done_label,
620 profile_probability::very_likely ());
621 expand_arith_set_overflow (lhs, target);
622 emit_label (done_label);
623 }
624 int prec = TYPE_PRECISION (TREE_TYPE (TREE_TYPE (lhs)));
625 int tgtprec = GET_MODE_PRECISION (tgtmode);
626 if (prec < tgtprec)
627 {
628 rtx_code_label *done_label = gen_label_rtx ();
629 int uns = TYPE_UNSIGNED (TREE_TYPE (TREE_TYPE (lhs)));
630 res = lres;
631 if (uns)
632 {
633 rtx mask
634 = immed_wide_int_const (wi::shifted_mask (0, prec, false, tgtprec),
635 tgtmode);
636 lres = expand_simple_binop (tgtmode, AND, res, mask, NULL_RTX,
637 true, OPTAB_LIB_WIDEN);
638 }
639 else
640 {
641 lres = expand_shift (LSHIFT_EXPR, tgtmode, res, tgtprec - prec,
642 NULL_RTX, 1);
643 lres = expand_shift (RSHIFT_EXPR, tgtmode, lres, tgtprec - prec,
644 NULL_RTX, 0);
645 }
646 do_compare_rtx_and_jump (res, lres,
647 EQ, true, tgtmode, NULL_RTX, NULL, done_label,
648 profile_probability::very_likely ());
649 expand_arith_set_overflow (lhs, target);
650 emit_label (done_label);
651 }
652 write_complex_part (target, lres, false);
653 }
654
655 /* Helper for expand_*_overflow. Store RES into TARGET. */
656
657 static void
expand_ubsan_result_store(rtx target,rtx res)658 expand_ubsan_result_store (rtx target, rtx res)
659 {
660 if (GET_CODE (target) == SUBREG && SUBREG_PROMOTED_VAR_P (target))
661 /* If this is a scalar in a register that is stored in a wider mode
662 than the declared mode, compute the result into its declared mode
663 and then convert to the wider mode. Our value is the computed
664 expression. */
665 convert_move (SUBREG_REG (target), res, SUBREG_PROMOTED_SIGN (target));
666 else
667 emit_move_insn (target, res);
668 }
669
670 /* Add sub/add overflow checking to the statement STMT.
671 CODE says whether the operation is +, or -. */
672
673 static void
expand_addsub_overflow(location_t loc,tree_code code,tree lhs,tree arg0,tree arg1,bool unsr_p,bool uns0_p,bool uns1_p,bool is_ubsan,tree * datap)674 expand_addsub_overflow (location_t loc, tree_code code, tree lhs,
675 tree arg0, tree arg1, bool unsr_p, bool uns0_p,
676 bool uns1_p, bool is_ubsan, tree *datap)
677 {
678 rtx res, target = NULL_RTX;
679 tree fn;
680 rtx_code_label *done_label = gen_label_rtx ();
681 rtx_code_label *do_error = gen_label_rtx ();
682 do_pending_stack_adjust ();
683 rtx op0 = expand_normal (arg0);
684 rtx op1 = expand_normal (arg1);
685 scalar_int_mode mode = SCALAR_INT_TYPE_MODE (TREE_TYPE (arg0));
686 int prec = GET_MODE_PRECISION (mode);
687 rtx sgn = immed_wide_int_const (wi::min_value (prec, SIGNED), mode);
688 bool do_xor = false;
689
690 if (is_ubsan)
691 gcc_assert (!unsr_p && !uns0_p && !uns1_p);
692
693 if (lhs)
694 {
695 target = expand_expr (lhs, NULL_RTX, VOIDmode, EXPAND_WRITE);
696 if (!is_ubsan)
697 write_complex_part (target, const0_rtx, true);
698 }
699
700 /* We assume both operands and result have the same precision
701 here (GET_MODE_BITSIZE (mode)), S stands for signed type
702 with that precision, U for unsigned type with that precision,
703 sgn for unsigned most significant bit in that precision.
704 s1 is signed first operand, u1 is unsigned first operand,
705 s2 is signed second operand, u2 is unsigned second operand,
706 sr is signed result, ur is unsigned result and the following
707 rules say how to compute result (which is always result of
708 the operands as if both were unsigned, cast to the right
709 signedness) and how to compute whether operation overflowed.
710
711 s1 + s2 -> sr
712 res = (S) ((U) s1 + (U) s2)
713 ovf = s2 < 0 ? res > s1 : res < s1 (or jump on overflow)
714 s1 - s2 -> sr
715 res = (S) ((U) s1 - (U) s2)
716 ovf = s2 < 0 ? res < s1 : res > s2 (or jump on overflow)
717 u1 + u2 -> ur
718 res = u1 + u2
719 ovf = res < u1 (or jump on carry, but RTL opts will handle it)
720 u1 - u2 -> ur
721 res = u1 - u2
722 ovf = res > u1 (or jump on carry, but RTL opts will handle it)
723 s1 + u2 -> sr
724 res = (S) ((U) s1 + u2)
725 ovf = ((U) res ^ sgn) < u2
726 s1 + u2 -> ur
727 t1 = (S) (u2 ^ sgn)
728 t2 = s1 + t1
729 res = (U) t2 ^ sgn
730 ovf = t1 < 0 ? t2 > s1 : t2 < s1 (or jump on overflow)
731 s1 - u2 -> sr
732 res = (S) ((U) s1 - u2)
733 ovf = u2 > ((U) s1 ^ sgn)
734 s1 - u2 -> ur
735 res = (U) s1 - u2
736 ovf = s1 < 0 || u2 > (U) s1
737 u1 - s2 -> sr
738 res = u1 - (U) s2
739 ovf = u1 >= ((U) s2 ^ sgn)
740 u1 - s2 -> ur
741 t1 = u1 ^ sgn
742 t2 = t1 - (U) s2
743 res = t2 ^ sgn
744 ovf = s2 < 0 ? (S) t2 < (S) t1 : (S) t2 > (S) t1 (or jump on overflow)
745 s1 + s2 -> ur
746 res = (U) s1 + (U) s2
747 ovf = s2 < 0 ? (s1 | (S) res) < 0) : (s1 & (S) res) < 0)
748 u1 + u2 -> sr
749 res = (S) (u1 + u2)
750 ovf = (U) res < u2 || res < 0
751 u1 - u2 -> sr
752 res = (S) (u1 - u2)
753 ovf = u1 >= u2 ? res < 0 : res >= 0
754 s1 - s2 -> ur
755 res = (U) s1 - (U) s2
756 ovf = s2 >= 0 ? ((s1 | (S) res) < 0) : ((s1 & (S) res) < 0) */
757
758 if (code == PLUS_EXPR && uns0_p && !uns1_p)
759 {
760 /* PLUS_EXPR is commutative, if operand signedness differs,
761 canonicalize to the first operand being signed and second
762 unsigned to simplify following code. */
763 std::swap (op0, op1);
764 std::swap (arg0, arg1);
765 uns0_p = false;
766 uns1_p = true;
767 }
768
769 /* u1 +- u2 -> ur */
770 if (uns0_p && uns1_p && unsr_p)
771 {
772 insn_code icode = optab_handler (code == PLUS_EXPR ? uaddv4_optab
773 : usubv4_optab, mode);
774 if (icode != CODE_FOR_nothing)
775 {
776 struct expand_operand ops[4];
777 rtx_insn *last = get_last_insn ();
778
779 res = gen_reg_rtx (mode);
780 create_output_operand (&ops[0], res, mode);
781 create_input_operand (&ops[1], op0, mode);
782 create_input_operand (&ops[2], op1, mode);
783 create_fixed_operand (&ops[3], do_error);
784 if (maybe_expand_insn (icode, 4, ops))
785 {
786 last = get_last_insn ();
787 if (profile_status_for_fn (cfun) != PROFILE_ABSENT
788 && JUMP_P (last)
789 && any_condjump_p (last)
790 && !find_reg_note (last, REG_BR_PROB, 0))
791 add_reg_br_prob_note (last,
792 profile_probability::very_unlikely ());
793 emit_jump (done_label);
794 goto do_error_label;
795 }
796
797 delete_insns_since (last);
798 }
799
800 /* Compute the operation. On RTL level, the addition is always
801 unsigned. */
802 res = expand_binop (mode, code == PLUS_EXPR ? add_optab : sub_optab,
803 op0, op1, NULL_RTX, false, OPTAB_LIB_WIDEN);
804 rtx tem = op0;
805 /* For PLUS_EXPR, the operation is commutative, so we can pick
806 operand to compare against. For prec <= BITS_PER_WORD, I think
807 preferring REG operand is better over CONST_INT, because
808 the CONST_INT might enlarge the instruction or CSE would need
809 to figure out we'd already loaded it into a register before.
810 For prec > BITS_PER_WORD, I think CONST_INT might be more beneficial,
811 as then the multi-word comparison can be perhaps simplified. */
812 if (code == PLUS_EXPR
813 && (prec <= BITS_PER_WORD
814 ? (CONST_SCALAR_INT_P (op0) && REG_P (op1))
815 : CONST_SCALAR_INT_P (op1)))
816 tem = op1;
817 do_compare_rtx_and_jump (res, tem, code == PLUS_EXPR ? GEU : LEU,
818 true, mode, NULL_RTX, NULL, done_label,
819 profile_probability::very_likely ());
820 goto do_error_label;
821 }
822
823 /* s1 +- u2 -> sr */
824 if (!uns0_p && uns1_p && !unsr_p)
825 {
826 /* Compute the operation. On RTL level, the addition is always
827 unsigned. */
828 res = expand_binop (mode, code == PLUS_EXPR ? add_optab : sub_optab,
829 op0, op1, NULL_RTX, false, OPTAB_LIB_WIDEN);
830 rtx tem = expand_binop (mode, add_optab,
831 code == PLUS_EXPR ? res : op0, sgn,
832 NULL_RTX, false, OPTAB_LIB_WIDEN);
833 do_compare_rtx_and_jump (tem, op1, GEU, true, mode, NULL_RTX, NULL,
834 done_label, profile_probability::very_likely ());
835 goto do_error_label;
836 }
837
838 /* s1 + u2 -> ur */
839 if (code == PLUS_EXPR && !uns0_p && uns1_p && unsr_p)
840 {
841 op1 = expand_binop (mode, add_optab, op1, sgn, NULL_RTX, false,
842 OPTAB_LIB_WIDEN);
843 /* As we've changed op1, we have to avoid using the value range
844 for the original argument. */
845 arg1 = error_mark_node;
846 do_xor = true;
847 goto do_signed;
848 }
849
850 /* u1 - s2 -> ur */
851 if (code == MINUS_EXPR && uns0_p && !uns1_p && unsr_p)
852 {
853 op0 = expand_binop (mode, add_optab, op0, sgn, NULL_RTX, false,
854 OPTAB_LIB_WIDEN);
855 /* As we've changed op0, we have to avoid using the value range
856 for the original argument. */
857 arg0 = error_mark_node;
858 do_xor = true;
859 goto do_signed;
860 }
861
862 /* s1 - u2 -> ur */
863 if (code == MINUS_EXPR && !uns0_p && uns1_p && unsr_p)
864 {
865 /* Compute the operation. On RTL level, the addition is always
866 unsigned. */
867 res = expand_binop (mode, sub_optab, op0, op1, NULL_RTX, false,
868 OPTAB_LIB_WIDEN);
869 int pos_neg = get_range_pos_neg (arg0);
870 if (pos_neg == 2)
871 /* If ARG0 is known to be always negative, this is always overflow. */
872 emit_jump (do_error);
873 else if (pos_neg == 3)
874 /* If ARG0 is not known to be always positive, check at runtime. */
875 do_compare_rtx_and_jump (op0, const0_rtx, LT, false, mode, NULL_RTX,
876 NULL, do_error, profile_probability::very_unlikely ());
877 do_compare_rtx_and_jump (op1, op0, LEU, true, mode, NULL_RTX, NULL,
878 done_label, profile_probability::very_likely ());
879 goto do_error_label;
880 }
881
882 /* u1 - s2 -> sr */
883 if (code == MINUS_EXPR && uns0_p && !uns1_p && !unsr_p)
884 {
885 /* Compute the operation. On RTL level, the addition is always
886 unsigned. */
887 res = expand_binop (mode, sub_optab, op0, op1, NULL_RTX, false,
888 OPTAB_LIB_WIDEN);
889 rtx tem = expand_binop (mode, add_optab, op1, sgn, NULL_RTX, false,
890 OPTAB_LIB_WIDEN);
891 do_compare_rtx_and_jump (op0, tem, LTU, true, mode, NULL_RTX, NULL,
892 done_label, profile_probability::very_likely ());
893 goto do_error_label;
894 }
895
896 /* u1 + u2 -> sr */
897 if (code == PLUS_EXPR && uns0_p && uns1_p && !unsr_p)
898 {
899 /* Compute the operation. On RTL level, the addition is always
900 unsigned. */
901 res = expand_binop (mode, add_optab, op0, op1, NULL_RTX, false,
902 OPTAB_LIB_WIDEN);
903 do_compare_rtx_and_jump (res, const0_rtx, LT, false, mode, NULL_RTX,
904 NULL, do_error, profile_probability::very_unlikely ());
905 rtx tem = op1;
906 /* The operation is commutative, so we can pick operand to compare
907 against. For prec <= BITS_PER_WORD, I think preferring REG operand
908 is better over CONST_INT, because the CONST_INT might enlarge the
909 instruction or CSE would need to figure out we'd already loaded it
910 into a register before. For prec > BITS_PER_WORD, I think CONST_INT
911 might be more beneficial, as then the multi-word comparison can be
912 perhaps simplified. */
913 if (prec <= BITS_PER_WORD
914 ? (CONST_SCALAR_INT_P (op1) && REG_P (op0))
915 : CONST_SCALAR_INT_P (op0))
916 tem = op0;
917 do_compare_rtx_and_jump (res, tem, GEU, true, mode, NULL_RTX, NULL,
918 done_label, profile_probability::very_likely ());
919 goto do_error_label;
920 }
921
922 /* s1 +- s2 -> ur */
923 if (!uns0_p && !uns1_p && unsr_p)
924 {
925 /* Compute the operation. On RTL level, the addition is always
926 unsigned. */
927 res = expand_binop (mode, code == PLUS_EXPR ? add_optab : sub_optab,
928 op0, op1, NULL_RTX, false, OPTAB_LIB_WIDEN);
929 int pos_neg = get_range_pos_neg (arg1);
930 if (code == PLUS_EXPR)
931 {
932 int pos_neg0 = get_range_pos_neg (arg0);
933 if (pos_neg0 != 3 && pos_neg == 3)
934 {
935 std::swap (op0, op1);
936 pos_neg = pos_neg0;
937 }
938 }
939 rtx tem;
940 if (pos_neg != 3)
941 {
942 tem = expand_binop (mode, ((pos_neg == 1) ^ (code == MINUS_EXPR))
943 ? and_optab : ior_optab,
944 op0, res, NULL_RTX, false, OPTAB_LIB_WIDEN);
945 do_compare_rtx_and_jump (tem, const0_rtx, GE, false, mode, NULL,
946 NULL, done_label, profile_probability::very_likely ());
947 }
948 else
949 {
950 rtx_code_label *do_ior_label = gen_label_rtx ();
951 do_compare_rtx_and_jump (op1, const0_rtx,
952 code == MINUS_EXPR ? GE : LT, false, mode,
953 NULL_RTX, NULL, do_ior_label,
954 profile_probability::even ());
955 tem = expand_binop (mode, and_optab, op0, res, NULL_RTX, false,
956 OPTAB_LIB_WIDEN);
957 do_compare_rtx_and_jump (tem, const0_rtx, GE, false, mode, NULL_RTX,
958 NULL, done_label, profile_probability::very_likely ());
959 emit_jump (do_error);
960 emit_label (do_ior_label);
961 tem = expand_binop (mode, ior_optab, op0, res, NULL_RTX, false,
962 OPTAB_LIB_WIDEN);
963 do_compare_rtx_and_jump (tem, const0_rtx, GE, false, mode, NULL_RTX,
964 NULL, done_label, profile_probability::very_likely ());
965 }
966 goto do_error_label;
967 }
968
969 /* u1 - u2 -> sr */
970 if (code == MINUS_EXPR && uns0_p && uns1_p && !unsr_p)
971 {
972 /* Compute the operation. On RTL level, the addition is always
973 unsigned. */
974 res = expand_binop (mode, sub_optab, op0, op1, NULL_RTX, false,
975 OPTAB_LIB_WIDEN);
976 rtx_code_label *op0_geu_op1 = gen_label_rtx ();
977 do_compare_rtx_and_jump (op0, op1, GEU, true, mode, NULL_RTX, NULL,
978 op0_geu_op1, profile_probability::even ());
979 do_compare_rtx_and_jump (res, const0_rtx, LT, false, mode, NULL_RTX,
980 NULL, done_label, profile_probability::very_likely ());
981 emit_jump (do_error);
982 emit_label (op0_geu_op1);
983 do_compare_rtx_and_jump (res, const0_rtx, GE, false, mode, NULL_RTX,
984 NULL, done_label, profile_probability::very_likely ());
985 goto do_error_label;
986 }
987
988 gcc_assert (!uns0_p && !uns1_p && !unsr_p);
989
990 /* s1 +- s2 -> sr */
991 do_signed:
992 {
993 insn_code icode = optab_handler (code == PLUS_EXPR ? addv4_optab
994 : subv4_optab, mode);
995 if (icode != CODE_FOR_nothing)
996 {
997 struct expand_operand ops[4];
998 rtx_insn *last = get_last_insn ();
999
1000 res = gen_reg_rtx (mode);
1001 create_output_operand (&ops[0], res, mode);
1002 create_input_operand (&ops[1], op0, mode);
1003 create_input_operand (&ops[2], op1, mode);
1004 create_fixed_operand (&ops[3], do_error);
1005 if (maybe_expand_insn (icode, 4, ops))
1006 {
1007 last = get_last_insn ();
1008 if (profile_status_for_fn (cfun) != PROFILE_ABSENT
1009 && JUMP_P (last)
1010 && any_condjump_p (last)
1011 && !find_reg_note (last, REG_BR_PROB, 0))
1012 add_reg_br_prob_note (last,
1013 profile_probability::very_unlikely ());
1014 emit_jump (done_label);
1015 goto do_error_label;
1016 }
1017
1018 delete_insns_since (last);
1019 }
1020
1021 /* Compute the operation. On RTL level, the addition is always
1022 unsigned. */
1023 res = expand_binop (mode, code == PLUS_EXPR ? add_optab : sub_optab,
1024 op0, op1, NULL_RTX, false, OPTAB_LIB_WIDEN);
1025
1026 /* If we can prove that one of the arguments (for MINUS_EXPR only
1027 the second operand, as subtraction is not commutative) is always
1028 non-negative or always negative, we can do just one comparison
1029 and conditional jump. */
1030 int pos_neg = get_range_pos_neg (arg1);
1031 if (code == PLUS_EXPR)
1032 {
1033 int pos_neg0 = get_range_pos_neg (arg0);
1034 if (pos_neg0 != 3 && pos_neg == 3)
1035 {
1036 std::swap (op0, op1);
1037 pos_neg = pos_neg0;
1038 }
1039 }
1040
1041 /* Addition overflows if and only if the two operands have the same sign,
1042 and the result has the opposite sign. Subtraction overflows if and
1043 only if the two operands have opposite sign, and the subtrahend has
1044 the same sign as the result. Here 0 is counted as positive. */
1045 if (pos_neg == 3)
1046 {
1047 /* Compute op0 ^ op1 (operands have opposite sign). */
1048 rtx op_xor = expand_binop (mode, xor_optab, op0, op1, NULL_RTX, false,
1049 OPTAB_LIB_WIDEN);
1050
1051 /* Compute res ^ op1 (result and 2nd operand have opposite sign). */
1052 rtx res_xor = expand_binop (mode, xor_optab, res, op1, NULL_RTX, false,
1053 OPTAB_LIB_WIDEN);
1054
1055 rtx tem;
1056 if (code == PLUS_EXPR)
1057 {
1058 /* Compute (res ^ op1) & ~(op0 ^ op1). */
1059 tem = expand_unop (mode, one_cmpl_optab, op_xor, NULL_RTX, false);
1060 tem = expand_binop (mode, and_optab, res_xor, tem, NULL_RTX, false,
1061 OPTAB_LIB_WIDEN);
1062 }
1063 else
1064 {
1065 /* Compute (op0 ^ op1) & ~(res ^ op1). */
1066 tem = expand_unop (mode, one_cmpl_optab, res_xor, NULL_RTX, false);
1067 tem = expand_binop (mode, and_optab, op_xor, tem, NULL_RTX, false,
1068 OPTAB_LIB_WIDEN);
1069 }
1070
1071 /* No overflow if the result has bit sign cleared. */
1072 do_compare_rtx_and_jump (tem, const0_rtx, GE, false, mode, NULL_RTX,
1073 NULL, done_label, profile_probability::very_likely ());
1074 }
1075
1076 /* Compare the result of the operation with the first operand.
1077 No overflow for addition if second operand is positive and result
1078 is larger or second operand is negative and result is smaller.
1079 Likewise for subtraction with sign of second operand flipped. */
1080 else
1081 do_compare_rtx_and_jump (res, op0,
1082 (pos_neg == 1) ^ (code == MINUS_EXPR) ? GE : LE,
1083 false, mode, NULL_RTX, NULL, done_label,
1084 profile_probability::very_likely ());
1085 }
1086
1087 do_error_label:
1088 emit_label (do_error);
1089 if (is_ubsan)
1090 {
1091 /* Expand the ubsan builtin call. */
1092 push_temp_slots ();
1093 fn = ubsan_build_overflow_builtin (code, loc, TREE_TYPE (arg0),
1094 arg0, arg1, datap);
1095 expand_normal (fn);
1096 pop_temp_slots ();
1097 do_pending_stack_adjust ();
1098 }
1099 else if (lhs)
1100 expand_arith_set_overflow (lhs, target);
1101
1102 /* We're done. */
1103 emit_label (done_label);
1104
1105 if (lhs)
1106 {
1107 if (is_ubsan)
1108 expand_ubsan_result_store (target, res);
1109 else
1110 {
1111 if (do_xor)
1112 res = expand_binop (mode, add_optab, res, sgn, NULL_RTX, false,
1113 OPTAB_LIB_WIDEN);
1114
1115 expand_arith_overflow_result_store (lhs, target, mode, res);
1116 }
1117 }
1118 }
1119
1120 /* Add negate overflow checking to the statement STMT. */
1121
1122 static void
expand_neg_overflow(location_t loc,tree lhs,tree arg1,bool is_ubsan,tree * datap)1123 expand_neg_overflow (location_t loc, tree lhs, tree arg1, bool is_ubsan,
1124 tree *datap)
1125 {
1126 rtx res, op1;
1127 tree fn;
1128 rtx_code_label *done_label, *do_error;
1129 rtx target = NULL_RTX;
1130
1131 done_label = gen_label_rtx ();
1132 do_error = gen_label_rtx ();
1133
1134 do_pending_stack_adjust ();
1135 op1 = expand_normal (arg1);
1136
1137 scalar_int_mode mode = SCALAR_INT_TYPE_MODE (TREE_TYPE (arg1));
1138 if (lhs)
1139 {
1140 target = expand_expr (lhs, NULL_RTX, VOIDmode, EXPAND_WRITE);
1141 if (!is_ubsan)
1142 write_complex_part (target, const0_rtx, true);
1143 }
1144
1145 enum insn_code icode = optab_handler (negv3_optab, mode);
1146 if (icode != CODE_FOR_nothing)
1147 {
1148 struct expand_operand ops[3];
1149 rtx_insn *last = get_last_insn ();
1150
1151 res = gen_reg_rtx (mode);
1152 create_output_operand (&ops[0], res, mode);
1153 create_input_operand (&ops[1], op1, mode);
1154 create_fixed_operand (&ops[2], do_error);
1155 if (maybe_expand_insn (icode, 3, ops))
1156 {
1157 last = get_last_insn ();
1158 if (profile_status_for_fn (cfun) != PROFILE_ABSENT
1159 && JUMP_P (last)
1160 && any_condjump_p (last)
1161 && !find_reg_note (last, REG_BR_PROB, 0))
1162 add_reg_br_prob_note (last,
1163 profile_probability::very_unlikely ());
1164 emit_jump (done_label);
1165 }
1166 else
1167 {
1168 delete_insns_since (last);
1169 icode = CODE_FOR_nothing;
1170 }
1171 }
1172
1173 if (icode == CODE_FOR_nothing)
1174 {
1175 /* Compute the operation. On RTL level, the addition is always
1176 unsigned. */
1177 res = expand_unop (mode, neg_optab, op1, NULL_RTX, false);
1178
1179 /* Compare the operand with the most negative value. */
1180 rtx minv = expand_normal (TYPE_MIN_VALUE (TREE_TYPE (arg1)));
1181 do_compare_rtx_and_jump (op1, minv, NE, true, mode, NULL_RTX, NULL,
1182 done_label, profile_probability::very_likely ());
1183 }
1184
1185 emit_label (do_error);
1186 if (is_ubsan)
1187 {
1188 /* Expand the ubsan builtin call. */
1189 push_temp_slots ();
1190 fn = ubsan_build_overflow_builtin (NEGATE_EXPR, loc, TREE_TYPE (arg1),
1191 arg1, NULL_TREE, datap);
1192 expand_normal (fn);
1193 pop_temp_slots ();
1194 do_pending_stack_adjust ();
1195 }
1196 else if (lhs)
1197 expand_arith_set_overflow (lhs, target);
1198
1199 /* We're done. */
1200 emit_label (done_label);
1201
1202 if (lhs)
1203 {
1204 if (is_ubsan)
1205 expand_ubsan_result_store (target, res);
1206 else
1207 expand_arith_overflow_result_store (lhs, target, mode, res);
1208 }
1209 }
1210
1211 /* Return true if UNS WIDEN_MULT_EXPR with result mode WMODE and operand
1212 mode MODE can be expanded without using a libcall. */
1213
1214 static bool
can_widen_mult_without_libcall(scalar_int_mode wmode,scalar_int_mode mode,rtx op0,rtx op1,bool uns)1215 can_widen_mult_without_libcall (scalar_int_mode wmode, scalar_int_mode mode,
1216 rtx op0, rtx op1, bool uns)
1217 {
1218 if (find_widening_optab_handler (umul_widen_optab, wmode, mode)
1219 != CODE_FOR_nothing)
1220 return true;
1221
1222 if (find_widening_optab_handler (smul_widen_optab, wmode, mode)
1223 != CODE_FOR_nothing)
1224 return true;
1225
1226 rtx_insn *last = get_last_insn ();
1227 if (CONSTANT_P (op0))
1228 op0 = convert_modes (wmode, mode, op0, uns);
1229 else
1230 op0 = gen_raw_REG (wmode, LAST_VIRTUAL_REGISTER + 1);
1231 if (CONSTANT_P (op1))
1232 op1 = convert_modes (wmode, mode, op1, uns);
1233 else
1234 op1 = gen_raw_REG (wmode, LAST_VIRTUAL_REGISTER + 2);
1235 rtx ret = expand_mult (wmode, op0, op1, NULL_RTX, uns, true);
1236 delete_insns_since (last);
1237 return ret != NULL_RTX;
1238 }
1239
1240 /* Add mul overflow checking to the statement STMT. */
1241
1242 static void
expand_mul_overflow(location_t loc,tree lhs,tree arg0,tree arg1,bool unsr_p,bool uns0_p,bool uns1_p,bool is_ubsan,tree * datap)1243 expand_mul_overflow (location_t loc, tree lhs, tree arg0, tree arg1,
1244 bool unsr_p, bool uns0_p, bool uns1_p, bool is_ubsan,
1245 tree *datap)
1246 {
1247 rtx res, op0, op1;
1248 tree fn, type;
1249 rtx_code_label *done_label, *do_error;
1250 rtx target = NULL_RTX;
1251 signop sign;
1252 enum insn_code icode;
1253
1254 done_label = gen_label_rtx ();
1255 do_error = gen_label_rtx ();
1256
1257 do_pending_stack_adjust ();
1258 op0 = expand_normal (arg0);
1259 op1 = expand_normal (arg1);
1260
1261 scalar_int_mode mode = SCALAR_INT_TYPE_MODE (TREE_TYPE (arg0));
1262 bool uns = unsr_p;
1263 if (lhs)
1264 {
1265 target = expand_expr (lhs, NULL_RTX, VOIDmode, EXPAND_WRITE);
1266 if (!is_ubsan)
1267 write_complex_part (target, const0_rtx, true);
1268 }
1269
1270 if (is_ubsan)
1271 gcc_assert (!unsr_p && !uns0_p && !uns1_p);
1272
1273 /* We assume both operands and result have the same precision
1274 here (GET_MODE_BITSIZE (mode)), S stands for signed type
1275 with that precision, U for unsigned type with that precision,
1276 sgn for unsigned most significant bit in that precision.
1277 s1 is signed first operand, u1 is unsigned first operand,
1278 s2 is signed second operand, u2 is unsigned second operand,
1279 sr is signed result, ur is unsigned result and the following
1280 rules say how to compute result (which is always result of
1281 the operands as if both were unsigned, cast to the right
1282 signedness) and how to compute whether operation overflowed.
1283 main_ovf (false) stands for jump on signed multiplication
1284 overflow or the main algorithm with uns == false.
1285 main_ovf (true) stands for jump on unsigned multiplication
1286 overflow or the main algorithm with uns == true.
1287
1288 s1 * s2 -> sr
1289 res = (S) ((U) s1 * (U) s2)
1290 ovf = main_ovf (false)
1291 u1 * u2 -> ur
1292 res = u1 * u2
1293 ovf = main_ovf (true)
1294 s1 * u2 -> ur
1295 res = (U) s1 * u2
1296 ovf = (s1 < 0 && u2) || main_ovf (true)
1297 u1 * u2 -> sr
1298 res = (S) (u1 * u2)
1299 ovf = res < 0 || main_ovf (true)
1300 s1 * u2 -> sr
1301 res = (S) ((U) s1 * u2)
1302 ovf = (S) u2 >= 0 ? main_ovf (false)
1303 : (s1 != 0 && (s1 != -1 || u2 != (U) res))
1304 s1 * s2 -> ur
1305 t1 = (s1 & s2) < 0 ? (-(U) s1) : ((U) s1)
1306 t2 = (s1 & s2) < 0 ? (-(U) s2) : ((U) s2)
1307 res = t1 * t2
1308 ovf = (s1 ^ s2) < 0 ? (s1 && s2) : main_ovf (true) */
1309
1310 if (uns0_p && !uns1_p)
1311 {
1312 /* Multiplication is commutative, if operand signedness differs,
1313 canonicalize to the first operand being signed and second
1314 unsigned to simplify following code. */
1315 std::swap (op0, op1);
1316 std::swap (arg0, arg1);
1317 uns0_p = false;
1318 uns1_p = true;
1319 }
1320
1321 int pos_neg0 = get_range_pos_neg (arg0);
1322 int pos_neg1 = get_range_pos_neg (arg1);
1323
1324 /* s1 * u2 -> ur */
1325 if (!uns0_p && uns1_p && unsr_p)
1326 {
1327 switch (pos_neg0)
1328 {
1329 case 1:
1330 /* If s1 is non-negative, just perform normal u1 * u2 -> ur. */
1331 goto do_main;
1332 case 2:
1333 /* If s1 is negative, avoid the main code, just multiply and
1334 signal overflow if op1 is not 0. */
1335 struct separate_ops ops;
1336 ops.code = MULT_EXPR;
1337 ops.type = TREE_TYPE (arg1);
1338 ops.op0 = make_tree (ops.type, op0);
1339 ops.op1 = make_tree (ops.type, op1);
1340 ops.op2 = NULL_TREE;
1341 ops.location = loc;
1342 res = expand_expr_real_2 (&ops, NULL_RTX, mode, EXPAND_NORMAL);
1343 do_compare_rtx_and_jump (op1, const0_rtx, EQ, true, mode, NULL_RTX,
1344 NULL, done_label, profile_probability::very_likely ());
1345 goto do_error_label;
1346 case 3:
1347 rtx_code_label *do_main_label;
1348 do_main_label = gen_label_rtx ();
1349 do_compare_rtx_and_jump (op0, const0_rtx, GE, false, mode, NULL_RTX,
1350 NULL, do_main_label, profile_probability::very_likely ());
1351 do_compare_rtx_and_jump (op1, const0_rtx, EQ, true, mode, NULL_RTX,
1352 NULL, do_main_label, profile_probability::very_likely ());
1353 expand_arith_set_overflow (lhs, target);
1354 emit_label (do_main_label);
1355 goto do_main;
1356 default:
1357 gcc_unreachable ();
1358 }
1359 }
1360
1361 /* u1 * u2 -> sr */
1362 if (uns0_p && uns1_p && !unsr_p)
1363 {
1364 uns = true;
1365 /* Rest of handling of this case after res is computed. */
1366 goto do_main;
1367 }
1368
1369 /* s1 * u2 -> sr */
1370 if (!uns0_p && uns1_p && !unsr_p)
1371 {
1372 switch (pos_neg1)
1373 {
1374 case 1:
1375 goto do_main;
1376 case 2:
1377 /* If (S) u2 is negative (i.e. u2 is larger than maximum of S,
1378 avoid the main code, just multiply and signal overflow
1379 unless 0 * u2 or -1 * ((U) Smin). */
1380 struct separate_ops ops;
1381 ops.code = MULT_EXPR;
1382 ops.type = TREE_TYPE (arg1);
1383 ops.op0 = make_tree (ops.type, op0);
1384 ops.op1 = make_tree (ops.type, op1);
1385 ops.op2 = NULL_TREE;
1386 ops.location = loc;
1387 res = expand_expr_real_2 (&ops, NULL_RTX, mode, EXPAND_NORMAL);
1388 do_compare_rtx_and_jump (op0, const0_rtx, EQ, true, mode, NULL_RTX,
1389 NULL, done_label, profile_probability::very_likely ());
1390 do_compare_rtx_and_jump (op0, constm1_rtx, NE, true, mode, NULL_RTX,
1391 NULL, do_error, profile_probability::very_unlikely ());
1392 int prec;
1393 prec = GET_MODE_PRECISION (mode);
1394 rtx sgn;
1395 sgn = immed_wide_int_const (wi::min_value (prec, SIGNED), mode);
1396 do_compare_rtx_and_jump (op1, sgn, EQ, true, mode, NULL_RTX,
1397 NULL, done_label, profile_probability::very_likely ());
1398 goto do_error_label;
1399 case 3:
1400 /* Rest of handling of this case after res is computed. */
1401 goto do_main;
1402 default:
1403 gcc_unreachable ();
1404 }
1405 }
1406
1407 /* s1 * s2 -> ur */
1408 if (!uns0_p && !uns1_p && unsr_p)
1409 {
1410 rtx tem;
1411 switch (pos_neg0 | pos_neg1)
1412 {
1413 case 1: /* Both operands known to be non-negative. */
1414 goto do_main;
1415 case 2: /* Both operands known to be negative. */
1416 op0 = expand_unop (mode, neg_optab, op0, NULL_RTX, false);
1417 op1 = expand_unop (mode, neg_optab, op1, NULL_RTX, false);
1418 /* Avoid looking at arg0/arg1 ranges, as we've changed
1419 the arguments. */
1420 arg0 = error_mark_node;
1421 arg1 = error_mark_node;
1422 goto do_main;
1423 case 3:
1424 if ((pos_neg0 ^ pos_neg1) == 3)
1425 {
1426 /* If one operand is known to be negative and the other
1427 non-negative, this overflows always, unless the non-negative
1428 one is 0. Just do normal multiply and set overflow
1429 unless one of the operands is 0. */
1430 struct separate_ops ops;
1431 ops.code = MULT_EXPR;
1432 ops.type
1433 = build_nonstandard_integer_type (GET_MODE_PRECISION (mode),
1434 1);
1435 ops.op0 = make_tree (ops.type, op0);
1436 ops.op1 = make_tree (ops.type, op1);
1437 ops.op2 = NULL_TREE;
1438 ops.location = loc;
1439 res = expand_expr_real_2 (&ops, NULL_RTX, mode, EXPAND_NORMAL);
1440 do_compare_rtx_and_jump (pos_neg0 == 1 ? op0 : op1, const0_rtx, EQ,
1441 true, mode, NULL_RTX, NULL, done_label,
1442 profile_probability::very_likely ());
1443 goto do_error_label;
1444 }
1445 /* The general case, do all the needed comparisons at runtime. */
1446 rtx_code_label *do_main_label, *after_negate_label;
1447 rtx rop0, rop1;
1448 rop0 = gen_reg_rtx (mode);
1449 rop1 = gen_reg_rtx (mode);
1450 emit_move_insn (rop0, op0);
1451 emit_move_insn (rop1, op1);
1452 op0 = rop0;
1453 op1 = rop1;
1454 do_main_label = gen_label_rtx ();
1455 after_negate_label = gen_label_rtx ();
1456 tem = expand_binop (mode, and_optab, op0, op1, NULL_RTX, false,
1457 OPTAB_LIB_WIDEN);
1458 do_compare_rtx_and_jump (tem, const0_rtx, GE, false, mode, NULL_RTX,
1459 NULL, after_negate_label, profile_probability::very_likely ());
1460 /* Both arguments negative here, negate them and continue with
1461 normal unsigned overflow checking multiplication. */
1462 emit_move_insn (op0, expand_unop (mode, neg_optab, op0,
1463 NULL_RTX, false));
1464 emit_move_insn (op1, expand_unop (mode, neg_optab, op1,
1465 NULL_RTX, false));
1466 /* Avoid looking at arg0/arg1 ranges, as we might have changed
1467 the arguments. */
1468 arg0 = error_mark_node;
1469 arg1 = error_mark_node;
1470 emit_jump (do_main_label);
1471 emit_label (after_negate_label);
1472 tem = expand_binop (mode, xor_optab, op0, op1, NULL_RTX, false,
1473 OPTAB_LIB_WIDEN);
1474 do_compare_rtx_and_jump (tem, const0_rtx, GE, false, mode, NULL_RTX,
1475 NULL, do_main_label,
1476 profile_probability::very_likely ());
1477 /* One argument is negative here, the other positive. This
1478 overflows always, unless one of the arguments is 0. But
1479 if e.g. s2 is 0, (U) s1 * 0 doesn't overflow, whatever s1
1480 is, thus we can keep do_main code oring in overflow as is. */
1481 if (pos_neg0 != 2)
1482 do_compare_rtx_and_jump (op0, const0_rtx, EQ, true, mode, NULL_RTX,
1483 NULL, do_main_label,
1484 profile_probability::very_unlikely ());
1485 if (pos_neg1 != 2)
1486 do_compare_rtx_and_jump (op1, const0_rtx, EQ, true, mode, NULL_RTX,
1487 NULL, do_main_label,
1488 profile_probability::very_unlikely ());
1489 expand_arith_set_overflow (lhs, target);
1490 emit_label (do_main_label);
1491 goto do_main;
1492 default:
1493 gcc_unreachable ();
1494 }
1495 }
1496
1497 do_main:
1498 type = build_nonstandard_integer_type (GET_MODE_PRECISION (mode), uns);
1499 sign = uns ? UNSIGNED : SIGNED;
1500 icode = optab_handler (uns ? umulv4_optab : mulv4_optab, mode);
1501 if (uns
1502 && (integer_pow2p (arg0) || integer_pow2p (arg1))
1503 && (optimize_insn_for_speed_p () || icode == CODE_FOR_nothing))
1504 {
1505 /* Optimize unsigned multiplication by power of 2 constant
1506 using 2 shifts, one for result, one to extract the shifted
1507 out bits to see if they are all zero.
1508 Don't do this if optimizing for size and we have umulv4_optab,
1509 in that case assume multiplication will be shorter.
1510 This is heuristics based on the single target that provides
1511 umulv4 right now (i?86/x86_64), if further targets add it, this
1512 might need to be revisited.
1513 Cases where both operands are constant should be folded already
1514 during GIMPLE, and cases where one operand is constant but not
1515 power of 2 are questionable, either the WIDEN_MULT_EXPR case
1516 below can be done without multiplication, just by shifts and adds,
1517 or we'd need to divide the result (and hope it actually doesn't
1518 really divide nor multiply) and compare the result of the division
1519 with the original operand. */
1520 rtx opn0 = op0;
1521 rtx opn1 = op1;
1522 tree argn0 = arg0;
1523 tree argn1 = arg1;
1524 if (integer_pow2p (arg0))
1525 {
1526 std::swap (opn0, opn1);
1527 std::swap (argn0, argn1);
1528 }
1529 int cnt = tree_log2 (argn1);
1530 if (cnt >= 0 && cnt < GET_MODE_PRECISION (mode))
1531 {
1532 rtx upper = const0_rtx;
1533 res = expand_shift (LSHIFT_EXPR, mode, opn0, cnt, NULL_RTX, uns);
1534 if (cnt != 0)
1535 upper = expand_shift (RSHIFT_EXPR, mode, opn0,
1536 GET_MODE_PRECISION (mode) - cnt,
1537 NULL_RTX, uns);
1538 do_compare_rtx_and_jump (upper, const0_rtx, EQ, true, mode,
1539 NULL_RTX, NULL, done_label,
1540 profile_probability::very_likely ());
1541 goto do_error_label;
1542 }
1543 }
1544 if (icode != CODE_FOR_nothing)
1545 {
1546 struct expand_operand ops[4];
1547 rtx_insn *last = get_last_insn ();
1548
1549 res = gen_reg_rtx (mode);
1550 create_output_operand (&ops[0], res, mode);
1551 create_input_operand (&ops[1], op0, mode);
1552 create_input_operand (&ops[2], op1, mode);
1553 create_fixed_operand (&ops[3], do_error);
1554 if (maybe_expand_insn (icode, 4, ops))
1555 {
1556 last = get_last_insn ();
1557 if (profile_status_for_fn (cfun) != PROFILE_ABSENT
1558 && JUMP_P (last)
1559 && any_condjump_p (last)
1560 && !find_reg_note (last, REG_BR_PROB, 0))
1561 add_reg_br_prob_note (last,
1562 profile_probability::very_unlikely ());
1563 emit_jump (done_label);
1564 }
1565 else
1566 {
1567 delete_insns_since (last);
1568 icode = CODE_FOR_nothing;
1569 }
1570 }
1571
1572 if (icode == CODE_FOR_nothing)
1573 {
1574 struct separate_ops ops;
1575 int prec = GET_MODE_PRECISION (mode);
1576 scalar_int_mode hmode, wmode;
1577 ops.op0 = make_tree (type, op0);
1578 ops.op1 = make_tree (type, op1);
1579 ops.op2 = NULL_TREE;
1580 ops.location = loc;
1581
1582 /* Optimize unsigned overflow check where we don't use the
1583 multiplication result, just whether overflow happened.
1584 If we can do MULT_HIGHPART_EXPR, that followed by
1585 comparison of the result against zero is cheapest.
1586 We'll still compute res, but it should be DCEd later. */
1587 use_operand_p use;
1588 gimple *use_stmt;
1589 if (!is_ubsan
1590 && lhs
1591 && uns
1592 && !(uns0_p && uns1_p && !unsr_p)
1593 && can_mult_highpart_p (mode, uns) == 1
1594 && single_imm_use (lhs, &use, &use_stmt)
1595 && is_gimple_assign (use_stmt)
1596 && gimple_assign_rhs_code (use_stmt) == IMAGPART_EXPR)
1597 goto highpart;
1598
1599 if (GET_MODE_2XWIDER_MODE (mode).exists (&wmode)
1600 && targetm.scalar_mode_supported_p (wmode)
1601 && can_widen_mult_without_libcall (wmode, mode, op0, op1, uns))
1602 {
1603 twoxwider:
1604 ops.code = WIDEN_MULT_EXPR;
1605 ops.type
1606 = build_nonstandard_integer_type (GET_MODE_PRECISION (wmode), uns);
1607
1608 res = expand_expr_real_2 (&ops, NULL_RTX, wmode, EXPAND_NORMAL);
1609 rtx hipart = expand_shift (RSHIFT_EXPR, wmode, res, prec,
1610 NULL_RTX, uns);
1611 hipart = convert_modes (mode, wmode, hipart, uns);
1612 res = convert_modes (mode, wmode, res, uns);
1613 if (uns)
1614 /* For the unsigned multiplication, there was overflow if
1615 HIPART is non-zero. */
1616 do_compare_rtx_and_jump (hipart, const0_rtx, EQ, true, mode,
1617 NULL_RTX, NULL, done_label,
1618 profile_probability::very_likely ());
1619 else
1620 {
1621 rtx signbit = expand_shift (RSHIFT_EXPR, mode, res, prec - 1,
1622 NULL_RTX, 0);
1623 /* RES is low half of the double width result, HIPART
1624 the high half. There was overflow if
1625 HIPART is different from RES < 0 ? -1 : 0. */
1626 do_compare_rtx_and_jump (signbit, hipart, EQ, true, mode,
1627 NULL_RTX, NULL, done_label,
1628 profile_probability::very_likely ());
1629 }
1630 }
1631 else if (can_mult_highpart_p (mode, uns) == 1)
1632 {
1633 highpart:
1634 ops.code = MULT_HIGHPART_EXPR;
1635 ops.type = type;
1636
1637 rtx hipart = expand_expr_real_2 (&ops, NULL_RTX, mode,
1638 EXPAND_NORMAL);
1639 ops.code = MULT_EXPR;
1640 res = expand_expr_real_2 (&ops, NULL_RTX, mode, EXPAND_NORMAL);
1641 if (uns)
1642 /* For the unsigned multiplication, there was overflow if
1643 HIPART is non-zero. */
1644 do_compare_rtx_and_jump (hipart, const0_rtx, EQ, true, mode,
1645 NULL_RTX, NULL, done_label,
1646 profile_probability::very_likely ());
1647 else
1648 {
1649 rtx signbit = expand_shift (RSHIFT_EXPR, mode, res, prec - 1,
1650 NULL_RTX, 0);
1651 /* RES is low half of the double width result, HIPART
1652 the high half. There was overflow if
1653 HIPART is different from RES < 0 ? -1 : 0. */
1654 do_compare_rtx_and_jump (signbit, hipart, EQ, true, mode,
1655 NULL_RTX, NULL, done_label,
1656 profile_probability::very_likely ());
1657 }
1658
1659 }
1660 else if (int_mode_for_size (prec / 2, 1).exists (&hmode)
1661 && 2 * GET_MODE_PRECISION (hmode) == prec)
1662 {
1663 rtx_code_label *large_op0 = gen_label_rtx ();
1664 rtx_code_label *small_op0_large_op1 = gen_label_rtx ();
1665 rtx_code_label *one_small_one_large = gen_label_rtx ();
1666 rtx_code_label *both_ops_large = gen_label_rtx ();
1667 rtx_code_label *after_hipart_neg = uns ? NULL : gen_label_rtx ();
1668 rtx_code_label *after_lopart_neg = uns ? NULL : gen_label_rtx ();
1669 rtx_code_label *do_overflow = gen_label_rtx ();
1670 rtx_code_label *hipart_different = uns ? NULL : gen_label_rtx ();
1671
1672 unsigned int hprec = GET_MODE_PRECISION (hmode);
1673 rtx hipart0 = expand_shift (RSHIFT_EXPR, mode, op0, hprec,
1674 NULL_RTX, uns);
1675 hipart0 = convert_modes (hmode, mode, hipart0, uns);
1676 rtx lopart0 = convert_modes (hmode, mode, op0, uns);
1677 rtx signbit0 = const0_rtx;
1678 if (!uns)
1679 signbit0 = expand_shift (RSHIFT_EXPR, hmode, lopart0, hprec - 1,
1680 NULL_RTX, 0);
1681 rtx hipart1 = expand_shift (RSHIFT_EXPR, mode, op1, hprec,
1682 NULL_RTX, uns);
1683 hipart1 = convert_modes (hmode, mode, hipart1, uns);
1684 rtx lopart1 = convert_modes (hmode, mode, op1, uns);
1685 rtx signbit1 = const0_rtx;
1686 if (!uns)
1687 signbit1 = expand_shift (RSHIFT_EXPR, hmode, lopart1, hprec - 1,
1688 NULL_RTX, 0);
1689
1690 res = gen_reg_rtx (mode);
1691
1692 /* True if op0 resp. op1 are known to be in the range of
1693 halfstype. */
1694 bool op0_small_p = false;
1695 bool op1_small_p = false;
1696 /* True if op0 resp. op1 are known to have all zeros or all ones
1697 in the upper half of bits, but are not known to be
1698 op{0,1}_small_p. */
1699 bool op0_medium_p = false;
1700 bool op1_medium_p = false;
1701 /* -1 if op{0,1} is known to be negative, 0 if it is known to be
1702 nonnegative, 1 if unknown. */
1703 int op0_sign = 1;
1704 int op1_sign = 1;
1705
1706 if (pos_neg0 == 1)
1707 op0_sign = 0;
1708 else if (pos_neg0 == 2)
1709 op0_sign = -1;
1710 if (pos_neg1 == 1)
1711 op1_sign = 0;
1712 else if (pos_neg1 == 2)
1713 op1_sign = -1;
1714
1715 unsigned int mprec0 = prec;
1716 if (arg0 != error_mark_node)
1717 mprec0 = get_min_precision (arg0, sign);
1718 if (mprec0 <= hprec)
1719 op0_small_p = true;
1720 else if (!uns && mprec0 <= hprec + 1)
1721 op0_medium_p = true;
1722 unsigned int mprec1 = prec;
1723 if (arg1 != error_mark_node)
1724 mprec1 = get_min_precision (arg1, sign);
1725 if (mprec1 <= hprec)
1726 op1_small_p = true;
1727 else if (!uns && mprec1 <= hprec + 1)
1728 op1_medium_p = true;
1729
1730 int smaller_sign = 1;
1731 int larger_sign = 1;
1732 if (op0_small_p)
1733 {
1734 smaller_sign = op0_sign;
1735 larger_sign = op1_sign;
1736 }
1737 else if (op1_small_p)
1738 {
1739 smaller_sign = op1_sign;
1740 larger_sign = op0_sign;
1741 }
1742 else if (op0_sign == op1_sign)
1743 {
1744 smaller_sign = op0_sign;
1745 larger_sign = op0_sign;
1746 }
1747
1748 if (!op0_small_p)
1749 do_compare_rtx_and_jump (signbit0, hipart0, NE, true, hmode,
1750 NULL_RTX, NULL, large_op0,
1751 profile_probability::unlikely ());
1752
1753 if (!op1_small_p)
1754 do_compare_rtx_and_jump (signbit1, hipart1, NE, true, hmode,
1755 NULL_RTX, NULL, small_op0_large_op1,
1756 profile_probability::unlikely ());
1757
1758 /* If both op0 and op1 are sign (!uns) or zero (uns) extended from
1759 hmode to mode, the multiplication will never overflow. We can
1760 do just one hmode x hmode => mode widening multiplication. */
1761 tree halfstype = build_nonstandard_integer_type (hprec, uns);
1762 ops.op0 = make_tree (halfstype, lopart0);
1763 ops.op1 = make_tree (halfstype, lopart1);
1764 ops.code = WIDEN_MULT_EXPR;
1765 ops.type = type;
1766 rtx thisres
1767 = expand_expr_real_2 (&ops, NULL_RTX, mode, EXPAND_NORMAL);
1768 emit_move_insn (res, thisres);
1769 emit_jump (done_label);
1770
1771 emit_label (small_op0_large_op1);
1772
1773 /* If op0 is sign (!uns) or zero (uns) extended from hmode to mode,
1774 but op1 is not, just swap the arguments and handle it as op1
1775 sign/zero extended, op0 not. */
1776 rtx larger = gen_reg_rtx (mode);
1777 rtx hipart = gen_reg_rtx (hmode);
1778 rtx lopart = gen_reg_rtx (hmode);
1779 emit_move_insn (larger, op1);
1780 emit_move_insn (hipart, hipart1);
1781 emit_move_insn (lopart, lopart0);
1782 emit_jump (one_small_one_large);
1783
1784 emit_label (large_op0);
1785
1786 if (!op1_small_p)
1787 do_compare_rtx_and_jump (signbit1, hipart1, NE, true, hmode,
1788 NULL_RTX, NULL, both_ops_large,
1789 profile_probability::unlikely ());
1790
1791 /* If op1 is sign (!uns) or zero (uns) extended from hmode to mode,
1792 but op0 is not, prepare larger, hipart and lopart pseudos and
1793 handle it together with small_op0_large_op1. */
1794 emit_move_insn (larger, op0);
1795 emit_move_insn (hipart, hipart0);
1796 emit_move_insn (lopart, lopart1);
1797
1798 emit_label (one_small_one_large);
1799
1800 /* lopart is the low part of the operand that is sign extended
1801 to mode, larger is the other operand, hipart is the
1802 high part of larger and lopart0 and lopart1 are the low parts
1803 of both operands.
1804 We perform lopart0 * lopart1 and lopart * hipart widening
1805 multiplications. */
1806 tree halfutype = build_nonstandard_integer_type (hprec, 1);
1807 ops.op0 = make_tree (halfutype, lopart0);
1808 ops.op1 = make_tree (halfutype, lopart1);
1809 rtx lo0xlo1
1810 = expand_expr_real_2 (&ops, NULL_RTX, mode, EXPAND_NORMAL);
1811
1812 ops.op0 = make_tree (halfutype, lopart);
1813 ops.op1 = make_tree (halfutype, hipart);
1814 rtx loxhi = gen_reg_rtx (mode);
1815 rtx tem = expand_expr_real_2 (&ops, NULL_RTX, mode, EXPAND_NORMAL);
1816 emit_move_insn (loxhi, tem);
1817
1818 if (!uns)
1819 {
1820 /* if (hipart < 0) loxhi -= lopart << (bitsize / 2); */
1821 if (larger_sign == 0)
1822 emit_jump (after_hipart_neg);
1823 else if (larger_sign != -1)
1824 do_compare_rtx_and_jump (hipart, const0_rtx, GE, false, hmode,
1825 NULL_RTX, NULL, after_hipart_neg,
1826 profile_probability::even ());
1827
1828 tem = convert_modes (mode, hmode, lopart, 1);
1829 tem = expand_shift (LSHIFT_EXPR, mode, tem, hprec, NULL_RTX, 1);
1830 tem = expand_simple_binop (mode, MINUS, loxhi, tem, NULL_RTX,
1831 1, OPTAB_WIDEN);
1832 emit_move_insn (loxhi, tem);
1833
1834 emit_label (after_hipart_neg);
1835
1836 /* if (lopart < 0) loxhi -= larger; */
1837 if (smaller_sign == 0)
1838 emit_jump (after_lopart_neg);
1839 else if (smaller_sign != -1)
1840 do_compare_rtx_and_jump (lopart, const0_rtx, GE, false, hmode,
1841 NULL_RTX, NULL, after_lopart_neg,
1842 profile_probability::even ());
1843
1844 tem = expand_simple_binop (mode, MINUS, loxhi, larger, NULL_RTX,
1845 1, OPTAB_WIDEN);
1846 emit_move_insn (loxhi, tem);
1847
1848 emit_label (after_lopart_neg);
1849 }
1850
1851 /* loxhi += (uns) lo0xlo1 >> (bitsize / 2); */
1852 tem = expand_shift (RSHIFT_EXPR, mode, lo0xlo1, hprec, NULL_RTX, 1);
1853 tem = expand_simple_binop (mode, PLUS, loxhi, tem, NULL_RTX,
1854 1, OPTAB_WIDEN);
1855 emit_move_insn (loxhi, tem);
1856
1857 /* if (loxhi >> (bitsize / 2)
1858 == (hmode) loxhi >> (bitsize / 2 - 1)) (if !uns)
1859 if (loxhi >> (bitsize / 2) == 0 (if uns). */
1860 rtx hipartloxhi = expand_shift (RSHIFT_EXPR, mode, loxhi, hprec,
1861 NULL_RTX, 0);
1862 hipartloxhi = convert_modes (hmode, mode, hipartloxhi, 0);
1863 rtx signbitloxhi = const0_rtx;
1864 if (!uns)
1865 signbitloxhi = expand_shift (RSHIFT_EXPR, hmode,
1866 convert_modes (hmode, mode,
1867 loxhi, 0),
1868 hprec - 1, NULL_RTX, 0);
1869
1870 do_compare_rtx_and_jump (signbitloxhi, hipartloxhi, NE, true, hmode,
1871 NULL_RTX, NULL, do_overflow,
1872 profile_probability::very_unlikely ());
1873
1874 /* res = (loxhi << (bitsize / 2)) | (hmode) lo0xlo1; */
1875 rtx loxhishifted = expand_shift (LSHIFT_EXPR, mode, loxhi, hprec,
1876 NULL_RTX, 1);
1877 tem = convert_modes (mode, hmode,
1878 convert_modes (hmode, mode, lo0xlo1, 1), 1);
1879
1880 tem = expand_simple_binop (mode, IOR, loxhishifted, tem, res,
1881 1, OPTAB_WIDEN);
1882 if (tem != res)
1883 emit_move_insn (res, tem);
1884 emit_jump (done_label);
1885
1886 emit_label (both_ops_large);
1887
1888 /* If both operands are large (not sign (!uns) or zero (uns)
1889 extended from hmode), then perform the full multiplication
1890 which will be the result of the operation.
1891 The only cases which don't overflow are for signed multiplication
1892 some cases where both hipart0 and highpart1 are 0 or -1.
1893 For unsigned multiplication when high parts are both non-zero
1894 this overflows always. */
1895 ops.code = MULT_EXPR;
1896 ops.op0 = make_tree (type, op0);
1897 ops.op1 = make_tree (type, op1);
1898 tem = expand_expr_real_2 (&ops, NULL_RTX, mode, EXPAND_NORMAL);
1899 emit_move_insn (res, tem);
1900
1901 if (!uns)
1902 {
1903 if (!op0_medium_p)
1904 {
1905 tem = expand_simple_binop (hmode, PLUS, hipart0, const1_rtx,
1906 NULL_RTX, 1, OPTAB_WIDEN);
1907 do_compare_rtx_and_jump (tem, const1_rtx, GTU, true, hmode,
1908 NULL_RTX, NULL, do_error,
1909 profile_probability::very_unlikely ());
1910 }
1911
1912 if (!op1_medium_p)
1913 {
1914 tem = expand_simple_binop (hmode, PLUS, hipart1, const1_rtx,
1915 NULL_RTX, 1, OPTAB_WIDEN);
1916 do_compare_rtx_and_jump (tem, const1_rtx, GTU, true, hmode,
1917 NULL_RTX, NULL, do_error,
1918 profile_probability::very_unlikely ());
1919 }
1920
1921 /* At this point hipart{0,1} are both in [-1, 0]. If they are
1922 the same, overflow happened if res is non-positive, if they
1923 are different, overflow happened if res is positive. */
1924 if (op0_sign != 1 && op1_sign != 1 && op0_sign != op1_sign)
1925 emit_jump (hipart_different);
1926 else if (op0_sign == 1 || op1_sign == 1)
1927 do_compare_rtx_and_jump (hipart0, hipart1, NE, true, hmode,
1928 NULL_RTX, NULL, hipart_different,
1929 profile_probability::even ());
1930
1931 do_compare_rtx_and_jump (res, const0_rtx, LE, false, mode,
1932 NULL_RTX, NULL, do_error,
1933 profile_probability::very_unlikely ());
1934 emit_jump (done_label);
1935
1936 emit_label (hipart_different);
1937
1938 do_compare_rtx_and_jump (res, const0_rtx, GE, false, mode,
1939 NULL_RTX, NULL, do_error,
1940 profile_probability::very_unlikely ());
1941 emit_jump (done_label);
1942 }
1943
1944 emit_label (do_overflow);
1945
1946 /* Overflow, do full multiplication and fallthru into do_error. */
1947 ops.op0 = make_tree (type, op0);
1948 ops.op1 = make_tree (type, op1);
1949 tem = expand_expr_real_2 (&ops, NULL_RTX, mode, EXPAND_NORMAL);
1950 emit_move_insn (res, tem);
1951 }
1952 else if (GET_MODE_2XWIDER_MODE (mode).exists (&wmode)
1953 && targetm.scalar_mode_supported_p (wmode))
1954 /* Even emitting a libcall is better than not detecting overflow
1955 at all. */
1956 goto twoxwider;
1957 else
1958 {
1959 gcc_assert (!is_ubsan);
1960 ops.code = MULT_EXPR;
1961 ops.type = type;
1962 res = expand_expr_real_2 (&ops, NULL_RTX, mode, EXPAND_NORMAL);
1963 emit_jump (done_label);
1964 }
1965 }
1966
1967 do_error_label:
1968 emit_label (do_error);
1969 if (is_ubsan)
1970 {
1971 /* Expand the ubsan builtin call. */
1972 push_temp_slots ();
1973 fn = ubsan_build_overflow_builtin (MULT_EXPR, loc, TREE_TYPE (arg0),
1974 arg0, arg1, datap);
1975 expand_normal (fn);
1976 pop_temp_slots ();
1977 do_pending_stack_adjust ();
1978 }
1979 else if (lhs)
1980 expand_arith_set_overflow (lhs, target);
1981
1982 /* We're done. */
1983 emit_label (done_label);
1984
1985 /* u1 * u2 -> sr */
1986 if (uns0_p && uns1_p && !unsr_p)
1987 {
1988 rtx_code_label *all_done_label = gen_label_rtx ();
1989 do_compare_rtx_and_jump (res, const0_rtx, GE, false, mode, NULL_RTX,
1990 NULL, all_done_label, profile_probability::very_likely ());
1991 expand_arith_set_overflow (lhs, target);
1992 emit_label (all_done_label);
1993 }
1994
1995 /* s1 * u2 -> sr */
1996 if (!uns0_p && uns1_p && !unsr_p && pos_neg1 == 3)
1997 {
1998 rtx_code_label *all_done_label = gen_label_rtx ();
1999 rtx_code_label *set_noovf = gen_label_rtx ();
2000 do_compare_rtx_and_jump (op1, const0_rtx, GE, false, mode, NULL_RTX,
2001 NULL, all_done_label, profile_probability::very_likely ());
2002 expand_arith_set_overflow (lhs, target);
2003 do_compare_rtx_and_jump (op0, const0_rtx, EQ, true, mode, NULL_RTX,
2004 NULL, set_noovf, profile_probability::very_likely ());
2005 do_compare_rtx_and_jump (op0, constm1_rtx, NE, true, mode, NULL_RTX,
2006 NULL, all_done_label, profile_probability::very_unlikely ());
2007 do_compare_rtx_and_jump (op1, res, NE, true, mode, NULL_RTX, NULL,
2008 all_done_label, profile_probability::very_unlikely ());
2009 emit_label (set_noovf);
2010 write_complex_part (target, const0_rtx, true);
2011 emit_label (all_done_label);
2012 }
2013
2014 if (lhs)
2015 {
2016 if (is_ubsan)
2017 expand_ubsan_result_store (target, res);
2018 else
2019 expand_arith_overflow_result_store (lhs, target, mode, res);
2020 }
2021 }
2022
2023 /* Expand UBSAN_CHECK_* internal function if it has vector operands. */
2024
2025 static void
expand_vector_ubsan_overflow(location_t loc,enum tree_code code,tree lhs,tree arg0,tree arg1)2026 expand_vector_ubsan_overflow (location_t loc, enum tree_code code, tree lhs,
2027 tree arg0, tree arg1)
2028 {
2029 poly_uint64 cnt = TYPE_VECTOR_SUBPARTS (TREE_TYPE (arg0));
2030 rtx_code_label *loop_lab = NULL;
2031 rtx cntvar = NULL_RTX;
2032 tree cntv = NULL_TREE;
2033 tree eltype = TREE_TYPE (TREE_TYPE (arg0));
2034 tree sz = TYPE_SIZE (eltype);
2035 tree data = NULL_TREE;
2036 tree resv = NULL_TREE;
2037 rtx lhsr = NULL_RTX;
2038 rtx resvr = NULL_RTX;
2039 unsigned HOST_WIDE_INT const_cnt = 0;
2040 bool use_loop_p = (!cnt.is_constant (&const_cnt) || const_cnt > 4);
2041
2042 if (lhs)
2043 {
2044 optab op;
2045 lhsr = expand_expr (lhs, NULL_RTX, VOIDmode, EXPAND_WRITE);
2046 if (!VECTOR_MODE_P (GET_MODE (lhsr))
2047 || (op = optab_for_tree_code (code, TREE_TYPE (arg0),
2048 optab_default)) == unknown_optab
2049 || (optab_handler (op, TYPE_MODE (TREE_TYPE (arg0)))
2050 == CODE_FOR_nothing))
2051 {
2052 if (MEM_P (lhsr))
2053 resv = make_tree (TREE_TYPE (lhs), lhsr);
2054 else
2055 {
2056 resvr = assign_temp (TREE_TYPE (lhs), 1, 1);
2057 resv = make_tree (TREE_TYPE (lhs), resvr);
2058 }
2059 }
2060 }
2061 if (use_loop_p)
2062 {
2063 do_pending_stack_adjust ();
2064 loop_lab = gen_label_rtx ();
2065 cntvar = gen_reg_rtx (TYPE_MODE (sizetype));
2066 cntv = make_tree (sizetype, cntvar);
2067 emit_move_insn (cntvar, const0_rtx);
2068 emit_label (loop_lab);
2069 }
2070 if (TREE_CODE (arg0) != VECTOR_CST)
2071 {
2072 rtx arg0r = expand_normal (arg0);
2073 arg0 = make_tree (TREE_TYPE (arg0), arg0r);
2074 }
2075 if (TREE_CODE (arg1) != VECTOR_CST)
2076 {
2077 rtx arg1r = expand_normal (arg1);
2078 arg1 = make_tree (TREE_TYPE (arg1), arg1r);
2079 }
2080 for (unsigned int i = 0; i < (use_loop_p ? 1 : const_cnt); i++)
2081 {
2082 tree op0, op1, res = NULL_TREE;
2083 if (use_loop_p)
2084 {
2085 tree atype = build_array_type_nelts (eltype, cnt);
2086 op0 = uniform_vector_p (arg0);
2087 if (op0 == NULL_TREE)
2088 {
2089 op0 = fold_build1_loc (loc, VIEW_CONVERT_EXPR, atype, arg0);
2090 op0 = build4_loc (loc, ARRAY_REF, eltype, op0, cntv,
2091 NULL_TREE, NULL_TREE);
2092 }
2093 op1 = uniform_vector_p (arg1);
2094 if (op1 == NULL_TREE)
2095 {
2096 op1 = fold_build1_loc (loc, VIEW_CONVERT_EXPR, atype, arg1);
2097 op1 = build4_loc (loc, ARRAY_REF, eltype, op1, cntv,
2098 NULL_TREE, NULL_TREE);
2099 }
2100 if (resv)
2101 {
2102 res = fold_build1_loc (loc, VIEW_CONVERT_EXPR, atype, resv);
2103 res = build4_loc (loc, ARRAY_REF, eltype, res, cntv,
2104 NULL_TREE, NULL_TREE);
2105 }
2106 }
2107 else
2108 {
2109 tree bitpos = bitsize_int (tree_to_uhwi (sz) * i);
2110 op0 = fold_build3_loc (loc, BIT_FIELD_REF, eltype, arg0, sz, bitpos);
2111 op1 = fold_build3_loc (loc, BIT_FIELD_REF, eltype, arg1, sz, bitpos);
2112 if (resv)
2113 res = fold_build3_loc (loc, BIT_FIELD_REF, eltype, resv, sz,
2114 bitpos);
2115 }
2116 switch (code)
2117 {
2118 case PLUS_EXPR:
2119 expand_addsub_overflow (loc, PLUS_EXPR, res, op0, op1,
2120 false, false, false, true, &data);
2121 break;
2122 case MINUS_EXPR:
2123 if (use_loop_p ? integer_zerop (arg0) : integer_zerop (op0))
2124 expand_neg_overflow (loc, res, op1, true, &data);
2125 else
2126 expand_addsub_overflow (loc, MINUS_EXPR, res, op0, op1,
2127 false, false, false, true, &data);
2128 break;
2129 case MULT_EXPR:
2130 expand_mul_overflow (loc, res, op0, op1, false, false, false,
2131 true, &data);
2132 break;
2133 default:
2134 gcc_unreachable ();
2135 }
2136 }
2137 if (use_loop_p)
2138 {
2139 struct separate_ops ops;
2140 ops.code = PLUS_EXPR;
2141 ops.type = TREE_TYPE (cntv);
2142 ops.op0 = cntv;
2143 ops.op1 = build_int_cst (TREE_TYPE (cntv), 1);
2144 ops.op2 = NULL_TREE;
2145 ops.location = loc;
2146 rtx ret = expand_expr_real_2 (&ops, cntvar, TYPE_MODE (sizetype),
2147 EXPAND_NORMAL);
2148 if (ret != cntvar)
2149 emit_move_insn (cntvar, ret);
2150 rtx cntrtx = gen_int_mode (cnt, TYPE_MODE (sizetype));
2151 do_compare_rtx_and_jump (cntvar, cntrtx, NE, false,
2152 TYPE_MODE (sizetype), NULL_RTX, NULL, loop_lab,
2153 profile_probability::very_likely ());
2154 }
2155 if (lhs && resv == NULL_TREE)
2156 {
2157 struct separate_ops ops;
2158 ops.code = code;
2159 ops.type = TREE_TYPE (arg0);
2160 ops.op0 = arg0;
2161 ops.op1 = arg1;
2162 ops.op2 = NULL_TREE;
2163 ops.location = loc;
2164 rtx ret = expand_expr_real_2 (&ops, lhsr, TYPE_MODE (TREE_TYPE (arg0)),
2165 EXPAND_NORMAL);
2166 if (ret != lhsr)
2167 emit_move_insn (lhsr, ret);
2168 }
2169 else if (resvr)
2170 emit_move_insn (lhsr, resvr);
2171 }
2172
2173 /* Expand UBSAN_CHECK_ADD call STMT. */
2174
2175 static void
expand_UBSAN_CHECK_ADD(internal_fn,gcall * stmt)2176 expand_UBSAN_CHECK_ADD (internal_fn, gcall *stmt)
2177 {
2178 location_t loc = gimple_location (stmt);
2179 tree lhs = gimple_call_lhs (stmt);
2180 tree arg0 = gimple_call_arg (stmt, 0);
2181 tree arg1 = gimple_call_arg (stmt, 1);
2182 if (VECTOR_TYPE_P (TREE_TYPE (arg0)))
2183 expand_vector_ubsan_overflow (loc, PLUS_EXPR, lhs, arg0, arg1);
2184 else
2185 expand_addsub_overflow (loc, PLUS_EXPR, lhs, arg0, arg1,
2186 false, false, false, true, NULL);
2187 }
2188
2189 /* Expand UBSAN_CHECK_SUB call STMT. */
2190
2191 static void
expand_UBSAN_CHECK_SUB(internal_fn,gcall * stmt)2192 expand_UBSAN_CHECK_SUB (internal_fn, gcall *stmt)
2193 {
2194 location_t loc = gimple_location (stmt);
2195 tree lhs = gimple_call_lhs (stmt);
2196 tree arg0 = gimple_call_arg (stmt, 0);
2197 tree arg1 = gimple_call_arg (stmt, 1);
2198 if (VECTOR_TYPE_P (TREE_TYPE (arg0)))
2199 expand_vector_ubsan_overflow (loc, MINUS_EXPR, lhs, arg0, arg1);
2200 else if (integer_zerop (arg0))
2201 expand_neg_overflow (loc, lhs, arg1, true, NULL);
2202 else
2203 expand_addsub_overflow (loc, MINUS_EXPR, lhs, arg0, arg1,
2204 false, false, false, true, NULL);
2205 }
2206
2207 /* Expand UBSAN_CHECK_MUL call STMT. */
2208
2209 static void
expand_UBSAN_CHECK_MUL(internal_fn,gcall * stmt)2210 expand_UBSAN_CHECK_MUL (internal_fn, gcall *stmt)
2211 {
2212 location_t loc = gimple_location (stmt);
2213 tree lhs = gimple_call_lhs (stmt);
2214 tree arg0 = gimple_call_arg (stmt, 0);
2215 tree arg1 = gimple_call_arg (stmt, 1);
2216 if (VECTOR_TYPE_P (TREE_TYPE (arg0)))
2217 expand_vector_ubsan_overflow (loc, MULT_EXPR, lhs, arg0, arg1);
2218 else
2219 expand_mul_overflow (loc, lhs, arg0, arg1, false, false, false, true,
2220 NULL);
2221 }
2222
2223 /* Helper function for {ADD,SUB,MUL}_OVERFLOW call stmt expansion. */
2224
2225 static void
expand_arith_overflow(enum tree_code code,gimple * stmt)2226 expand_arith_overflow (enum tree_code code, gimple *stmt)
2227 {
2228 tree lhs = gimple_call_lhs (stmt);
2229 if (lhs == NULL_TREE)
2230 return;
2231 tree arg0 = gimple_call_arg (stmt, 0);
2232 tree arg1 = gimple_call_arg (stmt, 1);
2233 tree type = TREE_TYPE (TREE_TYPE (lhs));
2234 int uns0_p = TYPE_UNSIGNED (TREE_TYPE (arg0));
2235 int uns1_p = TYPE_UNSIGNED (TREE_TYPE (arg1));
2236 int unsr_p = TYPE_UNSIGNED (type);
2237 int prec0 = TYPE_PRECISION (TREE_TYPE (arg0));
2238 int prec1 = TYPE_PRECISION (TREE_TYPE (arg1));
2239 int precres = TYPE_PRECISION (type);
2240 location_t loc = gimple_location (stmt);
2241 if (!uns0_p && get_range_pos_neg (arg0) == 1)
2242 uns0_p = true;
2243 if (!uns1_p && get_range_pos_neg (arg1) == 1)
2244 uns1_p = true;
2245 int pr = get_min_precision (arg0, uns0_p ? UNSIGNED : SIGNED);
2246 prec0 = MIN (prec0, pr);
2247 pr = get_min_precision (arg1, uns1_p ? UNSIGNED : SIGNED);
2248 prec1 = MIN (prec1, pr);
2249
2250 /* If uns0_p && uns1_p, precop is minimum needed precision
2251 of unsigned type to hold the exact result, otherwise
2252 precop is minimum needed precision of signed type to
2253 hold the exact result. */
2254 int precop;
2255 if (code == MULT_EXPR)
2256 precop = prec0 + prec1 + (uns0_p != uns1_p);
2257 else
2258 {
2259 if (uns0_p == uns1_p)
2260 precop = MAX (prec0, prec1) + 1;
2261 else if (uns0_p)
2262 precop = MAX (prec0 + 1, prec1) + 1;
2263 else
2264 precop = MAX (prec0, prec1 + 1) + 1;
2265 }
2266 int orig_precres = precres;
2267
2268 do
2269 {
2270 if ((uns0_p && uns1_p)
2271 ? ((precop + !unsr_p) <= precres
2272 /* u1 - u2 -> ur can overflow, no matter what precision
2273 the result has. */
2274 && (code != MINUS_EXPR || !unsr_p))
2275 : (!unsr_p && precop <= precres))
2276 {
2277 /* The infinity precision result will always fit into result. */
2278 rtx target = expand_expr (lhs, NULL_RTX, VOIDmode, EXPAND_WRITE);
2279 write_complex_part (target, const0_rtx, true);
2280 scalar_int_mode mode = SCALAR_INT_TYPE_MODE (type);
2281 struct separate_ops ops;
2282 ops.code = code;
2283 ops.type = type;
2284 ops.op0 = fold_convert_loc (loc, type, arg0);
2285 ops.op1 = fold_convert_loc (loc, type, arg1);
2286 ops.op2 = NULL_TREE;
2287 ops.location = loc;
2288 rtx tem = expand_expr_real_2 (&ops, NULL_RTX, mode, EXPAND_NORMAL);
2289 expand_arith_overflow_result_store (lhs, target, mode, tem);
2290 return;
2291 }
2292
2293 /* For operations with low precision, if target doesn't have them, start
2294 with precres widening right away, otherwise do it only if the most
2295 simple cases can't be used. */
2296 const int min_precision = targetm.min_arithmetic_precision ();
2297 if (orig_precres == precres && precres < min_precision)
2298 ;
2299 else if ((uns0_p && uns1_p && unsr_p && prec0 <= precres
2300 && prec1 <= precres)
2301 || ((!uns0_p || !uns1_p) && !unsr_p
2302 && prec0 + uns0_p <= precres
2303 && prec1 + uns1_p <= precres))
2304 {
2305 arg0 = fold_convert_loc (loc, type, arg0);
2306 arg1 = fold_convert_loc (loc, type, arg1);
2307 switch (code)
2308 {
2309 case MINUS_EXPR:
2310 if (integer_zerop (arg0) && !unsr_p)
2311 {
2312 expand_neg_overflow (loc, lhs, arg1, false, NULL);
2313 return;
2314 }
2315 /* FALLTHRU */
2316 case PLUS_EXPR:
2317 expand_addsub_overflow (loc, code, lhs, arg0, arg1, unsr_p,
2318 unsr_p, unsr_p, false, NULL);
2319 return;
2320 case MULT_EXPR:
2321 expand_mul_overflow (loc, lhs, arg0, arg1, unsr_p,
2322 unsr_p, unsr_p, false, NULL);
2323 return;
2324 default:
2325 gcc_unreachable ();
2326 }
2327 }
2328
2329 /* For sub-word operations, retry with a wider type first. */
2330 if (orig_precres == precres && precop <= BITS_PER_WORD)
2331 {
2332 int p = MAX (min_precision, precop);
2333 scalar_int_mode m = smallest_int_mode_for_size (p);
2334 tree optype = build_nonstandard_integer_type (GET_MODE_PRECISION (m),
2335 uns0_p && uns1_p
2336 && unsr_p);
2337 p = TYPE_PRECISION (optype);
2338 if (p > precres)
2339 {
2340 precres = p;
2341 unsr_p = TYPE_UNSIGNED (optype);
2342 type = optype;
2343 continue;
2344 }
2345 }
2346
2347 if (prec0 <= precres && prec1 <= precres)
2348 {
2349 tree types[2];
2350 if (unsr_p)
2351 {
2352 types[0] = build_nonstandard_integer_type (precres, 0);
2353 types[1] = type;
2354 }
2355 else
2356 {
2357 types[0] = type;
2358 types[1] = build_nonstandard_integer_type (precres, 1);
2359 }
2360 arg0 = fold_convert_loc (loc, types[uns0_p], arg0);
2361 arg1 = fold_convert_loc (loc, types[uns1_p], arg1);
2362 if (code != MULT_EXPR)
2363 expand_addsub_overflow (loc, code, lhs, arg0, arg1, unsr_p,
2364 uns0_p, uns1_p, false, NULL);
2365 else
2366 expand_mul_overflow (loc, lhs, arg0, arg1, unsr_p,
2367 uns0_p, uns1_p, false, NULL);
2368 return;
2369 }
2370
2371 /* Retry with a wider type. */
2372 if (orig_precres == precres)
2373 {
2374 int p = MAX (prec0, prec1);
2375 scalar_int_mode m = smallest_int_mode_for_size (p);
2376 tree optype = build_nonstandard_integer_type (GET_MODE_PRECISION (m),
2377 uns0_p && uns1_p
2378 && unsr_p);
2379 p = TYPE_PRECISION (optype);
2380 if (p > precres)
2381 {
2382 precres = p;
2383 unsr_p = TYPE_UNSIGNED (optype);
2384 type = optype;
2385 continue;
2386 }
2387 }
2388
2389 gcc_unreachable ();
2390 }
2391 while (1);
2392 }
2393
2394 /* Expand ADD_OVERFLOW STMT. */
2395
2396 static void
expand_ADD_OVERFLOW(internal_fn,gcall * stmt)2397 expand_ADD_OVERFLOW (internal_fn, gcall *stmt)
2398 {
2399 expand_arith_overflow (PLUS_EXPR, stmt);
2400 }
2401
2402 /* Expand SUB_OVERFLOW STMT. */
2403
2404 static void
expand_SUB_OVERFLOW(internal_fn,gcall * stmt)2405 expand_SUB_OVERFLOW (internal_fn, gcall *stmt)
2406 {
2407 expand_arith_overflow (MINUS_EXPR, stmt);
2408 }
2409
2410 /* Expand MUL_OVERFLOW STMT. */
2411
2412 static void
expand_MUL_OVERFLOW(internal_fn,gcall * stmt)2413 expand_MUL_OVERFLOW (internal_fn, gcall *stmt)
2414 {
2415 expand_arith_overflow (MULT_EXPR, stmt);
2416 }
2417
2418 /* This should get folded in tree-vectorizer.c. */
2419
2420 static void
expand_LOOP_VECTORIZED(internal_fn,gcall *)2421 expand_LOOP_VECTORIZED (internal_fn, gcall *)
2422 {
2423 gcc_unreachable ();
2424 }
2425
2426 /* This should get folded in tree-vectorizer.c. */
2427
2428 static void
expand_LOOP_DIST_ALIAS(internal_fn,gcall *)2429 expand_LOOP_DIST_ALIAS (internal_fn, gcall *)
2430 {
2431 gcc_unreachable ();
2432 }
2433
2434 /* Return a memory reference of type TYPE for argument INDEX of STMT.
2435 Use argument INDEX + 1 to derive the second (TBAA) operand. */
2436
2437 static tree
expand_call_mem_ref(tree type,gcall * stmt,int index)2438 expand_call_mem_ref (tree type, gcall *stmt, int index)
2439 {
2440 tree addr = gimple_call_arg (stmt, index);
2441 tree alias_ptr_type = TREE_TYPE (gimple_call_arg (stmt, index + 1));
2442 unsigned int align = tree_to_shwi (gimple_call_arg (stmt, index + 1));
2443 if (TYPE_ALIGN (type) != align)
2444 type = build_aligned_type (type, align);
2445
2446 tree tmp = addr;
2447 if (TREE_CODE (tmp) == SSA_NAME)
2448 {
2449 gimple *def = SSA_NAME_DEF_STMT (tmp);
2450 if (gimple_assign_single_p (def))
2451 tmp = gimple_assign_rhs1 (def);
2452 }
2453
2454 if (TREE_CODE (tmp) == ADDR_EXPR)
2455 {
2456 tree mem = TREE_OPERAND (tmp, 0);
2457 if (TREE_CODE (mem) == TARGET_MEM_REF
2458 && types_compatible_p (TREE_TYPE (mem), type))
2459 {
2460 tree offset = TMR_OFFSET (mem);
2461 if (type != TREE_TYPE (mem)
2462 || alias_ptr_type != TREE_TYPE (offset)
2463 || !integer_zerop (offset))
2464 {
2465 mem = copy_node (mem);
2466 TMR_OFFSET (mem) = wide_int_to_tree (alias_ptr_type,
2467 wi::to_poly_wide (offset));
2468 TREE_TYPE (mem) = type;
2469 }
2470 return mem;
2471 }
2472 }
2473
2474 return fold_build2 (MEM_REF, type, addr, build_int_cst (alias_ptr_type, 0));
2475 }
2476
2477 /* Expand MASK_LOAD{,_LANES} call STMT using optab OPTAB. */
2478
2479 static void
expand_mask_load_optab_fn(internal_fn,gcall * stmt,convert_optab optab)2480 expand_mask_load_optab_fn (internal_fn, gcall *stmt, convert_optab optab)
2481 {
2482 struct expand_operand ops[3];
2483 tree type, lhs, rhs, maskt;
2484 rtx mem, target, mask;
2485 insn_code icode;
2486
2487 maskt = gimple_call_arg (stmt, 2);
2488 lhs = gimple_call_lhs (stmt);
2489 if (lhs == NULL_TREE)
2490 return;
2491 type = TREE_TYPE (lhs);
2492 rhs = expand_call_mem_ref (type, stmt, 0);
2493
2494 if (optab == vec_mask_load_lanes_optab)
2495 icode = get_multi_vector_move (type, optab);
2496 else
2497 icode = convert_optab_handler (optab, TYPE_MODE (type),
2498 TYPE_MODE (TREE_TYPE (maskt)));
2499
2500 mem = expand_expr (rhs, NULL_RTX, VOIDmode, EXPAND_WRITE);
2501 gcc_assert (MEM_P (mem));
2502 mask = expand_normal (maskt);
2503 target = expand_expr (lhs, NULL_RTX, VOIDmode, EXPAND_WRITE);
2504 create_output_operand (&ops[0], target, TYPE_MODE (type));
2505 create_fixed_operand (&ops[1], mem);
2506 create_input_operand (&ops[2], mask, TYPE_MODE (TREE_TYPE (maskt)));
2507 expand_insn (icode, 3, ops);
2508 }
2509
2510 #define expand_mask_load_lanes_optab_fn expand_mask_load_optab_fn
2511
2512 /* Expand MASK_STORE{,_LANES} call STMT using optab OPTAB. */
2513
2514 static void
expand_mask_store_optab_fn(internal_fn,gcall * stmt,convert_optab optab)2515 expand_mask_store_optab_fn (internal_fn, gcall *stmt, convert_optab optab)
2516 {
2517 struct expand_operand ops[3];
2518 tree type, lhs, rhs, maskt;
2519 rtx mem, reg, mask;
2520 insn_code icode;
2521
2522 maskt = gimple_call_arg (stmt, 2);
2523 rhs = gimple_call_arg (stmt, 3);
2524 type = TREE_TYPE (rhs);
2525 lhs = expand_call_mem_ref (type, stmt, 0);
2526
2527 if (optab == vec_mask_store_lanes_optab)
2528 icode = get_multi_vector_move (type, optab);
2529 else
2530 icode = convert_optab_handler (optab, TYPE_MODE (type),
2531 TYPE_MODE (TREE_TYPE (maskt)));
2532
2533 mem = expand_expr (lhs, NULL_RTX, VOIDmode, EXPAND_WRITE);
2534 gcc_assert (MEM_P (mem));
2535 mask = expand_normal (maskt);
2536 reg = expand_normal (rhs);
2537 create_fixed_operand (&ops[0], mem);
2538 create_input_operand (&ops[1], reg, TYPE_MODE (type));
2539 create_input_operand (&ops[2], mask, TYPE_MODE (TREE_TYPE (maskt)));
2540 expand_insn (icode, 3, ops);
2541 }
2542
2543 #define expand_mask_store_lanes_optab_fn expand_mask_store_optab_fn
2544
2545 static void
expand_ABNORMAL_DISPATCHER(internal_fn,gcall *)2546 expand_ABNORMAL_DISPATCHER (internal_fn, gcall *)
2547 {
2548 }
2549
2550 static void
expand_BUILTIN_EXPECT(internal_fn,gcall * stmt)2551 expand_BUILTIN_EXPECT (internal_fn, gcall *stmt)
2552 {
2553 /* When guessing was done, the hints should be already stripped away. */
2554 gcc_assert (!flag_guess_branch_prob || optimize == 0 || seen_error ());
2555
2556 rtx target;
2557 tree lhs = gimple_call_lhs (stmt);
2558 if (lhs)
2559 target = expand_expr (lhs, NULL_RTX, VOIDmode, EXPAND_WRITE);
2560 else
2561 target = const0_rtx;
2562 rtx val = expand_expr (gimple_call_arg (stmt, 0), target, VOIDmode, EXPAND_NORMAL);
2563 if (lhs && val != target)
2564 emit_move_insn (target, val);
2565 }
2566
2567 /* IFN_VA_ARG is supposed to be expanded at pass_stdarg. So this dummy function
2568 should never be called. */
2569
2570 static void
expand_VA_ARG(internal_fn,gcall *)2571 expand_VA_ARG (internal_fn, gcall *)
2572 {
2573 gcc_unreachable ();
2574 }
2575
2576 /* IFN_VEC_CONVERT is supposed to be expanded at pass_lower_vector. So this
2577 dummy function should never be called. */
2578
2579 static void
expand_VEC_CONVERT(internal_fn,gcall *)2580 expand_VEC_CONVERT (internal_fn, gcall *)
2581 {
2582 gcc_unreachable ();
2583 }
2584
2585 /* Expand the IFN_UNIQUE function according to its first argument. */
2586
2587 static void
expand_UNIQUE(internal_fn,gcall * stmt)2588 expand_UNIQUE (internal_fn, gcall *stmt)
2589 {
2590 rtx pattern = NULL_RTX;
2591 enum ifn_unique_kind kind
2592 = (enum ifn_unique_kind) TREE_INT_CST_LOW (gimple_call_arg (stmt, 0));
2593
2594 switch (kind)
2595 {
2596 default:
2597 gcc_unreachable ();
2598
2599 case IFN_UNIQUE_UNSPEC:
2600 if (targetm.have_unique ())
2601 pattern = targetm.gen_unique ();
2602 break;
2603
2604 case IFN_UNIQUE_OACC_FORK:
2605 case IFN_UNIQUE_OACC_JOIN:
2606 if (targetm.have_oacc_fork () && targetm.have_oacc_join ())
2607 {
2608 tree lhs = gimple_call_lhs (stmt);
2609 rtx target = const0_rtx;
2610
2611 if (lhs)
2612 target = expand_expr (lhs, NULL_RTX, VOIDmode, EXPAND_WRITE);
2613
2614 rtx data_dep = expand_normal (gimple_call_arg (stmt, 1));
2615 rtx axis = expand_normal (gimple_call_arg (stmt, 2));
2616
2617 if (kind == IFN_UNIQUE_OACC_FORK)
2618 pattern = targetm.gen_oacc_fork (target, data_dep, axis);
2619 else
2620 pattern = targetm.gen_oacc_join (target, data_dep, axis);
2621 }
2622 else
2623 gcc_unreachable ();
2624 break;
2625 }
2626
2627 if (pattern)
2628 emit_insn (pattern);
2629 }
2630
2631 /* The size of an OpenACC compute dimension. */
2632
2633 static void
expand_GOACC_DIM_SIZE(internal_fn,gcall * stmt)2634 expand_GOACC_DIM_SIZE (internal_fn, gcall *stmt)
2635 {
2636 tree lhs = gimple_call_lhs (stmt);
2637
2638 if (!lhs)
2639 return;
2640
2641 rtx target = expand_expr (lhs, NULL_RTX, VOIDmode, EXPAND_WRITE);
2642 if (targetm.have_oacc_dim_size ())
2643 {
2644 rtx dim = expand_expr (gimple_call_arg (stmt, 0), NULL_RTX,
2645 VOIDmode, EXPAND_NORMAL);
2646 emit_insn (targetm.gen_oacc_dim_size (target, dim));
2647 }
2648 else
2649 emit_move_insn (target, GEN_INT (1));
2650 }
2651
2652 /* The position of an OpenACC execution engine along one compute axis. */
2653
2654 static void
expand_GOACC_DIM_POS(internal_fn,gcall * stmt)2655 expand_GOACC_DIM_POS (internal_fn, gcall *stmt)
2656 {
2657 tree lhs = gimple_call_lhs (stmt);
2658
2659 if (!lhs)
2660 return;
2661
2662 rtx target = expand_expr (lhs, NULL_RTX, VOIDmode, EXPAND_WRITE);
2663 if (targetm.have_oacc_dim_pos ())
2664 {
2665 rtx dim = expand_expr (gimple_call_arg (stmt, 0), NULL_RTX,
2666 VOIDmode, EXPAND_NORMAL);
2667 emit_insn (targetm.gen_oacc_dim_pos (target, dim));
2668 }
2669 else
2670 emit_move_insn (target, const0_rtx);
2671 }
2672
2673 /* This is expanded by oacc_device_lower pass. */
2674
2675 static void
expand_GOACC_LOOP(internal_fn,gcall *)2676 expand_GOACC_LOOP (internal_fn, gcall *)
2677 {
2678 gcc_unreachable ();
2679 }
2680
2681 /* This is expanded by oacc_device_lower pass. */
2682
2683 static void
expand_GOACC_REDUCTION(internal_fn,gcall *)2684 expand_GOACC_REDUCTION (internal_fn, gcall *)
2685 {
2686 gcc_unreachable ();
2687 }
2688
2689 /* This is expanded by oacc_device_lower pass. */
2690
2691 static void
expand_GOACC_TILE(internal_fn,gcall *)2692 expand_GOACC_TILE (internal_fn, gcall *)
2693 {
2694 gcc_unreachable ();
2695 }
2696
2697 /* Set errno to EDOM. */
2698
2699 static void
expand_SET_EDOM(internal_fn,gcall *)2700 expand_SET_EDOM (internal_fn, gcall *)
2701 {
2702 #ifdef TARGET_EDOM
2703 #ifdef GEN_ERRNO_RTX
2704 rtx errno_rtx = GEN_ERRNO_RTX;
2705 #else
2706 rtx errno_rtx = gen_rtx_MEM (word_mode, gen_rtx_SYMBOL_REF (Pmode, "errno"));
2707 #endif
2708 emit_move_insn (errno_rtx,
2709 gen_int_mode (TARGET_EDOM, GET_MODE (errno_rtx)));
2710 #else
2711 gcc_unreachable ();
2712 #endif
2713 }
2714
2715 /* Expand atomic bit test and set. */
2716
2717 static void
expand_ATOMIC_BIT_TEST_AND_SET(internal_fn,gcall * call)2718 expand_ATOMIC_BIT_TEST_AND_SET (internal_fn, gcall *call)
2719 {
2720 expand_ifn_atomic_bit_test_and (call);
2721 }
2722
2723 /* Expand atomic bit test and complement. */
2724
2725 static void
expand_ATOMIC_BIT_TEST_AND_COMPLEMENT(internal_fn,gcall * call)2726 expand_ATOMIC_BIT_TEST_AND_COMPLEMENT (internal_fn, gcall *call)
2727 {
2728 expand_ifn_atomic_bit_test_and (call);
2729 }
2730
2731 /* Expand atomic bit test and reset. */
2732
2733 static void
expand_ATOMIC_BIT_TEST_AND_RESET(internal_fn,gcall * call)2734 expand_ATOMIC_BIT_TEST_AND_RESET (internal_fn, gcall *call)
2735 {
2736 expand_ifn_atomic_bit_test_and (call);
2737 }
2738
2739 /* Expand atomic bit test and set. */
2740
2741 static void
expand_ATOMIC_COMPARE_EXCHANGE(internal_fn,gcall * call)2742 expand_ATOMIC_COMPARE_EXCHANGE (internal_fn, gcall *call)
2743 {
2744 expand_ifn_atomic_compare_exchange (call);
2745 }
2746
2747 /* Expand LAUNDER to assignment, lhs = arg0. */
2748
2749 static void
expand_LAUNDER(internal_fn,gcall * call)2750 expand_LAUNDER (internal_fn, gcall *call)
2751 {
2752 tree lhs = gimple_call_lhs (call);
2753
2754 if (!lhs)
2755 return;
2756
2757 expand_assignment (lhs, gimple_call_arg (call, 0), false);
2758 }
2759
2760 /* Expand {MASK_,}SCATTER_STORE{S,U} call CALL using optab OPTAB. */
2761
2762 static void
expand_scatter_store_optab_fn(internal_fn,gcall * stmt,direct_optab optab)2763 expand_scatter_store_optab_fn (internal_fn, gcall *stmt, direct_optab optab)
2764 {
2765 internal_fn ifn = gimple_call_internal_fn (stmt);
2766 int rhs_index = internal_fn_stored_value_index (ifn);
2767 int mask_index = internal_fn_mask_index (ifn);
2768 tree base = gimple_call_arg (stmt, 0);
2769 tree offset = gimple_call_arg (stmt, 1);
2770 tree scale = gimple_call_arg (stmt, 2);
2771 tree rhs = gimple_call_arg (stmt, rhs_index);
2772
2773 rtx base_rtx = expand_normal (base);
2774 rtx offset_rtx = expand_normal (offset);
2775 HOST_WIDE_INT scale_int = tree_to_shwi (scale);
2776 rtx rhs_rtx = expand_normal (rhs);
2777
2778 struct expand_operand ops[6];
2779 int i = 0;
2780 create_address_operand (&ops[i++], base_rtx);
2781 create_input_operand (&ops[i++], offset_rtx, TYPE_MODE (TREE_TYPE (offset)));
2782 create_integer_operand (&ops[i++], TYPE_UNSIGNED (TREE_TYPE (offset)));
2783 create_integer_operand (&ops[i++], scale_int);
2784 create_input_operand (&ops[i++], rhs_rtx, TYPE_MODE (TREE_TYPE (rhs)));
2785 if (mask_index >= 0)
2786 {
2787 tree mask = gimple_call_arg (stmt, mask_index);
2788 rtx mask_rtx = expand_normal (mask);
2789 create_input_operand (&ops[i++], mask_rtx, TYPE_MODE (TREE_TYPE (mask)));
2790 }
2791
2792 insn_code icode = direct_optab_handler (optab, TYPE_MODE (TREE_TYPE (rhs)));
2793 expand_insn (icode, i, ops);
2794 }
2795
2796 /* Expand {MASK_,}GATHER_LOAD call CALL using optab OPTAB. */
2797
2798 static void
expand_gather_load_optab_fn(internal_fn,gcall * stmt,direct_optab optab)2799 expand_gather_load_optab_fn (internal_fn, gcall *stmt, direct_optab optab)
2800 {
2801 tree lhs = gimple_call_lhs (stmt);
2802 tree base = gimple_call_arg (stmt, 0);
2803 tree offset = gimple_call_arg (stmt, 1);
2804 tree scale = gimple_call_arg (stmt, 2);
2805
2806 rtx lhs_rtx = expand_expr (lhs, NULL_RTX, VOIDmode, EXPAND_WRITE);
2807 rtx base_rtx = expand_normal (base);
2808 rtx offset_rtx = expand_normal (offset);
2809 HOST_WIDE_INT scale_int = tree_to_shwi (scale);
2810
2811 int i = 0;
2812 struct expand_operand ops[6];
2813 create_output_operand (&ops[i++], lhs_rtx, TYPE_MODE (TREE_TYPE (lhs)));
2814 create_address_operand (&ops[i++], base_rtx);
2815 create_input_operand (&ops[i++], offset_rtx, TYPE_MODE (TREE_TYPE (offset)));
2816 create_integer_operand (&ops[i++], TYPE_UNSIGNED (TREE_TYPE (offset)));
2817 create_integer_operand (&ops[i++], scale_int);
2818 if (optab == mask_gather_load_optab)
2819 {
2820 tree mask = gimple_call_arg (stmt, 3);
2821 rtx mask_rtx = expand_normal (mask);
2822 create_input_operand (&ops[i++], mask_rtx, TYPE_MODE (TREE_TYPE (mask)));
2823 }
2824 insn_code icode = direct_optab_handler (optab, TYPE_MODE (TREE_TYPE (lhs)));
2825 expand_insn (icode, i, ops);
2826 }
2827
2828 /* Expand DIVMOD() using:
2829 a) optab handler for udivmod/sdivmod if it is available.
2830 b) If optab_handler doesn't exist, generate call to
2831 target-specific divmod libfunc. */
2832
2833 static void
expand_DIVMOD(internal_fn,gcall * call_stmt)2834 expand_DIVMOD (internal_fn, gcall *call_stmt)
2835 {
2836 tree lhs = gimple_call_lhs (call_stmt);
2837 tree arg0 = gimple_call_arg (call_stmt, 0);
2838 tree arg1 = gimple_call_arg (call_stmt, 1);
2839
2840 gcc_assert (TREE_CODE (TREE_TYPE (lhs)) == COMPLEX_TYPE);
2841 tree type = TREE_TYPE (TREE_TYPE (lhs));
2842 machine_mode mode = TYPE_MODE (type);
2843 bool unsignedp = TYPE_UNSIGNED (type);
2844 optab tab = (unsignedp) ? udivmod_optab : sdivmod_optab;
2845
2846 rtx op0 = expand_normal (arg0);
2847 rtx op1 = expand_normal (arg1);
2848 rtx target = expand_expr (lhs, NULL_RTX, VOIDmode, EXPAND_WRITE);
2849
2850 rtx quotient, remainder, libfunc;
2851
2852 /* Check if optab_handler exists for divmod_optab for given mode. */
2853 if (optab_handler (tab, mode) != CODE_FOR_nothing)
2854 {
2855 quotient = gen_reg_rtx (mode);
2856 remainder = gen_reg_rtx (mode);
2857 expand_twoval_binop (tab, op0, op1, quotient, remainder, unsignedp);
2858 }
2859
2860 /* Generate call to divmod libfunc if it exists. */
2861 else if ((libfunc = optab_libfunc (tab, mode)) != NULL_RTX)
2862 targetm.expand_divmod_libfunc (libfunc, mode, op0, op1,
2863 "ient, &remainder);
2864
2865 else
2866 gcc_unreachable ();
2867
2868 /* Wrap the return value (quotient, remainder) within COMPLEX_EXPR. */
2869 expand_expr (build2 (COMPLEX_EXPR, TREE_TYPE (lhs),
2870 make_tree (TREE_TYPE (arg0), quotient),
2871 make_tree (TREE_TYPE (arg1), remainder)),
2872 target, VOIDmode, EXPAND_NORMAL);
2873 }
2874
2875 /* Expand a NOP. */
2876
2877 static void
expand_NOP(internal_fn,gcall *)2878 expand_NOP (internal_fn, gcall *)
2879 {
2880 /* Nothing. But it shouldn't really prevail. */
2881 }
2882
2883 /* Expand a call to FN using the operands in STMT. FN has a single
2884 output operand and NARGS input operands. */
2885
2886 static void
expand_direct_optab_fn(internal_fn fn,gcall * stmt,direct_optab optab,unsigned int nargs)2887 expand_direct_optab_fn (internal_fn fn, gcall *stmt, direct_optab optab,
2888 unsigned int nargs)
2889 {
2890 expand_operand *ops = XALLOCAVEC (expand_operand, nargs + 1);
2891
2892 tree_pair types = direct_internal_fn_types (fn, stmt);
2893 insn_code icode = direct_optab_handler (optab, TYPE_MODE (types.first));
2894 gcc_assert (icode != CODE_FOR_nothing);
2895
2896 tree lhs = gimple_call_lhs (stmt);
2897 rtx lhs_rtx = NULL_RTX;
2898 if (lhs)
2899 lhs_rtx = expand_expr (lhs, NULL_RTX, VOIDmode, EXPAND_WRITE);
2900
2901 /* Do not assign directly to a promoted subreg, since there is no
2902 guarantee that the instruction will leave the upper bits of the
2903 register in the state required by SUBREG_PROMOTED_SIGN. */
2904 rtx dest = lhs_rtx;
2905 if (dest && GET_CODE (dest) == SUBREG && SUBREG_PROMOTED_VAR_P (dest))
2906 dest = NULL_RTX;
2907
2908 create_output_operand (&ops[0], dest, insn_data[icode].operand[0].mode);
2909
2910 for (unsigned int i = 0; i < nargs; ++i)
2911 {
2912 tree rhs = gimple_call_arg (stmt, i);
2913 tree rhs_type = TREE_TYPE (rhs);
2914 rtx rhs_rtx = expand_normal (rhs);
2915 if (INTEGRAL_TYPE_P (rhs_type))
2916 create_convert_operand_from (&ops[i + 1], rhs_rtx,
2917 TYPE_MODE (rhs_type),
2918 TYPE_UNSIGNED (rhs_type));
2919 else
2920 create_input_operand (&ops[i + 1], rhs_rtx, TYPE_MODE (rhs_type));
2921 }
2922
2923 expand_insn (icode, nargs + 1, ops);
2924 if (lhs_rtx && !rtx_equal_p (lhs_rtx, ops[0].value))
2925 {
2926 /* If the return value has an integral type, convert the instruction
2927 result to that type. This is useful for things that return an
2928 int regardless of the size of the input. If the instruction result
2929 is smaller than required, assume that it is signed.
2930
2931 If the return value has a nonintegral type, its mode must match
2932 the instruction result. */
2933 if (GET_CODE (lhs_rtx) == SUBREG && SUBREG_PROMOTED_VAR_P (lhs_rtx))
2934 {
2935 /* If this is a scalar in a register that is stored in a wider
2936 mode than the declared mode, compute the result into its
2937 declared mode and then convert to the wider mode. */
2938 gcc_checking_assert (INTEGRAL_TYPE_P (TREE_TYPE (lhs)));
2939 rtx tmp = convert_to_mode (GET_MODE (lhs_rtx), ops[0].value, 0);
2940 convert_move (SUBREG_REG (lhs_rtx), tmp,
2941 SUBREG_PROMOTED_SIGN (lhs_rtx));
2942 }
2943 else if (GET_MODE (lhs_rtx) == GET_MODE (ops[0].value))
2944 emit_move_insn (lhs_rtx, ops[0].value);
2945 else
2946 {
2947 gcc_checking_assert (INTEGRAL_TYPE_P (TREE_TYPE (lhs)));
2948 convert_move (lhs_rtx, ops[0].value, 0);
2949 }
2950 }
2951 }
2952
2953 /* Expand WHILE_ULT call STMT using optab OPTAB. */
2954
2955 static void
expand_while_optab_fn(internal_fn,gcall * stmt,convert_optab optab)2956 expand_while_optab_fn (internal_fn, gcall *stmt, convert_optab optab)
2957 {
2958 expand_operand ops[3];
2959 tree rhs_type[2];
2960
2961 tree lhs = gimple_call_lhs (stmt);
2962 tree lhs_type = TREE_TYPE (lhs);
2963 rtx lhs_rtx = expand_expr (lhs, NULL_RTX, VOIDmode, EXPAND_WRITE);
2964 create_output_operand (&ops[0], lhs_rtx, TYPE_MODE (lhs_type));
2965
2966 for (unsigned int i = 0; i < 2; ++i)
2967 {
2968 tree rhs = gimple_call_arg (stmt, i);
2969 rhs_type[i] = TREE_TYPE (rhs);
2970 rtx rhs_rtx = expand_normal (rhs);
2971 create_input_operand (&ops[i + 1], rhs_rtx, TYPE_MODE (rhs_type[i]));
2972 }
2973
2974 insn_code icode = convert_optab_handler (optab, TYPE_MODE (rhs_type[0]),
2975 TYPE_MODE (lhs_type));
2976
2977 expand_insn (icode, 3, ops);
2978 if (!rtx_equal_p (lhs_rtx, ops[0].value))
2979 emit_move_insn (lhs_rtx, ops[0].value);
2980 }
2981
2982 /* Expanders for optabs that can use expand_direct_optab_fn. */
2983
2984 #define expand_unary_optab_fn(FN, STMT, OPTAB) \
2985 expand_direct_optab_fn (FN, STMT, OPTAB, 1)
2986
2987 #define expand_binary_optab_fn(FN, STMT, OPTAB) \
2988 expand_direct_optab_fn (FN, STMT, OPTAB, 2)
2989
2990 #define expand_ternary_optab_fn(FN, STMT, OPTAB) \
2991 expand_direct_optab_fn (FN, STMT, OPTAB, 3)
2992
2993 #define expand_cond_unary_optab_fn(FN, STMT, OPTAB) \
2994 expand_direct_optab_fn (FN, STMT, OPTAB, 3)
2995
2996 #define expand_cond_binary_optab_fn(FN, STMT, OPTAB) \
2997 expand_direct_optab_fn (FN, STMT, OPTAB, 4)
2998
2999 #define expand_cond_ternary_optab_fn(FN, STMT, OPTAB) \
3000 expand_direct_optab_fn (FN, STMT, OPTAB, 5)
3001
3002 #define expand_fold_extract_optab_fn(FN, STMT, OPTAB) \
3003 expand_direct_optab_fn (FN, STMT, OPTAB, 3)
3004
3005 #define expand_fold_left_optab_fn(FN, STMT, OPTAB) \
3006 expand_direct_optab_fn (FN, STMT, OPTAB, 2)
3007
3008 /* RETURN_TYPE and ARGS are a return type and argument list that are
3009 in principle compatible with FN (which satisfies direct_internal_fn_p).
3010 Return the types that should be used to determine whether the
3011 target supports FN. */
3012
3013 tree_pair
direct_internal_fn_types(internal_fn fn,tree return_type,tree * args)3014 direct_internal_fn_types (internal_fn fn, tree return_type, tree *args)
3015 {
3016 const direct_internal_fn_info &info = direct_internal_fn (fn);
3017 tree type0 = (info.type0 < 0 ? return_type : TREE_TYPE (args[info.type0]));
3018 tree type1 = (info.type1 < 0 ? return_type : TREE_TYPE (args[info.type1]));
3019 return tree_pair (type0, type1);
3020 }
3021
3022 /* CALL is a call whose return type and arguments are in principle
3023 compatible with FN (which satisfies direct_internal_fn_p). Return the
3024 types that should be used to determine whether the target supports FN. */
3025
3026 tree_pair
direct_internal_fn_types(internal_fn fn,gcall * call)3027 direct_internal_fn_types (internal_fn fn, gcall *call)
3028 {
3029 const direct_internal_fn_info &info = direct_internal_fn (fn);
3030 tree op0 = (info.type0 < 0
3031 ? gimple_call_lhs (call)
3032 : gimple_call_arg (call, info.type0));
3033 tree op1 = (info.type1 < 0
3034 ? gimple_call_lhs (call)
3035 : gimple_call_arg (call, info.type1));
3036 return tree_pair (TREE_TYPE (op0), TREE_TYPE (op1));
3037 }
3038
3039 /* Return true if OPTAB is supported for TYPES (whose modes should be
3040 the same) when the optimization type is OPT_TYPE. Used for simple
3041 direct optabs. */
3042
3043 static bool
direct_optab_supported_p(direct_optab optab,tree_pair types,optimization_type opt_type)3044 direct_optab_supported_p (direct_optab optab, tree_pair types,
3045 optimization_type opt_type)
3046 {
3047 machine_mode mode = TYPE_MODE (types.first);
3048 gcc_checking_assert (mode == TYPE_MODE (types.second));
3049 return direct_optab_handler (optab, mode, opt_type) != CODE_FOR_nothing;
3050 }
3051
3052 /* Return true if OPTAB is supported for TYPES, where the first type
3053 is the destination and the second type is the source. Used for
3054 convert optabs. */
3055
3056 static bool
convert_optab_supported_p(convert_optab optab,tree_pair types,optimization_type opt_type)3057 convert_optab_supported_p (convert_optab optab, tree_pair types,
3058 optimization_type opt_type)
3059 {
3060 return (convert_optab_handler (optab, TYPE_MODE (types.first),
3061 TYPE_MODE (types.second), opt_type)
3062 != CODE_FOR_nothing);
3063 }
3064
3065 /* Return true if load/store lanes optab OPTAB is supported for
3066 array type TYPES.first when the optimization type is OPT_TYPE. */
3067
3068 static bool
multi_vector_optab_supported_p(convert_optab optab,tree_pair types,optimization_type opt_type)3069 multi_vector_optab_supported_p (convert_optab optab, tree_pair types,
3070 optimization_type opt_type)
3071 {
3072 gcc_assert (TREE_CODE (types.first) == ARRAY_TYPE);
3073 machine_mode imode = TYPE_MODE (types.first);
3074 machine_mode vmode = TYPE_MODE (TREE_TYPE (types.first));
3075 return (convert_optab_handler (optab, imode, vmode, opt_type)
3076 != CODE_FOR_nothing);
3077 }
3078
3079 #define direct_unary_optab_supported_p direct_optab_supported_p
3080 #define direct_binary_optab_supported_p direct_optab_supported_p
3081 #define direct_ternary_optab_supported_p direct_optab_supported_p
3082 #define direct_cond_unary_optab_supported_p direct_optab_supported_p
3083 #define direct_cond_binary_optab_supported_p direct_optab_supported_p
3084 #define direct_cond_ternary_optab_supported_p direct_optab_supported_p
3085 #define direct_mask_load_optab_supported_p direct_optab_supported_p
3086 #define direct_load_lanes_optab_supported_p multi_vector_optab_supported_p
3087 #define direct_mask_load_lanes_optab_supported_p multi_vector_optab_supported_p
3088 #define direct_gather_load_optab_supported_p direct_optab_supported_p
3089 #define direct_mask_store_optab_supported_p direct_optab_supported_p
3090 #define direct_store_lanes_optab_supported_p multi_vector_optab_supported_p
3091 #define direct_mask_store_lanes_optab_supported_p multi_vector_optab_supported_p
3092 #define direct_scatter_store_optab_supported_p direct_optab_supported_p
3093 #define direct_while_optab_supported_p convert_optab_supported_p
3094 #define direct_fold_extract_optab_supported_p direct_optab_supported_p
3095 #define direct_fold_left_optab_supported_p direct_optab_supported_p
3096
3097 /* Return the optab used by internal function FN. */
3098
3099 static optab
direct_internal_fn_optab(internal_fn fn,tree_pair types)3100 direct_internal_fn_optab (internal_fn fn, tree_pair types)
3101 {
3102 switch (fn)
3103 {
3104 #define DEF_INTERNAL_FN(CODE, FLAGS, FNSPEC) \
3105 case IFN_##CODE: break;
3106 #define DEF_INTERNAL_OPTAB_FN(CODE, FLAGS, OPTAB, TYPE) \
3107 case IFN_##CODE: return OPTAB##_optab;
3108 #define DEF_INTERNAL_SIGNED_OPTAB_FN(CODE, FLAGS, SELECTOR, SIGNED_OPTAB, \
3109 UNSIGNED_OPTAB, TYPE) \
3110 case IFN_##CODE: return (TYPE_UNSIGNED (types.SELECTOR) \
3111 ? UNSIGNED_OPTAB ## _optab \
3112 : SIGNED_OPTAB ## _optab);
3113 #include "internal-fn.def"
3114
3115 case IFN_LAST:
3116 break;
3117 }
3118 gcc_unreachable ();
3119 }
3120
3121 /* Return the optab used by internal function FN. */
3122
3123 static optab
direct_internal_fn_optab(internal_fn fn)3124 direct_internal_fn_optab (internal_fn fn)
3125 {
3126 switch (fn)
3127 {
3128 #define DEF_INTERNAL_FN(CODE, FLAGS, FNSPEC) \
3129 case IFN_##CODE: break;
3130 #define DEF_INTERNAL_OPTAB_FN(CODE, FLAGS, OPTAB, TYPE) \
3131 case IFN_##CODE: return OPTAB##_optab;
3132 #include "internal-fn.def"
3133
3134 case IFN_LAST:
3135 break;
3136 }
3137 gcc_unreachable ();
3138 }
3139
3140 /* Return true if FN is supported for the types in TYPES when the
3141 optimization type is OPT_TYPE. The types are those associated with
3142 the "type0" and "type1" fields of FN's direct_internal_fn_info
3143 structure. */
3144
3145 bool
direct_internal_fn_supported_p(internal_fn fn,tree_pair types,optimization_type opt_type)3146 direct_internal_fn_supported_p (internal_fn fn, tree_pair types,
3147 optimization_type opt_type)
3148 {
3149 switch (fn)
3150 {
3151 #define DEF_INTERNAL_FN(CODE, FLAGS, FNSPEC) \
3152 case IFN_##CODE: break;
3153 #define DEF_INTERNAL_OPTAB_FN(CODE, FLAGS, OPTAB, TYPE) \
3154 case IFN_##CODE: \
3155 return direct_##TYPE##_optab_supported_p (OPTAB##_optab, types, \
3156 opt_type);
3157 #define DEF_INTERNAL_SIGNED_OPTAB_FN(CODE, FLAGS, SELECTOR, SIGNED_OPTAB, \
3158 UNSIGNED_OPTAB, TYPE) \
3159 case IFN_##CODE: \
3160 { \
3161 optab which_optab = (TYPE_UNSIGNED (types.SELECTOR) \
3162 ? UNSIGNED_OPTAB ## _optab \
3163 : SIGNED_OPTAB ## _optab); \
3164 return direct_##TYPE##_optab_supported_p (which_optab, types, \
3165 opt_type); \
3166 }
3167 #include "internal-fn.def"
3168
3169 case IFN_LAST:
3170 break;
3171 }
3172 gcc_unreachable ();
3173 }
3174
3175 /* Return true if FN is supported for type TYPE when the optimization
3176 type is OPT_TYPE. The caller knows that the "type0" and "type1"
3177 fields of FN's direct_internal_fn_info structure are the same. */
3178
3179 bool
direct_internal_fn_supported_p(internal_fn fn,tree type,optimization_type opt_type)3180 direct_internal_fn_supported_p (internal_fn fn, tree type,
3181 optimization_type opt_type)
3182 {
3183 const direct_internal_fn_info &info = direct_internal_fn (fn);
3184 gcc_checking_assert (info.type0 == info.type1);
3185 return direct_internal_fn_supported_p (fn, tree_pair (type, type), opt_type);
3186 }
3187
3188 /* Return true if the STMT is supported when the optimization type is OPT_TYPE,
3189 given that STMT is a call to a direct internal function. */
3190
3191 bool
direct_internal_fn_supported_p(gcall * stmt,optimization_type opt_type)3192 direct_internal_fn_supported_p (gcall *stmt, optimization_type opt_type)
3193 {
3194 internal_fn fn = gimple_call_internal_fn (stmt);
3195 tree_pair types = direct_internal_fn_types (fn, stmt);
3196 return direct_internal_fn_supported_p (fn, types, opt_type);
3197 }
3198
3199 /* If FN is commutative in two consecutive arguments, return the
3200 index of the first, otherwise return -1. */
3201
3202 int
first_commutative_argument(internal_fn fn)3203 first_commutative_argument (internal_fn fn)
3204 {
3205 switch (fn)
3206 {
3207 case IFN_FMA:
3208 case IFN_FMS:
3209 case IFN_FNMA:
3210 case IFN_FNMS:
3211 case IFN_AVG_FLOOR:
3212 case IFN_AVG_CEIL:
3213 case IFN_FMIN:
3214 case IFN_FMAX:
3215 return 0;
3216
3217 case IFN_COND_ADD:
3218 case IFN_COND_MUL:
3219 case IFN_COND_MIN:
3220 case IFN_COND_MAX:
3221 case IFN_COND_AND:
3222 case IFN_COND_IOR:
3223 case IFN_COND_XOR:
3224 case IFN_COND_FMA:
3225 case IFN_COND_FMS:
3226 case IFN_COND_FNMA:
3227 case IFN_COND_FNMS:
3228 return 1;
3229
3230 default:
3231 return -1;
3232 }
3233 }
3234
3235 /* Return true if IFN_SET_EDOM is supported. */
3236
3237 bool
set_edom_supported_p(void)3238 set_edom_supported_p (void)
3239 {
3240 #ifdef TARGET_EDOM
3241 return true;
3242 #else
3243 return false;
3244 #endif
3245 }
3246
3247 #define DEF_INTERNAL_OPTAB_FN(CODE, FLAGS, OPTAB, TYPE) \
3248 static void \
3249 expand_##CODE (internal_fn fn, gcall *stmt) \
3250 { \
3251 expand_##TYPE##_optab_fn (fn, stmt, OPTAB##_optab); \
3252 }
3253 #define DEF_INTERNAL_SIGNED_OPTAB_FN(CODE, FLAGS, SELECTOR, SIGNED_OPTAB, \
3254 UNSIGNED_OPTAB, TYPE) \
3255 static void \
3256 expand_##CODE (internal_fn fn, gcall *stmt) \
3257 { \
3258 tree_pair types = direct_internal_fn_types (fn, stmt); \
3259 optab which_optab = direct_internal_fn_optab (fn, types); \
3260 expand_##TYPE##_optab_fn (fn, stmt, which_optab); \
3261 }
3262 #include "internal-fn.def"
3263
3264 /* Routines to expand each internal function, indexed by function number.
3265 Each routine has the prototype:
3266
3267 expand_<NAME> (gcall *stmt)
3268
3269 where STMT is the statement that performs the call. */
3270 static void (*const internal_fn_expanders[]) (internal_fn, gcall *) = {
3271 #define DEF_INTERNAL_FN(CODE, FLAGS, FNSPEC) expand_##CODE,
3272 #include "internal-fn.def"
3273 0
3274 };
3275
3276 /* Invoke T(CODE, IFN) for each conditional function IFN that maps to a
3277 tree code CODE. */
3278 #define FOR_EACH_CODE_MAPPING(T) \
3279 T (PLUS_EXPR, IFN_COND_ADD) \
3280 T (MINUS_EXPR, IFN_COND_SUB) \
3281 T (MULT_EXPR, IFN_COND_MUL) \
3282 T (TRUNC_DIV_EXPR, IFN_COND_DIV) \
3283 T (TRUNC_MOD_EXPR, IFN_COND_MOD) \
3284 T (RDIV_EXPR, IFN_COND_RDIV) \
3285 T (MIN_EXPR, IFN_COND_MIN) \
3286 T (MAX_EXPR, IFN_COND_MAX) \
3287 T (BIT_AND_EXPR, IFN_COND_AND) \
3288 T (BIT_IOR_EXPR, IFN_COND_IOR) \
3289 T (BIT_XOR_EXPR, IFN_COND_XOR)
3290
3291 /* Return a function that only performs CODE when a certain condition is met
3292 and that uses a given fallback value otherwise. For example, if CODE is
3293 a binary operation associated with conditional function FN:
3294
3295 LHS = FN (COND, A, B, ELSE)
3296
3297 is equivalent to the C expression:
3298
3299 LHS = COND ? A CODE B : ELSE;
3300
3301 operating elementwise if the operands are vectors.
3302
3303 Return IFN_LAST if no such function exists. */
3304
3305 internal_fn
get_conditional_internal_fn(tree_code code)3306 get_conditional_internal_fn (tree_code code)
3307 {
3308 switch (code)
3309 {
3310 #define CASE(CODE, IFN) case CODE: return IFN;
3311 FOR_EACH_CODE_MAPPING(CASE)
3312 #undef CASE
3313 default:
3314 return IFN_LAST;
3315 }
3316 }
3317
3318 /* If IFN implements the conditional form of a tree code, return that
3319 tree code, otherwise return ERROR_MARK. */
3320
3321 tree_code
conditional_internal_fn_code(internal_fn ifn)3322 conditional_internal_fn_code (internal_fn ifn)
3323 {
3324 switch (ifn)
3325 {
3326 #define CASE(CODE, IFN) case IFN: return CODE;
3327 FOR_EACH_CODE_MAPPING(CASE)
3328 #undef CASE
3329 default:
3330 return ERROR_MARK;
3331 }
3332 }
3333
3334 /* Invoke T(IFN) for each internal function IFN that also has an
3335 IFN_COND_* form. */
3336 #define FOR_EACH_COND_FN_PAIR(T) \
3337 T (FMA) \
3338 T (FMS) \
3339 T (FNMA) \
3340 T (FNMS)
3341
3342 /* Return a function that only performs internal function FN when a
3343 certain condition is met and that uses a given fallback value otherwise.
3344 In other words, the returned function FN' is such that:
3345
3346 LHS = FN' (COND, A1, ... An, ELSE)
3347
3348 is equivalent to the C expression:
3349
3350 LHS = COND ? FN (A1, ..., An) : ELSE;
3351
3352 operating elementwise if the operands are vectors.
3353
3354 Return IFN_LAST if no such function exists. */
3355
3356 internal_fn
get_conditional_internal_fn(internal_fn fn)3357 get_conditional_internal_fn (internal_fn fn)
3358 {
3359 switch (fn)
3360 {
3361 #define CASE(NAME) case IFN_##NAME: return IFN_COND_##NAME;
3362 FOR_EACH_COND_FN_PAIR(CASE)
3363 #undef CASE
3364 default:
3365 return IFN_LAST;
3366 }
3367 }
3368
3369 /* If IFN implements the conditional form of an unconditional internal
3370 function, return that unconditional function, otherwise return IFN_LAST. */
3371
3372 internal_fn
get_unconditional_internal_fn(internal_fn ifn)3373 get_unconditional_internal_fn (internal_fn ifn)
3374 {
3375 switch (ifn)
3376 {
3377 #define CASE(NAME) case IFN_COND_##NAME: return IFN_##NAME;
3378 FOR_EACH_COND_FN_PAIR(CASE)
3379 #undef CASE
3380 default:
3381 return IFN_LAST;
3382 }
3383 }
3384
3385 /* Return true if STMT can be interpreted as a conditional tree code
3386 operation of the form:
3387
3388 LHS = COND ? OP (RHS1, ...) : ELSE;
3389
3390 operating elementwise if the operands are vectors. This includes
3391 the case of an all-true COND, so that the operation always happens.
3392
3393 When returning true, set:
3394
3395 - *COND_OUT to the condition COND, or to NULL_TREE if the condition
3396 is known to be all-true
3397 - *CODE_OUT to the tree code
3398 - OPS[I] to operand I of *CODE_OUT
3399 - *ELSE_OUT to the fallback value ELSE, or to NULL_TREE if the
3400 condition is known to be all true. */
3401
3402 bool
can_interpret_as_conditional_op_p(gimple * stmt,tree * cond_out,tree_code * code_out,tree (& ops)[3],tree * else_out)3403 can_interpret_as_conditional_op_p (gimple *stmt, tree *cond_out,
3404 tree_code *code_out,
3405 tree (&ops)[3], tree *else_out)
3406 {
3407 if (gassign *assign = dyn_cast <gassign *> (stmt))
3408 {
3409 *cond_out = NULL_TREE;
3410 *code_out = gimple_assign_rhs_code (assign);
3411 ops[0] = gimple_assign_rhs1 (assign);
3412 ops[1] = gimple_assign_rhs2 (assign);
3413 ops[2] = gimple_assign_rhs3 (assign);
3414 *else_out = NULL_TREE;
3415 return true;
3416 }
3417 if (gcall *call = dyn_cast <gcall *> (stmt))
3418 if (gimple_call_internal_p (call))
3419 {
3420 internal_fn ifn = gimple_call_internal_fn (call);
3421 tree_code code = conditional_internal_fn_code (ifn);
3422 if (code != ERROR_MARK)
3423 {
3424 *cond_out = gimple_call_arg (call, 0);
3425 *code_out = code;
3426 unsigned int nops = gimple_call_num_args (call) - 2;
3427 for (unsigned int i = 0; i < 3; ++i)
3428 ops[i] = i < nops ? gimple_call_arg (call, i + 1) : NULL_TREE;
3429 *else_out = gimple_call_arg (call, nops + 1);
3430 if (integer_truep (*cond_out))
3431 {
3432 *cond_out = NULL_TREE;
3433 *else_out = NULL_TREE;
3434 }
3435 return true;
3436 }
3437 }
3438 return false;
3439 }
3440
3441 /* Return true if IFN is some form of load from memory. */
3442
3443 bool
internal_load_fn_p(internal_fn fn)3444 internal_load_fn_p (internal_fn fn)
3445 {
3446 switch (fn)
3447 {
3448 case IFN_MASK_LOAD:
3449 case IFN_LOAD_LANES:
3450 case IFN_MASK_LOAD_LANES:
3451 case IFN_GATHER_LOAD:
3452 case IFN_MASK_GATHER_LOAD:
3453 return true;
3454
3455 default:
3456 return false;
3457 }
3458 }
3459
3460 /* Return true if IFN is some form of store to memory. */
3461
3462 bool
internal_store_fn_p(internal_fn fn)3463 internal_store_fn_p (internal_fn fn)
3464 {
3465 switch (fn)
3466 {
3467 case IFN_MASK_STORE:
3468 case IFN_STORE_LANES:
3469 case IFN_MASK_STORE_LANES:
3470 case IFN_SCATTER_STORE:
3471 case IFN_MASK_SCATTER_STORE:
3472 return true;
3473
3474 default:
3475 return false;
3476 }
3477 }
3478
3479 /* Return true if IFN is some form of gather load or scatter store. */
3480
3481 bool
internal_gather_scatter_fn_p(internal_fn fn)3482 internal_gather_scatter_fn_p (internal_fn fn)
3483 {
3484 switch (fn)
3485 {
3486 case IFN_GATHER_LOAD:
3487 case IFN_MASK_GATHER_LOAD:
3488 case IFN_SCATTER_STORE:
3489 case IFN_MASK_SCATTER_STORE:
3490 return true;
3491
3492 default:
3493 return false;
3494 }
3495 }
3496
3497 /* If FN takes a vector mask argument, return the index of that argument,
3498 otherwise return -1. */
3499
3500 int
internal_fn_mask_index(internal_fn fn)3501 internal_fn_mask_index (internal_fn fn)
3502 {
3503 switch (fn)
3504 {
3505 case IFN_MASK_LOAD:
3506 case IFN_MASK_LOAD_LANES:
3507 case IFN_MASK_STORE:
3508 case IFN_MASK_STORE_LANES:
3509 return 2;
3510
3511 case IFN_MASK_GATHER_LOAD:
3512 return 3;
3513
3514 case IFN_MASK_SCATTER_STORE:
3515 return 4;
3516
3517 default:
3518 return (conditional_internal_fn_code (fn) != ERROR_MARK
3519 || get_unconditional_internal_fn (fn) != IFN_LAST ? 0 : -1);
3520 }
3521 }
3522
3523 /* If FN takes a value that should be stored to memory, return the index
3524 of that argument, otherwise return -1. */
3525
3526 int
internal_fn_stored_value_index(internal_fn fn)3527 internal_fn_stored_value_index (internal_fn fn)
3528 {
3529 switch (fn)
3530 {
3531 case IFN_MASK_STORE:
3532 case IFN_SCATTER_STORE:
3533 case IFN_MASK_SCATTER_STORE:
3534 return 3;
3535
3536 default:
3537 return -1;
3538 }
3539 }
3540
3541 /* Return true if the target supports gather load or scatter store function
3542 IFN. For loads, VECTOR_TYPE is the vector type of the load result,
3543 while for stores it is the vector type of the stored data argument.
3544 MEMORY_ELEMENT_TYPE is the type of the memory elements being loaded
3545 or stored. OFFSET_SIGN is the sign of the offset argument, which is
3546 only relevant when the offset is narrower than an address. SCALE is
3547 the amount by which the offset should be multiplied *after* it has
3548 been extended to address width. */
3549
3550 bool
internal_gather_scatter_fn_supported_p(internal_fn ifn,tree vector_type,tree memory_element_type,signop offset_sign,int scale)3551 internal_gather_scatter_fn_supported_p (internal_fn ifn, tree vector_type,
3552 tree memory_element_type,
3553 signop offset_sign, int scale)
3554 {
3555 if (!tree_int_cst_equal (TYPE_SIZE (TREE_TYPE (vector_type)),
3556 TYPE_SIZE (memory_element_type)))
3557 return false;
3558 optab optab = direct_internal_fn_optab (ifn);
3559 insn_code icode = direct_optab_handler (optab, TYPE_MODE (vector_type));
3560 int output_ops = internal_load_fn_p (ifn) ? 1 : 0;
3561 return (icode != CODE_FOR_nothing
3562 && insn_operand_matches (icode, 2 + output_ops,
3563 GEN_INT (offset_sign == UNSIGNED))
3564 && insn_operand_matches (icode, 3 + output_ops,
3565 GEN_INT (scale)));
3566 }
3567
3568 /* Expand STMT as though it were a call to internal function FN. */
3569
3570 void
expand_internal_call(internal_fn fn,gcall * stmt)3571 expand_internal_call (internal_fn fn, gcall *stmt)
3572 {
3573 internal_fn_expanders[fn] (fn, stmt);
3574 }
3575
3576 /* Expand STMT, which is a call to internal function FN. */
3577
3578 void
expand_internal_call(gcall * stmt)3579 expand_internal_call (gcall *stmt)
3580 {
3581 expand_internal_call (gimple_call_internal_fn (stmt), stmt);
3582 }
3583
3584 /* If TYPE is a vector type, return true if IFN is a direct internal
3585 function that is supported for that type. If TYPE is a scalar type,
3586 return true if IFN is a direct internal function that is supported for
3587 the target's preferred vector version of TYPE. */
3588
3589 bool
vectorized_internal_fn_supported_p(internal_fn ifn,tree type)3590 vectorized_internal_fn_supported_p (internal_fn ifn, tree type)
3591 {
3592 scalar_mode smode;
3593 if (!VECTOR_TYPE_P (type) && is_a <scalar_mode> (TYPE_MODE (type), &smode))
3594 {
3595 machine_mode vmode = targetm.vectorize.preferred_simd_mode (smode);
3596 if (VECTOR_MODE_P (vmode))
3597 type = build_vector_type_for_mode (type, vmode);
3598 }
3599
3600 return (VECTOR_MODE_P (TYPE_MODE (type))
3601 && direct_internal_fn_supported_p (ifn, type, OPTIMIZE_FOR_SPEED));
3602 }
3603
3604 void
expand_PHI(internal_fn,gcall *)3605 expand_PHI (internal_fn, gcall *)
3606 {
3607 gcc_unreachable ();
3608 }
3609