1 /* Builtins' description for AArch64 SIMD architecture.
2 Copyright (C) 2011-2021 Free Software Foundation, Inc.
3 Contributed by ARM Ltd.
4
5 This file is part of GCC.
6
7 GCC is free software; you can redistribute it and/or modify it
8 under the terms of the GNU General Public License as published by
9 the Free Software Foundation; either version 3, or (at your option)
10 any later version.
11
12 GCC is distributed in the hope that it will be useful, but
13 WITHOUT ANY WARRANTY; without even the implied warranty of
14 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
15 General Public License for more details.
16
17 You should have received a copy of the GNU General Public License
18 along with GCC; see the file COPYING3. If not see
19 <http://www.gnu.org/licenses/>. */
20
21 #define IN_TARGET_CODE 1
22
23 #include "config.h"
24 #include "system.h"
25 #include "coretypes.h"
26 #include "tm.h"
27 #include "function.h"
28 #include "basic-block.h"
29 #include "rtl.h"
30 #include "tree.h"
31 #include "gimple.h"
32 #include "memmodel.h"
33 #include "tm_p.h"
34 #include "expmed.h"
35 #include "optabs.h"
36 #include "recog.h"
37 #include "diagnostic-core.h"
38 #include "fold-const.h"
39 #include "stor-layout.h"
40 #include "explow.h"
41 #include "expr.h"
42 #include "langhooks.h"
43 #include "gimple-iterator.h"
44 #include "case-cfn-macros.h"
45 #include "emit-rtl.h"
46 #include "stringpool.h"
47 #include "attribs.h"
48
49 #define v8qi_UP E_V8QImode
50 #define v4hi_UP E_V4HImode
51 #define v4hf_UP E_V4HFmode
52 #define v2si_UP E_V2SImode
53 #define v2sf_UP E_V2SFmode
54 #define v1df_UP E_V1DFmode
55 #define di_UP E_DImode
56 #define df_UP E_DFmode
57 #define v16qi_UP E_V16QImode
58 #define v8hi_UP E_V8HImode
59 #define v8hf_UP E_V8HFmode
60 #define v4si_UP E_V4SImode
61 #define v4sf_UP E_V4SFmode
62 #define v2di_UP E_V2DImode
63 #define v2df_UP E_V2DFmode
64 #define ti_UP E_TImode
65 #define oi_UP E_OImode
66 #define ci_UP E_CImode
67 #define xi_UP E_XImode
68 #define si_UP E_SImode
69 #define sf_UP E_SFmode
70 #define hi_UP E_HImode
71 #define hf_UP E_HFmode
72 #define qi_UP E_QImode
73 #define bf_UP E_BFmode
74 #define v4bf_UP E_V4BFmode
75 #define v8bf_UP E_V8BFmode
76 #define UP(X) X##_UP
77
78 #define SIMD_MAX_BUILTIN_ARGS 5
79
80 enum aarch64_type_qualifiers
81 {
82 /* T foo. */
83 qualifier_none = 0x0,
84 /* unsigned T foo. */
85 qualifier_unsigned = 0x1, /* 1 << 0 */
86 /* const T foo. */
87 qualifier_const = 0x2, /* 1 << 1 */
88 /* T *foo. */
89 qualifier_pointer = 0x4, /* 1 << 2 */
90 /* Used when expanding arguments if an operand could
91 be an immediate. */
92 qualifier_immediate = 0x8, /* 1 << 3 */
93 qualifier_maybe_immediate = 0x10, /* 1 << 4 */
94 /* void foo (...). */
95 qualifier_void = 0x20, /* 1 << 5 */
96 /* Some patterns may have internal operands, this qualifier is an
97 instruction to the initialisation code to skip this operand. */
98 qualifier_internal = 0x40, /* 1 << 6 */
99 /* Some builtins should use the T_*mode* encoded in a simd_builtin_datum
100 rather than using the type of the operand. */
101 qualifier_map_mode = 0x80, /* 1 << 7 */
102 /* qualifier_pointer | qualifier_map_mode */
103 qualifier_pointer_map_mode = 0x84,
104 /* qualifier_const | qualifier_pointer | qualifier_map_mode */
105 qualifier_const_pointer_map_mode = 0x86,
106 /* Polynomial types. */
107 qualifier_poly = 0x100,
108 /* Lane indices - must be in range, and flipped for bigendian. */
109 qualifier_lane_index = 0x200,
110 /* Lane indices for single lane structure loads and stores. */
111 qualifier_struct_load_store_lane_index = 0x400,
112 /* Lane indices selected in pairs. - must be in range, and flipped for
113 bigendian. */
114 qualifier_lane_pair_index = 0x800,
115 /* Lane indices selected in quadtuplets. - must be in range, and flipped for
116 bigendian. */
117 qualifier_lane_quadtup_index = 0x1000,
118 };
119
120 /* Flags that describe what a function might do. */
121 const unsigned int FLAG_NONE = 0U;
122 const unsigned int FLAG_READ_FPCR = 1U << 0;
123 const unsigned int FLAG_RAISE_FP_EXCEPTIONS = 1U << 1;
124 const unsigned int FLAG_READ_MEMORY = 1U << 2;
125 const unsigned int FLAG_PREFETCH_MEMORY = 1U << 3;
126 const unsigned int FLAG_WRITE_MEMORY = 1U << 4;
127
128 /* Not all FP intrinsics raise FP exceptions or read FPCR register,
129 use this flag to suppress it. */
130 const unsigned int FLAG_AUTO_FP = 1U << 5;
131
132 const unsigned int FLAG_FP = FLAG_READ_FPCR | FLAG_RAISE_FP_EXCEPTIONS;
133 const unsigned int FLAG_ALL = FLAG_READ_FPCR | FLAG_RAISE_FP_EXCEPTIONS
134 | FLAG_READ_MEMORY | FLAG_PREFETCH_MEMORY | FLAG_WRITE_MEMORY;
135 const unsigned int FLAG_STORE = FLAG_WRITE_MEMORY | FLAG_AUTO_FP;
136 const unsigned int FLAG_LOAD = FLAG_READ_MEMORY | FLAG_AUTO_FP;
137
138 typedef struct
139 {
140 const char *name;
141 machine_mode mode;
142 const enum insn_code code;
143 unsigned int fcode;
144 enum aarch64_type_qualifiers *qualifiers;
145 unsigned int flags;
146 } aarch64_simd_builtin_datum;
147
148 static enum aarch64_type_qualifiers
149 aarch64_types_unop_qualifiers[SIMD_MAX_BUILTIN_ARGS]
150 = { qualifier_none, qualifier_none };
151 #define TYPES_UNOP (aarch64_types_unop_qualifiers)
152 static enum aarch64_type_qualifiers
153 aarch64_types_unopu_qualifiers[SIMD_MAX_BUILTIN_ARGS]
154 = { qualifier_unsigned, qualifier_unsigned };
155 #define TYPES_UNOPU (aarch64_types_unopu_qualifiers)
156 static enum aarch64_type_qualifiers
157 aarch64_types_unopus_qualifiers[SIMD_MAX_BUILTIN_ARGS]
158 = { qualifier_unsigned, qualifier_none };
159 #define TYPES_UNOPUS (aarch64_types_unopus_qualifiers)
160 static enum aarch64_type_qualifiers
161 aarch64_types_binop_qualifiers[SIMD_MAX_BUILTIN_ARGS]
162 = { qualifier_none, qualifier_none, qualifier_maybe_immediate };
163 #define TYPES_BINOP (aarch64_types_binop_qualifiers)
164 static enum aarch64_type_qualifiers
165 aarch64_types_binopu_qualifiers[SIMD_MAX_BUILTIN_ARGS]
166 = { qualifier_unsigned, qualifier_unsigned, qualifier_unsigned };
167 #define TYPES_BINOPU (aarch64_types_binopu_qualifiers)
168 static enum aarch64_type_qualifiers
169 aarch64_types_binop_uus_qualifiers[SIMD_MAX_BUILTIN_ARGS]
170 = { qualifier_unsigned, qualifier_unsigned, qualifier_none };
171 #define TYPES_BINOP_UUS (aarch64_types_binop_uus_qualifiers)
172 static enum aarch64_type_qualifiers
173 aarch64_types_binop_ssu_qualifiers[SIMD_MAX_BUILTIN_ARGS]
174 = { qualifier_none, qualifier_none, qualifier_unsigned };
175 #define TYPES_BINOP_SSU (aarch64_types_binop_ssu_qualifiers)
176 static enum aarch64_type_qualifiers
177 aarch64_types_binop_uss_qualifiers[SIMD_MAX_BUILTIN_ARGS]
178 = { qualifier_unsigned, qualifier_none, qualifier_none };
179 #define TYPES_BINOP_USS (aarch64_types_binop_uss_qualifiers)
180 static enum aarch64_type_qualifiers
181 aarch64_types_binopp_qualifiers[SIMD_MAX_BUILTIN_ARGS]
182 = { qualifier_poly, qualifier_poly, qualifier_poly };
183 #define TYPES_BINOPP (aarch64_types_binopp_qualifiers)
184
185 static enum aarch64_type_qualifiers
186 aarch64_types_ternop_qualifiers[SIMD_MAX_BUILTIN_ARGS]
187 = { qualifier_none, qualifier_none, qualifier_none, qualifier_none };
188 #define TYPES_TERNOP (aarch64_types_ternop_qualifiers)
189 static enum aarch64_type_qualifiers
190 aarch64_types_ternop_lane_qualifiers[SIMD_MAX_BUILTIN_ARGS]
191 = { qualifier_none, qualifier_none, qualifier_none, qualifier_lane_index };
192 #define TYPES_TERNOP_LANE (aarch64_types_ternop_lane_qualifiers)
193 static enum aarch64_type_qualifiers
194 aarch64_types_ternopu_qualifiers[SIMD_MAX_BUILTIN_ARGS]
195 = { qualifier_unsigned, qualifier_unsigned,
196 qualifier_unsigned, qualifier_unsigned };
197 #define TYPES_TERNOPU (aarch64_types_ternopu_qualifiers)
198 static enum aarch64_type_qualifiers
199 aarch64_types_ternopu_lane_qualifiers[SIMD_MAX_BUILTIN_ARGS]
200 = { qualifier_unsigned, qualifier_unsigned,
201 qualifier_unsigned, qualifier_lane_index };
202 #define TYPES_TERNOPU_LANE (aarch64_types_ternopu_lane_qualifiers)
203 static enum aarch64_type_qualifiers
204 aarch64_types_ternopu_imm_qualifiers[SIMD_MAX_BUILTIN_ARGS]
205 = { qualifier_unsigned, qualifier_unsigned,
206 qualifier_unsigned, qualifier_immediate };
207 #define TYPES_TERNOPUI (aarch64_types_ternopu_imm_qualifiers)
208 static enum aarch64_type_qualifiers
209 aarch64_types_ternop_ssus_qualifiers[SIMD_MAX_BUILTIN_ARGS]
210 = { qualifier_none, qualifier_none, qualifier_unsigned, qualifier_none };
211 #define TYPES_TERNOP_SSUS (aarch64_types_ternop_ssus_qualifiers)
212
213
214 static enum aarch64_type_qualifiers
215 aarch64_types_quadop_lane_pair_qualifiers[SIMD_MAX_BUILTIN_ARGS]
216 = { qualifier_none, qualifier_none, qualifier_none,
217 qualifier_none, qualifier_lane_pair_index };
218 #define TYPES_QUADOP_LANE_PAIR (aarch64_types_quadop_lane_pair_qualifiers)
219 static enum aarch64_type_qualifiers
220 aarch64_types_quadop_lane_qualifiers[SIMD_MAX_BUILTIN_ARGS]
221 = { qualifier_none, qualifier_none, qualifier_none,
222 qualifier_none, qualifier_lane_index };
223 #define TYPES_QUADOP_LANE (aarch64_types_quadop_lane_qualifiers)
224 static enum aarch64_type_qualifiers
225 aarch64_types_quadopu_lane_qualifiers[SIMD_MAX_BUILTIN_ARGS]
226 = { qualifier_unsigned, qualifier_unsigned, qualifier_unsigned,
227 qualifier_unsigned, qualifier_lane_index };
228 #define TYPES_QUADOPU_LANE (aarch64_types_quadopu_lane_qualifiers)
229
230 static enum aarch64_type_qualifiers
231 aarch64_types_quadopssus_lane_quadtup_qualifiers[SIMD_MAX_BUILTIN_ARGS]
232 = { qualifier_none, qualifier_none, qualifier_unsigned,
233 qualifier_none, qualifier_lane_quadtup_index };
234 #define TYPES_QUADOPSSUS_LANE_QUADTUP \
235 (aarch64_types_quadopssus_lane_quadtup_qualifiers)
236 static enum aarch64_type_qualifiers
237 aarch64_types_quadopsssu_lane_quadtup_qualifiers[SIMD_MAX_BUILTIN_ARGS]
238 = { qualifier_none, qualifier_none, qualifier_none,
239 qualifier_unsigned, qualifier_lane_quadtup_index };
240 #define TYPES_QUADOPSSSU_LANE_QUADTUP \
241 (aarch64_types_quadopsssu_lane_quadtup_qualifiers)
242
243 static enum aarch64_type_qualifiers
244 aarch64_types_quadopu_imm_qualifiers[SIMD_MAX_BUILTIN_ARGS]
245 = { qualifier_unsigned, qualifier_unsigned, qualifier_unsigned,
246 qualifier_unsigned, qualifier_immediate };
247 #define TYPES_QUADOPUI (aarch64_types_quadopu_imm_qualifiers)
248
249 static enum aarch64_type_qualifiers
250 aarch64_types_binop_imm_p_qualifiers[SIMD_MAX_BUILTIN_ARGS]
251 = { qualifier_poly, qualifier_none, qualifier_immediate };
252 #define TYPES_GETREGP (aarch64_types_binop_imm_p_qualifiers)
253 static enum aarch64_type_qualifiers
254 aarch64_types_binop_imm_qualifiers[SIMD_MAX_BUILTIN_ARGS]
255 = { qualifier_none, qualifier_none, qualifier_immediate };
256 #define TYPES_GETREG (aarch64_types_binop_imm_qualifiers)
257 #define TYPES_SHIFTIMM (aarch64_types_binop_imm_qualifiers)
258 static enum aarch64_type_qualifiers
259 aarch64_types_shift_to_unsigned_qualifiers[SIMD_MAX_BUILTIN_ARGS]
260 = { qualifier_unsigned, qualifier_none, qualifier_immediate };
261 #define TYPES_SHIFTIMM_USS (aarch64_types_shift_to_unsigned_qualifiers)
262 static enum aarch64_type_qualifiers
263 aarch64_types_fcvt_from_unsigned_qualifiers[SIMD_MAX_BUILTIN_ARGS]
264 = { qualifier_none, qualifier_unsigned, qualifier_immediate };
265 #define TYPES_FCVTIMM_SUS (aarch64_types_fcvt_from_unsigned_qualifiers)
266 static enum aarch64_type_qualifiers
267 aarch64_types_unsigned_shift_qualifiers[SIMD_MAX_BUILTIN_ARGS]
268 = { qualifier_unsigned, qualifier_unsigned, qualifier_immediate };
269 #define TYPES_USHIFTIMM (aarch64_types_unsigned_shift_qualifiers)
270 #define TYPES_USHIFT2IMM (aarch64_types_ternopu_imm_qualifiers)
271 static enum aarch64_type_qualifiers
272 aarch64_types_shift2_to_unsigned_qualifiers[SIMD_MAX_BUILTIN_ARGS]
273 = { qualifier_unsigned, qualifier_unsigned, qualifier_none, qualifier_immediate };
274 #define TYPES_SHIFT2IMM_UUSS (aarch64_types_shift2_to_unsigned_qualifiers)
275
276 static enum aarch64_type_qualifiers
277 aarch64_types_ternop_s_imm_p_qualifiers[SIMD_MAX_BUILTIN_ARGS]
278 = { qualifier_none, qualifier_none, qualifier_poly, qualifier_immediate};
279 #define TYPES_SETREGP (aarch64_types_ternop_s_imm_p_qualifiers)
280 static enum aarch64_type_qualifiers
281 aarch64_types_ternop_s_imm_qualifiers[SIMD_MAX_BUILTIN_ARGS]
282 = { qualifier_none, qualifier_none, qualifier_none, qualifier_immediate};
283 #define TYPES_SETREG (aarch64_types_ternop_s_imm_qualifiers)
284 #define TYPES_SHIFTINSERT (aarch64_types_ternop_s_imm_qualifiers)
285 #define TYPES_SHIFTACC (aarch64_types_ternop_s_imm_qualifiers)
286 #define TYPES_SHIFT2IMM (aarch64_types_ternop_s_imm_qualifiers)
287
288 static enum aarch64_type_qualifiers
289 aarch64_types_ternop_p_imm_qualifiers[SIMD_MAX_BUILTIN_ARGS]
290 = { qualifier_poly, qualifier_poly, qualifier_poly, qualifier_immediate};
291 #define TYPES_SHIFTINSERTP (aarch64_types_ternop_p_imm_qualifiers)
292
293 static enum aarch64_type_qualifiers
294 aarch64_types_unsigned_shiftacc_qualifiers[SIMD_MAX_BUILTIN_ARGS]
295 = { qualifier_unsigned, qualifier_unsigned, qualifier_unsigned,
296 qualifier_immediate };
297 #define TYPES_USHIFTACC (aarch64_types_unsigned_shiftacc_qualifiers)
298
299
300 static enum aarch64_type_qualifiers
301 aarch64_types_combine_qualifiers[SIMD_MAX_BUILTIN_ARGS]
302 = { qualifier_none, qualifier_none, qualifier_none };
303 #define TYPES_COMBINE (aarch64_types_combine_qualifiers)
304
305 static enum aarch64_type_qualifiers
306 aarch64_types_combine_p_qualifiers[SIMD_MAX_BUILTIN_ARGS]
307 = { qualifier_poly, qualifier_poly, qualifier_poly };
308 #define TYPES_COMBINEP (aarch64_types_combine_p_qualifiers)
309
310 static enum aarch64_type_qualifiers
311 aarch64_types_load1_qualifiers[SIMD_MAX_BUILTIN_ARGS]
312 = { qualifier_none, qualifier_const_pointer_map_mode };
313 #define TYPES_LOAD1 (aarch64_types_load1_qualifiers)
314 #define TYPES_LOADSTRUCT (aarch64_types_load1_qualifiers)
315 static enum aarch64_type_qualifiers
316 aarch64_types_loadstruct_lane_qualifiers[SIMD_MAX_BUILTIN_ARGS]
317 = { qualifier_none, qualifier_const_pointer_map_mode,
318 qualifier_none, qualifier_struct_load_store_lane_index };
319 #define TYPES_LOADSTRUCT_LANE (aarch64_types_loadstruct_lane_qualifiers)
320
321 static enum aarch64_type_qualifiers
322 aarch64_types_bsl_p_qualifiers[SIMD_MAX_BUILTIN_ARGS]
323 = { qualifier_poly, qualifier_unsigned,
324 qualifier_poly, qualifier_poly };
325 #define TYPES_BSL_P (aarch64_types_bsl_p_qualifiers)
326 static enum aarch64_type_qualifiers
327 aarch64_types_bsl_s_qualifiers[SIMD_MAX_BUILTIN_ARGS]
328 = { qualifier_none, qualifier_unsigned,
329 qualifier_none, qualifier_none };
330 #define TYPES_BSL_S (aarch64_types_bsl_s_qualifiers)
331 static enum aarch64_type_qualifiers
332 aarch64_types_bsl_u_qualifiers[SIMD_MAX_BUILTIN_ARGS]
333 = { qualifier_unsigned, qualifier_unsigned,
334 qualifier_unsigned, qualifier_unsigned };
335 #define TYPES_BSL_U (aarch64_types_bsl_u_qualifiers)
336
337 /* The first argument (return type) of a store should be void type,
338 which we represent with qualifier_void. Their first operand will be
339 a DImode pointer to the location to store to, so we must use
340 qualifier_map_mode | qualifier_pointer to build a pointer to the
341 element type of the vector. */
342 static enum aarch64_type_qualifiers
343 aarch64_types_store1_p_qualifiers[SIMD_MAX_BUILTIN_ARGS]
344 = { qualifier_void, qualifier_pointer_map_mode, qualifier_poly };
345 #define TYPES_STORE1P (aarch64_types_store1_p_qualifiers)
346 static enum aarch64_type_qualifiers
347 aarch64_types_store1_qualifiers[SIMD_MAX_BUILTIN_ARGS]
348 = { qualifier_void, qualifier_pointer_map_mode, qualifier_none };
349 #define TYPES_STORE1 (aarch64_types_store1_qualifiers)
350 #define TYPES_STORESTRUCT (aarch64_types_store1_qualifiers)
351 static enum aarch64_type_qualifiers
352 aarch64_types_storestruct_lane_qualifiers[SIMD_MAX_BUILTIN_ARGS]
353 = { qualifier_void, qualifier_pointer_map_mode,
354 qualifier_none, qualifier_struct_load_store_lane_index };
355 #define TYPES_STORESTRUCT_LANE (aarch64_types_storestruct_lane_qualifiers)
356
357 #define CF0(N, X) CODE_FOR_aarch64_##N##X
358 #define CF1(N, X) CODE_FOR_##N##X##1
359 #define CF2(N, X) CODE_FOR_##N##X##2
360 #define CF3(N, X) CODE_FOR_##N##X##3
361 #define CF4(N, X) CODE_FOR_##N##X##4
362 #define CF10(N, X) CODE_FOR_##N##X
363
364 #define VAR1(T, N, MAP, FLAG, A) \
365 {#N #A, UP (A), CF##MAP (N, A), 0, TYPES_##T, FLAG_##FLAG},
366 #define VAR2(T, N, MAP, FLAG, A, B) \
367 VAR1 (T, N, MAP, FLAG, A) \
368 VAR1 (T, N, MAP, FLAG, B)
369 #define VAR3(T, N, MAP, FLAG, A, B, C) \
370 VAR2 (T, N, MAP, FLAG, A, B) \
371 VAR1 (T, N, MAP, FLAG, C)
372 #define VAR4(T, N, MAP, FLAG, A, B, C, D) \
373 VAR3 (T, N, MAP, FLAG, A, B, C) \
374 VAR1 (T, N, MAP, FLAG, D)
375 #define VAR5(T, N, MAP, FLAG, A, B, C, D, E) \
376 VAR4 (T, N, MAP, FLAG, A, B, C, D) \
377 VAR1 (T, N, MAP, FLAG, E)
378 #define VAR6(T, N, MAP, FLAG, A, B, C, D, E, F) \
379 VAR5 (T, N, MAP, FLAG, A, B, C, D, E) \
380 VAR1 (T, N, MAP, FLAG, F)
381 #define VAR7(T, N, MAP, FLAG, A, B, C, D, E, F, G) \
382 VAR6 (T, N, MAP, FLAG, A, B, C, D, E, F) \
383 VAR1 (T, N, MAP, FLAG, G)
384 #define VAR8(T, N, MAP, FLAG, A, B, C, D, E, F, G, H) \
385 VAR7 (T, N, MAP, FLAG, A, B, C, D, E, F, G) \
386 VAR1 (T, N, MAP, FLAG, H)
387 #define VAR9(T, N, MAP, FLAG, A, B, C, D, E, F, G, H, I) \
388 VAR8 (T, N, MAP, FLAG, A, B, C, D, E, F, G, H) \
389 VAR1 (T, N, MAP, FLAG, I)
390 #define VAR10(T, N, MAP, FLAG, A, B, C, D, E, F, G, H, I, J) \
391 VAR9 (T, N, MAP, FLAG, A, B, C, D, E, F, G, H, I) \
392 VAR1 (T, N, MAP, FLAG, J)
393 #define VAR11(T, N, MAP, FLAG, A, B, C, D, E, F, G, H, I, J, K) \
394 VAR10 (T, N, MAP, FLAG, A, B, C, D, E, F, G, H, I, J) \
395 VAR1 (T, N, MAP, FLAG, K)
396 #define VAR12(T, N, MAP, FLAG, A, B, C, D, E, F, G, H, I, J, K, L) \
397 VAR11 (T, N, MAP, FLAG, A, B, C, D, E, F, G, H, I, J, K) \
398 VAR1 (T, N, MAP, FLAG, L)
399 #define VAR13(T, N, MAP, FLAG, A, B, C, D, E, F, G, H, I, J, K, L, M) \
400 VAR12 (T, N, MAP, FLAG, A, B, C, D, E, F, G, H, I, J, K, L) \
401 VAR1 (T, N, MAP, FLAG, M)
402 #define VAR14(T, X, MAP, FLAG, A, B, C, D, E, F, G, H, I, J, K, L, M, N) \
403 VAR13 (T, X, MAP, FLAG, A, B, C, D, E, F, G, H, I, J, K, L, M) \
404 VAR1 (T, X, MAP, FLAG, N)
405 #define VAR15(T, X, MAP, FLAG, A, B, C, D, E, F, G, H, I, J, K, L, M, N, O) \
406 VAR14 (T, X, MAP, FLAG, A, B, C, D, E, F, G, H, I, J, K, L, M, N) \
407 VAR1 (T, X, MAP, FLAG, O)
408 #define VAR16(T, X, MAP, FLAG, A, B, C, D, E, F, G, H, I, J, K, L, M, N, O, P) \
409 VAR15 (T, X, MAP, FLAG, A, B, C, D, E, F, G, H, I, J, K, L, M, N, O) \
410 VAR1 (T, X, MAP, FLAG, P)
411
412 #include "aarch64-builtin-iterators.h"
413
414 static aarch64_simd_builtin_datum aarch64_simd_builtin_data[] = {
415 #include "aarch64-simd-builtins.def"
416 };
417
418 /* There's only 8 CRC32 builtins. Probably not worth their own .def file. */
419 #define AARCH64_CRC32_BUILTINS \
420 CRC32_BUILTIN (crc32b, QI) \
421 CRC32_BUILTIN (crc32h, HI) \
422 CRC32_BUILTIN (crc32w, SI) \
423 CRC32_BUILTIN (crc32x, DI) \
424 CRC32_BUILTIN (crc32cb, QI) \
425 CRC32_BUILTIN (crc32ch, HI) \
426 CRC32_BUILTIN (crc32cw, SI) \
427 CRC32_BUILTIN (crc32cx, DI)
428
429 /* The next 8 FCMLA instrinsics require some special handling compared the
430 normal simd intrinsics. */
431 #define AARCH64_SIMD_FCMLA_LANEQ_BUILTINS \
432 FCMLA_LANEQ_BUILTIN (0, v2sf, fcmla, V2SF, false) \
433 FCMLA_LANEQ_BUILTIN (90, v2sf, fcmla, V2SF, false) \
434 FCMLA_LANEQ_BUILTIN (180, v2sf, fcmla, V2SF, false) \
435 FCMLA_LANEQ_BUILTIN (270, v2sf, fcmla, V2SF, false) \
436 FCMLA_LANEQ_BUILTIN (0, v4hf, fcmla_laneq, V4HF, true) \
437 FCMLA_LANEQ_BUILTIN (90, v4hf, fcmla_laneq, V4HF, true) \
438 FCMLA_LANEQ_BUILTIN (180, v4hf, fcmla_laneq, V4HF, true) \
439 FCMLA_LANEQ_BUILTIN (270, v4hf, fcmla_laneq, V4HF, true) \
440
441 typedef struct
442 {
443 const char *name;
444 machine_mode mode;
445 const enum insn_code icode;
446 unsigned int fcode;
447 } aarch64_crc_builtin_datum;
448
449 /* Hold information about how to expand the FCMLA_LANEQ builtins. */
450 typedef struct
451 {
452 const char *name;
453 machine_mode mode;
454 const enum insn_code icode;
455 unsigned int fcode;
456 bool lane;
457 } aarch64_fcmla_laneq_builtin_datum;
458
459 #define CRC32_BUILTIN(N, M) \
460 AARCH64_BUILTIN_##N,
461
462 #define FCMLA_LANEQ_BUILTIN(I, N, X, M, T) \
463 AARCH64_SIMD_BUILTIN_FCMLA_LANEQ##I##_##M,
464
465 #undef VAR1
466 #define VAR1(T, N, MAP, FLAG, A) \
467 AARCH64_SIMD_BUILTIN_##T##_##N##A,
468
469 enum aarch64_builtins
470 {
471 AARCH64_BUILTIN_MIN,
472
473 AARCH64_BUILTIN_GET_FPCR,
474 AARCH64_BUILTIN_SET_FPCR,
475 AARCH64_BUILTIN_GET_FPSR,
476 AARCH64_BUILTIN_SET_FPSR,
477
478 AARCH64_BUILTIN_GET_FPCR64,
479 AARCH64_BUILTIN_SET_FPCR64,
480 AARCH64_BUILTIN_GET_FPSR64,
481 AARCH64_BUILTIN_SET_FPSR64,
482
483 AARCH64_BUILTIN_RSQRT_DF,
484 AARCH64_BUILTIN_RSQRT_SF,
485 AARCH64_BUILTIN_RSQRT_V2DF,
486 AARCH64_BUILTIN_RSQRT_V2SF,
487 AARCH64_BUILTIN_RSQRT_V4SF,
488 AARCH64_SIMD_BUILTIN_BASE,
489 AARCH64_SIMD_BUILTIN_LANE_CHECK,
490 #include "aarch64-simd-builtins.def"
491 /* The first enum element which is based on an insn_data pattern. */
492 AARCH64_SIMD_PATTERN_START = AARCH64_SIMD_BUILTIN_LANE_CHECK + 1,
493 AARCH64_SIMD_BUILTIN_MAX = AARCH64_SIMD_PATTERN_START
494 + ARRAY_SIZE (aarch64_simd_builtin_data) - 1,
495 AARCH64_CRC32_BUILTIN_BASE,
496 AARCH64_CRC32_BUILTINS
497 AARCH64_CRC32_BUILTIN_MAX,
498 /* ARMv8.3-A Pointer Authentication Builtins. */
499 AARCH64_PAUTH_BUILTIN_AUTIA1716,
500 AARCH64_PAUTH_BUILTIN_PACIA1716,
501 AARCH64_PAUTH_BUILTIN_AUTIB1716,
502 AARCH64_PAUTH_BUILTIN_PACIB1716,
503 AARCH64_PAUTH_BUILTIN_XPACLRI,
504 /* Special cased Armv8.3-A Complex FMA by Lane quad Builtins. */
505 AARCH64_SIMD_FCMLA_LANEQ_BUILTIN_BASE,
506 AARCH64_SIMD_FCMLA_LANEQ_BUILTINS
507 /* Builtin for Arm8.3-a Javascript conversion instruction. */
508 AARCH64_JSCVT,
509 /* TME builtins. */
510 AARCH64_TME_BUILTIN_TSTART,
511 AARCH64_TME_BUILTIN_TCOMMIT,
512 AARCH64_TME_BUILTIN_TTEST,
513 AARCH64_TME_BUILTIN_TCANCEL,
514 /* Armv8.5-a RNG instruction builtins. */
515 AARCH64_BUILTIN_RNG_RNDR,
516 AARCH64_BUILTIN_RNG_RNDRRS,
517 /* MEMTAG builtins. */
518 AARCH64_MEMTAG_BUILTIN_START,
519 AARCH64_MEMTAG_BUILTIN_IRG,
520 AARCH64_MEMTAG_BUILTIN_GMI,
521 AARCH64_MEMTAG_BUILTIN_SUBP,
522 AARCH64_MEMTAG_BUILTIN_INC_TAG,
523 AARCH64_MEMTAG_BUILTIN_SET_TAG,
524 AARCH64_MEMTAG_BUILTIN_GET_TAG,
525 AARCH64_MEMTAG_BUILTIN_END,
526 AARCH64_BUILTIN_MAX
527 };
528
529 #undef CRC32_BUILTIN
530 #define CRC32_BUILTIN(N, M) \
531 {"__builtin_aarch64_"#N, E_##M##mode, CODE_FOR_aarch64_##N, AARCH64_BUILTIN_##N},
532
533 static aarch64_crc_builtin_datum aarch64_crc_builtin_data[] = {
534 AARCH64_CRC32_BUILTINS
535 };
536
537
538 #undef FCMLA_LANEQ_BUILTIN
539 #define FCMLA_LANEQ_BUILTIN(I, N, X, M, T) \
540 {"__builtin_aarch64_fcmla_laneq"#I#N, E_##M##mode, CODE_FOR_aarch64_##X##I##N, \
541 AARCH64_SIMD_BUILTIN_FCMLA_LANEQ##I##_##M, T},
542
543 /* This structure contains how to manage the mapping form the builtin to the
544 instruction to generate in the backend and how to invoke the instruction. */
545 static aarch64_fcmla_laneq_builtin_datum aarch64_fcmla_lane_builtin_data[] = {
546 AARCH64_SIMD_FCMLA_LANEQ_BUILTINS
547 };
548
549 #undef CRC32_BUILTIN
550
551 static GTY(()) tree aarch64_builtin_decls[AARCH64_BUILTIN_MAX];
552
553 #define NUM_DREG_TYPES 6
554 #define NUM_QREG_TYPES 6
555
556 /* Internal scalar builtin types. These types are used to support
557 neon intrinsic builtins. They are _not_ user-visible types. Therefore
558 the mangling for these types are implementation defined. */
559 const char *aarch64_scalar_builtin_types[] = {
560 "__builtin_aarch64_simd_qi",
561 "__builtin_aarch64_simd_hi",
562 "__builtin_aarch64_simd_si",
563 "__builtin_aarch64_simd_hf",
564 "__builtin_aarch64_simd_sf",
565 "__builtin_aarch64_simd_di",
566 "__builtin_aarch64_simd_df",
567 "__builtin_aarch64_simd_poly8",
568 "__builtin_aarch64_simd_poly16",
569 "__builtin_aarch64_simd_poly64",
570 "__builtin_aarch64_simd_poly128",
571 "__builtin_aarch64_simd_ti",
572 "__builtin_aarch64_simd_uqi",
573 "__builtin_aarch64_simd_uhi",
574 "__builtin_aarch64_simd_usi",
575 "__builtin_aarch64_simd_udi",
576 "__builtin_aarch64_simd_ei",
577 "__builtin_aarch64_simd_oi",
578 "__builtin_aarch64_simd_ci",
579 "__builtin_aarch64_simd_xi",
580 "__builtin_aarch64_simd_bf",
581 NULL
582 };
583
584 #define ENTRY(E, M, Q, G) E,
585 enum aarch64_simd_type
586 {
587 #include "aarch64-simd-builtin-types.def"
588 ARM_NEON_H_TYPES_LAST
589 };
590 #undef ENTRY
591
592 struct aarch64_simd_type_info
593 {
594 enum aarch64_simd_type type;
595
596 /* Internal type name. */
597 const char *name;
598
599 /* Internal type name(mangled). The mangled names conform to the
600 AAPCS64 (see "Procedure Call Standard for the ARM 64-bit Architecture",
601 Appendix A). To qualify for emission with the mangled names defined in
602 that document, a vector type must not only be of the correct mode but also
603 be of the correct internal AdvSIMD vector type (e.g. __Int8x8_t); these
604 types are registered by aarch64_init_simd_builtin_types (). In other
605 words, vector types defined in other ways e.g. via vector_size attribute
606 will get default mangled names. */
607 const char *mangle;
608
609 /* Internal type. */
610 tree itype;
611
612 /* Element type. */
613 tree eltype;
614
615 /* Machine mode the internal type maps to. */
616 enum machine_mode mode;
617
618 /* Qualifiers. */
619 enum aarch64_type_qualifiers q;
620 };
621
622 #define ENTRY(E, M, Q, G) \
623 {E, "__" #E, #G "__" #E, NULL_TREE, NULL_TREE, E_##M##mode, qualifier_##Q},
624 static struct aarch64_simd_type_info aarch64_simd_types [] = {
625 #include "aarch64-simd-builtin-types.def"
626 };
627 #undef ENTRY
628
629 static tree aarch64_simd_intOI_type_node = NULL_TREE;
630 static tree aarch64_simd_intCI_type_node = NULL_TREE;
631 static tree aarch64_simd_intXI_type_node = NULL_TREE;
632
633 /* The user-visible __fp16 type, and a pointer to that type. Used
634 across the back-end. */
635 tree aarch64_fp16_type_node = NULL_TREE;
636 tree aarch64_fp16_ptr_type_node = NULL_TREE;
637
638 /* Back-end node type for brain float (bfloat) types. */
639 tree aarch64_bf16_type_node = NULL_TREE;
640 tree aarch64_bf16_ptr_type_node = NULL_TREE;
641
642 /* Wrapper around add_builtin_function. NAME is the name of the built-in
643 function, TYPE is the function type, CODE is the function subcode
644 (relative to AARCH64_BUILTIN_GENERAL), and ATTRS is the function
645 attributes. */
646 static tree
647 aarch64_general_add_builtin (const char *name, tree type, unsigned int code,
648 tree attrs = NULL_TREE)
649 {
650 code = (code << AARCH64_BUILTIN_SHIFT) | AARCH64_BUILTIN_GENERAL;
651 return add_builtin_function (name, type, code, BUILT_IN_MD,
652 NULL, attrs);
653 }
654
655 static const char *
aarch64_mangle_builtin_scalar_type(const_tree type)656 aarch64_mangle_builtin_scalar_type (const_tree type)
657 {
658 int i = 0;
659
660 while (aarch64_scalar_builtin_types[i] != NULL)
661 {
662 const char *name = aarch64_scalar_builtin_types[i];
663
664 if (TREE_CODE (TYPE_NAME (type)) == TYPE_DECL
665 && DECL_NAME (TYPE_NAME (type))
666 && !strcmp (IDENTIFIER_POINTER (DECL_NAME (TYPE_NAME (type))), name))
667 return aarch64_scalar_builtin_types[i];
668 i++;
669 }
670 return NULL;
671 }
672
673 static const char *
aarch64_mangle_builtin_vector_type(const_tree type)674 aarch64_mangle_builtin_vector_type (const_tree type)
675 {
676 tree attrs = TYPE_ATTRIBUTES (type);
677 if (tree attr = lookup_attribute ("Advanced SIMD type", attrs))
678 {
679 tree mangled_name = TREE_VALUE (TREE_VALUE (attr));
680 return IDENTIFIER_POINTER (mangled_name);
681 }
682
683 return NULL;
684 }
685
686 const char *
aarch64_general_mangle_builtin_type(const_tree type)687 aarch64_general_mangle_builtin_type (const_tree type)
688 {
689 const char *mangle;
690 /* Walk through all the AArch64 builtins types tables to filter out the
691 incoming type. */
692 if ((mangle = aarch64_mangle_builtin_vector_type (type))
693 || (mangle = aarch64_mangle_builtin_scalar_type (type)))
694 return mangle;
695
696 return NULL;
697 }
698
699 static tree
aarch64_simd_builtin_std_type(machine_mode mode,enum aarch64_type_qualifiers q)700 aarch64_simd_builtin_std_type (machine_mode mode,
701 enum aarch64_type_qualifiers q)
702 {
703 #define QUAL_TYPE(M) \
704 ((q == qualifier_none) ? int##M##_type_node : unsigned_int##M##_type_node);
705 switch (mode)
706 {
707 case E_QImode:
708 return QUAL_TYPE (QI);
709 case E_HImode:
710 return QUAL_TYPE (HI);
711 case E_SImode:
712 return QUAL_TYPE (SI);
713 case E_DImode:
714 return QUAL_TYPE (DI);
715 case E_TImode:
716 return QUAL_TYPE (TI);
717 case E_OImode:
718 return aarch64_simd_intOI_type_node;
719 case E_CImode:
720 return aarch64_simd_intCI_type_node;
721 case E_XImode:
722 return aarch64_simd_intXI_type_node;
723 case E_HFmode:
724 return aarch64_fp16_type_node;
725 case E_SFmode:
726 return float_type_node;
727 case E_DFmode:
728 return double_type_node;
729 case E_BFmode:
730 return aarch64_bf16_type_node;
731 default:
732 gcc_unreachable ();
733 }
734 #undef QUAL_TYPE
735 }
736
737 static tree
aarch64_lookup_simd_builtin_type(machine_mode mode,enum aarch64_type_qualifiers q)738 aarch64_lookup_simd_builtin_type (machine_mode mode,
739 enum aarch64_type_qualifiers q)
740 {
741 int i;
742 int nelts = sizeof (aarch64_simd_types) / sizeof (aarch64_simd_types[0]);
743
744 /* Non-poly scalar modes map to standard types not in the table. */
745 if (q != qualifier_poly && !VECTOR_MODE_P (mode))
746 return aarch64_simd_builtin_std_type (mode, q);
747
748 for (i = 0; i < nelts; i++)
749 if (aarch64_simd_types[i].mode == mode
750 && aarch64_simd_types[i].q == q)
751 return aarch64_simd_types[i].itype;
752
753 return NULL_TREE;
754 }
755
756 static tree
aarch64_simd_builtin_type(machine_mode mode,bool unsigned_p,bool poly_p)757 aarch64_simd_builtin_type (machine_mode mode,
758 bool unsigned_p, bool poly_p)
759 {
760 if (poly_p)
761 return aarch64_lookup_simd_builtin_type (mode, qualifier_poly);
762 else if (unsigned_p)
763 return aarch64_lookup_simd_builtin_type (mode, qualifier_unsigned);
764 else
765 return aarch64_lookup_simd_builtin_type (mode, qualifier_none);
766 }
767
768 static void
aarch64_init_simd_builtin_types(void)769 aarch64_init_simd_builtin_types (void)
770 {
771 int i;
772 int nelts = sizeof (aarch64_simd_types) / sizeof (aarch64_simd_types[0]);
773 tree tdecl;
774
775 /* Init all the element types built by the front-end. */
776 aarch64_simd_types[Int8x8_t].eltype = intQI_type_node;
777 aarch64_simd_types[Int8x16_t].eltype = intQI_type_node;
778 aarch64_simd_types[Int16x4_t].eltype = intHI_type_node;
779 aarch64_simd_types[Int16x8_t].eltype = intHI_type_node;
780 aarch64_simd_types[Int32x2_t].eltype = intSI_type_node;
781 aarch64_simd_types[Int32x4_t].eltype = intSI_type_node;
782 aarch64_simd_types[Int64x1_t].eltype = intDI_type_node;
783 aarch64_simd_types[Int64x2_t].eltype = intDI_type_node;
784 aarch64_simd_types[Uint8x8_t].eltype = unsigned_intQI_type_node;
785 aarch64_simd_types[Uint8x16_t].eltype = unsigned_intQI_type_node;
786 aarch64_simd_types[Uint16x4_t].eltype = unsigned_intHI_type_node;
787 aarch64_simd_types[Uint16x8_t].eltype = unsigned_intHI_type_node;
788 aarch64_simd_types[Uint32x2_t].eltype = unsigned_intSI_type_node;
789 aarch64_simd_types[Uint32x4_t].eltype = unsigned_intSI_type_node;
790 aarch64_simd_types[Uint64x1_t].eltype = unsigned_intDI_type_node;
791 aarch64_simd_types[Uint64x2_t].eltype = unsigned_intDI_type_node;
792
793 /* Poly types are a world of their own. */
794 aarch64_simd_types[Poly8_t].eltype = aarch64_simd_types[Poly8_t].itype =
795 build_distinct_type_copy (unsigned_intQI_type_node);
796 /* Prevent front-ends from transforming Poly8_t arrays into string
797 literals. */
798 TYPE_STRING_FLAG (aarch64_simd_types[Poly8_t].eltype) = false;
799
800 aarch64_simd_types[Poly16_t].eltype = aarch64_simd_types[Poly16_t].itype =
801 build_distinct_type_copy (unsigned_intHI_type_node);
802 aarch64_simd_types[Poly64_t].eltype = aarch64_simd_types[Poly64_t].itype =
803 build_distinct_type_copy (unsigned_intDI_type_node);
804 aarch64_simd_types[Poly128_t].eltype = aarch64_simd_types[Poly128_t].itype =
805 build_distinct_type_copy (unsigned_intTI_type_node);
806 /* Init poly vector element types with scalar poly types. */
807 aarch64_simd_types[Poly8x8_t].eltype = aarch64_simd_types[Poly8_t].itype;
808 aarch64_simd_types[Poly8x16_t].eltype = aarch64_simd_types[Poly8_t].itype;
809 aarch64_simd_types[Poly16x4_t].eltype = aarch64_simd_types[Poly16_t].itype;
810 aarch64_simd_types[Poly16x8_t].eltype = aarch64_simd_types[Poly16_t].itype;
811 aarch64_simd_types[Poly64x1_t].eltype = aarch64_simd_types[Poly64_t].itype;
812 aarch64_simd_types[Poly64x2_t].eltype = aarch64_simd_types[Poly64_t].itype;
813
814 /* Continue with standard types. */
815 aarch64_simd_types[Float16x4_t].eltype = aarch64_fp16_type_node;
816 aarch64_simd_types[Float16x8_t].eltype = aarch64_fp16_type_node;
817 aarch64_simd_types[Float32x2_t].eltype = float_type_node;
818 aarch64_simd_types[Float32x4_t].eltype = float_type_node;
819 aarch64_simd_types[Float64x1_t].eltype = double_type_node;
820 aarch64_simd_types[Float64x2_t].eltype = double_type_node;
821
822 /* Init Bfloat vector types with underlying __bf16 type. */
823 aarch64_simd_types[Bfloat16x4_t].eltype = aarch64_bf16_type_node;
824 aarch64_simd_types[Bfloat16x8_t].eltype = aarch64_bf16_type_node;
825
826 for (i = 0; i < nelts; i++)
827 {
828 tree eltype = aarch64_simd_types[i].eltype;
829 machine_mode mode = aarch64_simd_types[i].mode;
830
831 if (aarch64_simd_types[i].itype == NULL)
832 {
833 tree type = build_vector_type (eltype, GET_MODE_NUNITS (mode));
834 type = build_distinct_type_copy (type);
835 SET_TYPE_STRUCTURAL_EQUALITY (type);
836
837 tree mangled_name = get_identifier (aarch64_simd_types[i].mangle);
838 tree value = tree_cons (NULL_TREE, mangled_name, NULL_TREE);
839 TYPE_ATTRIBUTES (type)
840 = tree_cons (get_identifier ("Advanced SIMD type"), value,
841 TYPE_ATTRIBUTES (type));
842 aarch64_simd_types[i].itype = type;
843 }
844
845 tdecl = add_builtin_type (aarch64_simd_types[i].name,
846 aarch64_simd_types[i].itype);
847 TYPE_NAME (aarch64_simd_types[i].itype) = tdecl;
848 }
849
850 #define AARCH64_BUILD_SIGNED_TYPE(mode) \
851 make_signed_type (GET_MODE_PRECISION (mode));
852 aarch64_simd_intOI_type_node = AARCH64_BUILD_SIGNED_TYPE (OImode);
853 aarch64_simd_intCI_type_node = AARCH64_BUILD_SIGNED_TYPE (CImode);
854 aarch64_simd_intXI_type_node = AARCH64_BUILD_SIGNED_TYPE (XImode);
855 #undef AARCH64_BUILD_SIGNED_TYPE
856
857 tdecl = add_builtin_type
858 ("__builtin_aarch64_simd_oi" , aarch64_simd_intOI_type_node);
859 TYPE_NAME (aarch64_simd_intOI_type_node) = tdecl;
860 tdecl = add_builtin_type
861 ("__builtin_aarch64_simd_ci" , aarch64_simd_intCI_type_node);
862 TYPE_NAME (aarch64_simd_intCI_type_node) = tdecl;
863 tdecl = add_builtin_type
864 ("__builtin_aarch64_simd_xi" , aarch64_simd_intXI_type_node);
865 TYPE_NAME (aarch64_simd_intXI_type_node) = tdecl;
866 }
867
868 static void
aarch64_init_simd_builtin_scalar_types(void)869 aarch64_init_simd_builtin_scalar_types (void)
870 {
871 /* Define typedefs for all the standard scalar types. */
872 (*lang_hooks.types.register_builtin_type) (intQI_type_node,
873 "__builtin_aarch64_simd_qi");
874 (*lang_hooks.types.register_builtin_type) (intHI_type_node,
875 "__builtin_aarch64_simd_hi");
876 (*lang_hooks.types.register_builtin_type) (aarch64_fp16_type_node,
877 "__builtin_aarch64_simd_hf");
878 (*lang_hooks.types.register_builtin_type) (intSI_type_node,
879 "__builtin_aarch64_simd_si");
880 (*lang_hooks.types.register_builtin_type) (float_type_node,
881 "__builtin_aarch64_simd_sf");
882 (*lang_hooks.types.register_builtin_type) (intDI_type_node,
883 "__builtin_aarch64_simd_di");
884 (*lang_hooks.types.register_builtin_type) (double_type_node,
885 "__builtin_aarch64_simd_df");
886 (*lang_hooks.types.register_builtin_type) (unsigned_intQI_type_node,
887 "__builtin_aarch64_simd_poly8");
888 (*lang_hooks.types.register_builtin_type) (unsigned_intHI_type_node,
889 "__builtin_aarch64_simd_poly16");
890 (*lang_hooks.types.register_builtin_type) (unsigned_intDI_type_node,
891 "__builtin_aarch64_simd_poly64");
892 (*lang_hooks.types.register_builtin_type) (unsigned_intTI_type_node,
893 "__builtin_aarch64_simd_poly128");
894 (*lang_hooks.types.register_builtin_type) (intTI_type_node,
895 "__builtin_aarch64_simd_ti");
896 (*lang_hooks.types.register_builtin_type) (aarch64_bf16_type_node,
897 "__builtin_aarch64_simd_bf");
898 /* Unsigned integer types for various mode sizes. */
899 (*lang_hooks.types.register_builtin_type) (unsigned_intQI_type_node,
900 "__builtin_aarch64_simd_uqi");
901 (*lang_hooks.types.register_builtin_type) (unsigned_intHI_type_node,
902 "__builtin_aarch64_simd_uhi");
903 (*lang_hooks.types.register_builtin_type) (unsigned_intSI_type_node,
904 "__builtin_aarch64_simd_usi");
905 (*lang_hooks.types.register_builtin_type) (unsigned_intDI_type_node,
906 "__builtin_aarch64_simd_udi");
907 }
908
909 /* Return a set of FLAG_* flags that describe what the function could do,
910 taking the command-line flags into account. */
911 static unsigned int
aarch64_call_properties(aarch64_simd_builtin_datum * d)912 aarch64_call_properties (aarch64_simd_builtin_datum *d)
913 {
914 unsigned int flags = d->flags;
915
916 if (!(flags & FLAG_AUTO_FP) && FLOAT_MODE_P (d->mode))
917 flags |= FLAG_FP;
918
919 /* -fno-trapping-math means that we can assume any FP exceptions
920 are not user-visible. */
921 if (!flag_trapping_math)
922 flags &= ~FLAG_RAISE_FP_EXCEPTIONS;
923
924 return flags;
925 }
926
927 /* Return true if calls to the function could modify some form of
928 global state. */
929 static bool
aarch64_modifies_global_state_p(aarch64_simd_builtin_datum * d)930 aarch64_modifies_global_state_p (aarch64_simd_builtin_datum *d)
931 {
932 unsigned int flags = aarch64_call_properties (d);
933
934 if (flags & FLAG_RAISE_FP_EXCEPTIONS)
935 return true;
936
937 if (flags & FLAG_PREFETCH_MEMORY)
938 return true;
939
940 return flags & FLAG_WRITE_MEMORY;
941 }
942
943 /* Return true if calls to the function could read some form of
944 global state. */
945 static bool
aarch64_reads_global_state_p(aarch64_simd_builtin_datum * d)946 aarch64_reads_global_state_p (aarch64_simd_builtin_datum *d)
947 {
948 unsigned int flags = aarch64_call_properties (d);
949
950 if (flags & FLAG_READ_FPCR)
951 return true;
952
953 return flags & FLAG_READ_MEMORY;
954 }
955
956 /* Return true if calls to the function could raise a signal. */
957 static bool
aarch64_could_trap_p(aarch64_simd_builtin_datum * d)958 aarch64_could_trap_p (aarch64_simd_builtin_datum *d)
959 {
960 unsigned int flags = aarch64_call_properties (d);
961
962 if (flags & FLAG_RAISE_FP_EXCEPTIONS)
963 return true;
964
965 if (flags & (FLAG_READ_MEMORY | FLAG_WRITE_MEMORY))
966 return true;
967
968 return false;
969 }
970
971 /* Add attribute NAME to ATTRS. */
972 static tree
aarch64_add_attribute(const char * name,tree attrs)973 aarch64_add_attribute (const char *name, tree attrs)
974 {
975 return tree_cons (get_identifier (name), NULL_TREE, attrs);
976 }
977
978 /* Return the appropriate function attributes. */
979 static tree
aarch64_get_attributes(aarch64_simd_builtin_datum * d)980 aarch64_get_attributes (aarch64_simd_builtin_datum *d)
981 {
982 tree attrs = NULL_TREE;
983
984 if (!aarch64_modifies_global_state_p (d))
985 {
986 if (aarch64_reads_global_state_p (d))
987 attrs = aarch64_add_attribute ("pure", attrs);
988 else
989 attrs = aarch64_add_attribute ("const", attrs);
990 }
991
992 if (!flag_non_call_exceptions || !aarch64_could_trap_p (d))
993 attrs = aarch64_add_attribute ("nothrow", attrs);
994
995 return aarch64_add_attribute ("leaf", attrs);
996 }
997
998 static bool aarch64_simd_builtins_initialized_p = false;
999
1000 /* Due to the architecture not providing lane variant of the lane instructions
1001 for fcmla we can't use the standard simd builtin expansion code, but we
1002 still want the majority of the validation that would normally be done. */
1003
1004 void
aarch64_init_fcmla_laneq_builtins(void)1005 aarch64_init_fcmla_laneq_builtins (void)
1006 {
1007 unsigned int i = 0;
1008
1009 for (i = 0; i < ARRAY_SIZE (aarch64_fcmla_lane_builtin_data); ++i)
1010 {
1011 aarch64_fcmla_laneq_builtin_datum* d
1012 = &aarch64_fcmla_lane_builtin_data[i];
1013 tree argtype = aarch64_lookup_simd_builtin_type (d->mode, qualifier_none);
1014 machine_mode quadmode = GET_MODE_2XWIDER_MODE (d->mode).require ();
1015 tree quadtype
1016 = aarch64_lookup_simd_builtin_type (quadmode, qualifier_none);
1017 tree lanetype
1018 = aarch64_simd_builtin_std_type (SImode, qualifier_lane_pair_index);
1019 tree ftype = build_function_type_list (argtype, argtype, argtype,
1020 quadtype, lanetype, NULL_TREE);
1021 tree fndecl = aarch64_general_add_builtin (d->name, ftype, d->fcode);
1022
1023 aarch64_builtin_decls[d->fcode] = fndecl;
1024 }
1025 }
1026
1027 void
aarch64_init_simd_builtins(void)1028 aarch64_init_simd_builtins (void)
1029 {
1030 unsigned int i, fcode = AARCH64_SIMD_PATTERN_START;
1031
1032 if (aarch64_simd_builtins_initialized_p)
1033 return;
1034
1035 aarch64_simd_builtins_initialized_p = true;
1036
1037 aarch64_init_simd_builtin_types ();
1038
1039 /* Strong-typing hasn't been implemented for all AdvSIMD builtin intrinsics.
1040 Therefore we need to preserve the old __builtin scalar types. It can be
1041 removed once all the intrinsics become strongly typed using the qualifier
1042 system. */
1043 aarch64_init_simd_builtin_scalar_types ();
1044
1045 tree lane_check_fpr = build_function_type_list (void_type_node,
1046 size_type_node,
1047 size_type_node,
1048 intSI_type_node,
1049 NULL);
1050 aarch64_builtin_decls[AARCH64_SIMD_BUILTIN_LANE_CHECK]
1051 = aarch64_general_add_builtin ("__builtin_aarch64_im_lane_boundsi",
1052 lane_check_fpr,
1053 AARCH64_SIMD_BUILTIN_LANE_CHECK);
1054
1055 for (i = 0; i < ARRAY_SIZE (aarch64_simd_builtin_data); i++, fcode++)
1056 {
1057 bool print_type_signature_p = false;
1058 char type_signature[SIMD_MAX_BUILTIN_ARGS + 1] = { 0 };
1059 aarch64_simd_builtin_datum *d = &aarch64_simd_builtin_data[i];
1060 char namebuf[60];
1061 tree ftype = NULL;
1062 tree fndecl = NULL;
1063
1064 d->fcode = fcode;
1065
1066 /* We must track two variables here. op_num is
1067 the operand number as in the RTL pattern. This is
1068 required to access the mode (e.g. V4SF mode) of the
1069 argument, from which the base type can be derived.
1070 arg_num is an index in to the qualifiers data, which
1071 gives qualifiers to the type (e.g. const unsigned).
1072 The reason these two variables may differ by one is the
1073 void return type. While all return types take the 0th entry
1074 in the qualifiers array, there is no operand for them in the
1075 RTL pattern. */
1076 int op_num = insn_data[d->code].n_operands - 1;
1077 int arg_num = d->qualifiers[0] & qualifier_void
1078 ? op_num + 1
1079 : op_num;
1080 tree return_type = void_type_node, args = void_list_node;
1081 tree eltype;
1082
1083 /* Build a function type directly from the insn_data for this
1084 builtin. The build_function_type () function takes care of
1085 removing duplicates for us. */
1086 for (; op_num >= 0; arg_num--, op_num--)
1087 {
1088 machine_mode op_mode = insn_data[d->code].operand[op_num].mode;
1089 enum aarch64_type_qualifiers qualifiers = d->qualifiers[arg_num];
1090
1091 if (qualifiers & qualifier_unsigned)
1092 {
1093 type_signature[op_num] = 'u';
1094 print_type_signature_p = true;
1095 }
1096 else if (qualifiers & qualifier_poly)
1097 {
1098 type_signature[op_num] = 'p';
1099 print_type_signature_p = true;
1100 }
1101 else
1102 type_signature[op_num] = 's';
1103
1104 /* Skip an internal operand for vget_{low, high}. */
1105 if (qualifiers & qualifier_internal)
1106 continue;
1107
1108 /* Some builtins have different user-facing types
1109 for certain arguments, encoded in d->mode. */
1110 if (qualifiers & qualifier_map_mode)
1111 op_mode = d->mode;
1112
1113 /* For pointers, we want a pointer to the basic type
1114 of the vector. */
1115 if (qualifiers & qualifier_pointer && VECTOR_MODE_P (op_mode))
1116 op_mode = GET_MODE_INNER (op_mode);
1117
1118 eltype = aarch64_simd_builtin_type
1119 (op_mode,
1120 (qualifiers & qualifier_unsigned) != 0,
1121 (qualifiers & qualifier_poly) != 0);
1122 gcc_assert (eltype != NULL);
1123
1124 /* Add qualifiers. */
1125 if (qualifiers & qualifier_const)
1126 eltype = build_qualified_type (eltype, TYPE_QUAL_CONST);
1127
1128 if (qualifiers & qualifier_pointer)
1129 eltype = build_pointer_type (eltype);
1130
1131 /* If we have reached arg_num == 0, we are at a non-void
1132 return type. Otherwise, we are still processing
1133 arguments. */
1134 if (arg_num == 0)
1135 return_type = eltype;
1136 else
1137 args = tree_cons (NULL_TREE, eltype, args);
1138 }
1139
1140 ftype = build_function_type (return_type, args);
1141
1142 gcc_assert (ftype != NULL);
1143
1144 if (print_type_signature_p)
1145 snprintf (namebuf, sizeof (namebuf), "__builtin_aarch64_%s_%s",
1146 d->name, type_signature);
1147 else
1148 snprintf (namebuf, sizeof (namebuf), "__builtin_aarch64_%s",
1149 d->name);
1150
1151 tree attrs = aarch64_get_attributes (d);
1152
1153 fndecl = aarch64_general_add_builtin (namebuf, ftype, fcode, attrs);
1154 aarch64_builtin_decls[fcode] = fndecl;
1155 }
1156
1157 /* Initialize the remaining fcmla_laneq intrinsics. */
1158 aarch64_init_fcmla_laneq_builtins ();
1159 }
1160
1161 static void
aarch64_init_crc32_builtins()1162 aarch64_init_crc32_builtins ()
1163 {
1164 tree usi_type = aarch64_simd_builtin_std_type (SImode, qualifier_unsigned);
1165 unsigned int i = 0;
1166
1167 for (i = 0; i < ARRAY_SIZE (aarch64_crc_builtin_data); ++i)
1168 {
1169 aarch64_crc_builtin_datum* d = &aarch64_crc_builtin_data[i];
1170 tree argtype = aarch64_simd_builtin_std_type (d->mode,
1171 qualifier_unsigned);
1172 tree ftype = build_function_type_list (usi_type, usi_type, argtype, NULL_TREE);
1173 tree fndecl = aarch64_general_add_builtin (d->name, ftype, d->fcode);
1174
1175 aarch64_builtin_decls[d->fcode] = fndecl;
1176 }
1177 }
1178
1179 /* Add builtins for reciprocal square root. */
1180
1181 void
aarch64_init_builtin_rsqrt(void)1182 aarch64_init_builtin_rsqrt (void)
1183 {
1184 tree fndecl = NULL;
1185 tree ftype = NULL;
1186
1187 tree V2SF_type_node = build_vector_type (float_type_node, 2);
1188 tree V2DF_type_node = build_vector_type (double_type_node, 2);
1189 tree V4SF_type_node = build_vector_type (float_type_node, 4);
1190
1191 struct builtin_decls_data
1192 {
1193 tree type_node;
1194 const char *builtin_name;
1195 int function_code;
1196 };
1197
1198 builtin_decls_data bdda[] =
1199 {
1200 { double_type_node, "__builtin_aarch64_rsqrt_df", AARCH64_BUILTIN_RSQRT_DF },
1201 { float_type_node, "__builtin_aarch64_rsqrt_sf", AARCH64_BUILTIN_RSQRT_SF },
1202 { V2DF_type_node, "__builtin_aarch64_rsqrt_v2df", AARCH64_BUILTIN_RSQRT_V2DF },
1203 { V2SF_type_node, "__builtin_aarch64_rsqrt_v2sf", AARCH64_BUILTIN_RSQRT_V2SF },
1204 { V4SF_type_node, "__builtin_aarch64_rsqrt_v4sf", AARCH64_BUILTIN_RSQRT_V4SF }
1205 };
1206
1207 builtin_decls_data *bdd = bdda;
1208 builtin_decls_data *bdd_end = bdd + (sizeof (bdda) / sizeof (builtin_decls_data));
1209
1210 for (; bdd < bdd_end; bdd++)
1211 {
1212 ftype = build_function_type_list (bdd->type_node, bdd->type_node, NULL_TREE);
1213 fndecl = aarch64_general_add_builtin (bdd->builtin_name,
1214 ftype, bdd->function_code);
1215 aarch64_builtin_decls[bdd->function_code] = fndecl;
1216 }
1217 }
1218
1219 /* Initialize the backend types that support the user-visible __fp16
1220 type, also initialize a pointer to that type, to be used when
1221 forming HFAs. */
1222
1223 static void
aarch64_init_fp16_types(void)1224 aarch64_init_fp16_types (void)
1225 {
1226 aarch64_fp16_type_node = make_node (REAL_TYPE);
1227 TYPE_PRECISION (aarch64_fp16_type_node) = 16;
1228 layout_type (aarch64_fp16_type_node);
1229
1230 (*lang_hooks.types.register_builtin_type) (aarch64_fp16_type_node, "__fp16");
1231 aarch64_fp16_ptr_type_node = build_pointer_type (aarch64_fp16_type_node);
1232 }
1233
1234 /* Initialize the backend REAL_TYPE type supporting bfloat types. */
1235 static void
aarch64_init_bf16_types(void)1236 aarch64_init_bf16_types (void)
1237 {
1238 aarch64_bf16_type_node = make_node (REAL_TYPE);
1239 TYPE_PRECISION (aarch64_bf16_type_node) = 16;
1240 SET_TYPE_MODE (aarch64_bf16_type_node, BFmode);
1241 layout_type (aarch64_bf16_type_node);
1242
1243 lang_hooks.types.register_builtin_type (aarch64_bf16_type_node, "__bf16");
1244 aarch64_bf16_ptr_type_node = build_pointer_type (aarch64_bf16_type_node);
1245 }
1246
1247 /* Pointer authentication builtins that will become NOP on legacy platform.
1248 Currently, these builtins are for internal use only (libgcc EH unwinder). */
1249
1250 void
aarch64_init_pauth_hint_builtins(void)1251 aarch64_init_pauth_hint_builtins (void)
1252 {
1253 /* Pointer Authentication builtins. */
1254 tree ftype_pointer_auth
1255 = build_function_type_list (ptr_type_node, ptr_type_node,
1256 unsigned_intDI_type_node, NULL_TREE);
1257 tree ftype_pointer_strip
1258 = build_function_type_list (ptr_type_node, ptr_type_node, NULL_TREE);
1259
1260 aarch64_builtin_decls[AARCH64_PAUTH_BUILTIN_AUTIA1716]
1261 = aarch64_general_add_builtin ("__builtin_aarch64_autia1716",
1262 ftype_pointer_auth,
1263 AARCH64_PAUTH_BUILTIN_AUTIA1716);
1264 aarch64_builtin_decls[AARCH64_PAUTH_BUILTIN_PACIA1716]
1265 = aarch64_general_add_builtin ("__builtin_aarch64_pacia1716",
1266 ftype_pointer_auth,
1267 AARCH64_PAUTH_BUILTIN_PACIA1716);
1268 aarch64_builtin_decls[AARCH64_PAUTH_BUILTIN_AUTIB1716]
1269 = aarch64_general_add_builtin ("__builtin_aarch64_autib1716",
1270 ftype_pointer_auth,
1271 AARCH64_PAUTH_BUILTIN_AUTIB1716);
1272 aarch64_builtin_decls[AARCH64_PAUTH_BUILTIN_PACIB1716]
1273 = aarch64_general_add_builtin ("__builtin_aarch64_pacib1716",
1274 ftype_pointer_auth,
1275 AARCH64_PAUTH_BUILTIN_PACIB1716);
1276 aarch64_builtin_decls[AARCH64_PAUTH_BUILTIN_XPACLRI]
1277 = aarch64_general_add_builtin ("__builtin_aarch64_xpaclri",
1278 ftype_pointer_strip,
1279 AARCH64_PAUTH_BUILTIN_XPACLRI);
1280 }
1281
1282 /* Initialize the transactional memory extension (TME) builtins. */
1283 static void
aarch64_init_tme_builtins(void)1284 aarch64_init_tme_builtins (void)
1285 {
1286 tree ftype_uint64_void
1287 = build_function_type_list (uint64_type_node, NULL);
1288 tree ftype_void_void
1289 = build_function_type_list (void_type_node, NULL);
1290 tree ftype_void_uint64
1291 = build_function_type_list (void_type_node, uint64_type_node, NULL);
1292
1293 aarch64_builtin_decls[AARCH64_TME_BUILTIN_TSTART]
1294 = aarch64_general_add_builtin ("__builtin_aarch64_tstart",
1295 ftype_uint64_void,
1296 AARCH64_TME_BUILTIN_TSTART);
1297 aarch64_builtin_decls[AARCH64_TME_BUILTIN_TTEST]
1298 = aarch64_general_add_builtin ("__builtin_aarch64_ttest",
1299 ftype_uint64_void,
1300 AARCH64_TME_BUILTIN_TTEST);
1301 aarch64_builtin_decls[AARCH64_TME_BUILTIN_TCOMMIT]
1302 = aarch64_general_add_builtin ("__builtin_aarch64_tcommit",
1303 ftype_void_void,
1304 AARCH64_TME_BUILTIN_TCOMMIT);
1305 aarch64_builtin_decls[AARCH64_TME_BUILTIN_TCANCEL]
1306 = aarch64_general_add_builtin ("__builtin_aarch64_tcancel",
1307 ftype_void_uint64,
1308 AARCH64_TME_BUILTIN_TCANCEL);
1309 }
1310
1311 /* Add builtins for Random Number instructions. */
1312
1313 static void
aarch64_init_rng_builtins(void)1314 aarch64_init_rng_builtins (void)
1315 {
1316 tree unsigned_ptr_type = build_pointer_type (unsigned_intDI_type_node);
1317 tree ftype
1318 = build_function_type_list (integer_type_node, unsigned_ptr_type, NULL);
1319 aarch64_builtin_decls[AARCH64_BUILTIN_RNG_RNDR]
1320 = aarch64_general_add_builtin ("__builtin_aarch64_rndr", ftype,
1321 AARCH64_BUILTIN_RNG_RNDR);
1322 aarch64_builtin_decls[AARCH64_BUILTIN_RNG_RNDRRS]
1323 = aarch64_general_add_builtin ("__builtin_aarch64_rndrrs", ftype,
1324 AARCH64_BUILTIN_RNG_RNDRRS);
1325 }
1326
1327 /* Initialize the memory tagging extension (MTE) builtins. */
1328 struct
1329 {
1330 tree ftype;
1331 enum insn_code icode;
1332 } aarch64_memtag_builtin_data[AARCH64_MEMTAG_BUILTIN_END -
1333 AARCH64_MEMTAG_BUILTIN_START - 1];
1334
1335 static void
aarch64_init_memtag_builtins(void)1336 aarch64_init_memtag_builtins (void)
1337 {
1338 tree fntype = NULL;
1339
1340 #define AARCH64_INIT_MEMTAG_BUILTINS_DECL(F, N, I, T) \
1341 aarch64_builtin_decls[AARCH64_MEMTAG_BUILTIN_##F] \
1342 = aarch64_general_add_builtin ("__builtin_aarch64_memtag_"#N, \
1343 T, AARCH64_MEMTAG_BUILTIN_##F); \
1344 aarch64_memtag_builtin_data[AARCH64_MEMTAG_BUILTIN_##F - \
1345 AARCH64_MEMTAG_BUILTIN_START - 1] = \
1346 {T, CODE_FOR_##I};
1347
1348 fntype = build_function_type_list (ptr_type_node, ptr_type_node,
1349 uint64_type_node, NULL);
1350 AARCH64_INIT_MEMTAG_BUILTINS_DECL (IRG, irg, irg, fntype);
1351
1352 fntype = build_function_type_list (uint64_type_node, ptr_type_node,
1353 uint64_type_node, NULL);
1354 AARCH64_INIT_MEMTAG_BUILTINS_DECL (GMI, gmi, gmi, fntype);
1355
1356 fntype = build_function_type_list (ptrdiff_type_node, ptr_type_node,
1357 ptr_type_node, NULL);
1358 AARCH64_INIT_MEMTAG_BUILTINS_DECL (SUBP, subp, subp, fntype);
1359
1360 fntype = build_function_type_list (ptr_type_node, ptr_type_node,
1361 unsigned_type_node, NULL);
1362 AARCH64_INIT_MEMTAG_BUILTINS_DECL (INC_TAG, inc_tag, addg, fntype);
1363
1364 fntype = build_function_type_list (void_type_node, ptr_type_node, NULL);
1365 AARCH64_INIT_MEMTAG_BUILTINS_DECL (SET_TAG, set_tag, stg, fntype);
1366
1367 fntype = build_function_type_list (ptr_type_node, ptr_type_node, NULL);
1368 AARCH64_INIT_MEMTAG_BUILTINS_DECL (GET_TAG, get_tag, ldg, fntype);
1369
1370 #undef AARCH64_INIT_MEMTAG_BUILTINS_DECL
1371 }
1372
1373 /* Initialize fpsr fpcr getters and setters. */
1374
1375 static void
aarch64_init_fpsr_fpcr_builtins(void)1376 aarch64_init_fpsr_fpcr_builtins (void)
1377 {
1378 tree ftype_set
1379 = build_function_type_list (void_type_node, unsigned_type_node, NULL);
1380 tree ftype_get
1381 = build_function_type_list (unsigned_type_node, NULL);
1382
1383 aarch64_builtin_decls[AARCH64_BUILTIN_GET_FPCR]
1384 = aarch64_general_add_builtin ("__builtin_aarch64_get_fpcr",
1385 ftype_get,
1386 AARCH64_BUILTIN_GET_FPCR);
1387 aarch64_builtin_decls[AARCH64_BUILTIN_SET_FPCR]
1388 = aarch64_general_add_builtin ("__builtin_aarch64_set_fpcr",
1389 ftype_set,
1390 AARCH64_BUILTIN_SET_FPCR);
1391 aarch64_builtin_decls[AARCH64_BUILTIN_GET_FPSR]
1392 = aarch64_general_add_builtin ("__builtin_aarch64_get_fpsr",
1393 ftype_get,
1394 AARCH64_BUILTIN_GET_FPSR);
1395 aarch64_builtin_decls[AARCH64_BUILTIN_SET_FPSR]
1396 = aarch64_general_add_builtin ("__builtin_aarch64_set_fpsr",
1397 ftype_set,
1398 AARCH64_BUILTIN_SET_FPSR);
1399
1400 ftype_set
1401 = build_function_type_list (void_type_node, long_long_unsigned_type_node,
1402 NULL);
1403 ftype_get
1404 = build_function_type_list (long_long_unsigned_type_node, NULL);
1405
1406 aarch64_builtin_decls[AARCH64_BUILTIN_GET_FPCR64]
1407 = aarch64_general_add_builtin ("__builtin_aarch64_get_fpcr64",
1408 ftype_get,
1409 AARCH64_BUILTIN_GET_FPCR64);
1410 aarch64_builtin_decls[AARCH64_BUILTIN_SET_FPCR64]
1411 = aarch64_general_add_builtin ("__builtin_aarch64_set_fpcr64",
1412 ftype_set,
1413 AARCH64_BUILTIN_SET_FPCR64);
1414 aarch64_builtin_decls[AARCH64_BUILTIN_GET_FPSR64]
1415 = aarch64_general_add_builtin ("__builtin_aarch64_get_fpsr64",
1416 ftype_get,
1417 AARCH64_BUILTIN_GET_FPSR64);
1418 aarch64_builtin_decls[AARCH64_BUILTIN_SET_FPSR64]
1419 = aarch64_general_add_builtin ("__builtin_aarch64_set_fpsr64",
1420 ftype_set,
1421 AARCH64_BUILTIN_SET_FPSR64);
1422 }
1423
1424 /* Initialize all builtins in the AARCH64_BUILTIN_GENERAL group. */
1425
1426 void
aarch64_general_init_builtins(void)1427 aarch64_general_init_builtins (void)
1428 {
1429 aarch64_init_fpsr_fpcr_builtins ();
1430
1431 aarch64_init_fp16_types ();
1432
1433 aarch64_init_bf16_types ();
1434
1435 if (TARGET_SIMD)
1436 aarch64_init_simd_builtins ();
1437
1438 aarch64_init_crc32_builtins ();
1439 aarch64_init_builtin_rsqrt ();
1440 aarch64_init_rng_builtins ();
1441
1442 tree ftype_jcvt
1443 = build_function_type_list (intSI_type_node, double_type_node, NULL);
1444 aarch64_builtin_decls[AARCH64_JSCVT]
1445 = aarch64_general_add_builtin ("__builtin_aarch64_jcvtzs", ftype_jcvt,
1446 AARCH64_JSCVT);
1447
1448 /* Initialize pointer authentication builtins which are backed by instructions
1449 in NOP encoding space.
1450
1451 NOTE: these builtins are supposed to be used by libgcc unwinder only, as
1452 there is no support on return address signing under ILP32, we don't
1453 register them. */
1454 if (!TARGET_ILP32)
1455 aarch64_init_pauth_hint_builtins ();
1456
1457 if (TARGET_TME)
1458 aarch64_init_tme_builtins ();
1459
1460 if (TARGET_MEMTAG)
1461 aarch64_init_memtag_builtins ();
1462 }
1463
1464 /* Implement TARGET_BUILTIN_DECL for the AARCH64_BUILTIN_GENERAL group. */
1465 tree
aarch64_general_builtin_decl(unsigned code,bool)1466 aarch64_general_builtin_decl (unsigned code, bool)
1467 {
1468 if (code >= AARCH64_BUILTIN_MAX)
1469 return error_mark_node;
1470
1471 return aarch64_builtin_decls[code];
1472 }
1473
1474 typedef enum
1475 {
1476 SIMD_ARG_COPY_TO_REG,
1477 SIMD_ARG_CONSTANT,
1478 SIMD_ARG_LANE_INDEX,
1479 SIMD_ARG_STRUCT_LOAD_STORE_LANE_INDEX,
1480 SIMD_ARG_LANE_PAIR_INDEX,
1481 SIMD_ARG_LANE_QUADTUP_INDEX,
1482 SIMD_ARG_STOP
1483 } builtin_simd_arg;
1484
1485
1486 static rtx
aarch64_simd_expand_args(rtx target,int icode,int have_retval,tree exp,builtin_simd_arg * args,machine_mode builtin_mode)1487 aarch64_simd_expand_args (rtx target, int icode, int have_retval,
1488 tree exp, builtin_simd_arg *args,
1489 machine_mode builtin_mode)
1490 {
1491 rtx pat;
1492 rtx op[SIMD_MAX_BUILTIN_ARGS + 1]; /* First element for result operand. */
1493 int opc = 0;
1494
1495 if (have_retval)
1496 {
1497 machine_mode tmode = insn_data[icode].operand[0].mode;
1498 if (!target
1499 || GET_MODE (target) != tmode
1500 || !(*insn_data[icode].operand[0].predicate) (target, tmode))
1501 target = gen_reg_rtx (tmode);
1502 op[opc++] = target;
1503 }
1504
1505 for (;;)
1506 {
1507 builtin_simd_arg thisarg = args[opc - have_retval];
1508
1509 if (thisarg == SIMD_ARG_STOP)
1510 break;
1511 else
1512 {
1513 tree arg = CALL_EXPR_ARG (exp, opc - have_retval);
1514 machine_mode mode = insn_data[icode].operand[opc].mode;
1515 op[opc] = expand_normal (arg);
1516
1517 switch (thisarg)
1518 {
1519 case SIMD_ARG_COPY_TO_REG:
1520 if (POINTER_TYPE_P (TREE_TYPE (arg)))
1521 op[opc] = convert_memory_address (Pmode, op[opc]);
1522 /*gcc_assert (GET_MODE (op[opc]) == mode); */
1523 if (!(*insn_data[icode].operand[opc].predicate)
1524 (op[opc], mode))
1525 op[opc] = copy_to_mode_reg (mode, op[opc]);
1526 break;
1527
1528 case SIMD_ARG_STRUCT_LOAD_STORE_LANE_INDEX:
1529 gcc_assert (opc > 1);
1530 if (CONST_INT_P (op[opc]))
1531 {
1532 unsigned int nunits
1533 = GET_MODE_NUNITS (builtin_mode).to_constant ();
1534 aarch64_simd_lane_bounds (op[opc], 0, nunits, exp);
1535 /* Keep to GCC-vector-extension lane indices in the RTL. */
1536 op[opc] = aarch64_endian_lane_rtx (builtin_mode,
1537 INTVAL (op[opc]));
1538 }
1539 goto constant_arg;
1540
1541 case SIMD_ARG_LANE_INDEX:
1542 /* Must be a previous operand into which this is an index. */
1543 gcc_assert (opc > 0);
1544 if (CONST_INT_P (op[opc]))
1545 {
1546 machine_mode vmode = insn_data[icode].operand[opc - 1].mode;
1547 unsigned int nunits
1548 = GET_MODE_NUNITS (vmode).to_constant ();
1549 aarch64_simd_lane_bounds (op[opc], 0, nunits, exp);
1550 /* Keep to GCC-vector-extension lane indices in the RTL. */
1551 op[opc] = aarch64_endian_lane_rtx (vmode, INTVAL (op[opc]));
1552 }
1553 /* If the lane index isn't a constant then error out. */
1554 goto constant_arg;
1555
1556 case SIMD_ARG_LANE_PAIR_INDEX:
1557 /* Must be a previous operand into which this is an index and
1558 index is restricted to nunits / 2. */
1559 gcc_assert (opc > 0);
1560 if (CONST_INT_P (op[opc]))
1561 {
1562 machine_mode vmode = insn_data[icode].operand[opc - 1].mode;
1563 unsigned int nunits
1564 = GET_MODE_NUNITS (vmode).to_constant ();
1565 aarch64_simd_lane_bounds (op[opc], 0, nunits / 2, exp);
1566 /* Keep to GCC-vector-extension lane indices in the RTL. */
1567 int lane = INTVAL (op[opc]);
1568 op[opc] = gen_int_mode (ENDIAN_LANE_N (nunits / 2, lane),
1569 SImode);
1570 }
1571 /* If the lane index isn't a constant then error out. */
1572 goto constant_arg;
1573 case SIMD_ARG_LANE_QUADTUP_INDEX:
1574 /* Must be a previous operand into which this is an index and
1575 index is restricted to nunits / 4. */
1576 gcc_assert (opc > 0);
1577 if (CONST_INT_P (op[opc]))
1578 {
1579 machine_mode vmode = insn_data[icode].operand[opc - 1].mode;
1580 unsigned int nunits
1581 = GET_MODE_NUNITS (vmode).to_constant ();
1582 aarch64_simd_lane_bounds (op[opc], 0, nunits / 4, exp);
1583 /* Keep to GCC-vector-extension lane indices in the RTL. */
1584 int lane = INTVAL (op[opc]);
1585 op[opc] = gen_int_mode (ENDIAN_LANE_N (nunits / 4, lane),
1586 SImode);
1587 }
1588 /* If the lane index isn't a constant then error out. */
1589 goto constant_arg;
1590 case SIMD_ARG_CONSTANT:
1591 constant_arg:
1592 if (!(*insn_data[icode].operand[opc].predicate)
1593 (op[opc], mode))
1594 {
1595 error ("%Kargument %d must be a constant immediate",
1596 exp, opc + 1 - have_retval);
1597 return const0_rtx;
1598 }
1599 break;
1600
1601 case SIMD_ARG_STOP:
1602 gcc_unreachable ();
1603 }
1604
1605 opc++;
1606 }
1607 }
1608
1609 switch (opc)
1610 {
1611 case 1:
1612 pat = GEN_FCN (icode) (op[0]);
1613 break;
1614
1615 case 2:
1616 pat = GEN_FCN (icode) (op[0], op[1]);
1617 break;
1618
1619 case 3:
1620 pat = GEN_FCN (icode) (op[0], op[1], op[2]);
1621 break;
1622
1623 case 4:
1624 pat = GEN_FCN (icode) (op[0], op[1], op[2], op[3]);
1625 break;
1626
1627 case 5:
1628 pat = GEN_FCN (icode) (op[0], op[1], op[2], op[3], op[4]);
1629 break;
1630
1631 case 6:
1632 pat = GEN_FCN (icode) (op[0], op[1], op[2], op[3], op[4], op[5]);
1633 break;
1634
1635 default:
1636 gcc_unreachable ();
1637 }
1638
1639 if (!pat)
1640 return NULL_RTX;
1641
1642 emit_insn (pat);
1643
1644 return target;
1645 }
1646
1647 /* Expand an AArch64 AdvSIMD builtin(intrinsic). */
1648 rtx
aarch64_simd_expand_builtin(int fcode,tree exp,rtx target)1649 aarch64_simd_expand_builtin (int fcode, tree exp, rtx target)
1650 {
1651 if (fcode == AARCH64_SIMD_BUILTIN_LANE_CHECK)
1652 {
1653 rtx totalsize = expand_normal (CALL_EXPR_ARG (exp, 0));
1654 rtx elementsize = expand_normal (CALL_EXPR_ARG (exp, 1));
1655 if (CONST_INT_P (totalsize) && CONST_INT_P (elementsize)
1656 && UINTVAL (elementsize) != 0
1657 && UINTVAL (totalsize) != 0)
1658 {
1659 rtx lane_idx = expand_normal (CALL_EXPR_ARG (exp, 2));
1660 if (CONST_INT_P (lane_idx))
1661 aarch64_simd_lane_bounds (lane_idx, 0,
1662 UINTVAL (totalsize)
1663 / UINTVAL (elementsize),
1664 exp);
1665 else
1666 error ("%Klane index must be a constant immediate", exp);
1667 }
1668 else
1669 error ("%Ktotal size and element size must be a non-zero constant immediate", exp);
1670 /* Don't generate any RTL. */
1671 return const0_rtx;
1672 }
1673 aarch64_simd_builtin_datum *d =
1674 &aarch64_simd_builtin_data[fcode - AARCH64_SIMD_PATTERN_START];
1675 enum insn_code icode = d->code;
1676 builtin_simd_arg args[SIMD_MAX_BUILTIN_ARGS + 1];
1677 int num_args = insn_data[d->code].n_operands;
1678 int is_void = 0;
1679 int k;
1680
1681 is_void = !!(d->qualifiers[0] & qualifier_void);
1682
1683 num_args += is_void;
1684
1685 for (k = 1; k < num_args; k++)
1686 {
1687 /* We have four arrays of data, each indexed in a different fashion.
1688 qualifiers - element 0 always describes the function return type.
1689 operands - element 0 is either the operand for return value (if
1690 the function has a non-void return type) or the operand for the
1691 first argument.
1692 expr_args - element 0 always holds the first argument.
1693 args - element 0 is always used for the return type. */
1694 int qualifiers_k = k;
1695 int operands_k = k - is_void;
1696 int expr_args_k = k - 1;
1697
1698 if (d->qualifiers[qualifiers_k] & qualifier_lane_index)
1699 args[k] = SIMD_ARG_LANE_INDEX;
1700 else if (d->qualifiers[qualifiers_k] & qualifier_lane_pair_index)
1701 args[k] = SIMD_ARG_LANE_PAIR_INDEX;
1702 else if (d->qualifiers[qualifiers_k] & qualifier_lane_quadtup_index)
1703 args[k] = SIMD_ARG_LANE_QUADTUP_INDEX;
1704 else if (d->qualifiers[qualifiers_k] & qualifier_struct_load_store_lane_index)
1705 args[k] = SIMD_ARG_STRUCT_LOAD_STORE_LANE_INDEX;
1706 else if (d->qualifiers[qualifiers_k] & qualifier_immediate)
1707 args[k] = SIMD_ARG_CONSTANT;
1708 else if (d->qualifiers[qualifiers_k] & qualifier_maybe_immediate)
1709 {
1710 rtx arg
1711 = expand_normal (CALL_EXPR_ARG (exp,
1712 (expr_args_k)));
1713 /* Handle constants only if the predicate allows it. */
1714 bool op_const_int_p =
1715 (CONST_INT_P (arg)
1716 && (*insn_data[icode].operand[operands_k].predicate)
1717 (arg, insn_data[icode].operand[operands_k].mode));
1718 args[k] = op_const_int_p ? SIMD_ARG_CONSTANT : SIMD_ARG_COPY_TO_REG;
1719 }
1720 else
1721 args[k] = SIMD_ARG_COPY_TO_REG;
1722
1723 }
1724 args[k] = SIMD_ARG_STOP;
1725
1726 /* The interface to aarch64_simd_expand_args expects a 0 if
1727 the function is void, and a 1 if it is not. */
1728 return aarch64_simd_expand_args
1729 (target, icode, !is_void, exp, &args[1], d->mode);
1730 }
1731
1732 rtx
aarch64_crc32_expand_builtin(int fcode,tree exp,rtx target)1733 aarch64_crc32_expand_builtin (int fcode, tree exp, rtx target)
1734 {
1735 rtx pat;
1736 aarch64_crc_builtin_datum *d
1737 = &aarch64_crc_builtin_data[fcode - (AARCH64_CRC32_BUILTIN_BASE + 1)];
1738 enum insn_code icode = d->icode;
1739 tree arg0 = CALL_EXPR_ARG (exp, 0);
1740 tree arg1 = CALL_EXPR_ARG (exp, 1);
1741 rtx op0 = expand_normal (arg0);
1742 rtx op1 = expand_normal (arg1);
1743 machine_mode tmode = insn_data[icode].operand[0].mode;
1744 machine_mode mode0 = insn_data[icode].operand[1].mode;
1745 machine_mode mode1 = insn_data[icode].operand[2].mode;
1746
1747 if (! target
1748 || GET_MODE (target) != tmode
1749 || ! (*insn_data[icode].operand[0].predicate) (target, tmode))
1750 target = gen_reg_rtx (tmode);
1751
1752 gcc_assert ((GET_MODE (op0) == mode0 || GET_MODE (op0) == VOIDmode)
1753 && (GET_MODE (op1) == mode1 || GET_MODE (op1) == VOIDmode));
1754
1755 if (! (*insn_data[icode].operand[1].predicate) (op0, mode0))
1756 op0 = copy_to_mode_reg (mode0, op0);
1757 if (! (*insn_data[icode].operand[2].predicate) (op1, mode1))
1758 op1 = copy_to_mode_reg (mode1, op1);
1759
1760 pat = GEN_FCN (icode) (target, op0, op1);
1761 if (!pat)
1762 return NULL_RTX;
1763
1764 emit_insn (pat);
1765 return target;
1766 }
1767
1768 /* Function to expand reciprocal square root builtins. */
1769
1770 static rtx
aarch64_expand_builtin_rsqrt(int fcode,tree exp,rtx target)1771 aarch64_expand_builtin_rsqrt (int fcode, tree exp, rtx target)
1772 {
1773 tree arg0 = CALL_EXPR_ARG (exp, 0);
1774 rtx op0 = expand_normal (arg0);
1775
1776 rtx (*gen) (rtx, rtx);
1777
1778 switch (fcode)
1779 {
1780 case AARCH64_BUILTIN_RSQRT_DF:
1781 gen = gen_rsqrtdf2;
1782 break;
1783 case AARCH64_BUILTIN_RSQRT_SF:
1784 gen = gen_rsqrtsf2;
1785 break;
1786 case AARCH64_BUILTIN_RSQRT_V2DF:
1787 gen = gen_rsqrtv2df2;
1788 break;
1789 case AARCH64_BUILTIN_RSQRT_V2SF:
1790 gen = gen_rsqrtv2sf2;
1791 break;
1792 case AARCH64_BUILTIN_RSQRT_V4SF:
1793 gen = gen_rsqrtv4sf2;
1794 break;
1795 default: gcc_unreachable ();
1796 }
1797
1798 if (!target)
1799 target = gen_reg_rtx (GET_MODE (op0));
1800
1801 emit_insn (gen (target, op0));
1802
1803 return target;
1804 }
1805
1806 /* Expand a FCMLA lane expression EXP with code FCODE and
1807 result going to TARGET if that is convenient. */
1808
1809 rtx
aarch64_expand_fcmla_builtin(tree exp,rtx target,int fcode)1810 aarch64_expand_fcmla_builtin (tree exp, rtx target, int fcode)
1811 {
1812 int bcode = fcode - AARCH64_SIMD_FCMLA_LANEQ_BUILTIN_BASE - 1;
1813 aarch64_fcmla_laneq_builtin_datum* d
1814 = &aarch64_fcmla_lane_builtin_data[bcode];
1815 machine_mode quadmode = GET_MODE_2XWIDER_MODE (d->mode).require ();
1816 rtx op0 = force_reg (d->mode, expand_normal (CALL_EXPR_ARG (exp, 0)));
1817 rtx op1 = force_reg (d->mode, expand_normal (CALL_EXPR_ARG (exp, 1)));
1818 rtx op2 = force_reg (quadmode, expand_normal (CALL_EXPR_ARG (exp, 2)));
1819 tree tmp = CALL_EXPR_ARG (exp, 3);
1820 rtx lane_idx = expand_expr (tmp, NULL_RTX, VOIDmode, EXPAND_INITIALIZER);
1821
1822 /* Validate that the lane index is a constant. */
1823 if (!CONST_INT_P (lane_idx))
1824 {
1825 error ("%Kargument %d must be a constant immediate", exp, 4);
1826 return const0_rtx;
1827 }
1828
1829 /* Validate that the index is within the expected range. */
1830 int nunits = GET_MODE_NUNITS (quadmode).to_constant ();
1831 aarch64_simd_lane_bounds (lane_idx, 0, nunits / 2, exp);
1832
1833 /* Generate the correct register and mode. */
1834 int lane = INTVAL (lane_idx);
1835
1836 if (lane < nunits / 4)
1837 op2 = simplify_gen_subreg (d->mode, op2, quadmode,
1838 subreg_lowpart_offset (d->mode, quadmode));
1839 else
1840 {
1841 /* Select the upper 64 bits, either a V2SF or V4HF, this however
1842 is quite messy, as the operation required even though simple
1843 doesn't have a simple RTL pattern, and seems it's quite hard to
1844 define using a single RTL pattern. The target generic version
1845 gen_highpart_mode generates code that isn't optimal. */
1846 rtx temp1 = gen_reg_rtx (d->mode);
1847 rtx temp2 = gen_reg_rtx (DImode);
1848 temp1 = simplify_gen_subreg (d->mode, op2, quadmode,
1849 subreg_lowpart_offset (d->mode, quadmode));
1850 temp1 = simplify_gen_subreg (V2DImode, temp1, d->mode, 0);
1851 if (BYTES_BIG_ENDIAN)
1852 emit_insn (gen_aarch64_get_lanev2di (temp2, temp1, const0_rtx));
1853 else
1854 emit_insn (gen_aarch64_get_lanev2di (temp2, temp1, const1_rtx));
1855 op2 = simplify_gen_subreg (d->mode, temp2, GET_MODE (temp2), 0);
1856
1857 /* And recalculate the index. */
1858 lane -= nunits / 4;
1859 }
1860
1861 /* Keep to GCC-vector-extension lane indices in the RTL, only nunits / 4
1862 (max nunits in range check) are valid. Which means only 0-1, so we
1863 only need to know the order in a V2mode. */
1864 lane_idx = aarch64_endian_lane_rtx (V2DImode, lane);
1865
1866 if (!target
1867 || !REG_P (target)
1868 || GET_MODE (target) != d->mode)
1869 target = gen_reg_rtx (d->mode);
1870
1871 rtx pat = NULL_RTX;
1872
1873 if (d->lane)
1874 pat = GEN_FCN (d->icode) (target, op0, op1, op2, lane_idx);
1875 else
1876 pat = GEN_FCN (d->icode) (target, op0, op1, op2);
1877
1878 if (!pat)
1879 return NULL_RTX;
1880
1881 emit_insn (pat);
1882 return target;
1883 }
1884
1885 /* Function to expand an expression EXP which calls one of the Transactional
1886 Memory Extension (TME) builtins FCODE with the result going to TARGET. */
1887 static rtx
aarch64_expand_builtin_tme(int fcode,tree exp,rtx target)1888 aarch64_expand_builtin_tme (int fcode, tree exp, rtx target)
1889 {
1890 switch (fcode)
1891 {
1892 case AARCH64_TME_BUILTIN_TSTART:
1893 target = gen_reg_rtx (DImode);
1894 emit_insn (GEN_FCN (CODE_FOR_tstart) (target));
1895 break;
1896
1897 case AARCH64_TME_BUILTIN_TTEST:
1898 target = gen_reg_rtx (DImode);
1899 emit_insn (GEN_FCN (CODE_FOR_ttest) (target));
1900 break;
1901
1902 case AARCH64_TME_BUILTIN_TCOMMIT:
1903 emit_insn (GEN_FCN (CODE_FOR_tcommit) ());
1904 break;
1905
1906 case AARCH64_TME_BUILTIN_TCANCEL:
1907 {
1908 tree arg0 = CALL_EXPR_ARG (exp, 0);
1909 rtx op0 = expand_normal (arg0);
1910 if (CONST_INT_P (op0) && UINTVAL (op0) <= 65536)
1911 emit_insn (GEN_FCN (CODE_FOR_tcancel) (op0));
1912 else
1913 {
1914 error ("%Kargument must be a 16-bit constant immediate", exp);
1915 return const0_rtx;
1916 }
1917 }
1918 break;
1919
1920 default :
1921 gcc_unreachable ();
1922 }
1923 return target;
1924 }
1925
1926 /* Expand a random number builtin EXP with code FCODE, putting the result
1927 int TARGET. If IGNORE is true the return value is ignored. */
1928
1929 rtx
aarch64_expand_rng_builtin(tree exp,rtx target,int fcode,int ignore)1930 aarch64_expand_rng_builtin (tree exp, rtx target, int fcode, int ignore)
1931 {
1932 rtx pat;
1933 enum insn_code icode;
1934 if (fcode == AARCH64_BUILTIN_RNG_RNDR)
1935 icode = CODE_FOR_aarch64_rndr;
1936 else if (fcode == AARCH64_BUILTIN_RNG_RNDRRS)
1937 icode = CODE_FOR_aarch64_rndrrs;
1938 else
1939 gcc_unreachable ();
1940
1941 rtx rand = gen_reg_rtx (DImode);
1942 pat = GEN_FCN (icode) (rand);
1943 if (!pat)
1944 return NULL_RTX;
1945
1946 tree arg0 = CALL_EXPR_ARG (exp, 0);
1947 rtx res_addr = expand_normal (arg0);
1948 res_addr = convert_memory_address (Pmode, res_addr);
1949 rtx res_mem = gen_rtx_MEM (DImode, res_addr);
1950 emit_insn (pat);
1951 emit_move_insn (res_mem, rand);
1952 /* If the status result is unused don't generate the CSET code. */
1953 if (ignore)
1954 return target;
1955
1956 rtx cc_reg = gen_rtx_REG (CC_Zmode, CC_REGNUM);
1957 rtx cmp_rtx = gen_rtx_fmt_ee (EQ, SImode, cc_reg, const0_rtx);
1958 emit_insn (gen_aarch64_cstoresi (target, cmp_rtx, cc_reg));
1959 return target;
1960 }
1961
1962 /* Expand an expression EXP that calls a MEMTAG built-in FCODE
1963 with result going to TARGET. */
1964 static rtx
aarch64_expand_builtin_memtag(int fcode,tree exp,rtx target)1965 aarch64_expand_builtin_memtag (int fcode, tree exp, rtx target)
1966 {
1967 if (TARGET_ILP32)
1968 {
1969 error ("Memory Tagging Extension does not support %<-mabi=ilp32%>");
1970 return const0_rtx;
1971 }
1972
1973 rtx pat = NULL;
1974 enum insn_code icode = aarch64_memtag_builtin_data[fcode -
1975 AARCH64_MEMTAG_BUILTIN_START - 1].icode;
1976
1977 rtx op0 = expand_normal (CALL_EXPR_ARG (exp, 0));
1978 machine_mode mode0 = GET_MODE (op0);
1979 op0 = force_reg (mode0 == VOIDmode ? DImode : mode0, op0);
1980 op0 = convert_to_mode (DImode, op0, true);
1981
1982 switch (fcode)
1983 {
1984 case AARCH64_MEMTAG_BUILTIN_IRG:
1985 case AARCH64_MEMTAG_BUILTIN_GMI:
1986 case AARCH64_MEMTAG_BUILTIN_SUBP:
1987 case AARCH64_MEMTAG_BUILTIN_INC_TAG:
1988 {
1989 if (! target
1990 || GET_MODE (target) != DImode
1991 || ! (*insn_data[icode].operand[0].predicate) (target, DImode))
1992 target = gen_reg_rtx (DImode);
1993
1994 if (fcode == AARCH64_MEMTAG_BUILTIN_INC_TAG)
1995 {
1996 rtx op1 = expand_normal (CALL_EXPR_ARG (exp, 1));
1997
1998 if ((*insn_data[icode].operand[3].predicate) (op1, QImode))
1999 {
2000 pat = GEN_FCN (icode) (target, op0, const0_rtx, op1);
2001 break;
2002 }
2003 error ("%Kargument %d must be a constant immediate "
2004 "in range [0,15]", exp, 2);
2005 return const0_rtx;
2006 }
2007 else
2008 {
2009 rtx op1 = expand_normal (CALL_EXPR_ARG (exp, 1));
2010 machine_mode mode1 = GET_MODE (op1);
2011 op1 = force_reg (mode1 == VOIDmode ? DImode : mode1, op1);
2012 op1 = convert_to_mode (DImode, op1, true);
2013 pat = GEN_FCN (icode) (target, op0, op1);
2014 }
2015 break;
2016 }
2017 case AARCH64_MEMTAG_BUILTIN_GET_TAG:
2018 target = op0;
2019 pat = GEN_FCN (icode) (target, op0, const0_rtx);
2020 break;
2021 case AARCH64_MEMTAG_BUILTIN_SET_TAG:
2022 pat = GEN_FCN (icode) (op0, op0, const0_rtx);
2023 break;
2024 default:
2025 gcc_unreachable();
2026 }
2027
2028 if (!pat)
2029 return NULL_RTX;
2030
2031 emit_insn (pat);
2032 return target;
2033 }
2034
2035 /* Expand an expression EXP as fpsr or fpcr setter (depending on
2036 UNSPEC) using MODE. */
2037 static void
aarch64_expand_fpsr_fpcr_setter(int unspec,machine_mode mode,tree exp)2038 aarch64_expand_fpsr_fpcr_setter (int unspec, machine_mode mode, tree exp)
2039 {
2040 tree arg = CALL_EXPR_ARG (exp, 0);
2041 rtx op = force_reg (mode, expand_normal (arg));
2042 emit_insn (gen_aarch64_set (unspec, mode, op));
2043 }
2044
2045 /* Expand a fpsr or fpcr getter (depending on UNSPEC) using MODE.
2046 Return the target. */
2047 static rtx
aarch64_expand_fpsr_fpcr_getter(enum insn_code icode,machine_mode mode,rtx target)2048 aarch64_expand_fpsr_fpcr_getter (enum insn_code icode, machine_mode mode,
2049 rtx target)
2050 {
2051 expand_operand op;
2052 create_output_operand (&op, target, mode);
2053 expand_insn (icode, 1, &op);
2054 return op.value;
2055 }
2056
2057 /* Expand an expression EXP that calls built-in function FCODE,
2058 with result going to TARGET if that's convenient. IGNORE is true
2059 if the result of the builtin is ignored. */
2060 rtx
aarch64_general_expand_builtin(unsigned int fcode,tree exp,rtx target,int ignore)2061 aarch64_general_expand_builtin (unsigned int fcode, tree exp, rtx target,
2062 int ignore)
2063 {
2064 int icode;
2065 rtx op0;
2066 tree arg0;
2067
2068 switch (fcode)
2069 {
2070 case AARCH64_BUILTIN_GET_FPCR:
2071 return aarch64_expand_fpsr_fpcr_getter (CODE_FOR_aarch64_get_fpcrsi,
2072 SImode, target);
2073 case AARCH64_BUILTIN_SET_FPCR:
2074 aarch64_expand_fpsr_fpcr_setter (UNSPECV_SET_FPCR, SImode, exp);
2075 return target;
2076 case AARCH64_BUILTIN_GET_FPSR:
2077 return aarch64_expand_fpsr_fpcr_getter (CODE_FOR_aarch64_get_fpsrsi,
2078 SImode, target);
2079 case AARCH64_BUILTIN_SET_FPSR:
2080 aarch64_expand_fpsr_fpcr_setter (UNSPECV_SET_FPSR, SImode, exp);
2081 return target;
2082 case AARCH64_BUILTIN_GET_FPCR64:
2083 return aarch64_expand_fpsr_fpcr_getter (CODE_FOR_aarch64_get_fpcrdi,
2084 DImode, target);
2085 case AARCH64_BUILTIN_SET_FPCR64:
2086 aarch64_expand_fpsr_fpcr_setter (UNSPECV_SET_FPCR, DImode, exp);
2087 return target;
2088 case AARCH64_BUILTIN_GET_FPSR64:
2089 return aarch64_expand_fpsr_fpcr_getter (CODE_FOR_aarch64_get_fpsrdi,
2090 DImode, target);
2091 case AARCH64_BUILTIN_SET_FPSR64:
2092 aarch64_expand_fpsr_fpcr_setter (UNSPECV_SET_FPSR, DImode, exp);
2093 return target;
2094 case AARCH64_PAUTH_BUILTIN_AUTIA1716:
2095 case AARCH64_PAUTH_BUILTIN_PACIA1716:
2096 case AARCH64_PAUTH_BUILTIN_AUTIB1716:
2097 case AARCH64_PAUTH_BUILTIN_PACIB1716:
2098 case AARCH64_PAUTH_BUILTIN_XPACLRI:
2099 arg0 = CALL_EXPR_ARG (exp, 0);
2100 op0 = force_reg (Pmode, expand_normal (arg0));
2101
2102 if (fcode == AARCH64_PAUTH_BUILTIN_XPACLRI)
2103 {
2104 rtx lr = gen_rtx_REG (Pmode, R30_REGNUM);
2105 icode = CODE_FOR_xpaclri;
2106 emit_move_insn (lr, op0);
2107 emit_insn (GEN_FCN (icode) ());
2108 return lr;
2109 }
2110 else
2111 {
2112 tree arg1 = CALL_EXPR_ARG (exp, 1);
2113 rtx op1 = force_reg (Pmode, expand_normal (arg1));
2114 switch (fcode)
2115 {
2116 case AARCH64_PAUTH_BUILTIN_AUTIA1716:
2117 icode = CODE_FOR_autia1716;
2118 break;
2119 case AARCH64_PAUTH_BUILTIN_AUTIB1716:
2120 icode = CODE_FOR_autib1716;
2121 break;
2122 case AARCH64_PAUTH_BUILTIN_PACIA1716:
2123 icode = CODE_FOR_pacia1716;
2124 break;
2125 case AARCH64_PAUTH_BUILTIN_PACIB1716:
2126 icode = CODE_FOR_pacib1716;
2127 break;
2128 default:
2129 icode = 0;
2130 gcc_unreachable ();
2131 }
2132
2133 rtx x16_reg = gen_rtx_REG (Pmode, R16_REGNUM);
2134 rtx x17_reg = gen_rtx_REG (Pmode, R17_REGNUM);
2135 emit_move_insn (x17_reg, op0);
2136 emit_move_insn (x16_reg, op1);
2137 emit_insn (GEN_FCN (icode) ());
2138 return x17_reg;
2139 }
2140
2141 case AARCH64_JSCVT:
2142 {
2143 expand_operand ops[2];
2144 create_output_operand (&ops[0], target, SImode);
2145 op0 = expand_normal (CALL_EXPR_ARG (exp, 0));
2146 create_input_operand (&ops[1], op0, DFmode);
2147 expand_insn (CODE_FOR_aarch64_fjcvtzs, 2, ops);
2148 return ops[0].value;
2149 }
2150
2151 case AARCH64_SIMD_BUILTIN_FCMLA_LANEQ0_V2SF:
2152 case AARCH64_SIMD_BUILTIN_FCMLA_LANEQ90_V2SF:
2153 case AARCH64_SIMD_BUILTIN_FCMLA_LANEQ180_V2SF:
2154 case AARCH64_SIMD_BUILTIN_FCMLA_LANEQ270_V2SF:
2155 case AARCH64_SIMD_BUILTIN_FCMLA_LANEQ0_V4HF:
2156 case AARCH64_SIMD_BUILTIN_FCMLA_LANEQ90_V4HF:
2157 case AARCH64_SIMD_BUILTIN_FCMLA_LANEQ180_V4HF:
2158 case AARCH64_SIMD_BUILTIN_FCMLA_LANEQ270_V4HF:
2159 return aarch64_expand_fcmla_builtin (exp, target, fcode);
2160 case AARCH64_BUILTIN_RNG_RNDR:
2161 case AARCH64_BUILTIN_RNG_RNDRRS:
2162 return aarch64_expand_rng_builtin (exp, target, fcode, ignore);
2163 }
2164
2165 if (fcode >= AARCH64_SIMD_BUILTIN_BASE && fcode <= AARCH64_SIMD_BUILTIN_MAX)
2166 return aarch64_simd_expand_builtin (fcode, exp, target);
2167 else if (fcode >= AARCH64_CRC32_BUILTIN_BASE && fcode <= AARCH64_CRC32_BUILTIN_MAX)
2168 return aarch64_crc32_expand_builtin (fcode, exp, target);
2169
2170 if (fcode == AARCH64_BUILTIN_RSQRT_DF
2171 || fcode == AARCH64_BUILTIN_RSQRT_SF
2172 || fcode == AARCH64_BUILTIN_RSQRT_V2DF
2173 || fcode == AARCH64_BUILTIN_RSQRT_V2SF
2174 || fcode == AARCH64_BUILTIN_RSQRT_V4SF)
2175 return aarch64_expand_builtin_rsqrt (fcode, exp, target);
2176
2177 if (fcode == AARCH64_TME_BUILTIN_TSTART
2178 || fcode == AARCH64_TME_BUILTIN_TCOMMIT
2179 || fcode == AARCH64_TME_BUILTIN_TTEST
2180 || fcode == AARCH64_TME_BUILTIN_TCANCEL)
2181 return aarch64_expand_builtin_tme (fcode, exp, target);
2182
2183 if (fcode >= AARCH64_MEMTAG_BUILTIN_START
2184 && fcode <= AARCH64_MEMTAG_BUILTIN_END)
2185 return aarch64_expand_builtin_memtag (fcode, exp, target);
2186
2187 gcc_unreachable ();
2188 }
2189
2190 tree
aarch64_builtin_vectorized_function(unsigned int fn,tree type_out,tree type_in)2191 aarch64_builtin_vectorized_function (unsigned int fn, tree type_out,
2192 tree type_in)
2193 {
2194 machine_mode in_mode, out_mode;
2195
2196 if (TREE_CODE (type_out) != VECTOR_TYPE
2197 || TREE_CODE (type_in) != VECTOR_TYPE)
2198 return NULL_TREE;
2199
2200 out_mode = TYPE_MODE (type_out);
2201 in_mode = TYPE_MODE (type_in);
2202
2203 #undef AARCH64_CHECK_BUILTIN_MODE
2204 #define AARCH64_CHECK_BUILTIN_MODE(C, N) 1
2205 #define AARCH64_FIND_FRINT_VARIANT(N) \
2206 (AARCH64_CHECK_BUILTIN_MODE (2, D) \
2207 ? aarch64_builtin_decls[AARCH64_SIMD_BUILTIN_UNOP_##N##v2df] \
2208 : (AARCH64_CHECK_BUILTIN_MODE (4, S) \
2209 ? aarch64_builtin_decls[AARCH64_SIMD_BUILTIN_UNOP_##N##v4sf] \
2210 : (AARCH64_CHECK_BUILTIN_MODE (2, S) \
2211 ? aarch64_builtin_decls[AARCH64_SIMD_BUILTIN_UNOP_##N##v2sf] \
2212 : NULL_TREE)))
2213 switch (fn)
2214 {
2215 #undef AARCH64_CHECK_BUILTIN_MODE
2216 #define AARCH64_CHECK_BUILTIN_MODE(C, N) \
2217 (out_mode == V##C##N##Fmode && in_mode == V##C##N##Fmode)
2218 CASE_CFN_FLOOR:
2219 return AARCH64_FIND_FRINT_VARIANT (floor);
2220 CASE_CFN_CEIL:
2221 return AARCH64_FIND_FRINT_VARIANT (ceil);
2222 CASE_CFN_TRUNC:
2223 return AARCH64_FIND_FRINT_VARIANT (btrunc);
2224 CASE_CFN_ROUND:
2225 return AARCH64_FIND_FRINT_VARIANT (round);
2226 CASE_CFN_NEARBYINT:
2227 return AARCH64_FIND_FRINT_VARIANT (nearbyint);
2228 CASE_CFN_SQRT:
2229 return AARCH64_FIND_FRINT_VARIANT (sqrt);
2230 #undef AARCH64_CHECK_BUILTIN_MODE
2231 #define AARCH64_CHECK_BUILTIN_MODE(C, N) \
2232 (out_mode == V##C##SImode && in_mode == V##C##N##Imode)
2233 CASE_CFN_CLZ:
2234 {
2235 if (AARCH64_CHECK_BUILTIN_MODE (4, S))
2236 return aarch64_builtin_decls[AARCH64_SIMD_BUILTIN_UNOP_clzv4si];
2237 return NULL_TREE;
2238 }
2239 CASE_CFN_CTZ:
2240 {
2241 if (AARCH64_CHECK_BUILTIN_MODE (2, S))
2242 return aarch64_builtin_decls[AARCH64_SIMD_BUILTIN_UNOP_ctzv2si];
2243 else if (AARCH64_CHECK_BUILTIN_MODE (4, S))
2244 return aarch64_builtin_decls[AARCH64_SIMD_BUILTIN_UNOP_ctzv4si];
2245 return NULL_TREE;
2246 }
2247 #undef AARCH64_CHECK_BUILTIN_MODE
2248 #define AARCH64_CHECK_BUILTIN_MODE(C, N) \
2249 (out_mode == V##C##N##Imode && in_mode == V##C##N##Fmode)
2250 CASE_CFN_IFLOOR:
2251 CASE_CFN_LFLOOR:
2252 CASE_CFN_LLFLOOR:
2253 {
2254 enum aarch64_builtins builtin;
2255 if (AARCH64_CHECK_BUILTIN_MODE (2, D))
2256 builtin = AARCH64_SIMD_BUILTIN_UNOP_lfloorv2dfv2di;
2257 else if (AARCH64_CHECK_BUILTIN_MODE (4, S))
2258 builtin = AARCH64_SIMD_BUILTIN_UNOP_lfloorv4sfv4si;
2259 else if (AARCH64_CHECK_BUILTIN_MODE (2, S))
2260 builtin = AARCH64_SIMD_BUILTIN_UNOP_lfloorv2sfv2si;
2261 else
2262 return NULL_TREE;
2263
2264 return aarch64_builtin_decls[builtin];
2265 }
2266 CASE_CFN_ICEIL:
2267 CASE_CFN_LCEIL:
2268 CASE_CFN_LLCEIL:
2269 {
2270 enum aarch64_builtins builtin;
2271 if (AARCH64_CHECK_BUILTIN_MODE (2, D))
2272 builtin = AARCH64_SIMD_BUILTIN_UNOP_lceilv2dfv2di;
2273 else if (AARCH64_CHECK_BUILTIN_MODE (4, S))
2274 builtin = AARCH64_SIMD_BUILTIN_UNOP_lceilv4sfv4si;
2275 else if (AARCH64_CHECK_BUILTIN_MODE (2, S))
2276 builtin = AARCH64_SIMD_BUILTIN_UNOP_lceilv2sfv2si;
2277 else
2278 return NULL_TREE;
2279
2280 return aarch64_builtin_decls[builtin];
2281 }
2282 CASE_CFN_IROUND:
2283 CASE_CFN_LROUND:
2284 CASE_CFN_LLROUND:
2285 {
2286 enum aarch64_builtins builtin;
2287 if (AARCH64_CHECK_BUILTIN_MODE (2, D))
2288 builtin = AARCH64_SIMD_BUILTIN_UNOP_lroundv2dfv2di;
2289 else if (AARCH64_CHECK_BUILTIN_MODE (4, S))
2290 builtin = AARCH64_SIMD_BUILTIN_UNOP_lroundv4sfv4si;
2291 else if (AARCH64_CHECK_BUILTIN_MODE (2, S))
2292 builtin = AARCH64_SIMD_BUILTIN_UNOP_lroundv2sfv2si;
2293 else
2294 return NULL_TREE;
2295
2296 return aarch64_builtin_decls[builtin];
2297 }
2298 default:
2299 return NULL_TREE;
2300 }
2301
2302 return NULL_TREE;
2303 }
2304
2305 /* Return builtin for reciprocal square root. */
2306
2307 tree
aarch64_general_builtin_rsqrt(unsigned int fn)2308 aarch64_general_builtin_rsqrt (unsigned int fn)
2309 {
2310 if (fn == AARCH64_SIMD_BUILTIN_UNOP_sqrtv2df)
2311 return aarch64_builtin_decls[AARCH64_BUILTIN_RSQRT_V2DF];
2312 if (fn == AARCH64_SIMD_BUILTIN_UNOP_sqrtv2sf)
2313 return aarch64_builtin_decls[AARCH64_BUILTIN_RSQRT_V2SF];
2314 if (fn == AARCH64_SIMD_BUILTIN_UNOP_sqrtv4sf)
2315 return aarch64_builtin_decls[AARCH64_BUILTIN_RSQRT_V4SF];
2316 return NULL_TREE;
2317 }
2318
2319 #undef VAR1
2320 #define VAR1(T, N, MAP, FLAG, A) \
2321 case AARCH64_SIMD_BUILTIN_##T##_##N##A:
2322
2323 /* Try to fold a call to the built-in function with subcode FCODE. The
2324 function is passed the N_ARGS arguments in ARGS and it returns a value
2325 of type TYPE. Return the new expression on success and NULL_TREE on
2326 failure. */
2327 tree
aarch64_general_fold_builtin(unsigned int fcode,tree type,unsigned int n_args ATTRIBUTE_UNUSED,tree * args)2328 aarch64_general_fold_builtin (unsigned int fcode, tree type,
2329 unsigned int n_args ATTRIBUTE_UNUSED, tree *args)
2330 {
2331 switch (fcode)
2332 {
2333 BUILTIN_VDQF (UNOP, abs, 2, ALL)
2334 return fold_build1 (ABS_EXPR, type, args[0]);
2335 VAR1 (UNOP, floatv2si, 2, ALL, v2sf)
2336 VAR1 (UNOP, floatv4si, 2, ALL, v4sf)
2337 VAR1 (UNOP, floatv2di, 2, ALL, v2df)
2338 return fold_build1 (FLOAT_EXPR, type, args[0]);
2339 default:
2340 break;
2341 }
2342
2343 return NULL_TREE;
2344 }
2345
2346 /* Try to fold STMT, given that it's a call to the built-in function with
2347 subcode FCODE. Return the new statement on success and null on
2348 failure. */
2349 gimple *
aarch64_general_gimple_fold_builtin(unsigned int fcode,gcall * stmt)2350 aarch64_general_gimple_fold_builtin (unsigned int fcode, gcall *stmt)
2351 {
2352 gimple *new_stmt = NULL;
2353 unsigned nargs = gimple_call_num_args (stmt);
2354 tree *args = (nargs > 0
2355 ? gimple_call_arg_ptr (stmt, 0)
2356 : &error_mark_node);
2357
2358 /* We use gimple's IFN_REDUC_(PLUS|MIN|MAX)s for float, signed int
2359 and unsigned int; it will distinguish according to the types of
2360 the arguments to the __builtin. */
2361 switch (fcode)
2362 {
2363 BUILTIN_VALL (UNOP, reduc_plus_scal_, 10, ALL)
2364 new_stmt = gimple_build_call_internal (IFN_REDUC_PLUS,
2365 1, args[0]);
2366 gimple_call_set_lhs (new_stmt, gimple_call_lhs (stmt));
2367 break;
2368 BUILTIN_VDQIF (UNOP, reduc_smax_scal_, 10, ALL)
2369 BUILTIN_VDQ_BHSI (UNOPU, reduc_umax_scal_, 10, ALL)
2370 new_stmt = gimple_build_call_internal (IFN_REDUC_MAX,
2371 1, args[0]);
2372 gimple_call_set_lhs (new_stmt, gimple_call_lhs (stmt));
2373 break;
2374 BUILTIN_VDQIF (UNOP, reduc_smin_scal_, 10, ALL)
2375 BUILTIN_VDQ_BHSI (UNOPU, reduc_umin_scal_, 10, ALL)
2376 new_stmt = gimple_build_call_internal (IFN_REDUC_MIN,
2377 1, args[0]);
2378 gimple_call_set_lhs (new_stmt, gimple_call_lhs (stmt));
2379 break;
2380 BUILTIN_GPF (BINOP, fmulx, 0, ALL)
2381 {
2382 gcc_assert (nargs == 2);
2383 bool a0_cst_p = TREE_CODE (args[0]) == REAL_CST;
2384 bool a1_cst_p = TREE_CODE (args[1]) == REAL_CST;
2385 if (a0_cst_p || a1_cst_p)
2386 {
2387 if (a0_cst_p && a1_cst_p)
2388 {
2389 tree t0 = TREE_TYPE (args[0]);
2390 real_value a0 = (TREE_REAL_CST (args[0]));
2391 real_value a1 = (TREE_REAL_CST (args[1]));
2392 if (real_equal (&a1, &dconst0))
2393 std::swap (a0, a1);
2394 /* According to real_equal (), +0 equals -0. */
2395 if (real_equal (&a0, &dconst0) && real_isinf (&a1))
2396 {
2397 real_value res = dconst2;
2398 res.sign = a0.sign ^ a1.sign;
2399 new_stmt = gimple_build_assign (gimple_call_lhs (stmt),
2400 REAL_CST,
2401 build_real (t0, res));
2402 }
2403 else
2404 new_stmt = gimple_build_assign (gimple_call_lhs (stmt),
2405 MULT_EXPR,
2406 args[0], args[1]);
2407 }
2408 else /* a0_cst_p ^ a1_cst_p. */
2409 {
2410 real_value const_part = a0_cst_p
2411 ? TREE_REAL_CST (args[0]) : TREE_REAL_CST (args[1]);
2412 if (!real_equal (&const_part, &dconst0)
2413 && !real_isinf (&const_part))
2414 new_stmt = gimple_build_assign (gimple_call_lhs (stmt),
2415 MULT_EXPR, args[0],
2416 args[1]);
2417 }
2418 }
2419 if (new_stmt)
2420 {
2421 gimple_set_vuse (new_stmt, gimple_vuse (stmt));
2422 gimple_set_vdef (new_stmt, gimple_vdef (stmt));
2423 }
2424 break;
2425 }
2426 default:
2427 break;
2428 }
2429 return new_stmt;
2430 }
2431
2432 void
aarch64_atomic_assign_expand_fenv(tree * hold,tree * clear,tree * update)2433 aarch64_atomic_assign_expand_fenv (tree *hold, tree *clear, tree *update)
2434 {
2435 const unsigned AARCH64_FE_INVALID = 1;
2436 const unsigned AARCH64_FE_DIVBYZERO = 2;
2437 const unsigned AARCH64_FE_OVERFLOW = 4;
2438 const unsigned AARCH64_FE_UNDERFLOW = 8;
2439 const unsigned AARCH64_FE_INEXACT = 16;
2440 const unsigned HOST_WIDE_INT AARCH64_FE_ALL_EXCEPT = (AARCH64_FE_INVALID
2441 | AARCH64_FE_DIVBYZERO
2442 | AARCH64_FE_OVERFLOW
2443 | AARCH64_FE_UNDERFLOW
2444 | AARCH64_FE_INEXACT);
2445 const unsigned HOST_WIDE_INT AARCH64_FE_EXCEPT_SHIFT = 8;
2446 tree fenv_cr, fenv_sr, get_fpcr, set_fpcr, mask_cr, mask_sr;
2447 tree ld_fenv_cr, ld_fenv_sr, masked_fenv_cr, masked_fenv_sr, hold_fnclex_cr;
2448 tree hold_fnclex_sr, new_fenv_var, reload_fenv, restore_fnenv, get_fpsr, set_fpsr;
2449 tree update_call, atomic_feraiseexcept, hold_fnclex, masked_fenv, ld_fenv;
2450
2451 /* Generate the equivalence of :
2452 unsigned int fenv_cr;
2453 fenv_cr = __builtin_aarch64_get_fpcr ();
2454
2455 unsigned int fenv_sr;
2456 fenv_sr = __builtin_aarch64_get_fpsr ();
2457
2458 Now set all exceptions to non-stop
2459 unsigned int mask_cr
2460 = ~(AARCH64_FE_ALL_EXCEPT << AARCH64_FE_EXCEPT_SHIFT);
2461 unsigned int masked_cr;
2462 masked_cr = fenv_cr & mask_cr;
2463
2464 And clear all exception flags
2465 unsigned int maske_sr = ~AARCH64_FE_ALL_EXCEPT;
2466 unsigned int masked_cr;
2467 masked_sr = fenv_sr & mask_sr;
2468
2469 __builtin_aarch64_set_cr (masked_cr);
2470 __builtin_aarch64_set_sr (masked_sr); */
2471
2472 fenv_cr = create_tmp_var_raw (unsigned_type_node);
2473 fenv_sr = create_tmp_var_raw (unsigned_type_node);
2474
2475 get_fpcr = aarch64_builtin_decls[AARCH64_BUILTIN_GET_FPCR];
2476 set_fpcr = aarch64_builtin_decls[AARCH64_BUILTIN_SET_FPCR];
2477 get_fpsr = aarch64_builtin_decls[AARCH64_BUILTIN_GET_FPSR];
2478 set_fpsr = aarch64_builtin_decls[AARCH64_BUILTIN_SET_FPSR];
2479
2480 mask_cr = build_int_cst (unsigned_type_node,
2481 ~(AARCH64_FE_ALL_EXCEPT << AARCH64_FE_EXCEPT_SHIFT));
2482 mask_sr = build_int_cst (unsigned_type_node,
2483 ~(AARCH64_FE_ALL_EXCEPT));
2484
2485 ld_fenv_cr = build4 (TARGET_EXPR, unsigned_type_node,
2486 fenv_cr, build_call_expr (get_fpcr, 0),
2487 NULL_TREE, NULL_TREE);
2488 ld_fenv_sr = build4 (TARGET_EXPR, unsigned_type_node,
2489 fenv_sr, build_call_expr (get_fpsr, 0),
2490 NULL_TREE, NULL_TREE);
2491
2492 masked_fenv_cr = build2 (BIT_AND_EXPR, unsigned_type_node, fenv_cr, mask_cr);
2493 masked_fenv_sr = build2 (BIT_AND_EXPR, unsigned_type_node, fenv_sr, mask_sr);
2494
2495 hold_fnclex_cr = build_call_expr (set_fpcr, 1, masked_fenv_cr);
2496 hold_fnclex_sr = build_call_expr (set_fpsr, 1, masked_fenv_sr);
2497
2498 hold_fnclex = build2 (COMPOUND_EXPR, void_type_node, hold_fnclex_cr,
2499 hold_fnclex_sr);
2500 masked_fenv = build2 (COMPOUND_EXPR, void_type_node, masked_fenv_cr,
2501 masked_fenv_sr);
2502 ld_fenv = build2 (COMPOUND_EXPR, void_type_node, ld_fenv_cr, ld_fenv_sr);
2503
2504 *hold = build2 (COMPOUND_EXPR, void_type_node,
2505 build2 (COMPOUND_EXPR, void_type_node, masked_fenv, ld_fenv),
2506 hold_fnclex);
2507
2508 /* Store the value of masked_fenv to clear the exceptions:
2509 __builtin_aarch64_set_fpsr (masked_fenv_sr); */
2510
2511 *clear = build_call_expr (set_fpsr, 1, masked_fenv_sr);
2512
2513 /* Generate the equivalent of :
2514 unsigned int new_fenv_var;
2515 new_fenv_var = __builtin_aarch64_get_fpsr ();
2516
2517 __builtin_aarch64_set_fpsr (fenv_sr);
2518
2519 __atomic_feraiseexcept (new_fenv_var); */
2520
2521 new_fenv_var = create_tmp_var_raw (unsigned_type_node);
2522 reload_fenv = build4 (TARGET_EXPR, unsigned_type_node,
2523 new_fenv_var, build_call_expr (get_fpsr, 0),
2524 NULL_TREE, NULL_TREE);
2525 restore_fnenv = build_call_expr (set_fpsr, 1, fenv_sr);
2526 atomic_feraiseexcept = builtin_decl_implicit (BUILT_IN_ATOMIC_FERAISEEXCEPT);
2527 update_call = build_call_expr (atomic_feraiseexcept, 1,
2528 fold_convert (integer_type_node, new_fenv_var));
2529 *update = build2 (COMPOUND_EXPR, void_type_node,
2530 build2 (COMPOUND_EXPR, void_type_node,
2531 reload_fenv, restore_fnenv), update_call);
2532 }
2533
2534 /* Resolve overloaded MEMTAG build-in functions. */
2535 #define AARCH64_BUILTIN_SUBCODE(F) \
2536 (DECL_MD_FUNCTION_CODE (F) >> AARCH64_BUILTIN_SHIFT)
2537
2538 static tree
aarch64_resolve_overloaded_memtag(location_t loc,tree fndecl,void * pass_params)2539 aarch64_resolve_overloaded_memtag (location_t loc,
2540 tree fndecl, void *pass_params)
2541 {
2542 vec<tree, va_gc> *params = static_cast<vec<tree, va_gc> *> (pass_params);
2543 unsigned param_num = params ? params->length() : 0;
2544 unsigned int fcode = AARCH64_BUILTIN_SUBCODE (fndecl);
2545 tree inittype = aarch64_memtag_builtin_data[
2546 fcode - AARCH64_MEMTAG_BUILTIN_START - 1].ftype;
2547 unsigned arg_num = list_length (TYPE_ARG_TYPES (inittype)) - 1;
2548
2549 if (param_num != arg_num)
2550 {
2551 TREE_TYPE (fndecl) = inittype;
2552 return NULL_TREE;
2553 }
2554 tree retype = NULL;
2555
2556 if (fcode == AARCH64_MEMTAG_BUILTIN_SUBP)
2557 {
2558 tree t0 = TREE_TYPE ((*params)[0]);
2559 tree t1 = TREE_TYPE ((*params)[1]);
2560
2561 if (t0 == error_mark_node || TREE_CODE (t0) != POINTER_TYPE)
2562 t0 = ptr_type_node;
2563 if (t1 == error_mark_node || TREE_CODE (t1) != POINTER_TYPE)
2564 t1 = ptr_type_node;
2565
2566 if (TYPE_MODE (t0) != DImode)
2567 warning_at (loc, 1, "expected 64-bit address but argument 1 is %d-bit",
2568 (int)tree_to_shwi (DECL_SIZE ((*params)[0])));
2569
2570 if (TYPE_MODE (t1) != DImode)
2571 warning_at (loc, 1, "expected 64-bit address but argument 2 is %d-bit",
2572 (int)tree_to_shwi (DECL_SIZE ((*params)[1])));
2573
2574 retype = build_function_type_list (ptrdiff_type_node, t0, t1, NULL);
2575 }
2576 else
2577 {
2578 tree t0 = TREE_TYPE ((*params)[0]);
2579
2580 if (t0 == error_mark_node || TREE_CODE (t0) != POINTER_TYPE)
2581 {
2582 TREE_TYPE (fndecl) = inittype;
2583 return NULL_TREE;
2584 }
2585
2586 if (TYPE_MODE (t0) != DImode)
2587 warning_at (loc, 1, "expected 64-bit address but argument 1 is %d-bit",
2588 (int)tree_to_shwi (DECL_SIZE ((*params)[0])));
2589
2590 switch (fcode)
2591 {
2592 case AARCH64_MEMTAG_BUILTIN_IRG:
2593 retype = build_function_type_list (t0, t0, uint64_type_node, NULL);
2594 break;
2595 case AARCH64_MEMTAG_BUILTIN_GMI:
2596 retype = build_function_type_list (uint64_type_node, t0,
2597 uint64_type_node, NULL);
2598 break;
2599 case AARCH64_MEMTAG_BUILTIN_INC_TAG:
2600 retype = build_function_type_list (t0, t0, unsigned_type_node, NULL);
2601 break;
2602 case AARCH64_MEMTAG_BUILTIN_SET_TAG:
2603 retype = build_function_type_list (void_type_node, t0, NULL);
2604 break;
2605 case AARCH64_MEMTAG_BUILTIN_GET_TAG:
2606 retype = build_function_type_list (t0, t0, NULL);
2607 break;
2608 default:
2609 return NULL_TREE;
2610 }
2611 }
2612
2613 if (!retype || retype == error_mark_node)
2614 TREE_TYPE (fndecl) = inittype;
2615 else
2616 TREE_TYPE (fndecl) = retype;
2617
2618 return NULL_TREE;
2619 }
2620
2621 /* Called at aarch64_resolve_overloaded_builtin in aarch64-c.c. */
2622 tree
aarch64_resolve_overloaded_builtin_general(location_t loc,tree function,void * pass_params)2623 aarch64_resolve_overloaded_builtin_general (location_t loc, tree function,
2624 void *pass_params)
2625 {
2626 unsigned int fcode = AARCH64_BUILTIN_SUBCODE (function);
2627
2628 if (fcode >= AARCH64_MEMTAG_BUILTIN_START
2629 && fcode <= AARCH64_MEMTAG_BUILTIN_END)
2630 return aarch64_resolve_overloaded_memtag(loc, function, pass_params);
2631
2632 return NULL_TREE;
2633 }
2634
2635 #undef AARCH64_CHECK_BUILTIN_MODE
2636 #undef AARCH64_FIND_FRINT_VARIANT
2637 #undef CF0
2638 #undef CF1
2639 #undef CF2
2640 #undef CF3
2641 #undef CF4
2642 #undef CF10
2643 #undef VAR1
2644 #undef VAR2
2645 #undef VAR3
2646 #undef VAR4
2647 #undef VAR5
2648 #undef VAR6
2649 #undef VAR7
2650 #undef VAR8
2651 #undef VAR9
2652 #undef VAR10
2653 #undef VAR11
2654
2655 #include "gt-aarch64-builtins.h"
2656