1 /* Copyright (c) 2009, 2010, 2011, 2012 ARM Ltd.
2
3 Permission is hereby granted, free of charge, to any person obtaining
4 a copy of this software and associated documentation files (the
5 ``Software''), to deal in the Software without restriction, including
6 without limitation the rights to use, copy, modify, merge, publish,
7 distribute, sublicense, and/or sell copies of the Software, and to
8 permit persons to whom the Software is furnished to do so, subject to
9 the following conditions:
10
11 The above copyright notice and this permission notice shall be
12 included in all copies or substantial portions of the Software.
13
14 THE SOFTWARE IS PROVIDED ``AS IS'', WITHOUT WARRANTY OF ANY KIND,
15 EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
16 MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
17 IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY
18 CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,
19 TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
20 SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. */
21
22 #include <stdio.h>
23
24 #include <ffi.h>
25 #include <ffi_common.h>
26
27 #include <stdlib.h>
28
29 /* Stack alignment requirement in bytes */
30 #if defined (__APPLE__)
31 #define AARCH64_STACK_ALIGN 1
32 #else
33 #define AARCH64_STACK_ALIGN 16
34 #endif
35
36 #define N_X_ARG_REG 8
37 #define N_V_ARG_REG 8
38
39 #define AARCH64_FFI_WITH_V (1 << AARCH64_FFI_WITH_V_BIT)
40
41 union _d
42 {
43 UINT64 d;
44 UINT32 s[2];
45 };
46
47 struct call_context
48 {
49 UINT64 x [AARCH64_N_XREG];
50 struct
51 {
52 union _d d[2];
53 } v [AARCH64_N_VREG];
54 };
55
56 #if defined (__clang__) && defined (__APPLE__)
57 extern void
58 sys_icache_invalidate (void *start, size_t len);
59 #endif
60
61 static inline void
ffi_clear_cache(void * start,void * end)62 ffi_clear_cache (void *start, void *end)
63 {
64 #if defined (__clang__) && defined (__APPLE__)
65 sys_icache_invalidate (start, (char *)end - (char *)start);
66 #elif defined (__GNUC__)
67 __builtin___clear_cache (start, end);
68 #else
69 #error "Missing builtin to flush instruction cache"
70 #endif
71 }
72
73 static void *
get_x_addr(struct call_context * context,unsigned n)74 get_x_addr (struct call_context *context, unsigned n)
75 {
76 return &context->x[n];
77 }
78
79 static void *
get_s_addr(struct call_context * context,unsigned n)80 get_s_addr (struct call_context *context, unsigned n)
81 {
82 #if defined __AARCH64EB__
83 return &context->v[n].d[1].s[1];
84 #else
85 return &context->v[n].d[0].s[0];
86 #endif
87 }
88
89 static void *
get_d_addr(struct call_context * context,unsigned n)90 get_d_addr (struct call_context *context, unsigned n)
91 {
92 #if defined __AARCH64EB__
93 return &context->v[n].d[1];
94 #else
95 return &context->v[n].d[0];
96 #endif
97 }
98
99 static void *
get_v_addr(struct call_context * context,unsigned n)100 get_v_addr (struct call_context *context, unsigned n)
101 {
102 return &context->v[n];
103 }
104
105 /* Return the memory location at which a basic type would reside
106 were it to have been stored in register n. */
107
108 static void *
get_basic_type_addr(unsigned short type,struct call_context * context,unsigned n)109 get_basic_type_addr (unsigned short type, struct call_context *context,
110 unsigned n)
111 {
112 switch (type)
113 {
114 case FFI_TYPE_FLOAT:
115 return get_s_addr (context, n);
116 case FFI_TYPE_DOUBLE:
117 return get_d_addr (context, n);
118 #if FFI_TYPE_DOUBLE != FFI_TYPE_LONGDOUBLE
119 case FFI_TYPE_LONGDOUBLE:
120 return get_v_addr (context, n);
121 #endif
122 case FFI_TYPE_UINT8:
123 case FFI_TYPE_SINT8:
124 case FFI_TYPE_UINT16:
125 case FFI_TYPE_SINT16:
126 case FFI_TYPE_UINT32:
127 case FFI_TYPE_SINT32:
128 case FFI_TYPE_INT:
129 case FFI_TYPE_POINTER:
130 case FFI_TYPE_UINT64:
131 case FFI_TYPE_SINT64:
132 return get_x_addr (context, n);
133 case FFI_TYPE_VOID:
134 return NULL;
135 default:
136 FFI_ASSERT (0);
137 return NULL;
138 }
139 }
140
141 /* Return the alignment width for each of the basic types. */
142
143 static size_t
get_basic_type_alignment(unsigned short type)144 get_basic_type_alignment (unsigned short type)
145 {
146 switch (type)
147 {
148 case FFI_TYPE_FLOAT:
149 case FFI_TYPE_DOUBLE:
150 return sizeof (UINT64);
151 #if FFI_TYPE_DOUBLE != FFI_TYPE_LONGDOUBLE
152 case FFI_TYPE_LONGDOUBLE:
153 return sizeof (long double);
154 #endif
155 case FFI_TYPE_UINT8:
156 case FFI_TYPE_SINT8:
157 #if defined (__APPLE__)
158 return sizeof (UINT8);
159 #endif
160 case FFI_TYPE_UINT16:
161 case FFI_TYPE_SINT16:
162 #if defined (__APPLE__)
163 return sizeof (UINT16);
164 #endif
165 case FFI_TYPE_UINT32:
166 case FFI_TYPE_INT:
167 case FFI_TYPE_SINT32:
168 #if defined (__APPLE__)
169 return sizeof (UINT32);
170 #endif
171 case FFI_TYPE_POINTER:
172 case FFI_TYPE_UINT64:
173 case FFI_TYPE_SINT64:
174 return sizeof (UINT64);
175
176 default:
177 FFI_ASSERT (0);
178 return 0;
179 }
180 }
181
182 /* Return the size in bytes for each of the basic types. */
183
184 static size_t
get_basic_type_size(unsigned short type)185 get_basic_type_size (unsigned short type)
186 {
187 switch (type)
188 {
189 case FFI_TYPE_FLOAT:
190 return sizeof (UINT32);
191 case FFI_TYPE_DOUBLE:
192 return sizeof (UINT64);
193 #if FFI_TYPE_DOUBLE != FFI_TYPE_LONGDOUBLE
194 case FFI_TYPE_LONGDOUBLE:
195 return sizeof (long double);
196 #endif
197 case FFI_TYPE_UINT8:
198 return sizeof (UINT8);
199 case FFI_TYPE_SINT8:
200 return sizeof (SINT8);
201 case FFI_TYPE_UINT16:
202 return sizeof (UINT16);
203 case FFI_TYPE_SINT16:
204 return sizeof (SINT16);
205 case FFI_TYPE_UINT32:
206 return sizeof (UINT32);
207 case FFI_TYPE_INT:
208 case FFI_TYPE_SINT32:
209 return sizeof (SINT32);
210 case FFI_TYPE_POINTER:
211 case FFI_TYPE_UINT64:
212 return sizeof (UINT64);
213 case FFI_TYPE_SINT64:
214 return sizeof (SINT64);
215
216 default:
217 FFI_ASSERT (0);
218 return 0;
219 }
220 }
221
222 extern void
223 ffi_call_SYSV (unsigned (*)(struct call_context *context, unsigned char *,
224 extended_cif *),
225 struct call_context *context,
226 extended_cif *,
227 size_t,
228 void (*fn)(void));
229
230 extern void
231 ffi_closure_SYSV (ffi_closure *);
232
233 /* Test for an FFI floating point representation. */
234
235 static unsigned
is_floating_type(unsigned short type)236 is_floating_type (unsigned short type)
237 {
238 return (type == FFI_TYPE_FLOAT || type == FFI_TYPE_DOUBLE
239 || type == FFI_TYPE_LONGDOUBLE);
240 }
241
242 /* Test for a homogeneous structure. */
243
244 static unsigned short
get_homogeneous_type(ffi_type * ty)245 get_homogeneous_type (ffi_type *ty)
246 {
247 if (ty->type == FFI_TYPE_STRUCT && ty->elements)
248 {
249 unsigned i;
250 unsigned short candidate_type
251 = get_homogeneous_type (ty->elements[0]);
252 for (i =1; ty->elements[i]; i++)
253 {
254 unsigned short iteration_type = 0;
255 /* If we have a nested struct, we must find its homogeneous type.
256 If that fits with our candidate type, we are still
257 homogeneous. */
258 if (ty->elements[i]->type == FFI_TYPE_STRUCT
259 && ty->elements[i]->elements)
260 {
261 iteration_type = get_homogeneous_type (ty->elements[i]);
262 }
263 else
264 {
265 iteration_type = ty->elements[i]->type;
266 }
267
268 /* If we are not homogeneous, return FFI_TYPE_STRUCT. */
269 if (candidate_type != iteration_type)
270 return FFI_TYPE_STRUCT;
271 }
272 return candidate_type;
273 }
274
275 /* Base case, we have no more levels of nesting, so we
276 are a basic type, and so, trivially homogeneous in that type. */
277 return ty->type;
278 }
279
280 /* Determine the number of elements within a STRUCT.
281
282 Note, we must handle nested structs.
283
284 If ty is not a STRUCT this function will return 0. */
285
286 static unsigned
element_count(ffi_type * ty)287 element_count (ffi_type *ty)
288 {
289 if (ty->type == FFI_TYPE_STRUCT && ty->elements)
290 {
291 unsigned n;
292 unsigned elems = 0;
293 for (n = 0; ty->elements[n]; n++)
294 {
295 if (ty->elements[n]->type == FFI_TYPE_STRUCT
296 && ty->elements[n]->elements)
297 elems += element_count (ty->elements[n]);
298 else
299 elems++;
300 }
301 return elems;
302 }
303 return 0;
304 }
305
306 /* Test for a homogeneous floating point aggregate.
307
308 A homogeneous floating point aggregate is a homogeneous aggregate of
309 a half- single- or double- precision floating point type with one
310 to four elements. Note that this includes nested structs of the
311 basic type. */
312
313 static int
is_hfa(ffi_type * ty)314 is_hfa (ffi_type *ty)
315 {
316 if (ty->type == FFI_TYPE_STRUCT
317 && ty->elements[0]
318 && is_floating_type (get_homogeneous_type (ty)))
319 {
320 unsigned n = element_count (ty);
321 return n >= 1 && n <= 4;
322 }
323 return 0;
324 }
325
326 /* Test if an ffi_type is a candidate for passing in a register.
327
328 This test does not check that sufficient registers of the
329 appropriate class are actually available, merely that IFF
330 sufficient registers are available then the argument will be passed
331 in register(s).
332
333 Note that an ffi_type that is deemed to be a register candidate
334 will always be returned in registers.
335
336 Returns 1 if a register candidate else 0. */
337
338 static int
is_register_candidate(ffi_type * ty)339 is_register_candidate (ffi_type *ty)
340 {
341 switch (ty->type)
342 {
343 case FFI_TYPE_VOID:
344 case FFI_TYPE_FLOAT:
345 case FFI_TYPE_DOUBLE:
346 #if FFI_TYPE_DOUBLE != FFI_TYPE_LONGDOUBLE
347 case FFI_TYPE_LONGDOUBLE:
348 #endif
349 case FFI_TYPE_UINT8:
350 case FFI_TYPE_UINT16:
351 case FFI_TYPE_UINT32:
352 case FFI_TYPE_UINT64:
353 case FFI_TYPE_POINTER:
354 case FFI_TYPE_SINT8:
355 case FFI_TYPE_SINT16:
356 case FFI_TYPE_SINT32:
357 case FFI_TYPE_INT:
358 case FFI_TYPE_SINT64:
359 return 1;
360
361 case FFI_TYPE_STRUCT:
362 if (is_hfa (ty))
363 {
364 return 1;
365 }
366 else if (ty->size > 16)
367 {
368 /* Too large. Will be replaced with a pointer to memory. The
369 pointer MAY be passed in a register, but the value will
370 not. This test specifically fails since the argument will
371 never be passed by value in registers. */
372 return 0;
373 }
374 else
375 {
376 /* Might be passed in registers depending on the number of
377 registers required. */
378 return (ty->size + 7) / 8 < N_X_ARG_REG;
379 }
380 break;
381
382 default:
383 FFI_ASSERT (0);
384 break;
385 }
386
387 return 0;
388 }
389
390 /* Test if an ffi_type argument or result is a candidate for a vector
391 register. */
392
393 static int
is_v_register_candidate(ffi_type * ty)394 is_v_register_candidate (ffi_type *ty)
395 {
396 return is_floating_type (ty->type)
397 || (ty->type == FFI_TYPE_STRUCT && is_hfa (ty));
398 }
399
400 /* Representation of the procedure call argument marshalling
401 state.
402
403 The terse state variable names match the names used in the AARCH64
404 PCS. */
405
406 struct arg_state
407 {
408 unsigned ngrn; /* Next general-purpose register number. */
409 unsigned nsrn; /* Next vector register number. */
410 size_t nsaa; /* Next stack offset. */
411
412 #if defined (__APPLE__)
413 unsigned allocating_variadic;
414 #endif
415 };
416
417 /* Initialize a procedure call argument marshalling state. */
418 static void
arg_init(struct arg_state * state,size_t call_frame_size)419 arg_init (struct arg_state *state, size_t call_frame_size)
420 {
421 state->ngrn = 0;
422 state->nsrn = 0;
423 state->nsaa = 0;
424
425 #if defined (__APPLE__)
426 state->allocating_variadic = 0;
427 #endif
428 }
429
430 /* Return the number of available consecutive core argument
431 registers. */
432
433 static unsigned
available_x(struct arg_state * state)434 available_x (struct arg_state *state)
435 {
436 return N_X_ARG_REG - state->ngrn;
437 }
438
439 /* Return the number of available consecutive vector argument
440 registers. */
441
442 static unsigned
available_v(struct arg_state * state)443 available_v (struct arg_state *state)
444 {
445 return N_V_ARG_REG - state->nsrn;
446 }
447
448 static void *
allocate_to_x(struct call_context * context,struct arg_state * state)449 allocate_to_x (struct call_context *context, struct arg_state *state)
450 {
451 FFI_ASSERT (state->ngrn < N_X_ARG_REG);
452 return get_x_addr (context, (state->ngrn)++);
453 }
454
455 static void *
allocate_to_s(struct call_context * context,struct arg_state * state)456 allocate_to_s (struct call_context *context, struct arg_state *state)
457 {
458 FFI_ASSERT (state->nsrn < N_V_ARG_REG);
459 return get_s_addr (context, (state->nsrn)++);
460 }
461
462 static void *
allocate_to_d(struct call_context * context,struct arg_state * state)463 allocate_to_d (struct call_context *context, struct arg_state *state)
464 {
465 FFI_ASSERT (state->nsrn < N_V_ARG_REG);
466 return get_d_addr (context, (state->nsrn)++);
467 }
468
469 static void *
allocate_to_v(struct call_context * context,struct arg_state * state)470 allocate_to_v (struct call_context *context, struct arg_state *state)
471 {
472 FFI_ASSERT (state->nsrn < N_V_ARG_REG);
473 return get_v_addr (context, (state->nsrn)++);
474 }
475
476 /* Allocate an aligned slot on the stack and return a pointer to it. */
477 static void *
allocate_to_stack(struct arg_state * state,void * stack,size_t alignment,size_t size)478 allocate_to_stack (struct arg_state *state, void *stack, size_t alignment,
479 size_t size)
480 {
481 void *allocation;
482
483 /* Round up the NSAA to the larger of 8 or the natural
484 alignment of the argument's type. */
485 state->nsaa = ALIGN (state->nsaa, alignment);
486 state->nsaa = ALIGN (state->nsaa, alignment);
487 #if defined (__APPLE__)
488 if (state->allocating_variadic)
489 state->nsaa = ALIGN (state->nsaa, 8);
490 #else
491 state->nsaa = ALIGN (state->nsaa, 8);
492 #endif
493
494 allocation = stack + state->nsaa;
495
496 state->nsaa += size;
497 return allocation;
498 }
499
500 static void
copy_basic_type(void * dest,void * source,unsigned short type)501 copy_basic_type (void *dest, void *source, unsigned short type)
502 {
503 /* This is necessary to ensure that basic types are copied
504 sign extended to 64-bits as libffi expects. */
505 switch (type)
506 {
507 case FFI_TYPE_FLOAT:
508 *(float *) dest = *(float *) source;
509 break;
510 case FFI_TYPE_DOUBLE:
511 *(double *) dest = *(double *) source;
512 break;
513 #if FFI_TYPE_DOUBLE != FFI_TYPE_LONGDOUBLE
514 case FFI_TYPE_LONGDOUBLE:
515 *(long double *) dest = *(long double *) source;
516 break;
517 #endif
518 case FFI_TYPE_UINT8:
519 *(ffi_arg *) dest = *(UINT8 *) source;
520 break;
521 case FFI_TYPE_SINT8:
522 *(ffi_sarg *) dest = *(SINT8 *) source;
523 break;
524 case FFI_TYPE_UINT16:
525 *(ffi_arg *) dest = *(UINT16 *) source;
526 break;
527 case FFI_TYPE_SINT16:
528 *(ffi_sarg *) dest = *(SINT16 *) source;
529 break;
530 case FFI_TYPE_UINT32:
531 *(ffi_arg *) dest = *(UINT32 *) source;
532 break;
533 case FFI_TYPE_INT:
534 case FFI_TYPE_SINT32:
535 *(ffi_sarg *) dest = *(SINT32 *) source;
536 break;
537 case FFI_TYPE_POINTER:
538 case FFI_TYPE_UINT64:
539 *(ffi_arg *) dest = *(UINT64 *) source;
540 break;
541 case FFI_TYPE_SINT64:
542 *(ffi_sarg *) dest = *(SINT64 *) source;
543 break;
544 case FFI_TYPE_VOID:
545 break;
546
547 default:
548 FFI_ASSERT (0);
549 }
550 }
551
552 static void
copy_hfa_to_reg_or_stack(void * memory,ffi_type * ty,struct call_context * context,unsigned char * stack,struct arg_state * state)553 copy_hfa_to_reg_or_stack (void *memory,
554 ffi_type *ty,
555 struct call_context *context,
556 unsigned char *stack,
557 struct arg_state *state)
558 {
559 unsigned elems = element_count (ty);
560 if (available_v (state) < elems)
561 {
562 /* There are insufficient V registers. Further V register allocations
563 are prevented, the NSAA is adjusted (by allocate_to_stack ())
564 and the argument is copied to memory at the adjusted NSAA. */
565 state->nsrn = N_V_ARG_REG;
566 memcpy (allocate_to_stack (state, stack, ty->alignment, ty->size),
567 memory,
568 ty->size);
569 }
570 else
571 {
572 int i;
573 unsigned short type = get_homogeneous_type (ty);
574 for (i = 0; i < elems; i++)
575 {
576 void *reg = allocate_to_v (context, state);
577 copy_basic_type (reg, memory, type);
578 memory += get_basic_type_size (type);
579 }
580 }
581 }
582
583 /* Either allocate an appropriate register for the argument type, or if
584 none are available, allocate a stack slot and return a pointer
585 to the allocated space. */
586
587 static void *
allocate_to_register_or_stack(struct call_context * context,unsigned char * stack,struct arg_state * state,unsigned short type)588 allocate_to_register_or_stack (struct call_context *context,
589 unsigned char *stack,
590 struct arg_state *state,
591 unsigned short type)
592 {
593 size_t alignment = get_basic_type_alignment (type);
594 size_t size = alignment;
595 switch (type)
596 {
597 case FFI_TYPE_FLOAT:
598 /* This is the only case for which the allocated stack size
599 should not match the alignment of the type. */
600 size = sizeof (UINT32);
601 /* Fall through. */
602 case FFI_TYPE_DOUBLE:
603 if (state->nsrn < N_V_ARG_REG)
604 return allocate_to_d (context, state);
605 state->nsrn = N_V_ARG_REG;
606 break;
607 #if FFI_TYPE_DOUBLE != FFI_TYPE_LONGDOUBLE
608 case FFI_TYPE_LONGDOUBLE:
609 if (state->nsrn < N_V_ARG_REG)
610 return allocate_to_v (context, state);
611 state->nsrn = N_V_ARG_REG;
612 break;
613 #endif
614 case FFI_TYPE_UINT8:
615 case FFI_TYPE_SINT8:
616 case FFI_TYPE_UINT16:
617 case FFI_TYPE_SINT16:
618 case FFI_TYPE_UINT32:
619 case FFI_TYPE_SINT32:
620 case FFI_TYPE_INT:
621 case FFI_TYPE_POINTER:
622 case FFI_TYPE_UINT64:
623 case FFI_TYPE_SINT64:
624 if (state->ngrn < N_X_ARG_REG)
625 return allocate_to_x (context, state);
626 state->ngrn = N_X_ARG_REG;
627 break;
628 default:
629 FFI_ASSERT (0);
630 }
631
632 return allocate_to_stack (state, stack, alignment, size);
633 }
634
635 /* Copy a value to an appropriate register, or if none are
636 available, to the stack. */
637
638 static void
copy_to_register_or_stack(struct call_context * context,unsigned char * stack,struct arg_state * state,void * value,unsigned short type)639 copy_to_register_or_stack (struct call_context *context,
640 unsigned char *stack,
641 struct arg_state *state,
642 void *value,
643 unsigned short type)
644 {
645 copy_basic_type (
646 allocate_to_register_or_stack (context, stack, state, type),
647 value,
648 type);
649 }
650
651 /* Marshall the arguments from FFI representation to procedure call
652 context and stack. */
653
654 static unsigned
aarch64_prep_args(struct call_context * context,unsigned char * stack,extended_cif * ecif)655 aarch64_prep_args (struct call_context *context, unsigned char *stack,
656 extended_cif *ecif)
657 {
658 int i;
659 struct arg_state state;
660
661 arg_init (&state, ALIGN(ecif->cif->bytes, 16));
662
663 for (i = 0; i < ecif->cif->nargs; i++)
664 {
665 ffi_type *ty = ecif->cif->arg_types[i];
666 switch (ty->type)
667 {
668 case FFI_TYPE_VOID:
669 FFI_ASSERT (0);
670 break;
671
672 /* If the argument is a basic type the argument is allocated to an
673 appropriate register, or if none are available, to the stack. */
674 case FFI_TYPE_FLOAT:
675 case FFI_TYPE_DOUBLE:
676 #if FFI_TYPE_DOUBLE != FFI_TYPE_LONGDOUBLE
677 case FFI_TYPE_LONGDOUBLE:
678 #endif
679 case FFI_TYPE_UINT8:
680 case FFI_TYPE_SINT8:
681 case FFI_TYPE_UINT16:
682 case FFI_TYPE_SINT16:
683 case FFI_TYPE_UINT32:
684 case FFI_TYPE_INT:
685 case FFI_TYPE_SINT32:
686 case FFI_TYPE_POINTER:
687 case FFI_TYPE_UINT64:
688 case FFI_TYPE_SINT64:
689 copy_to_register_or_stack (context, stack, &state,
690 ecif->avalue[i], ty->type);
691 break;
692
693 case FFI_TYPE_STRUCT:
694 if (is_hfa (ty))
695 {
696 copy_hfa_to_reg_or_stack (ecif->avalue[i], ty, context,
697 stack, &state);
698 }
699 else if (ty->size > 16)
700 {
701 /* If the argument is a composite type that is larger than 16
702 bytes, then the argument has been copied to memory, and
703 the argument is replaced by a pointer to the copy. */
704
705 copy_to_register_or_stack (context, stack, &state,
706 &(ecif->avalue[i]), FFI_TYPE_POINTER);
707 }
708 else if (available_x (&state) >= (ty->size + 7) / 8)
709 {
710 /* If the argument is a composite type and the size in
711 double-words is not more than the number of available
712 X registers, then the argument is copied into consecutive
713 X registers. */
714 int j;
715 for (j = 0; j < (ty->size + 7) / 8; j++)
716 {
717 memcpy (allocate_to_x (context, &state),
718 &(((UINT64 *) ecif->avalue[i])[j]),
719 sizeof (UINT64));
720 }
721 }
722 else
723 {
724 /* Otherwise, there are insufficient X registers. Further X
725 register allocations are prevented, the NSAA is adjusted
726 (by allocate_to_stack ()) and the argument is copied to
727 memory at the adjusted NSAA. */
728 state.ngrn = N_X_ARG_REG;
729
730 memcpy (allocate_to_stack (&state, stack, ty->alignment,
731 ty->size), ecif->avalue + i, ty->size);
732 }
733 break;
734
735 default:
736 FFI_ASSERT (0);
737 break;
738 }
739
740 #if defined (__APPLE__)
741 if (i + 1 == ecif->cif->aarch64_nfixedargs)
742 {
743 state.ngrn = N_X_ARG_REG;
744 state.nsrn = N_V_ARG_REG;
745
746 state.allocating_variadic = 1;
747 }
748 #endif
749 }
750
751 return ecif->cif->aarch64_flags;
752 }
753
754 ffi_status
ffi_prep_cif_machdep(ffi_cif * cif)755 ffi_prep_cif_machdep (ffi_cif *cif)
756 {
757 /* Round the stack up to a multiple of the stack alignment requirement. */
758 cif->bytes =
759 (cif->bytes + (AARCH64_STACK_ALIGN - 1)) & ~ (AARCH64_STACK_ALIGN - 1);
760
761 /* Initialize our flags. We are interested if this CIF will touch a
762 vector register, if so we will enable context save and load to
763 those registers, otherwise not. This is intended to be friendly
764 to lazy float context switching in the kernel. */
765 cif->aarch64_flags = 0;
766
767 if (is_v_register_candidate (cif->rtype))
768 {
769 cif->aarch64_flags |= AARCH64_FFI_WITH_V;
770 }
771 else
772 {
773 int i;
774 for (i = 0; i < cif->nargs; i++)
775 if (is_v_register_candidate (cif->arg_types[i]))
776 {
777 cif->aarch64_flags |= AARCH64_FFI_WITH_V;
778 break;
779 }
780 }
781
782 return FFI_OK;
783 }
784
785 #if defined (__APPLE__)
786
787 /* Perform Apple-specific cif processing for variadic calls */
ffi_prep_cif_machdep_var(ffi_cif * cif,unsigned int nfixedargs,unsigned int ntotalargs)788 ffi_status ffi_prep_cif_machdep_var(ffi_cif *cif,
789 unsigned int nfixedargs,
790 unsigned int ntotalargs)
791 {
792 cif->aarch64_nfixedargs = nfixedargs;
793
794 return ffi_prep_cif_machdep(cif);
795 }
796
797 #endif
798
799 /* Call a function with the provided arguments and capture the return
800 value. */
801 void
ffi_call(ffi_cif * cif,void (* fn)(void),void * rvalue,void ** avalue)802 ffi_call (ffi_cif *cif, void (*fn)(void), void *rvalue, void **avalue)
803 {
804 extended_cif ecif;
805
806 ecif.cif = cif;
807 ecif.avalue = avalue;
808 ecif.rvalue = rvalue;
809
810 switch (cif->abi)
811 {
812 case FFI_SYSV:
813 {
814 struct call_context context;
815 size_t stack_bytes;
816
817 /* Figure out the total amount of stack space we need, the
818 above call frame space needs to be 16 bytes aligned to
819 ensure correct alignment of the first object inserted in
820 that space hence the ALIGN applied to cif->bytes.*/
821 stack_bytes = ALIGN(cif->bytes, 16);
822
823 memset (&context, 0, sizeof (context));
824 if (is_register_candidate (cif->rtype))
825 {
826 ffi_call_SYSV (aarch64_prep_args, &context, &ecif, stack_bytes, fn);
827 switch (cif->rtype->type)
828 {
829 case FFI_TYPE_VOID:
830 case FFI_TYPE_FLOAT:
831 case FFI_TYPE_DOUBLE:
832 #if FFI_TYPE_DOUBLE != FFI_TYPE_LONGDOUBLE
833 case FFI_TYPE_LONGDOUBLE:
834 #endif
835 case FFI_TYPE_UINT8:
836 case FFI_TYPE_SINT8:
837 case FFI_TYPE_UINT16:
838 case FFI_TYPE_SINT16:
839 case FFI_TYPE_UINT32:
840 case FFI_TYPE_SINT32:
841 case FFI_TYPE_POINTER:
842 case FFI_TYPE_UINT64:
843 case FFI_TYPE_INT:
844 case FFI_TYPE_SINT64:
845 {
846 void *addr = get_basic_type_addr (cif->rtype->type,
847 &context, 0);
848 copy_basic_type (rvalue, addr, cif->rtype->type);
849 break;
850 }
851
852 case FFI_TYPE_STRUCT:
853 if (is_hfa (cif->rtype))
854 {
855 int j;
856 unsigned short type = get_homogeneous_type (cif->rtype);
857 unsigned elems = element_count (cif->rtype);
858 for (j = 0; j < elems; j++)
859 {
860 void *reg = get_basic_type_addr (type, &context, j);
861 copy_basic_type (rvalue, reg, type);
862 rvalue += get_basic_type_size (type);
863 }
864 }
865 else if ((cif->rtype->size + 7) / 8 < N_X_ARG_REG)
866 {
867 size_t size = ALIGN (cif->rtype->size, sizeof (UINT64));
868 memcpy (rvalue, get_x_addr (&context, 0), size);
869 }
870 else
871 {
872 FFI_ASSERT (0);
873 }
874 break;
875
876 default:
877 FFI_ASSERT (0);
878 break;
879 }
880 }
881 else
882 {
883 memcpy (get_x_addr (&context, 8), &rvalue, sizeof (UINT64));
884 ffi_call_SYSV (aarch64_prep_args, &context, &ecif,
885 stack_bytes, fn);
886 }
887 break;
888 }
889
890 default:
891 FFI_ASSERT (0);
892 break;
893 }
894 }
895
896 static unsigned char trampoline [] =
897 { 0x70, 0x00, 0x00, 0x58, /* ldr x16, 1f */
898 0x91, 0x00, 0x00, 0x10, /* adr x17, 2f */
899 0x00, 0x02, 0x1f, 0xd6 /* br x16 */
900 };
901
902 /* Build a trampoline. */
903
904 #define FFI_INIT_TRAMPOLINE(TRAMP,FUN,CTX,FLAGS) \
905 ({unsigned char *__tramp = (unsigned char*)(TRAMP); \
906 UINT64 __fun = (UINT64)(FUN); \
907 UINT64 __ctx = (UINT64)(CTX); \
908 UINT64 __flags = (UINT64)(FLAGS); \
909 memcpy (__tramp, trampoline, sizeof (trampoline)); \
910 memcpy (__tramp + 12, &__fun, sizeof (__fun)); \
911 memcpy (__tramp + 20, &__ctx, sizeof (__ctx)); \
912 memcpy (__tramp + 28, &__flags, sizeof (__flags)); \
913 ffi_clear_cache(__tramp, __tramp + FFI_TRAMPOLINE_SIZE); \
914 })
915
916 ffi_status
ffi_prep_closure_loc(ffi_closure * closure,ffi_cif * cif,void (* fun)(ffi_cif *,void *,void **,void *),void * user_data,void * codeloc)917 ffi_prep_closure_loc (ffi_closure* closure,
918 ffi_cif* cif,
919 void (*fun)(ffi_cif*,void*,void**,void*),
920 void *user_data,
921 void *codeloc)
922 {
923 if (cif->abi != FFI_SYSV)
924 return FFI_BAD_ABI;
925
926 FFI_INIT_TRAMPOLINE (&closure->tramp[0], &ffi_closure_SYSV, codeloc,
927 cif->aarch64_flags);
928
929 closure->cif = cif;
930 closure->user_data = user_data;
931 closure->fun = fun;
932
933 return FFI_OK;
934 }
935
936 /* Primary handler to setup and invoke a function within a closure.
937
938 A closure when invoked enters via the assembler wrapper
939 ffi_closure_SYSV(). The wrapper allocates a call context on the
940 stack, saves the interesting registers (from the perspective of
941 the calling convention) into the context then passes control to
942 ffi_closure_SYSV_inner() passing the saved context and a pointer to
943 the stack at the point ffi_closure_SYSV() was invoked.
944
945 On the return path the assembler wrapper will reload call context
946 registers.
947
948 ffi_closure_SYSV_inner() marshalls the call context into ffi value
949 descriptors, invokes the wrapped function, then marshalls the return
950 value back into the call context. */
951
952 void FFI_HIDDEN
ffi_closure_SYSV_inner(ffi_closure * closure,struct call_context * context,void * stack)953 ffi_closure_SYSV_inner (ffi_closure *closure, struct call_context *context,
954 void *stack)
955 {
956 ffi_cif *cif = closure->cif;
957 void **avalue = (void**) alloca (cif->nargs * sizeof (void*));
958 void *rvalue = NULL;
959 int i;
960 struct arg_state state;
961
962 arg_init (&state, ALIGN(cif->bytes, 16));
963
964 for (i = 0; i < cif->nargs; i++)
965 {
966 ffi_type *ty = cif->arg_types[i];
967
968 switch (ty->type)
969 {
970 case FFI_TYPE_VOID:
971 FFI_ASSERT (0);
972 break;
973
974 case FFI_TYPE_UINT8:
975 case FFI_TYPE_SINT8:
976 case FFI_TYPE_UINT16:
977 case FFI_TYPE_SINT16:
978 case FFI_TYPE_UINT32:
979 case FFI_TYPE_SINT32:
980 case FFI_TYPE_INT:
981 case FFI_TYPE_POINTER:
982 case FFI_TYPE_UINT64:
983 case FFI_TYPE_SINT64:
984 case FFI_TYPE_FLOAT:
985 case FFI_TYPE_DOUBLE:
986 #if FFI_TYPE_DOUBLE != FFI_TYPE_LONGDOUBLE
987 case FFI_TYPE_LONGDOUBLE:
988 avalue[i] = allocate_to_register_or_stack (context, stack,
989 &state, ty->type);
990 break;
991 #endif
992
993 case FFI_TYPE_STRUCT:
994 if (is_hfa (ty))
995 {
996 unsigned n = element_count (ty);
997 if (available_v (&state) < n)
998 {
999 state.nsrn = N_V_ARG_REG;
1000 avalue[i] = allocate_to_stack (&state, stack, ty->alignment,
1001 ty->size);
1002 }
1003 else
1004 {
1005 switch (get_homogeneous_type (ty))
1006 {
1007 case FFI_TYPE_FLOAT:
1008 {
1009 /* Eeek! We need a pointer to the structure,
1010 however the homogeneous float elements are
1011 being passed in individual S registers,
1012 therefore the structure is not represented as
1013 a contiguous sequence of bytes in our saved
1014 register context. We need to fake up a copy
1015 of the structure laid out in memory
1016 correctly. The fake can be tossed once the
1017 closure function has returned hence alloca()
1018 is sufficient. */
1019 int j;
1020 UINT32 *p = avalue[i] = alloca (ty->size);
1021 for (j = 0; j < element_count (ty); j++)
1022 memcpy (&p[j],
1023 allocate_to_s (context, &state),
1024 sizeof (*p));
1025 break;
1026 }
1027
1028 case FFI_TYPE_DOUBLE:
1029 {
1030 /* Eeek! We need a pointer to the structure,
1031 however the homogeneous float elements are
1032 being passed in individual S registers,
1033 therefore the structure is not represented as
1034 a contiguous sequence of bytes in our saved
1035 register context. We need to fake up a copy
1036 of the structure laid out in memory
1037 correctly. The fake can be tossed once the
1038 closure function has returned hence alloca()
1039 is sufficient. */
1040 int j;
1041 UINT64 *p = avalue[i] = alloca (ty->size);
1042 for (j = 0; j < element_count (ty); j++)
1043 memcpy (&p[j],
1044 allocate_to_d (context, &state),
1045 sizeof (*p));
1046 break;
1047 }
1048
1049 #if FFI_TYPE_DOUBLE != FFI_TYPE_LONGDOUBLE
1050 case FFI_TYPE_LONGDOUBLE:
1051 memcpy (&avalue[i],
1052 allocate_to_v (context, &state),
1053 sizeof (*avalue));
1054 break;
1055 #endif
1056
1057 default:
1058 FFI_ASSERT (0);
1059 break;
1060 }
1061 }
1062 }
1063 else if (ty->size > 16)
1064 {
1065 /* Replace Composite type of size greater than 16 with a
1066 pointer. */
1067 memcpy (&avalue[i],
1068 allocate_to_register_or_stack (context, stack,
1069 &state, FFI_TYPE_POINTER),
1070 sizeof (avalue[i]));
1071 }
1072 else if (available_x (&state) >= (ty->size + 7) / 8)
1073 {
1074 avalue[i] = get_x_addr (context, state.ngrn);
1075 state.ngrn += (ty->size + 7) / 8;
1076 }
1077 else
1078 {
1079 state.ngrn = N_X_ARG_REG;
1080
1081 avalue[i] = allocate_to_stack (&state, stack, ty->alignment,
1082 ty->size);
1083 }
1084 break;
1085
1086 default:
1087 FFI_ASSERT (0);
1088 break;
1089 }
1090 }
1091
1092 /* Figure out where the return value will be passed, either in
1093 registers or in a memory block allocated by the caller and passed
1094 in x8. */
1095
1096 if (is_register_candidate (cif->rtype))
1097 {
1098 /* Register candidates are *always* returned in registers. */
1099
1100 /* Allocate a scratchpad for the return value, we will let the
1101 callee scrible the result into the scratch pad then move the
1102 contents into the appropriate return value location for the
1103 call convention. */
1104 rvalue = alloca (cif->rtype->size);
1105 (closure->fun) (cif, rvalue, avalue, closure->user_data);
1106
1107 /* Copy the return value into the call context so that it is returned
1108 as expected to our caller. */
1109 switch (cif->rtype->type)
1110 {
1111 case FFI_TYPE_VOID:
1112 break;
1113
1114 case FFI_TYPE_UINT8:
1115 case FFI_TYPE_UINT16:
1116 case FFI_TYPE_UINT32:
1117 case FFI_TYPE_POINTER:
1118 case FFI_TYPE_UINT64:
1119 case FFI_TYPE_SINT8:
1120 case FFI_TYPE_SINT16:
1121 case FFI_TYPE_INT:
1122 case FFI_TYPE_SINT32:
1123 case FFI_TYPE_SINT64:
1124 case FFI_TYPE_FLOAT:
1125 case FFI_TYPE_DOUBLE:
1126 #if FFI_TYPE_DOUBLE != FFI_TYPE_LONGDOUBLE
1127 case FFI_TYPE_LONGDOUBLE:
1128 #endif
1129 {
1130 void *addr = get_basic_type_addr (cif->rtype->type, context, 0);
1131 copy_basic_type (addr, rvalue, cif->rtype->type);
1132 break;
1133 }
1134 case FFI_TYPE_STRUCT:
1135 if (is_hfa (cif->rtype))
1136 {
1137 int j;
1138 unsigned short type = get_homogeneous_type (cif->rtype);
1139 unsigned elems = element_count (cif->rtype);
1140 for (j = 0; j < elems; j++)
1141 {
1142 void *reg = get_basic_type_addr (type, context, j);
1143 copy_basic_type (reg, rvalue, type);
1144 rvalue += get_basic_type_size (type);
1145 }
1146 }
1147 else if ((cif->rtype->size + 7) / 8 < N_X_ARG_REG)
1148 {
1149 size_t size = ALIGN (cif->rtype->size, sizeof (UINT64)) ;
1150 memcpy (get_x_addr (context, 0), rvalue, size);
1151 }
1152 else
1153 {
1154 FFI_ASSERT (0);
1155 }
1156 break;
1157 default:
1158 FFI_ASSERT (0);
1159 break;
1160 }
1161 }
1162 else
1163 {
1164 memcpy (&rvalue, get_x_addr (context, 8), sizeof (UINT64));
1165 (closure->fun) (cif, rvalue, avalue, closure->user_data);
1166 }
1167 }
1168
1169