1 /* -----------------------------------------------------------------------
2 ffi64.c - Copyright (c) 2011, 2018 Anthony Green
3 Copyright (c) 2013 The Written Word, Inc.
4 Copyright (c) 2008, 2010 Red Hat, Inc.
5 Copyright (c) 2002, 2007 Bo Thorsen <bo@suse.de>
6
7 x86-64 Foreign Function Interface
8
9 Permission is hereby granted, free of charge, to any person obtaining
10 a copy of this software and associated documentation files (the
11 ``Software''), to deal in the Software without restriction, including
12 without limitation the rights to use, copy, modify, merge, publish,
13 distribute, sublicense, and/or sell copies of the Software, and to
14 permit persons to whom the Software is furnished to do so, subject to
15 the following conditions:
16
17 The above copyright notice and this permission notice shall be included
18 in all copies or substantial portions of the Software.
19
20 THE SOFTWARE IS PROVIDED ``AS IS'', WITHOUT WARRANTY OF ANY KIND,
21 EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
22 MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
23 NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT
24 HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
25 WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
26 OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
27 DEALINGS IN THE SOFTWARE.
28 ----------------------------------------------------------------------- */
29
30 #include <ffi.h>
31 #include <ffi_common.h>
32
33 #include <stdlib.h>
34 #include <stdarg.h>
35 #include <stdint.h>
36 #include "internal64.h"
37
38 #ifdef __x86_64__
39
40 #define MAX_GPR_REGS 6
41 #define MAX_SSE_REGS 8
42
43 #if defined(__INTEL_COMPILER)
44 #include "xmmintrin.h"
45 #define UINT128 __m128
46 #else
47 #if defined(__SUNPRO_C)
48 #include <sunmedia_types.h>
49 #define UINT128 __m128i
50 #else
51 #define UINT128 __int128_t
52 #endif
53 #endif
54
55 union big_int_union
56 {
57 UINT32 i32;
58 UINT64 i64;
59 UINT128 i128;
60 };
61
62 struct register_args
63 {
64 /* Registers for argument passing. */
65 UINT64 gpr[MAX_GPR_REGS];
66 union big_int_union sse[MAX_SSE_REGS];
67 UINT64 rax; /* ssecount */
68 UINT64 r10; /* static chain */
69 };
70
71 extern void ffi_call_unix64 (void *args, unsigned long bytes, unsigned flags,
72 void *raddr, void (*fnaddr)(void)) FFI_HIDDEN;
73
74 /* All reference to register classes here is identical to the code in
75 gcc/config/i386/i386.c. Do *not* change one without the other. */
76
77 /* Register class used for passing given 64bit part of the argument.
78 These represent classes as documented by the PS ABI, with the
79 exception of SSESF, SSEDF classes, that are basically SSE class,
80 just gcc will use SF or DFmode move instead of DImode to avoid
81 reformatting penalties.
82
83 Similary we play games with INTEGERSI_CLASS to use cheaper SImode moves
84 whenever possible (upper half does contain padding). */
85 enum x86_64_reg_class
86 {
87 X86_64_NO_CLASS,
88 X86_64_INTEGER_CLASS,
89 X86_64_INTEGERSI_CLASS,
90 X86_64_SSE_CLASS,
91 X86_64_SSESF_CLASS,
92 X86_64_SSEDF_CLASS,
93 X86_64_SSEUP_CLASS,
94 X86_64_X87_CLASS,
95 X86_64_X87UP_CLASS,
96 X86_64_COMPLEX_X87_CLASS,
97 X86_64_MEMORY_CLASS
98 };
99
100 #define MAX_CLASSES 4
101
102 #define SSE_CLASS_P(X) ((X) >= X86_64_SSE_CLASS && X <= X86_64_SSEUP_CLASS)
103
104 /* x86-64 register passing implementation. See x86-64 ABI for details. Goal
105 of this code is to classify each 8bytes of incoming argument by the register
106 class and assign registers accordingly. */
107
108 /* Return the union class of CLASS1 and CLASS2.
109 See the x86-64 PS ABI for details. */
110
111 static enum x86_64_reg_class
merge_classes(enum x86_64_reg_class class1,enum x86_64_reg_class class2)112 merge_classes (enum x86_64_reg_class class1, enum x86_64_reg_class class2)
113 {
114 /* Rule #1: If both classes are equal, this is the resulting class. */
115 if (class1 == class2)
116 return class1;
117
118 /* Rule #2: If one of the classes is NO_CLASS, the resulting class is
119 the other class. */
120 if (class1 == X86_64_NO_CLASS)
121 return class2;
122 if (class2 == X86_64_NO_CLASS)
123 return class1;
124
125 /* Rule #3: If one of the classes is MEMORY, the result is MEMORY. */
126 if (class1 == X86_64_MEMORY_CLASS || class2 == X86_64_MEMORY_CLASS)
127 return X86_64_MEMORY_CLASS;
128
129 /* Rule #4: If one of the classes is INTEGER, the result is INTEGER. */
130 if ((class1 == X86_64_INTEGERSI_CLASS && class2 == X86_64_SSESF_CLASS)
131 || (class2 == X86_64_INTEGERSI_CLASS && class1 == X86_64_SSESF_CLASS))
132 return X86_64_INTEGERSI_CLASS;
133 if (class1 == X86_64_INTEGER_CLASS || class1 == X86_64_INTEGERSI_CLASS
134 || class2 == X86_64_INTEGER_CLASS || class2 == X86_64_INTEGERSI_CLASS)
135 return X86_64_INTEGER_CLASS;
136
137 /* Rule #5: If one of the classes is X87, X87UP, or COMPLEX_X87 class,
138 MEMORY is used. */
139 if (class1 == X86_64_X87_CLASS
140 || class1 == X86_64_X87UP_CLASS
141 || class1 == X86_64_COMPLEX_X87_CLASS
142 || class2 == X86_64_X87_CLASS
143 || class2 == X86_64_X87UP_CLASS
144 || class2 == X86_64_COMPLEX_X87_CLASS)
145 return X86_64_MEMORY_CLASS;
146
147 /* Rule #6: Otherwise class SSE is used. */
148 return X86_64_SSE_CLASS;
149 }
150
151 /* Classify the argument of type TYPE and mode MODE.
152 CLASSES will be filled by the register class used to pass each word
153 of the operand. The number of words is returned. In case the parameter
154 should be passed in memory, 0 is returned. As a special case for zero
155 sized containers, classes[0] will be NO_CLASS and 1 is returned.
156
157 See the x86-64 PS ABI for details.
158 */
159 static size_t
classify_argument(ffi_type * type,enum x86_64_reg_class classes[],size_t byte_offset)160 classify_argument (ffi_type *type, enum x86_64_reg_class classes[],
161 size_t byte_offset)
162 {
163 switch (type->type)
164 {
165 case FFI_TYPE_UINT8:
166 case FFI_TYPE_SINT8:
167 case FFI_TYPE_UINT16:
168 case FFI_TYPE_SINT16:
169 case FFI_TYPE_UINT32:
170 case FFI_TYPE_SINT32:
171 case FFI_TYPE_UINT64:
172 case FFI_TYPE_SINT64:
173 case FFI_TYPE_POINTER:
174 do_integer:
175 {
176 size_t size = byte_offset + type->size;
177
178 if (size <= 4)
179 {
180 classes[0] = X86_64_INTEGERSI_CLASS;
181 return 1;
182 }
183 else if (size <= 8)
184 {
185 classes[0] = X86_64_INTEGER_CLASS;
186 return 1;
187 }
188 else if (size <= 12)
189 {
190 classes[0] = X86_64_INTEGER_CLASS;
191 classes[1] = X86_64_INTEGERSI_CLASS;
192 return 2;
193 }
194 else if (size <= 16)
195 {
196 classes[0] = classes[1] = X86_64_INTEGER_CLASS;
197 return 2;
198 }
199 else
200 FFI_ASSERT (0);
201 }
202 case FFI_TYPE_FLOAT:
203 if (!(byte_offset % 8))
204 classes[0] = X86_64_SSESF_CLASS;
205 else
206 classes[0] = X86_64_SSE_CLASS;
207 return 1;
208 case FFI_TYPE_DOUBLE:
209 classes[0] = X86_64_SSEDF_CLASS;
210 return 1;
211 #if FFI_TYPE_LONGDOUBLE != FFI_TYPE_DOUBLE
212 case FFI_TYPE_LONGDOUBLE:
213 classes[0] = X86_64_X87_CLASS;
214 classes[1] = X86_64_X87UP_CLASS;
215 return 2;
216 #endif
217 case FFI_TYPE_STRUCT:
218 {
219 const size_t UNITS_PER_WORD = 8;
220 size_t words = (type->size + UNITS_PER_WORD - 1) / UNITS_PER_WORD;
221 ffi_type **ptr;
222 unsigned int i;
223 enum x86_64_reg_class subclasses[MAX_CLASSES];
224
225 /* If the struct is larger than 32 bytes, pass it on the stack. */
226 if (type->size > 32)
227 return 0;
228
229 for (i = 0; i < words; i++)
230 classes[i] = X86_64_NO_CLASS;
231
232 /* Zero sized arrays or structures are NO_CLASS. We return 0 to
233 signalize memory class, so handle it as special case. */
234 if (!words)
235 {
236 case FFI_TYPE_VOID:
237 classes[0] = X86_64_NO_CLASS;
238 return 1;
239 }
240
241 /* Merge the fields of structure. */
242 for (ptr = type->elements; *ptr != NULL; ptr++)
243 {
244 size_t num;
245
246 byte_offset = FFI_ALIGN (byte_offset, (*ptr)->alignment);
247
248 num = classify_argument (*ptr, subclasses, byte_offset % 8);
249 if (num == 0)
250 return 0;
251 for (i = 0; i < num; i++)
252 {
253 size_t pos = byte_offset / 8;
254 classes[i + pos] =
255 merge_classes (subclasses[i], classes[i + pos]);
256 }
257
258 byte_offset += (*ptr)->size;
259 }
260
261 if (words > 2)
262 {
263 /* When size > 16 bytes, if the first one isn't
264 X86_64_SSE_CLASS or any other ones aren't
265 X86_64_SSEUP_CLASS, everything should be passed in
266 memory. */
267 if (classes[0] != X86_64_SSE_CLASS)
268 return 0;
269
270 for (i = 1; i < words; i++)
271 if (classes[i] != X86_64_SSEUP_CLASS)
272 return 0;
273 }
274
275 /* Final merger cleanup. */
276 for (i = 0; i < words; i++)
277 {
278 /* If one class is MEMORY, everything should be passed in
279 memory. */
280 if (classes[i] == X86_64_MEMORY_CLASS)
281 return 0;
282
283 /* The X86_64_SSEUP_CLASS should be always preceded by
284 X86_64_SSE_CLASS or X86_64_SSEUP_CLASS. */
285 if (i > 1 && classes[i] == X86_64_SSEUP_CLASS
286 && classes[i - 1] != X86_64_SSE_CLASS
287 && classes[i - 1] != X86_64_SSEUP_CLASS)
288 {
289 /* The first one should never be X86_64_SSEUP_CLASS. */
290 FFI_ASSERT (i != 0);
291 classes[i] = X86_64_SSE_CLASS;
292 }
293
294 /* If X86_64_X87UP_CLASS isn't preceded by X86_64_X87_CLASS,
295 everything should be passed in memory. */
296 if (i > 1 && classes[i] == X86_64_X87UP_CLASS
297 && (classes[i - 1] != X86_64_X87_CLASS))
298 {
299 /* The first one should never be X86_64_X87UP_CLASS. */
300 FFI_ASSERT (i != 0);
301 return 0;
302 }
303 }
304 return words;
305 }
306 case FFI_TYPE_COMPLEX:
307 {
308 ffi_type *inner = type->elements[0];
309 switch (inner->type)
310 {
311 case FFI_TYPE_INT:
312 case FFI_TYPE_UINT8:
313 case FFI_TYPE_SINT8:
314 case FFI_TYPE_UINT16:
315 case FFI_TYPE_SINT16:
316 case FFI_TYPE_UINT32:
317 case FFI_TYPE_SINT32:
318 case FFI_TYPE_UINT64:
319 case FFI_TYPE_SINT64:
320 goto do_integer;
321
322 case FFI_TYPE_FLOAT:
323 classes[0] = X86_64_SSE_CLASS;
324 if (byte_offset % 8)
325 {
326 classes[1] = X86_64_SSESF_CLASS;
327 return 2;
328 }
329 return 1;
330 case FFI_TYPE_DOUBLE:
331 classes[0] = classes[1] = X86_64_SSEDF_CLASS;
332 return 2;
333 #if FFI_TYPE_LONGDOUBLE != FFI_TYPE_DOUBLE
334 case FFI_TYPE_LONGDOUBLE:
335 classes[0] = X86_64_COMPLEX_X87_CLASS;
336 return 1;
337 #endif
338 }
339 }
340 }
341 abort();
342 }
343
344 /* Examine the argument and return set number of register required in each
345 class. Return zero iff parameter should be passed in memory, otherwise
346 the number of registers. */
347
348 static size_t
examine_argument(ffi_type * type,enum x86_64_reg_class classes[MAX_CLASSES],_Bool in_return,int * pngpr,int * pnsse)349 examine_argument (ffi_type *type, enum x86_64_reg_class classes[MAX_CLASSES],
350 _Bool in_return, int *pngpr, int *pnsse)
351 {
352 size_t n;
353 unsigned int i;
354 int ngpr, nsse;
355
356 n = classify_argument (type, classes, 0);
357 if (n == 0)
358 return 0;
359
360 ngpr = nsse = 0;
361 for (i = 0; i < n; ++i)
362 switch (classes[i])
363 {
364 case X86_64_INTEGER_CLASS:
365 case X86_64_INTEGERSI_CLASS:
366 ngpr++;
367 break;
368 case X86_64_SSE_CLASS:
369 case X86_64_SSESF_CLASS:
370 case X86_64_SSEDF_CLASS:
371 nsse++;
372 break;
373 case X86_64_NO_CLASS:
374 case X86_64_SSEUP_CLASS:
375 break;
376 case X86_64_X87_CLASS:
377 case X86_64_X87UP_CLASS:
378 case X86_64_COMPLEX_X87_CLASS:
379 return in_return != 0;
380 default:
381 abort ();
382 }
383
384 *pngpr = ngpr;
385 *pnsse = nsse;
386
387 return n;
388 }
389
390 /* Perform machine dependent cif processing. */
391
392 #ifndef __ILP32__
393 extern ffi_status
394 ffi_prep_cif_machdep_efi64(ffi_cif *cif);
395 #endif
396
397 ffi_status FFI_HIDDEN
ffi_prep_cif_machdep(ffi_cif * cif)398 ffi_prep_cif_machdep (ffi_cif *cif)
399 {
400 int gprcount, ssecount, i, avn, ngpr, nsse;
401 unsigned flags;
402 enum x86_64_reg_class classes[MAX_CLASSES];
403 size_t bytes, n, rtype_size;
404 ffi_type *rtype;
405
406 #ifndef __ILP32__
407 if (cif->abi == FFI_EFI64 || cif->abi == FFI_GNUW64)
408 return ffi_prep_cif_machdep_efi64(cif);
409 #endif
410 if (cif->abi != FFI_UNIX64)
411 return FFI_BAD_ABI;
412
413 gprcount = ssecount = 0;
414
415 rtype = cif->rtype;
416 rtype_size = rtype->size;
417 switch (rtype->type)
418 {
419 case FFI_TYPE_VOID:
420 flags = UNIX64_RET_VOID;
421 break;
422 case FFI_TYPE_UINT8:
423 flags = UNIX64_RET_UINT8;
424 break;
425 case FFI_TYPE_SINT8:
426 flags = UNIX64_RET_SINT8;
427 break;
428 case FFI_TYPE_UINT16:
429 flags = UNIX64_RET_UINT16;
430 break;
431 case FFI_TYPE_SINT16:
432 flags = UNIX64_RET_SINT16;
433 break;
434 case FFI_TYPE_UINT32:
435 flags = UNIX64_RET_UINT32;
436 break;
437 case FFI_TYPE_INT:
438 case FFI_TYPE_SINT32:
439 flags = UNIX64_RET_SINT32;
440 break;
441 case FFI_TYPE_UINT64:
442 case FFI_TYPE_SINT64:
443 flags = UNIX64_RET_INT64;
444 break;
445 case FFI_TYPE_POINTER:
446 flags = (sizeof(void *) == 4 ? UNIX64_RET_UINT32 : UNIX64_RET_INT64);
447 break;
448 case FFI_TYPE_FLOAT:
449 flags = UNIX64_RET_XMM32;
450 break;
451 case FFI_TYPE_DOUBLE:
452 flags = UNIX64_RET_XMM64;
453 break;
454 #if FFI_TYPE_LONGDOUBLE != FFI_TYPE_DOUBLE
455 case FFI_TYPE_LONGDOUBLE:
456 flags = UNIX64_RET_X87;
457 break;
458 #endif
459 case FFI_TYPE_STRUCT:
460 n = examine_argument (cif->rtype, classes, 1, &ngpr, &nsse);
461 if (n == 0)
462 {
463 /* The return value is passed in memory. A pointer to that
464 memory is the first argument. Allocate a register for it. */
465 gprcount++;
466 /* We don't have to do anything in asm for the return. */
467 flags = UNIX64_RET_VOID | UNIX64_FLAG_RET_IN_MEM;
468 }
469 else
470 {
471 _Bool sse0 = SSE_CLASS_P (classes[0]);
472
473 if (rtype_size == 4 && sse0)
474 flags = UNIX64_RET_XMM32;
475 else if (rtype_size == 8)
476 flags = sse0 ? UNIX64_RET_XMM64 : UNIX64_RET_INT64;
477 else
478 {
479 _Bool sse1 = n == 2 && SSE_CLASS_P (classes[1]);
480 if (sse0 && sse1)
481 flags = UNIX64_RET_ST_XMM0_XMM1;
482 else if (sse0)
483 flags = UNIX64_RET_ST_XMM0_RAX;
484 else if (sse1)
485 flags = UNIX64_RET_ST_RAX_XMM0;
486 else
487 flags = UNIX64_RET_ST_RAX_RDX;
488 flags |= rtype_size << UNIX64_SIZE_SHIFT;
489 }
490 }
491 break;
492 case FFI_TYPE_COMPLEX:
493 switch (rtype->elements[0]->type)
494 {
495 case FFI_TYPE_UINT8:
496 case FFI_TYPE_SINT8:
497 case FFI_TYPE_UINT16:
498 case FFI_TYPE_SINT16:
499 case FFI_TYPE_INT:
500 case FFI_TYPE_UINT32:
501 case FFI_TYPE_SINT32:
502 case FFI_TYPE_UINT64:
503 case FFI_TYPE_SINT64:
504 flags = UNIX64_RET_ST_RAX_RDX | ((unsigned) rtype_size << UNIX64_SIZE_SHIFT);
505 break;
506 case FFI_TYPE_FLOAT:
507 flags = UNIX64_RET_XMM64;
508 break;
509 case FFI_TYPE_DOUBLE:
510 flags = UNIX64_RET_ST_XMM0_XMM1 | (16 << UNIX64_SIZE_SHIFT);
511 break;
512 #if FFI_TYPE_LONGDOUBLE != FFI_TYPE_DOUBLE
513 case FFI_TYPE_LONGDOUBLE:
514 flags = UNIX64_RET_X87_2;
515 break;
516 #endif
517 default:
518 return FFI_BAD_TYPEDEF;
519 }
520 break;
521 default:
522 return FFI_BAD_TYPEDEF;
523 }
524
525 /* Go over all arguments and determine the way they should be passed.
526 If it's in a register and there is space for it, let that be so. If
527 not, add it's size to the stack byte count. */
528 for (bytes = 0, i = 0, avn = cif->nargs; i < avn; i++)
529 {
530 if (examine_argument (cif->arg_types[i], classes, 0, &ngpr, &nsse) == 0
531 || gprcount + ngpr > MAX_GPR_REGS
532 || ssecount + nsse > MAX_SSE_REGS)
533 {
534 long align = cif->arg_types[i]->alignment;
535
536 if (align < 8)
537 align = 8;
538
539 bytes = FFI_ALIGN (bytes, align);
540 bytes += cif->arg_types[i]->size;
541 }
542 else
543 {
544 gprcount += ngpr;
545 ssecount += nsse;
546 }
547 }
548 if (ssecount)
549 flags |= UNIX64_FLAG_XMM_ARGS;
550
551 cif->flags = flags;
552 cif->bytes = (unsigned) FFI_ALIGN (bytes, 8);
553
554 return FFI_OK;
555 }
556
557 #ifndef __SANITIZE_ADDRESS__
558 # ifdef __clang__
559 # if __has_feature(address_sanitizer)
560 # define __SANITIZE_ADDRESS__
561 # endif
562 # endif
563 #endif
564 #ifdef __SANITIZE_ADDRESS__
565 __attribute__((noinline,no_sanitize_address))
566 #endif
567 static void
ffi_call_int(ffi_cif * cif,void (* fn)(void),void * rvalue,void ** avalue,void * closure)568 ffi_call_int (ffi_cif *cif, void (*fn)(void), void *rvalue,
569 void **avalue, void *closure)
570 {
571 enum x86_64_reg_class classes[MAX_CLASSES];
572 char *stack, *argp;
573 ffi_type **arg_types;
574 int gprcount, ssecount, ngpr, nsse, i, avn, flags;
575 struct register_args *reg_args;
576
577 /* Can't call 32-bit mode from 64-bit mode. */
578 FFI_ASSERT (cif->abi == FFI_UNIX64);
579
580 /* If the return value is a struct and we don't have a return value
581 address then we need to make one. Otherwise we can ignore it. */
582 flags = cif->flags;
583 if (rvalue == NULL)
584 {
585 if (flags & UNIX64_FLAG_RET_IN_MEM)
586 rvalue = alloca (cif->rtype->size);
587 else
588 flags = UNIX64_RET_VOID;
589 }
590
591 /* Allocate the space for the arguments, plus 4 words of temp space. */
592 stack = alloca (sizeof (struct register_args) + cif->bytes + 4*8);
593 reg_args = (struct register_args *) stack;
594 argp = stack + sizeof (struct register_args);
595
596 reg_args->r10 = (uintptr_t) closure;
597
598 gprcount = ssecount = 0;
599
600 /* If the return value is passed in memory, add the pointer as the
601 first integer argument. */
602 if (flags & UNIX64_FLAG_RET_IN_MEM)
603 reg_args->gpr[gprcount++] = (unsigned long) rvalue;
604
605 avn = cif->nargs;
606 arg_types = cif->arg_types;
607
608 for (i = 0; i < avn; ++i)
609 {
610 size_t n, size = arg_types[i]->size;
611
612 n = examine_argument (arg_types[i], classes, 0, &ngpr, &nsse);
613 if (n == 0
614 || gprcount + ngpr > MAX_GPR_REGS
615 || ssecount + nsse > MAX_SSE_REGS)
616 {
617 long align = arg_types[i]->alignment;
618
619 /* Stack arguments are *always* at least 8 byte aligned. */
620 if (align < 8)
621 align = 8;
622
623 /* Pass this argument in memory. */
624 argp = (void *) FFI_ALIGN (argp, align);
625 memcpy (argp, avalue[i], size);
626 argp += size;
627 }
628 else
629 {
630 /* The argument is passed entirely in registers. */
631 char *a = (char *) avalue[i];
632 unsigned int j;
633
634 for (j = 0; j < n; j++, a += 8, size -= 8)
635 {
636 switch (classes[j])
637 {
638 case X86_64_NO_CLASS:
639 case X86_64_SSEUP_CLASS:
640 break;
641 case X86_64_INTEGER_CLASS:
642 case X86_64_INTEGERSI_CLASS:
643 /* Sign-extend integer arguments passed in general
644 purpose registers, to cope with the fact that
645 LLVM incorrectly assumes that this will be done
646 (the x86-64 PS ABI does not specify this). */
647 switch (arg_types[i]->type)
648 {
649 case FFI_TYPE_SINT8:
650 reg_args->gpr[gprcount] = (SINT64) *((SINT8 *) a);
651 break;
652 case FFI_TYPE_SINT16:
653 reg_args->gpr[gprcount] = (SINT64) *((SINT16 *) a);
654 break;
655 case FFI_TYPE_SINT32:
656 reg_args->gpr[gprcount] = (SINT64) *((SINT32 *) a);
657 break;
658 default:
659 reg_args->gpr[gprcount] = 0;
660 memcpy (®_args->gpr[gprcount], a, size);
661 }
662 gprcount++;
663 break;
664 case X86_64_SSE_CLASS:
665 case X86_64_SSEDF_CLASS:
666 memcpy (®_args->sse[ssecount++].i64, a, sizeof(UINT64));
667 break;
668 case X86_64_SSESF_CLASS:
669 memcpy (®_args->sse[ssecount++].i32, a, sizeof(UINT32));
670 break;
671 default:
672 abort();
673 }
674 }
675 }
676 }
677 reg_args->rax = ssecount;
678
679 ffi_call_unix64 (stack, cif->bytes + sizeof (struct register_args),
680 flags, rvalue, fn);
681 }
682
683 #ifndef __ILP32__
684 extern void
685 ffi_call_efi64(ffi_cif *cif, void (*fn)(void), void *rvalue, void **avalue);
686 #endif
687
688 void
ffi_call(ffi_cif * cif,void (* fn)(void),void * rvalue,void ** avalue)689 ffi_call (ffi_cif *cif, void (*fn)(void), void *rvalue, void **avalue)
690 {
691 #ifndef __ILP32__
692 if (cif->abi == FFI_EFI64 || cif->abi == FFI_GNUW64)
693 {
694 ffi_call_efi64(cif, fn, rvalue, avalue);
695 return;
696 }
697 #endif
698 ffi_call_int (cif, fn, rvalue, avalue, NULL);
699 }
700
701 #ifndef __ILP32__
702 extern void
703 ffi_call_go_efi64(ffi_cif *cif, void (*fn)(void), void *rvalue,
704 void **avalue, void *closure);
705 #endif
706
707 void
ffi_call_go(ffi_cif * cif,void (* fn)(void),void * rvalue,void ** avalue,void * closure)708 ffi_call_go (ffi_cif *cif, void (*fn)(void), void *rvalue,
709 void **avalue, void *closure)
710 {
711 #ifndef __ILP32__
712 if (cif->abi == FFI_EFI64 || cif->abi == FFI_GNUW64)
713 {
714 ffi_call_go_efi64(cif, fn, rvalue, avalue, closure);
715 return;
716 }
717 #endif
718 ffi_call_int (cif, fn, rvalue, avalue, closure);
719 }
720
721
722 extern void ffi_closure_unix64(void) FFI_HIDDEN;
723 extern void ffi_closure_unix64_sse(void) FFI_HIDDEN;
724
725 #ifndef __ILP32__
726 extern ffi_status
727 ffi_prep_closure_loc_efi64(ffi_closure* closure,
728 ffi_cif* cif,
729 void (*fun)(ffi_cif*, void*, void**, void*),
730 void *user_data,
731 void *codeloc);
732 #endif
733
734 ffi_status
ffi_prep_closure_loc(ffi_closure * closure,ffi_cif * cif,void (* fun)(ffi_cif *,void *,void **,void *),void * user_data,void * codeloc)735 ffi_prep_closure_loc (ffi_closure* closure,
736 ffi_cif* cif,
737 void (*fun)(ffi_cif*, void*, void**, void*),
738 void *user_data,
739 void *codeloc)
740 {
741 static const unsigned char trampoline[16] = {
742 /* leaq -0x7(%rip),%r10 # 0x0 */
743 0x4c, 0x8d, 0x15, 0xf9, 0xff, 0xff, 0xff,
744 /* jmpq *0x3(%rip) # 0x10 */
745 0xff, 0x25, 0x03, 0x00, 0x00, 0x00,
746 /* nopl (%rax) */
747 0x0f, 0x1f, 0x00
748 };
749 void (*dest)(void);
750 char *tramp = closure->tramp;
751
752 #ifndef __ILP32__
753 if (cif->abi == FFI_EFI64 || cif->abi == FFI_GNUW64)
754 return ffi_prep_closure_loc_efi64(closure, cif, fun, user_data, codeloc);
755 #endif
756 if (cif->abi != FFI_UNIX64)
757 return FFI_BAD_ABI;
758
759 if (cif->flags & UNIX64_FLAG_XMM_ARGS)
760 dest = ffi_closure_unix64_sse;
761 else
762 dest = ffi_closure_unix64;
763
764 memcpy (tramp, trampoline, sizeof(trampoline));
765 *(UINT64 *)(tramp + 16) = (uintptr_t)dest;
766
767 closure->cif = cif;
768 closure->fun = fun;
769 closure->user_data = user_data;
770
771 return FFI_OK;
772 }
773
774 int FFI_HIDDEN
ffi_closure_unix64_inner(ffi_cif * cif,void (* fun)(ffi_cif *,void *,void **,void *),void * user_data,void * rvalue,struct register_args * reg_args,char * argp)775 ffi_closure_unix64_inner(ffi_cif *cif,
776 void (*fun)(ffi_cif*, void*, void**, void*),
777 void *user_data,
778 void *rvalue,
779 struct register_args *reg_args,
780 char *argp)
781 {
782 void **avalue;
783 ffi_type **arg_types;
784 long i, avn;
785 int gprcount, ssecount, ngpr, nsse;
786 int flags;
787
788 avn = cif->nargs;
789 flags = cif->flags;
790 avalue = alloca(avn * sizeof(void *));
791 gprcount = ssecount = 0;
792
793 if (flags & UNIX64_FLAG_RET_IN_MEM)
794 {
795 /* On return, %rax will contain the address that was passed
796 by the caller in %rdi. */
797 void *r = (void *)(uintptr_t)reg_args->gpr[gprcount++];
798 *(void **)rvalue = r;
799 rvalue = r;
800 flags = (sizeof(void *) == 4 ? UNIX64_RET_UINT32 : UNIX64_RET_INT64);
801 }
802
803 arg_types = cif->arg_types;
804 for (i = 0; i < avn; ++i)
805 {
806 enum x86_64_reg_class classes[MAX_CLASSES];
807 size_t n;
808
809 n = examine_argument (arg_types[i], classes, 0, &ngpr, &nsse);
810 if (n == 0
811 || gprcount + ngpr > MAX_GPR_REGS
812 || ssecount + nsse > MAX_SSE_REGS)
813 {
814 long align = arg_types[i]->alignment;
815
816 /* Stack arguments are *always* at least 8 byte aligned. */
817 if (align < 8)
818 align = 8;
819
820 /* Pass this argument in memory. */
821 argp = (void *) FFI_ALIGN (argp, align);
822 avalue[i] = argp;
823 argp += arg_types[i]->size;
824 }
825 /* If the argument is in a single register, or two consecutive
826 integer registers, then we can use that address directly. */
827 else if (n == 1
828 || (n == 2 && !(SSE_CLASS_P (classes[0])
829 || SSE_CLASS_P (classes[1]))))
830 {
831 /* The argument is in a single register. */
832 if (SSE_CLASS_P (classes[0]))
833 {
834 avalue[i] = ®_args->sse[ssecount];
835 ssecount += n;
836 }
837 else
838 {
839 avalue[i] = ®_args->gpr[gprcount];
840 gprcount += n;
841 }
842 }
843 /* Otherwise, allocate space to make them consecutive. */
844 else
845 {
846 char *a = alloca (16);
847 unsigned int j;
848
849 avalue[i] = a;
850 for (j = 0; j < n; j++, a += 8)
851 {
852 if (SSE_CLASS_P (classes[j]))
853 memcpy (a, ®_args->sse[ssecount++], 8);
854 else
855 memcpy (a, ®_args->gpr[gprcount++], 8);
856 }
857 }
858 }
859
860 /* Invoke the closure. */
861 fun (cif, rvalue, avalue, user_data);
862
863 /* Tell assembly how to perform return type promotions. */
864 return flags;
865 }
866
867 extern void ffi_go_closure_unix64(void) FFI_HIDDEN;
868 extern void ffi_go_closure_unix64_sse(void) FFI_HIDDEN;
869
870 #ifndef __ILP32__
871 extern ffi_status
872 ffi_prep_go_closure_efi64(ffi_go_closure* closure, ffi_cif* cif,
873 void (*fun)(ffi_cif*, void*, void**, void*));
874 #endif
875
876 ffi_status
ffi_prep_go_closure(ffi_go_closure * closure,ffi_cif * cif,void (* fun)(ffi_cif *,void *,void **,void *))877 ffi_prep_go_closure (ffi_go_closure* closure, ffi_cif* cif,
878 void (*fun)(ffi_cif*, void*, void**, void*))
879 {
880 #ifndef __ILP32__
881 if (cif->abi == FFI_EFI64 || cif->abi == FFI_GNUW64)
882 return ffi_prep_go_closure_efi64(closure, cif, fun);
883 #endif
884 if (cif->abi != FFI_UNIX64)
885 return FFI_BAD_ABI;
886
887 closure->tramp = (cif->flags & UNIX64_FLAG_XMM_ARGS
888 ? ffi_go_closure_unix64_sse
889 : ffi_go_closure_unix64);
890 closure->cif = cif;
891 closure->fun = fun;
892
893 return FFI_OK;
894 }
895
896 #endif /* __x86_64__ */
897