1 /* -----------------------------------------------------------------------
2 ffi64.c - Copyright (c) 2013 The Written Word, Inc.
3 Copyright (c) 2011 Anthony Green
4 Copyright (c) 2008, 2010 Red Hat, Inc.
5 Copyright (c) 2002, 2007 Bo Thorsen <bo@suse.de>
6
7 x86-64 Foreign Function Interface
8
9 Permission is hereby granted, free of charge, to any person obtaining
10 a copy of this software and associated documentation files (the
11 ``Software''), to deal in the Software without restriction, including
12 without limitation the rights to use, copy, modify, merge, publish,
13 distribute, sublicense, and/or sell copies of the Software, and to
14 permit persons to whom the Software is furnished to do so, subject to
15 the following conditions:
16
17 The above copyright notice and this permission notice shall be included
18 in all copies or substantial portions of the Software.
19
20 THE SOFTWARE IS PROVIDED ``AS IS'', WITHOUT WARRANTY OF ANY KIND,
21 EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
22 MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
23 NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT
24 HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
25 WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
26 OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
27 DEALINGS IN THE SOFTWARE.
28 ----------------------------------------------------------------------- */
29
30 #include <ffi.h>
31 #include <ffi_common.h>
32
33 #include <stdlib.h>
34 #include <stdarg.h>
35
36 #ifdef __x86_64__
37
38 #define MAX_GPR_REGS 6
39 #define MAX_SSE_REGS 8
40
41 #if defined(__INTEL_COMPILER)
42 #include "xmmintrin.h"
43 #define UINT128 __m128
44 #else
45 #if defined(__SUNPRO_C)
46 #include <sunmedia_types.h>
47 #define UINT128 __m128i
48 #else
49 #define UINT128 __int128_t
50 #endif
51 #endif
52
53 union big_int_union
54 {
55 UINT32 i32;
56 UINT64 i64;
57 UINT128 i128;
58 };
59
60 struct register_args
61 {
62 /* Registers for argument passing. */
63 UINT64 gpr[MAX_GPR_REGS];
64 union big_int_union sse[MAX_SSE_REGS];
65 };
66
67 extern void ffi_call_unix64 (void *args, unsigned long bytes, unsigned flags,
68 void *raddr, void (*fnaddr)(void), unsigned ssecount);
69
70 /* All reference to register classes here is identical to the code in
71 gcc/config/i386/i386.c. Do *not* change one without the other. */
72
73 /* Register class used for passing given 64bit part of the argument.
74 These represent classes as documented by the PS ABI, with the
75 exception of SSESF, SSEDF classes, that are basically SSE class,
76 just gcc will use SF or DFmode move instead of DImode to avoid
77 reformatting penalties.
78
79 Similary we play games with INTEGERSI_CLASS to use cheaper SImode moves
80 whenever possible (upper half does contain padding). */
81 enum x86_64_reg_class
82 {
83 X86_64_NO_CLASS,
84 X86_64_INTEGER_CLASS,
85 X86_64_INTEGERSI_CLASS,
86 X86_64_SSE_CLASS,
87 X86_64_SSESF_CLASS,
88 X86_64_SSEDF_CLASS,
89 X86_64_SSEUP_CLASS,
90 X86_64_X87_CLASS,
91 X86_64_X87UP_CLASS,
92 X86_64_COMPLEX_X87_CLASS,
93 X86_64_MEMORY_CLASS
94 };
95
96 #define MAX_CLASSES 4
97
98 #define SSE_CLASS_P(X) ((X) >= X86_64_SSE_CLASS && X <= X86_64_SSEUP_CLASS)
99
100 /* x86-64 register passing implementation. See x86-64 ABI for details. Goal
101 of this code is to classify each 8bytes of incoming argument by the register
102 class and assign registers accordingly. */
103
104 /* Return the union class of CLASS1 and CLASS2.
105 See the x86-64 PS ABI for details. */
106
107 static enum x86_64_reg_class
merge_classes(enum x86_64_reg_class class1,enum x86_64_reg_class class2)108 merge_classes (enum x86_64_reg_class class1, enum x86_64_reg_class class2)
109 {
110 /* Rule #1: If both classes are equal, this is the resulting class. */
111 if (class1 == class2)
112 return class1;
113
114 /* Rule #2: If one of the classes is NO_CLASS, the resulting class is
115 the other class. */
116 if (class1 == X86_64_NO_CLASS)
117 return class2;
118 if (class2 == X86_64_NO_CLASS)
119 return class1;
120
121 /* Rule #3: If one of the classes is MEMORY, the result is MEMORY. */
122 if (class1 == X86_64_MEMORY_CLASS || class2 == X86_64_MEMORY_CLASS)
123 return X86_64_MEMORY_CLASS;
124
125 /* Rule #4: If one of the classes is INTEGER, the result is INTEGER. */
126 if ((class1 == X86_64_INTEGERSI_CLASS && class2 == X86_64_SSESF_CLASS)
127 || (class2 == X86_64_INTEGERSI_CLASS && class1 == X86_64_SSESF_CLASS))
128 return X86_64_INTEGERSI_CLASS;
129 if (class1 == X86_64_INTEGER_CLASS || class1 == X86_64_INTEGERSI_CLASS
130 || class2 == X86_64_INTEGER_CLASS || class2 == X86_64_INTEGERSI_CLASS)
131 return X86_64_INTEGER_CLASS;
132
133 /* Rule #5: If one of the classes is X87, X87UP, or COMPLEX_X87 class,
134 MEMORY is used. */
135 if (class1 == X86_64_X87_CLASS
136 || class1 == X86_64_X87UP_CLASS
137 || class1 == X86_64_COMPLEX_X87_CLASS
138 || class2 == X86_64_X87_CLASS
139 || class2 == X86_64_X87UP_CLASS
140 || class2 == X86_64_COMPLEX_X87_CLASS)
141 return X86_64_MEMORY_CLASS;
142
143 /* Rule #6: Otherwise class SSE is used. */
144 return X86_64_SSE_CLASS;
145 }
146
147 /* Classify the argument of type TYPE and mode MODE.
148 CLASSES will be filled by the register class used to pass each word
149 of the operand. The number of words is returned. In case the parameter
150 should be passed in memory, 0 is returned. As a special case for zero
151 sized containers, classes[0] will be NO_CLASS and 1 is returned.
152
153 See the x86-64 PS ABI for details.
154 */
155 static size_t
classify_argument(ffi_type * type,enum x86_64_reg_class classes[],size_t byte_offset)156 classify_argument (ffi_type *type, enum x86_64_reg_class classes[],
157 size_t byte_offset)
158 {
159 switch (type->type)
160 {
161 case FFI_TYPE_UINT8:
162 case FFI_TYPE_SINT8:
163 case FFI_TYPE_UINT16:
164 case FFI_TYPE_SINT16:
165 case FFI_TYPE_UINT32:
166 case FFI_TYPE_SINT32:
167 case FFI_TYPE_UINT64:
168 case FFI_TYPE_SINT64:
169 case FFI_TYPE_POINTER:
170 {
171 size_t size = byte_offset + type->size;
172
173 if (size <= 4)
174 {
175 classes[0] = X86_64_INTEGERSI_CLASS;
176 return 1;
177 }
178 else if (size <= 8)
179 {
180 classes[0] = X86_64_INTEGER_CLASS;
181 return 1;
182 }
183 else if (size <= 12)
184 {
185 classes[0] = X86_64_INTEGER_CLASS;
186 classes[1] = X86_64_INTEGERSI_CLASS;
187 return 2;
188 }
189 else if (size <= 16)
190 {
191 classes[0] = classes[1] = X86_64_INTEGERSI_CLASS;
192 return 2;
193 }
194 else
195 FFI_ASSERT (0);
196 }
197 case FFI_TYPE_FLOAT:
198 if (!(byte_offset % 8))
199 classes[0] = X86_64_SSESF_CLASS;
200 else
201 classes[0] = X86_64_SSE_CLASS;
202 return 1;
203 case FFI_TYPE_DOUBLE:
204 classes[0] = X86_64_SSEDF_CLASS;
205 return 1;
206 #if FFI_TYPE_LONGDOUBLE != FFI_TYPE_DOUBLE
207 case FFI_TYPE_LONGDOUBLE:
208 classes[0] = X86_64_X87_CLASS;
209 classes[1] = X86_64_X87UP_CLASS;
210 return 2;
211 #endif
212 case FFI_TYPE_STRUCT:
213 {
214 const size_t UNITS_PER_WORD = 8;
215 size_t words = (type->size + UNITS_PER_WORD - 1) / UNITS_PER_WORD;
216 ffi_type **ptr;
217 int i;
218 enum x86_64_reg_class subclasses[MAX_CLASSES];
219
220 /* If the struct is larger than 32 bytes, pass it on the stack. */
221 if (type->size > 32)
222 return 0;
223
224 for (i = 0; i < words; i++)
225 classes[i] = X86_64_NO_CLASS;
226
227 /* Zero sized arrays or structures are NO_CLASS. We return 0 to
228 signalize memory class, so handle it as special case. */
229 if (!words)
230 {
231 classes[0] = X86_64_NO_CLASS;
232 return 1;
233 }
234
235 /* Merge the fields of structure. */
236 for (ptr = type->elements; *ptr != NULL; ptr++)
237 {
238 size_t num;
239
240 byte_offset = ALIGN (byte_offset, (*ptr)->alignment);
241
242 num = classify_argument (*ptr, subclasses, byte_offset % 8);
243 if (num == 0)
244 return 0;
245 for (i = 0; i < num; i++)
246 {
247 size_t pos = byte_offset / 8;
248 classes[i + pos] =
249 merge_classes (subclasses[i], classes[i + pos]);
250 }
251
252 byte_offset += (*ptr)->size;
253 }
254
255 if (words > 2)
256 {
257 /* When size > 16 bytes, if the first one isn't
258 X86_64_SSE_CLASS or any other ones aren't
259 X86_64_SSEUP_CLASS, everything should be passed in
260 memory. */
261 if (classes[0] != X86_64_SSE_CLASS)
262 return 0;
263
264 for (i = 1; i < words; i++)
265 if (classes[i] != X86_64_SSEUP_CLASS)
266 return 0;
267 }
268
269 /* Final merger cleanup. */
270 for (i = 0; i < words; i++)
271 {
272 /* If one class is MEMORY, everything should be passed in
273 memory. */
274 if (classes[i] == X86_64_MEMORY_CLASS)
275 return 0;
276
277 /* The X86_64_SSEUP_CLASS should be always preceded by
278 X86_64_SSE_CLASS or X86_64_SSEUP_CLASS. */
279 if (classes[i] == X86_64_SSEUP_CLASS
280 && classes[i - 1] != X86_64_SSE_CLASS
281 && classes[i - 1] != X86_64_SSEUP_CLASS)
282 {
283 /* The first one should never be X86_64_SSEUP_CLASS. */
284 FFI_ASSERT (i != 0);
285 classes[i] = X86_64_SSE_CLASS;
286 }
287
288 /* If X86_64_X87UP_CLASS isn't preceded by X86_64_X87_CLASS,
289 everything should be passed in memory. */
290 if (classes[i] == X86_64_X87UP_CLASS
291 && (classes[i - 1] != X86_64_X87_CLASS))
292 {
293 /* The first one should never be X86_64_X87UP_CLASS. */
294 FFI_ASSERT (i != 0);
295 return 0;
296 }
297 }
298 return words;
299 }
300
301 default:
302 FFI_ASSERT(0);
303 }
304 return 0; /* Never reached. */
305 }
306
307 /* Examine the argument and return set number of register required in each
308 class. Return zero iff parameter should be passed in memory, otherwise
309 the number of registers. */
310
311 static size_t
examine_argument(ffi_type * type,enum x86_64_reg_class classes[MAX_CLASSES],_Bool in_return,int * pngpr,int * pnsse)312 examine_argument (ffi_type *type, enum x86_64_reg_class classes[MAX_CLASSES],
313 _Bool in_return, int *pngpr, int *pnsse)
314 {
315 size_t n;
316 int i, ngpr, nsse;
317
318 n = classify_argument (type, classes, 0);
319 if (n == 0)
320 return 0;
321
322 ngpr = nsse = 0;
323 for (i = 0; i < n; ++i)
324 switch (classes[i])
325 {
326 case X86_64_INTEGER_CLASS:
327 case X86_64_INTEGERSI_CLASS:
328 ngpr++;
329 break;
330 case X86_64_SSE_CLASS:
331 case X86_64_SSESF_CLASS:
332 case X86_64_SSEDF_CLASS:
333 nsse++;
334 break;
335 case X86_64_NO_CLASS:
336 case X86_64_SSEUP_CLASS:
337 break;
338 case X86_64_X87_CLASS:
339 case X86_64_X87UP_CLASS:
340 case X86_64_COMPLEX_X87_CLASS:
341 return in_return != 0;
342 default:
343 abort ();
344 }
345
346 *pngpr = ngpr;
347 *pnsse = nsse;
348
349 return n;
350 }
351
352 /* Perform machine dependent cif processing. */
353
354 ffi_status
ffi_prep_cif_machdep(ffi_cif * cif)355 ffi_prep_cif_machdep (ffi_cif *cif)
356 {
357 int gprcount, ssecount, i, avn, ngpr, nsse, flags;
358 enum x86_64_reg_class classes[MAX_CLASSES];
359 size_t bytes, n;
360
361 gprcount = ssecount = 0;
362
363 flags = cif->rtype->type;
364 if (flags != FFI_TYPE_VOID)
365 {
366 n = examine_argument (cif->rtype, classes, 1, &ngpr, &nsse);
367 if (n == 0)
368 {
369 /* The return value is passed in memory. A pointer to that
370 memory is the first argument. Allocate a register for it. */
371 gprcount++;
372 /* We don't have to do anything in asm for the return. */
373 flags = FFI_TYPE_VOID;
374 }
375 else if (flags == FFI_TYPE_STRUCT)
376 {
377 /* Mark which registers the result appears in. */
378 _Bool sse0 = SSE_CLASS_P (classes[0]);
379 _Bool sse1 = n == 2 && SSE_CLASS_P (classes[1]);
380 if (sse0 && !sse1)
381 flags |= 1 << 8;
382 else if (!sse0 && sse1)
383 flags |= 1 << 9;
384 else if (sse0 && sse1)
385 flags |= 1 << 10;
386 /* Mark the true size of the structure. */
387 flags |= cif->rtype->size << 12;
388 }
389 }
390
391 /* Go over all arguments and determine the way they should be passed.
392 If it's in a register and there is space for it, let that be so. If
393 not, add it's size to the stack byte count. */
394 for (bytes = 0, i = 0, avn = cif->nargs; i < avn; i++)
395 {
396 if (examine_argument (cif->arg_types[i], classes, 0, &ngpr, &nsse) == 0
397 || gprcount + ngpr > MAX_GPR_REGS
398 || ssecount + nsse > MAX_SSE_REGS)
399 {
400 long align = cif->arg_types[i]->alignment;
401
402 if (align < 8)
403 align = 8;
404
405 bytes = ALIGN (bytes, align);
406 bytes += cif->arg_types[i]->size;
407 }
408 else
409 {
410 gprcount += ngpr;
411 ssecount += nsse;
412 }
413 }
414 if (ssecount)
415 flags |= 1 << 11;
416 cif->flags = flags;
417 cif->bytes = (unsigned)ALIGN (bytes, 8);
418
419 return FFI_OK;
420 }
421
422 #ifndef __SANITIZE_ADDRESS__
423 # ifdef __clang__
424 # if __has_feature(address_sanitizer)
425 # define __SANITIZE_ADDRESS__
426 # endif
427 # endif
428 #endif
429 #ifdef __SANITIZE_ADDRESS__
430 __attribute__((noinline,no_sanitize_address))
431 #endif
432 void
ffi_call(ffi_cif * cif,void (* fn)(void),void * rvalue,void ** avalue)433 ffi_call (ffi_cif *cif, void (*fn)(void), void *rvalue, void **avalue)
434 {
435 enum x86_64_reg_class classes[MAX_CLASSES];
436 char *stack, *argp;
437 ffi_type **arg_types;
438 int gprcount, ssecount, ngpr, nsse, i, avn;
439 _Bool ret_in_memory;
440 struct register_args *reg_args;
441
442 /* Can't call 32-bit mode from 64-bit mode. */
443 FFI_ASSERT (cif->abi == FFI_UNIX64);
444
445 /* If the return value is a struct and we don't have a return value
446 address then we need to make one. Note the setting of flags to
447 VOID above in ffi_prep_cif_machdep. */
448 ret_in_memory = (cif->rtype->type == FFI_TYPE_STRUCT
449 && (cif->flags & 0xff) == FFI_TYPE_VOID);
450 if (rvalue == NULL && ret_in_memory)
451 rvalue = alloca (cif->rtype->size);
452
453 /* Allocate the space for the arguments, plus 4 words of temp space. */
454 stack = alloca (sizeof (struct register_args) + cif->bytes + 4*8);
455 reg_args = (struct register_args *) stack;
456 argp = stack + sizeof (struct register_args);
457
458 gprcount = ssecount = 0;
459
460 /* If the return value is passed in memory, add the pointer as the
461 first integer argument. */
462 if (ret_in_memory)
463 reg_args->gpr[gprcount++] = (unsigned long) rvalue;
464
465 avn = cif->nargs;
466 arg_types = cif->arg_types;
467
468 for (i = 0; i < avn; ++i)
469 {
470 size_t n, size = arg_types[i]->size;
471
472 n = examine_argument (arg_types[i], classes, 0, &ngpr, &nsse);
473 if (n == 0
474 || gprcount + ngpr > MAX_GPR_REGS
475 || ssecount + nsse > MAX_SSE_REGS)
476 {
477 long align = arg_types[i]->alignment;
478
479 /* Stack arguments are *always* at least 8 byte aligned. */
480 if (align < 8)
481 align = 8;
482
483 /* Pass this argument in memory. */
484 argp = (void *) ALIGN (argp, align);
485 memcpy (argp, avalue[i], size);
486 argp += size;
487 }
488 else
489 {
490 /* The argument is passed entirely in registers. */
491 char *a = (char *) avalue[i];
492 int j;
493
494 for (j = 0; j < n; j++, a += 8, size -= 8)
495 {
496 switch (classes[j])
497 {
498 case X86_64_INTEGER_CLASS:
499 case X86_64_INTEGERSI_CLASS:
500 /* Sign-extend integer arguments passed in general
501 purpose registers, to cope with the fact that
502 LLVM incorrectly assumes that this will be done
503 (the x86-64 PS ABI does not specify this). */
504 switch (arg_types[i]->type)
505 {
506 case FFI_TYPE_SINT8:
507 *(SINT64 *)®_args->gpr[gprcount] = (SINT64) *((SINT8 *) a);
508 break;
509 case FFI_TYPE_SINT16:
510 *(SINT64 *)®_args->gpr[gprcount] = (SINT64) *((SINT16 *) a);
511 break;
512 case FFI_TYPE_SINT32:
513 *(SINT64 *)®_args->gpr[gprcount] = (SINT64) *((SINT32 *) a);
514 break;
515 default:
516 reg_args->gpr[gprcount] = 0;
517 memcpy (®_args->gpr[gprcount], a, size < 8 ? size : 8);
518 }
519 gprcount++;
520 break;
521 case X86_64_SSE_CLASS:
522 case X86_64_SSEDF_CLASS:
523 reg_args->sse[ssecount++].i64 = *(UINT64 *) a;
524 break;
525 case X86_64_SSESF_CLASS:
526 reg_args->sse[ssecount++].i32 = *(UINT32 *) a;
527 break;
528 default:
529 abort();
530 }
531 }
532 }
533 }
534
535 ffi_call_unix64 (stack, cif->bytes + sizeof (struct register_args),
536 cif->flags, rvalue, fn, ssecount);
537 }
538
539
540 extern void ffi_closure_unix64(void);
541
542 ffi_status
ffi_prep_closure_loc(ffi_closure * closure,ffi_cif * cif,void (* fun)(ffi_cif *,void *,void **,void *),void * user_data,void * codeloc)543 ffi_prep_closure_loc (ffi_closure* closure,
544 ffi_cif* cif,
545 void (*fun)(ffi_cif*, void*, void**, void*),
546 void *user_data,
547 void *codeloc)
548 {
549 volatile unsigned short *tramp;
550
551 /* Sanity check on the cif ABI. */
552 {
553 int abi = cif->abi;
554 if (UNLIKELY (! (abi > FFI_FIRST_ABI && abi < FFI_LAST_ABI)))
555 return FFI_BAD_ABI;
556 }
557
558 tramp = (volatile unsigned short *) &closure->tramp[0];
559
560 tramp[0] = 0xbb49; /* mov <code>, %r11 */
561 *((unsigned long long * volatile) &tramp[1])
562 = (unsigned long) ffi_closure_unix64;
563 tramp[5] = 0xba49; /* mov <data>, %r10 */
564 *((unsigned long long * volatile) &tramp[6])
565 = (unsigned long) codeloc;
566
567 /* Set the carry bit iff the function uses any sse registers.
568 This is clc or stc, together with the first byte of the jmp. */
569 tramp[10] = cif->flags & (1 << 11) ? 0x49f9 : 0x49f8;
570
571 tramp[11] = 0xe3ff; /* jmp *%r11 */
572
573 closure->cif = cif;
574 closure->fun = fun;
575 closure->user_data = user_data;
576
577 return FFI_OK;
578 }
579
580 int
ffi_closure_unix64_inner(ffi_closure * closure,void * rvalue,struct register_args * reg_args,char * argp)581 ffi_closure_unix64_inner(ffi_closure *closure, void *rvalue,
582 struct register_args *reg_args, char *argp)
583 {
584 ffi_cif *cif;
585 void **avalue;
586 ffi_type **arg_types;
587 long i, avn;
588 int gprcount, ssecount, ngpr, nsse;
589 int ret;
590
591 cif = closure->cif;
592 avalue = alloca(cif->nargs * sizeof(void *));
593 gprcount = ssecount = 0;
594
595 ret = cif->rtype->type;
596 if (ret != FFI_TYPE_VOID)
597 {
598 enum x86_64_reg_class classes[MAX_CLASSES];
599 size_t n = examine_argument (cif->rtype, classes, 1, &ngpr, &nsse);
600 if (n == 0)
601 {
602 /* The return value goes in memory. Arrange for the closure
603 return value to go directly back to the original caller. */
604 rvalue = (void *) (unsigned long) reg_args->gpr[gprcount++];
605 /* We don't have to do anything in asm for the return. */
606 ret = FFI_TYPE_VOID;
607 }
608 else if (ret == FFI_TYPE_STRUCT && n == 2)
609 {
610 /* Mark which register the second word of the structure goes in. */
611 _Bool sse0 = SSE_CLASS_P (classes[0]);
612 _Bool sse1 = SSE_CLASS_P (classes[1]);
613 if (!sse0 && sse1)
614 ret |= 1 << 8;
615 else if (sse0 && !sse1)
616 ret |= 1 << 9;
617 }
618 }
619
620 avn = cif->nargs;
621 arg_types = cif->arg_types;
622
623 for (i = 0; i < avn; ++i)
624 {
625 enum x86_64_reg_class classes[MAX_CLASSES];
626 size_t n;
627
628 n = examine_argument (arg_types[i], classes, 0, &ngpr, &nsse);
629 if (n == 0
630 || gprcount + ngpr > MAX_GPR_REGS
631 || ssecount + nsse > MAX_SSE_REGS)
632 {
633 long align = arg_types[i]->alignment;
634
635 /* Stack arguments are *always* at least 8 byte aligned. */
636 if (align < 8)
637 align = 8;
638
639 /* Pass this argument in memory. */
640 argp = (void *) ALIGN (argp, align);
641 avalue[i] = argp;
642 argp += arg_types[i]->size;
643 }
644 /* If the argument is in a single register, or two consecutive
645 integer registers, then we can use that address directly. */
646 else if (n == 1
647 || (n == 2 && !(SSE_CLASS_P (classes[0])
648 || SSE_CLASS_P (classes[1]))))
649 {
650 /* The argument is in a single register. */
651 if (SSE_CLASS_P (classes[0]))
652 {
653 avalue[i] = ®_args->sse[ssecount];
654 ssecount += n;
655 }
656 else
657 {
658 avalue[i] = ®_args->gpr[gprcount];
659 gprcount += n;
660 }
661 }
662 /* Otherwise, allocate space to make them consecutive. */
663 else
664 {
665 char *a = alloca (16);
666 int j;
667
668 avalue[i] = a;
669 for (j = 0; j < n; j++, a += 8)
670 {
671 if (SSE_CLASS_P (classes[j]))
672 memcpy (a, ®_args->sse[ssecount++], 8);
673 else
674 memcpy (a, ®_args->gpr[gprcount++], 8);
675 }
676 }
677 }
678
679 /* Invoke the closure. */
680 closure->fun (cif, rvalue, avalue, closure->user_data);
681
682 /* Tell assembly how to perform return type promotions. */
683 return ret;
684 }
685
686 #endif /* __x86_64__ */
687