1 /* -----------------------------------------------------------------------
2 ffi_linux64.c - Copyright (C) 2013 IBM
3 Copyright (C) 2011 Anthony Green
4 Copyright (C) 2011 Kyle Moffett
5 Copyright (C) 2008 Red Hat, Inc
6 Copyright (C) 2007, 2008 Free Software Foundation, Inc
7 Copyright (c) 1998 Geoffrey Keating
8
9 PowerPC Foreign Function Interface
10
11 Permission is hereby granted, free of charge, to any person obtaining
12 a copy of this software and associated documentation files (the
13 ``Software''), to deal in the Software without restriction, including
14 without limitation the rights to use, copy, modify, merge, publish,
15 distribute, sublicense, and/or sell copies of the Software, and to
16 permit persons to whom the Software is furnished to do so, subject to
17 the following conditions:
18
19 The above copyright notice and this permission notice shall be included
20 in all copies or substantial portions of the Software.
21
22 THE SOFTWARE IS PROVIDED ``AS IS'', WITHOUT WARRANTY OF ANY KIND, EXPRESS
23 OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
24 MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
25 IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR
26 OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
27 ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
28 OTHER DEALINGS IN THE SOFTWARE.
29 ----------------------------------------------------------------------- */
30
31 #include "ffi.h"
32
33 #ifdef POWERPC64
34 #include "ffi_common.h"
35 #include "ffi_powerpc.h"
36
37
38 /* About the LINUX64 ABI. */
39 enum {
40 NUM_GPR_ARG_REGISTERS64 = 8,
41 NUM_FPR_ARG_REGISTERS64 = 13
42 };
43 enum { ASM_NEEDS_REGISTERS64 = 4 };
44
45
46 #if HAVE_LONG_DOUBLE_VARIANT && FFI_TYPE_LONGDOUBLE != FFI_TYPE_DOUBLE
47 /* Adjust size of ffi_type_longdouble. */
48 void FFI_HIDDEN
ffi_prep_types_linux64(ffi_abi abi)49 ffi_prep_types_linux64 (ffi_abi abi)
50 {
51 if ((abi & (FFI_LINUX | FFI_LINUX_LONG_DOUBLE_128)) == FFI_LINUX)
52 {
53 ffi_type_longdouble.size = 8;
54 ffi_type_longdouble.alignment = 8;
55 }
56 else
57 {
58 ffi_type_longdouble.size = 16;
59 ffi_type_longdouble.alignment = 16;
60 }
61 }
62 #endif
63
64
65 static unsigned int
discover_homogeneous_aggregate(const ffi_type * t,unsigned int * elnum)66 discover_homogeneous_aggregate (const ffi_type *t, unsigned int *elnum)
67 {
68 switch (t->type)
69 {
70 case FFI_TYPE_FLOAT:
71 case FFI_TYPE_DOUBLE:
72 *elnum = 1;
73 return (int) t->type;
74
75 case FFI_TYPE_STRUCT:;
76 {
77 unsigned int base_elt = 0, total_elnum = 0;
78 ffi_type **el = t->elements;
79 while (*el)
80 {
81 unsigned int el_elt, el_elnum = 0;
82 el_elt = discover_homogeneous_aggregate (*el, &el_elnum);
83 if (el_elt == 0
84 || (base_elt && base_elt != el_elt))
85 return 0;
86 base_elt = el_elt;
87 total_elnum += el_elnum;
88 #if _CALL_ELF == 2
89 if (total_elnum > 8)
90 return 0;
91 #else
92 if (total_elnum > 1)
93 return 0;
94 #endif
95 el++;
96 }
97 *elnum = total_elnum;
98 return base_elt;
99 }
100
101 default:
102 return 0;
103 }
104 }
105
106
107 /* Perform machine dependent cif processing */
108 static ffi_status
ffi_prep_cif_linux64_core(ffi_cif * cif)109 ffi_prep_cif_linux64_core (ffi_cif *cif)
110 {
111 ffi_type **ptr;
112 unsigned bytes;
113 unsigned i, fparg_count = 0, intarg_count = 0;
114 unsigned flags = cif->flags;
115 unsigned int elt, elnum;
116
117 #if FFI_TYPE_LONGDOUBLE == FFI_TYPE_DOUBLE
118 /* If compiled without long double support.. */
119 if ((cif->abi & FFI_LINUX_LONG_DOUBLE_128) != 0)
120 return FFI_BAD_ABI;
121 #endif
122
123 /* The machine-independent calculation of cif->bytes doesn't work
124 for us. Redo the calculation. */
125 #if _CALL_ELF == 2
126 /* Space for backchain, CR, LR, TOC and the asm's temp regs. */
127 bytes = (4 + ASM_NEEDS_REGISTERS64) * sizeof (long);
128
129 /* Space for the general registers. */
130 bytes += NUM_GPR_ARG_REGISTERS64 * sizeof (long);
131 #else
132 /* Space for backchain, CR, LR, cc/ld doubleword, TOC and the asm's temp
133 regs. */
134 bytes = (6 + ASM_NEEDS_REGISTERS64) * sizeof (long);
135
136 /* Space for the mandatory parm save area and general registers. */
137 bytes += 2 * NUM_GPR_ARG_REGISTERS64 * sizeof (long);
138 #endif
139
140 /* Return value handling. */
141 switch (cif->rtype->type)
142 {
143 #if FFI_TYPE_LONGDOUBLE != FFI_TYPE_DOUBLE
144 case FFI_TYPE_LONGDOUBLE:
145 if ((cif->abi & FFI_LINUX_LONG_DOUBLE_128) != 0)
146 flags |= FLAG_RETURNS_128BITS;
147 /* Fall through. */
148 #endif
149 case FFI_TYPE_DOUBLE:
150 flags |= FLAG_RETURNS_64BITS;
151 /* Fall through. */
152 case FFI_TYPE_FLOAT:
153 flags |= FLAG_RETURNS_FP;
154 break;
155
156 case FFI_TYPE_UINT128:
157 flags |= FLAG_RETURNS_128BITS;
158 /* Fall through. */
159 case FFI_TYPE_UINT64:
160 case FFI_TYPE_SINT64:
161 case FFI_TYPE_POINTER:
162 flags |= FLAG_RETURNS_64BITS;
163 break;
164
165 case FFI_TYPE_STRUCT:
166 #if _CALL_ELF == 2
167 elt = discover_homogeneous_aggregate (cif->rtype, &elnum);
168 if (elt)
169 {
170 if (elt == FFI_TYPE_DOUBLE)
171 flags |= FLAG_RETURNS_64BITS;
172 flags |= FLAG_RETURNS_FP | FLAG_RETURNS_SMST;
173 break;
174 }
175 if (cif->rtype->size <= 16)
176 {
177 flags |= FLAG_RETURNS_SMST;
178 break;
179 }
180 #endif
181 intarg_count++;
182 flags |= FLAG_RETVAL_REFERENCE;
183 /* Fall through. */
184 case FFI_TYPE_VOID:
185 flags |= FLAG_RETURNS_NOTHING;
186 break;
187
188 default:
189 /* Returns 32-bit integer, or similar. Nothing to do here. */
190 break;
191 }
192
193 for (ptr = cif->arg_types, i = cif->nargs; i > 0; i--, ptr++)
194 {
195 unsigned int align;
196
197 switch ((*ptr)->type)
198 {
199 #if FFI_TYPE_LONGDOUBLE != FFI_TYPE_DOUBLE
200 case FFI_TYPE_LONGDOUBLE:
201 if ((cif->abi & FFI_LINUX_LONG_DOUBLE_128) != 0)
202 {
203 fparg_count++;
204 intarg_count++;
205 }
206 /* Fall through. */
207 #endif
208 case FFI_TYPE_DOUBLE:
209 case FFI_TYPE_FLOAT:
210 fparg_count++;
211 intarg_count++;
212 if (fparg_count > NUM_FPR_ARG_REGISTERS64)
213 flags |= FLAG_ARG_NEEDS_PSAVE;
214 break;
215
216 case FFI_TYPE_STRUCT:
217 if ((cif->abi & FFI_LINUX_STRUCT_ALIGN) != 0)
218 {
219 align = (*ptr)->alignment;
220 if (align > 16)
221 align = 16;
222 align = align / 8;
223 if (align > 1)
224 intarg_count = ALIGN (intarg_count, align);
225 }
226 intarg_count += ((*ptr)->size + 7) / 8;
227 elt = discover_homogeneous_aggregate (*ptr, &elnum);
228 if (elt)
229 {
230 fparg_count += elnum;
231 if (fparg_count > NUM_FPR_ARG_REGISTERS64)
232 flags |= FLAG_ARG_NEEDS_PSAVE;
233 }
234 else
235 {
236 if (intarg_count > NUM_GPR_ARG_REGISTERS64)
237 flags |= FLAG_ARG_NEEDS_PSAVE;
238 }
239 break;
240
241 case FFI_TYPE_POINTER:
242 case FFI_TYPE_UINT64:
243 case FFI_TYPE_SINT64:
244 case FFI_TYPE_INT:
245 case FFI_TYPE_UINT32:
246 case FFI_TYPE_SINT32:
247 case FFI_TYPE_UINT16:
248 case FFI_TYPE_SINT16:
249 case FFI_TYPE_UINT8:
250 case FFI_TYPE_SINT8:
251 /* Everything else is passed as a 8-byte word in a GPR, either
252 the object itself or a pointer to it. */
253 intarg_count++;
254 if (intarg_count > NUM_GPR_ARG_REGISTERS64)
255 flags |= FLAG_ARG_NEEDS_PSAVE;
256 break;
257 default:
258 FFI_ASSERT (0);
259 }
260 }
261
262 if (fparg_count != 0)
263 flags |= FLAG_FP_ARGUMENTS;
264 if (intarg_count > 4)
265 flags |= FLAG_4_GPR_ARGUMENTS;
266
267 /* Space for the FPR registers, if needed. */
268 if (fparg_count != 0)
269 bytes += NUM_FPR_ARG_REGISTERS64 * sizeof (double);
270
271 /* Stack space. */
272 #if _CALL_ELF == 2
273 if ((flags & FLAG_ARG_NEEDS_PSAVE) != 0)
274 bytes += intarg_count * sizeof (long);
275 #else
276 if (intarg_count > NUM_GPR_ARG_REGISTERS64)
277 bytes += (intarg_count - NUM_GPR_ARG_REGISTERS64) * sizeof (long);
278 #endif
279
280 /* The stack space allocated needs to be a multiple of 16 bytes. */
281 bytes = (bytes + 15) & ~0xF;
282
283 cif->flags = flags;
284 cif->bytes = bytes;
285
286 return FFI_OK;
287 }
288
289 ffi_status FFI_HIDDEN
ffi_prep_cif_linux64(ffi_cif * cif)290 ffi_prep_cif_linux64 (ffi_cif *cif)
291 {
292 if ((cif->abi & FFI_LINUX) != 0)
293 cif->nfixedargs = cif->nargs;
294 #if _CALL_ELF != 2
295 else if (cif->abi == FFI_COMPAT_LINUX64)
296 {
297 /* This call is from old code. Don't touch cif->nfixedargs
298 since old code will be using a smaller cif. */
299 cif->flags |= FLAG_COMPAT;
300 /* Translate to new abi value. */
301 cif->abi = FFI_LINUX | FFI_LINUX_LONG_DOUBLE_128;
302 }
303 #endif
304 else
305 return FFI_BAD_ABI;
306 return ffi_prep_cif_linux64_core (cif);
307 }
308
309 ffi_status FFI_HIDDEN
ffi_prep_cif_linux64_var(ffi_cif * cif,unsigned int nfixedargs,unsigned int ntotalargs MAYBE_UNUSED)310 ffi_prep_cif_linux64_var (ffi_cif *cif,
311 unsigned int nfixedargs,
312 unsigned int ntotalargs MAYBE_UNUSED)
313 {
314 if ((cif->abi & FFI_LINUX) != 0)
315 cif->nfixedargs = nfixedargs;
316 #if _CALL_ELF != 2
317 else if (cif->abi == FFI_COMPAT_LINUX64)
318 {
319 /* This call is from old code. Don't touch cif->nfixedargs
320 since old code will be using a smaller cif. */
321 cif->flags |= FLAG_COMPAT;
322 /* Translate to new abi value. */
323 cif->abi = FFI_LINUX | FFI_LINUX_LONG_DOUBLE_128;
324 }
325 #endif
326 else
327 return FFI_BAD_ABI;
328 #if _CALL_ELF == 2
329 cif->flags |= FLAG_ARG_NEEDS_PSAVE;
330 #endif
331 return ffi_prep_cif_linux64_core (cif);
332 }
333
334
335 /* ffi_prep_args64 is called by the assembly routine once stack space
336 has been allocated for the function's arguments.
337
338 The stack layout we want looks like this:
339
340 | Ret addr from ffi_call_LINUX64 8bytes | higher addresses
341 |--------------------------------------------|
342 | CR save area 8bytes |
343 |--------------------------------------------|
344 | Previous backchain pointer 8 | stack pointer here
345 |--------------------------------------------|<+ <<< on entry to
346 | Saved r28-r31 4*8 | | ffi_call_LINUX64
347 |--------------------------------------------| |
348 | GPR registers r3-r10 8*8 | |
349 |--------------------------------------------| |
350 | FPR registers f1-f13 (optional) 13*8 | |
351 |--------------------------------------------| |
352 | Parameter save area | |
353 |--------------------------------------------| |
354 | TOC save area 8 | |
355 |--------------------------------------------| | stack |
356 | Linker doubleword 8 | | grows |
357 |--------------------------------------------| | down V
358 | Compiler doubleword 8 | |
359 |--------------------------------------------| | lower addresses
360 | Space for callee's LR 8 | |
361 |--------------------------------------------| |
362 | CR save area 8 | |
363 |--------------------------------------------| | stack pointer here
364 | Current backchain pointer 8 |-/ during
365 |--------------------------------------------| <<< ffi_call_LINUX64
366
367 */
368
369 void FFI_HIDDEN
ffi_prep_args64(extended_cif * ecif,unsigned long * const stack)370 ffi_prep_args64 (extended_cif *ecif, unsigned long *const stack)
371 {
372 const unsigned long bytes = ecif->cif->bytes;
373 const unsigned long flags = ecif->cif->flags;
374
375 typedef union
376 {
377 char *c;
378 unsigned long *ul;
379 float *f;
380 double *d;
381 size_t p;
382 } valp;
383
384 /* 'stacktop' points at the previous backchain pointer. */
385 valp stacktop;
386
387 /* 'next_arg' points at the space for gpr3, and grows upwards as
388 we use GPR registers, then continues at rest. */
389 valp gpr_base;
390 valp gpr_end;
391 valp rest;
392 valp next_arg;
393
394 /* 'fpr_base' points at the space for fpr3, and grows upwards as
395 we use FPR registers. */
396 valp fpr_base;
397 unsigned int fparg_count;
398
399 unsigned int i, words, nargs, nfixedargs;
400 ffi_type **ptr;
401 double double_tmp;
402 union
403 {
404 void **v;
405 char **c;
406 signed char **sc;
407 unsigned char **uc;
408 signed short **ss;
409 unsigned short **us;
410 signed int **si;
411 unsigned int **ui;
412 unsigned long **ul;
413 float **f;
414 double **d;
415 } p_argv;
416 unsigned long gprvalue;
417 unsigned long align;
418
419 stacktop.c = (char *) stack + bytes;
420 gpr_base.ul = stacktop.ul - ASM_NEEDS_REGISTERS64 - NUM_GPR_ARG_REGISTERS64;
421 gpr_end.ul = gpr_base.ul + NUM_GPR_ARG_REGISTERS64;
422 #if _CALL_ELF == 2
423 rest.ul = stack + 4 + NUM_GPR_ARG_REGISTERS64;
424 #else
425 rest.ul = stack + 6 + NUM_GPR_ARG_REGISTERS64;
426 #endif
427 fpr_base.d = gpr_base.d - NUM_FPR_ARG_REGISTERS64;
428 fparg_count = 0;
429 next_arg.ul = gpr_base.ul;
430
431 /* Check that everything starts aligned properly. */
432 FFI_ASSERT (((unsigned long) (char *) stack & 0xF) == 0);
433 FFI_ASSERT (((unsigned long) stacktop.c & 0xF) == 0);
434 FFI_ASSERT ((bytes & 0xF) == 0);
435
436 /* Deal with return values that are actually pass-by-reference. */
437 if (flags & FLAG_RETVAL_REFERENCE)
438 *next_arg.ul++ = (unsigned long) (char *) ecif->rvalue;
439
440 /* Now for the arguments. */
441 p_argv.v = ecif->avalue;
442 nargs = ecif->cif->nargs;
443 #if _CALL_ELF != 2
444 nfixedargs = (unsigned) -1;
445 if ((flags & FLAG_COMPAT) == 0)
446 #endif
447 nfixedargs = ecif->cif->nfixedargs;
448 for (ptr = ecif->cif->arg_types, i = 0;
449 i < nargs;
450 i++, ptr++, p_argv.v++)
451 {
452 unsigned int elt, elnum;
453
454 switch ((*ptr)->type)
455 {
456 #if FFI_TYPE_LONGDOUBLE != FFI_TYPE_DOUBLE
457 case FFI_TYPE_LONGDOUBLE:
458 if ((ecif->cif->abi & FFI_LINUX_LONG_DOUBLE_128) != 0)
459 {
460 double_tmp = (*p_argv.d)[0];
461 if (fparg_count < NUM_FPR_ARG_REGISTERS64 && i < nfixedargs)
462 {
463 *fpr_base.d++ = double_tmp;
464 # if _CALL_ELF != 2
465 if ((flags & FLAG_COMPAT) != 0)
466 *next_arg.d = double_tmp;
467 # endif
468 }
469 else
470 *next_arg.d = double_tmp;
471 if (++next_arg.ul == gpr_end.ul)
472 next_arg.ul = rest.ul;
473 fparg_count++;
474 double_tmp = (*p_argv.d)[1];
475 if (fparg_count < NUM_FPR_ARG_REGISTERS64 && i < nfixedargs)
476 {
477 *fpr_base.d++ = double_tmp;
478 # if _CALL_ELF != 2
479 if ((flags & FLAG_COMPAT) != 0)
480 *next_arg.d = double_tmp;
481 # endif
482 }
483 else
484 *next_arg.d = double_tmp;
485 if (++next_arg.ul == gpr_end.ul)
486 next_arg.ul = rest.ul;
487 fparg_count++;
488 FFI_ASSERT (__LDBL_MANT_DIG__ == 106);
489 FFI_ASSERT (flags & FLAG_FP_ARGUMENTS);
490 break;
491 }
492 /* Fall through. */
493 #endif
494 case FFI_TYPE_DOUBLE:
495 do_double:
496 double_tmp = **p_argv.d;
497 if (fparg_count < NUM_FPR_ARG_REGISTERS64 && i < nfixedargs)
498 {
499 *fpr_base.d++ = double_tmp;
500 #if _CALL_ELF != 2
501 if ((flags & FLAG_COMPAT) != 0)
502 *next_arg.d = double_tmp;
503 #endif
504 }
505 else
506 *next_arg.d = double_tmp;
507 if (++next_arg.ul == gpr_end.ul)
508 next_arg.ul = rest.ul;
509 fparg_count++;
510 FFI_ASSERT (flags & FLAG_FP_ARGUMENTS);
511 break;
512
513 case FFI_TYPE_FLOAT:
514 do_float:
515 double_tmp = **p_argv.f;
516 if (fparg_count < NUM_FPR_ARG_REGISTERS64 && i < nfixedargs)
517 {
518 *fpr_base.d++ = double_tmp;
519 #if _CALL_ELF != 2
520 if ((flags & FLAG_COMPAT) != 0)
521 {
522 # ifndef __LITTLE_ENDIAN__
523 next_arg.f[1] = (float) double_tmp;
524 # else
525 next_arg.f[0] = (float) double_tmp;
526 # endif
527 }
528 #endif
529 }
530 else
531 {
532 # ifndef __LITTLE_ENDIAN__
533 next_arg.f[1] = (float) double_tmp;
534 # else
535 next_arg.f[0] = (float) double_tmp;
536 # endif
537 }
538 if (++next_arg.ul == gpr_end.ul)
539 next_arg.ul = rest.ul;
540 fparg_count++;
541 FFI_ASSERT (flags & FLAG_FP_ARGUMENTS);
542 break;
543
544 case FFI_TYPE_STRUCT:
545 if ((ecif->cif->abi & FFI_LINUX_STRUCT_ALIGN) != 0)
546 {
547 align = (*ptr)->alignment;
548 if (align > 16)
549 align = 16;
550 if (align > 1)
551 next_arg.p = ALIGN (next_arg.p, align);
552 }
553 elt = discover_homogeneous_aggregate (*ptr, &elnum);
554 if (elt)
555 {
556 #if _CALL_ELF == 2
557 union {
558 void *v;
559 float *f;
560 double *d;
561 } arg;
562
563 arg.v = *p_argv.v;
564 if (elt == FFI_TYPE_FLOAT)
565 {
566 do
567 {
568 double_tmp = *arg.f++;
569 if (fparg_count < NUM_FPR_ARG_REGISTERS64
570 && i < nfixedargs)
571 *fpr_base.d++ = double_tmp;
572 else
573 *next_arg.f = (float) double_tmp;
574 if (++next_arg.f == gpr_end.f)
575 next_arg.f = rest.f;
576 fparg_count++;
577 }
578 while (--elnum != 0);
579 if ((next_arg.p & 3) != 0)
580 {
581 if (++next_arg.f == gpr_end.f)
582 next_arg.f = rest.f;
583 }
584 }
585 else
586 do
587 {
588 double_tmp = *arg.d++;
589 if (fparg_count < NUM_FPR_ARG_REGISTERS64 && i < nfixedargs)
590 *fpr_base.d++ = double_tmp;
591 else
592 *next_arg.d = double_tmp;
593 if (++next_arg.d == gpr_end.d)
594 next_arg.d = rest.d;
595 fparg_count++;
596 }
597 while (--elnum != 0);
598 #else
599 if (elt == FFI_TYPE_FLOAT)
600 goto do_float;
601 else
602 goto do_double;
603 #endif
604 }
605 else
606 {
607 words = ((*ptr)->size + 7) / 8;
608 if (next_arg.ul >= gpr_base.ul && next_arg.ul + words > gpr_end.ul)
609 {
610 size_t first = gpr_end.c - next_arg.c;
611 memcpy (next_arg.c, *p_argv.c, first);
612 memcpy (rest.c, *p_argv.c + first, (*ptr)->size - first);
613 next_arg.c = rest.c + words * 8 - first;
614 }
615 else
616 {
617 char *where = next_arg.c;
618
619 #ifndef __LITTLE_ENDIAN__
620 /* Structures with size less than eight bytes are passed
621 left-padded. */
622 if ((*ptr)->size < 8)
623 where += 8 - (*ptr)->size;
624 #endif
625 memcpy (where, *p_argv.c, (*ptr)->size);
626 next_arg.ul += words;
627 if (next_arg.ul == gpr_end.ul)
628 next_arg.ul = rest.ul;
629 }
630 }
631 break;
632
633 case FFI_TYPE_UINT8:
634 gprvalue = **p_argv.uc;
635 goto putgpr;
636 case FFI_TYPE_SINT8:
637 gprvalue = **p_argv.sc;
638 goto putgpr;
639 case FFI_TYPE_UINT16:
640 gprvalue = **p_argv.us;
641 goto putgpr;
642 case FFI_TYPE_SINT16:
643 gprvalue = **p_argv.ss;
644 goto putgpr;
645 case FFI_TYPE_UINT32:
646 gprvalue = **p_argv.ui;
647 goto putgpr;
648 case FFI_TYPE_INT:
649 case FFI_TYPE_SINT32:
650 gprvalue = **p_argv.si;
651 goto putgpr;
652
653 case FFI_TYPE_UINT64:
654 case FFI_TYPE_SINT64:
655 case FFI_TYPE_POINTER:
656 gprvalue = **p_argv.ul;
657 putgpr:
658 *next_arg.ul++ = gprvalue;
659 if (next_arg.ul == gpr_end.ul)
660 next_arg.ul = rest.ul;
661 break;
662 }
663 }
664
665 FFI_ASSERT (flags & FLAG_4_GPR_ARGUMENTS
666 || (next_arg.ul >= gpr_base.ul
667 && next_arg.ul <= gpr_base.ul + 4));
668 }
669
670
671 #if _CALL_ELF == 2
672 #define MIN_CACHE_LINE_SIZE 8
673
674 static void
flush_icache(char * wraddr,char * xaddr,int size)675 flush_icache (char *wraddr, char *xaddr, int size)
676 {
677 int i;
678 for (i = 0; i < size; i += MIN_CACHE_LINE_SIZE)
679 __asm__ volatile ("icbi 0,%0;" "dcbf 0,%1;"
680 : : "r" (xaddr + i), "r" (wraddr + i) : "memory");
681 __asm__ volatile ("icbi 0,%0;" "dcbf 0,%1;" "sync;" "isync;"
682 : : "r"(xaddr + size - 1), "r"(wraddr + size - 1)
683 : "memory");
684 }
685 #endif
686
687
688 ffi_status FFI_HIDDEN
ffi_prep_closure_loc_linux64(ffi_closure * closure,ffi_cif * cif,void (* fun)(ffi_cif *,void *,void **,void *),void * user_data,void * codeloc)689 ffi_prep_closure_loc_linux64 (ffi_closure *closure,
690 ffi_cif *cif,
691 void (*fun) (ffi_cif *, void *, void **, void *),
692 void *user_data,
693 void *codeloc)
694 {
695 #if _CALL_ELF == 2
696 unsigned int *tramp = (unsigned int *) &closure->tramp[0];
697
698 if (cif->abi < FFI_LINUX || cif->abi >= FFI_LAST_ABI)
699 return FFI_BAD_ABI;
700
701 tramp[0] = 0xe96c0018; /* 0: ld 11,2f-0b(12) */
702 tramp[1] = 0xe98c0010; /* ld 12,1f-0b(12) */
703 tramp[2] = 0x7d8903a6; /* mtctr 12 */
704 tramp[3] = 0x4e800420; /* bctr */
705 /* 1: .quad function_addr */
706 /* 2: .quad context */
707 *(void **) &tramp[4] = (void *) ffi_closure_LINUX64;
708 *(void **) &tramp[6] = codeloc;
709 flush_icache ((char *) tramp, (char *) codeloc, 4 * 4);
710 #else
711 void **tramp = (void **) &closure->tramp[0];
712
713 if (cif->abi < FFI_LINUX || cif->abi >= FFI_LAST_ABI)
714 return FFI_BAD_ABI;
715
716 /* Copy function address and TOC from ffi_closure_LINUX64 OPD. */
717 memcpy (&tramp[0], (void **) ffi_closure_LINUX64, sizeof (void *));
718 tramp[1] = codeloc;
719 memcpy (&tramp[2], (void **) ffi_closure_LINUX64 + 1, sizeof (void *));
720 #endif
721
722 closure->cif = cif;
723 closure->fun = fun;
724 closure->user_data = user_data;
725
726 return FFI_OK;
727 }
728
729
730 int FFI_HIDDEN
ffi_closure_helper_LINUX64(ffi_cif * cif,void (* fun)(ffi_cif *,void *,void **,void *),void * user_data,void * rvalue,unsigned long * pst,ffi_dblfl * pfr)731 ffi_closure_helper_LINUX64 (ffi_cif *cif,
732 void (*fun) (ffi_cif *, void *, void **, void *),
733 void *user_data,
734 void *rvalue,
735 unsigned long *pst,
736 ffi_dblfl *pfr)
737 {
738 /* rvalue is the pointer to space for return value in closure assembly */
739 /* pst is the pointer to parameter save area
740 (r3-r10 are stored into its first 8 slots by ffi_closure_LINUX64) */
741 /* pfr is the pointer to where f1-f13 are stored in ffi_closure_LINUX64 */
742
743 void **avalue;
744 ffi_type **arg_types;
745 unsigned long i, avn, nfixedargs;
746 ffi_dblfl *end_pfr = pfr + NUM_FPR_ARG_REGISTERS64;
747 unsigned long align;
748
749 avalue = alloca (cif->nargs * sizeof (void *));
750
751 /* Copy the caller's structure return value address so that the
752 closure returns the data directly to the caller. */
753 if (cif->rtype->type == FFI_TYPE_STRUCT
754 && (cif->flags & FLAG_RETURNS_SMST) == 0)
755 {
756 rvalue = (void *) *pst;
757 pst++;
758 }
759
760 i = 0;
761 avn = cif->nargs;
762 #if _CALL_ELF != 2
763 nfixedargs = (unsigned) -1;
764 if ((cif->flags & FLAG_COMPAT) == 0)
765 #endif
766 nfixedargs = cif->nfixedargs;
767 arg_types = cif->arg_types;
768
769 /* Grab the addresses of the arguments from the stack frame. */
770 while (i < avn)
771 {
772 unsigned int elt, elnum;
773
774 switch (arg_types[i]->type)
775 {
776 case FFI_TYPE_SINT8:
777 case FFI_TYPE_UINT8:
778 #ifndef __LITTLE_ENDIAN__
779 avalue[i] = (char *) pst + 7;
780 pst++;
781 break;
782 #endif
783
784 case FFI_TYPE_SINT16:
785 case FFI_TYPE_UINT16:
786 #ifndef __LITTLE_ENDIAN__
787 avalue[i] = (char *) pst + 6;
788 pst++;
789 break;
790 #endif
791
792 case FFI_TYPE_SINT32:
793 case FFI_TYPE_UINT32:
794 #ifndef __LITTLE_ENDIAN__
795 avalue[i] = (char *) pst + 4;
796 pst++;
797 break;
798 #endif
799
800 case FFI_TYPE_SINT64:
801 case FFI_TYPE_UINT64:
802 case FFI_TYPE_POINTER:
803 avalue[i] = pst;
804 pst++;
805 break;
806
807 case FFI_TYPE_STRUCT:
808 if ((cif->abi & FFI_LINUX_STRUCT_ALIGN) != 0)
809 {
810 align = arg_types[i]->alignment;
811 if (align > 16)
812 align = 16;
813 if (align > 1)
814 pst = (unsigned long *) ALIGN ((size_t) pst, align);
815 }
816 elt = discover_homogeneous_aggregate (arg_types[i], &elnum);
817 if (elt)
818 {
819 #if _CALL_ELF == 2
820 union {
821 void *v;
822 unsigned long *ul;
823 float *f;
824 double *d;
825 size_t p;
826 } to, from;
827
828 /* Repackage the aggregate from its parts. The
829 aggregate size is not greater than the space taken by
830 the registers so store back to the register/parameter
831 save arrays. */
832 if (pfr + elnum <= end_pfr)
833 to.v = pfr;
834 else
835 to.v = pst;
836
837 avalue[i] = to.v;
838 from.ul = pst;
839 if (elt == FFI_TYPE_FLOAT)
840 {
841 do
842 {
843 if (pfr < end_pfr && i < nfixedargs)
844 {
845 *to.f = (float) pfr->d;
846 pfr++;
847 }
848 else
849 *to.f = *from.f;
850 to.f++;
851 from.f++;
852 }
853 while (--elnum != 0);
854 }
855 else
856 {
857 do
858 {
859 if (pfr < end_pfr && i < nfixedargs)
860 {
861 *to.d = pfr->d;
862 pfr++;
863 }
864 else
865 *to.d = *from.d;
866 to.d++;
867 from.d++;
868 }
869 while (--elnum != 0);
870 }
871 #else
872 if (elt == FFI_TYPE_FLOAT)
873 goto do_float;
874 else
875 goto do_double;
876 #endif
877 }
878 else
879 {
880 #ifndef __LITTLE_ENDIAN__
881 /* Structures with size less than eight bytes are passed
882 left-padded. */
883 if (arg_types[i]->size < 8)
884 avalue[i] = (char *) pst + 8 - arg_types[i]->size;
885 else
886 #endif
887 avalue[i] = pst;
888 }
889 pst += (arg_types[i]->size + 7) / 8;
890 break;
891
892 #if FFI_TYPE_LONGDOUBLE != FFI_TYPE_DOUBLE
893 case FFI_TYPE_LONGDOUBLE:
894 if ((cif->abi & FFI_LINUX_LONG_DOUBLE_128) != 0)
895 {
896 if (pfr + 1 < end_pfr && i + 1 < nfixedargs)
897 {
898 avalue[i] = pfr;
899 pfr += 2;
900 }
901 else
902 {
903 if (pfr < end_pfr && i < nfixedargs)
904 {
905 /* Passed partly in f13 and partly on the stack.
906 Move it all to the stack. */
907 *pst = *(unsigned long *) pfr;
908 pfr++;
909 }
910 avalue[i] = pst;
911 }
912 pst += 2;
913 break;
914 }
915 /* Fall through. */
916 #endif
917 case FFI_TYPE_DOUBLE:
918 do_double:
919 /* On the outgoing stack all values are aligned to 8 */
920 /* there are 13 64bit floating point registers */
921
922 if (pfr < end_pfr && i < nfixedargs)
923 {
924 avalue[i] = pfr;
925 pfr++;
926 }
927 else
928 avalue[i] = pst;
929 pst++;
930 break;
931
932 case FFI_TYPE_FLOAT:
933 do_float:
934 if (pfr < end_pfr && i < nfixedargs)
935 {
936 /* Float values are stored as doubles in the
937 ffi_closure_LINUX64 code. Fix them here. */
938 pfr->f = (float) pfr->d;
939 avalue[i] = pfr;
940 pfr++;
941 }
942 else
943 {
944 #ifndef __LITTLE_ENDIAN__
945 avalue[i] = (char *) pst + 4;
946 #else
947 avalue[i] = pst;
948 #endif
949 }
950 pst++;
951 break;
952
953 default:
954 FFI_ASSERT (0);
955 }
956
957 i++;
958 }
959
960 (*fun) (cif, rvalue, avalue, user_data);
961
962 /* Tell ffi_closure_LINUX64 how to perform return type promotions. */
963 if ((cif->flags & FLAG_RETURNS_SMST) != 0)
964 {
965 if ((cif->flags & FLAG_RETURNS_FP) == 0)
966 return FFI_V2_TYPE_SMALL_STRUCT + cif->rtype->size - 1;
967 else if ((cif->flags & FLAG_RETURNS_64BITS) != 0)
968 return FFI_V2_TYPE_DOUBLE_HOMOG;
969 else
970 return FFI_V2_TYPE_FLOAT_HOMOG;
971 }
972 return cif->rtype->type;
973 }
974 #endif
975