1 #include "cado.h" // IWYU pragma: keep
2 // IWYU pragma: no_include <mm_malloc.h>
3 #include <stdio.h>
4 #include <stdlib.h>
5 #include <string.h>
6 #include <stdint.h>
7 #ifdef HAVE_SYS_MMAN_H
8 #include <sys/mman.h>
9 #endif
10 /* For MinGW Build */
11 #if defined(_WIN32) || defined(_WIN64)
12 #include <windows.h>
13 #endif
14 #include "macros.h"
15 #include "memory.h"
16 #include "portability.h"
17
18 #ifndef LARGE_PAGE_SIZE
19 #define LARGE_PAGE_SIZE (2UL*1024*1024)
20 #endif
21
22 void
malloc_check(const size_t x)23 *malloc_check (const size_t x)
24 {
25 void *p;
26 p = malloc (x);
27 if (p == NULL)
28 {
29 fprintf (stderr, "Error, malloc of %zu bytes failed\n", x);
30 fflush (stderr);
31 abort ();
32 }
33 return p;
34 }
35
36 #ifndef MAP_ANONYMOUS
37 #define MAP_ANONYMOUS MAP_ANON
38 #endif
39
40 /* Not everybody has posix_memalign. In order to provide a viable
41 * alternative, we need an ``aligned free'' matching the ``aligned
42 * malloc''. We rely on posix_memalign if it is available, or else fall
43 * back on ugly pointer arithmetic so as to guarantee alignment. Note
44 * that not providing the requested alignment can have some troublesome
45 * consequences. At best, a performance hit, at worst a segv (sse-2
46 * movdqa on a pentium4 causes a GPE if improperly aligned).
47 */
48
malloc_aligned(size_t size,size_t alignment)49 void *malloc_aligned(size_t size, size_t alignment)
50 {
51 #ifdef HAVE_POSIX_MEMALIGN
52 void *res = NULL;
53 int rc = posix_memalign(&res, alignment, size);
54 // ASSERT_ALWAYS(rc == 0);
55 DIE_ERRNO_DIAG(rc != 0, "malloc_aligned(%s)", "");
56 return res;
57 #else
58 char * res;
59 res = malloc(size + sizeof(size_t) + alignment);
60 res += sizeof(size_t);
61 size_t displ = alignment - ((uintptr_t) res) % alignment;
62 res += displ;
63 memcpy(res - sizeof(size_t), &displ, sizeof(size_t));
64 ASSERT_ALWAYS((((uintptr_t) res) % alignment) == 0);
65 return (void*) res;
66 #endif
67 }
68
69 /* Reallocate aligned memory.
70 p must have been allocated via malloc_aligned() or realloc_aligned().
71 old_size must be equal to the size parameter of malloc_aligned(), or
72 to the new_size parameter of realloc_aligned(), of the function that
73 allocated p. */
74
75 void *
realloc_aligned(void * p,const size_t old_size,const size_t new_size,const size_t alignment)76 realloc_aligned(void * p, const size_t old_size, const size_t new_size,
77 const size_t alignment)
78 {
79 #ifdef HAVE_POSIX_MEMALIGN
80 /* Alas, there is no posix_realloc_aligned(). Try to realloc(); if it
81 happens to result in the desired alignment, there is nothing left
82 to do. If it does not result in the desired alignment, then we
83 actually do two data copies: one as part of realloc(), and another
84 below. Let's hope this happens kinda rarely. */
85 p = realloc(p, new_size);
86 if (((uintptr_t) p) % alignment == 0) {
87 return p;
88 }
89 #else
90 /* Without posix_memalign(), we always alloc/copy/free */
91 #endif
92 /* We did not get the desired alignment, or we don't have posix_memalign().
93 Allocate new memory with the desired alignment and copy the data */
94 void * const alloc_p = malloc_aligned(new_size, alignment);
95 memcpy(alloc_p, p, MIN(old_size, new_size));
96 /* If we have posix_memalign(), then p was allocated by realloc() and can be
97 freed with free(), which is what free_aligned() does. If we don't have
98 posix_memalign(), then p was allocated by malloc_aligned() or
99 realloc_aligned(), so using free_aligned() is correct again. */
100 free_aligned(p);
101 return alloc_p;
102 }
103
104
free_aligned(void * p)105 void free_aligned(void * p)
106 {
107 #ifdef HAVE_POSIX_MEMALIGN
108 free((void *) p);
109 #else
110 if (p == NULL)
111 return;
112 const char * res = (const char *) p;
113 size_t displ;
114 memcpy(&displ, res - sizeof(size_t), sizeof(size_t));
115 res -= displ;
116 res -= sizeof(size_t);
117 free((void *)res);
118 #endif
119 }
120
121 /* if aligned_alloc is not provided, define it */
122 #ifndef HAVE_ALIGNED_ALLOC
123 void*
aligned_alloc(size_t alignment,size_t size)124 aligned_alloc (size_t alignment, size_t size)
125 {
126 return malloc_aligned (size, alignment);
127 }
128 #endif
129
malloc_pagealigned(size_t sz)130 void *malloc_pagealigned(size_t sz)
131 {
132 long ps = pagesize();
133 ASSERT_ALWAYS(ps > 0);
134 void *p = malloc_aligned (sz, ps);
135 ASSERT_ALWAYS(p != NULL);
136 return p;
137 }
138
free_pagealigned(void * p)139 void free_pagealigned(void * p)
140 {
141 free_aligned(p);
142 }
143