1 /*
2 Copyright (c) 2010, 2021, Oracle and/or its affiliates.
3
4 This program is free software; you can redistribute it and/or modify
5 it under the terms of the GNU General Public License, version 2.0,
6 as published by the Free Software Foundation.
7
8 This program is also distributed with certain software (including
9 but not limited to OpenSSL) that is licensed under separate terms,
10 as designated in a particular file or component or in included license
11 documentation. The authors of MySQL hereby grant you an additional
12 permission to link the program and your derivative works with the
13 separately licensed software that they have included with MySQL.
14
15 This program is distributed in the hope that it will be useful,
16 but WITHOUT ANY WARRANTY; without even the implied warranty of
17 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
18 GNU General Public License, version 2.0, for more details.
19
20 You should have received a copy of the GNU General Public License
21 along with this program; if not, write to the Free Software
22 Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA
23 */
24
25 #include <NdbNuma.h>
26 #include <ndb_global.h>
27
28 static int NDB_TRACE_NUMA = 0;
29
30 #if defined HAVE_DLFCN_H && defined HAVE_DLOPEN
31 #include <dlfcn.h>
32
33 /**
34 * Load libnuma using dlopen, not have to put link dependency on it...
35 * - handle fact that there are 2 versions of libnuma...
36 * use existance of symbol "numa_all_nodes_ptr" to use v2 abi
37 */
38 struct bitmask;
39 extern "C"
40 {
41 typedef int (* fun0)(void);
42 typedef void (* fun1)(struct bitmask*);
43 typedef void (* fun2)(int);
44 typedef int (* fun3)(int node, unsigned long * bug, int buflen);
45 typedef bitmask * (* fun4)();
46 typedef void (* fun5)(struct bitmask*);
47 typedef bitmask * (* fun6)(struct bitmask*);
48 };
49
50 class NdbNuma
51 {
52 public:
NdbNuma()53 NdbNuma() { handle = 0;}
~NdbNuma()54 ~NdbNuma() { if (handle) dlclose(handle); }
55
56 int open();
57 int build_cputonodemap();
58
59 void * handle;
60 fun0 numa_available;
61
62 fun0 numa_max_node;
63 fun0 numa_max_possible_node;
64 fun1 numa_set_interleave_mask;
65 fun2 numa_set_strict;
66 fun3 numa_node_to_cpus;
67 fun4 numa_allocate_nodemask;
68 fun5 numa_bitmask_free;
69 fun6 numa_bitmask_setall;
70
71 struct bitmask * numa_all_nodes;
72 struct bitmask * numa_all_nodes_ptr;
73 };
74
75 static
76 void*
my_dlopen(const char * name)77 my_dlopen(const char * name)
78 {
79 void * p = dlopen(name, RTLD_LAZY);
80 if (NDB_TRACE_NUMA)
81 {
82 if (p == 0)
83 printf("info: failed to load %s\n", name);
84 else
85 printf("info: loaded %s\n", name);
86 }
87 return p;
88 }
89
90 static
91 void*
my_dlsym(void * handle,const char * name)92 my_dlsym(void * handle, const char * name)
93 {
94 void * p = dlsym(handle, name);
95 if (NDB_TRACE_NUMA)
96 {
97 if (p != 0)
98 {
99 printf("info: %s OK\n", name);
100 }
101 else
102 {
103 printf("info: %s NOT FOUND\n", name);
104 }
105 }
106 return p;
107 }
108
109 int
open()110 NdbNuma::open()
111 {
112 handle = my_dlopen("libnuma.so");
113 if (handle == 0)
114 {
115 handle = my_dlopen("libnuma.so.1");
116 }
117 if (handle == 0)
118 {
119 return -1;
120 }
121
122 numa_available = (fun0)my_dlsym(handle, "numa_available");
123 if (numa_available == 0)
124 {
125 goto fail;
126 }
127
128 if ((* numa_available)() == -1)
129 {
130 if (NDB_TRACE_NUMA)
131 {
132 printf("info: numa_available() returns -1 => no numa support\n");
133 }
134 goto fail;
135 }
136
137 numa_max_node = (fun0)my_dlsym(handle, "numa_max_node");
138 numa_set_interleave_mask = (fun1)my_dlsym(handle, "numa_set_interleave_mask");
139 numa_set_strict = (fun2)my_dlsym(handle, "numa_set_strict");
140 numa_node_to_cpus = (fun3)my_dlsym(handle, "numa_node_to_cpus");
141 numa_all_nodes = (struct bitmask*)my_dlsym(handle, "numa_all_nodes");
142 numa_all_nodes_ptr = (struct bitmask*)my_dlsym(handle, "numa_all_nodes_ptr");
143 numa_allocate_nodemask = (fun4)my_dlsym(handle, "numa_allocate_nodemask");
144 numa_bitmask_free = (fun5)my_dlsym(handle, "numa_bitmask_free");
145 numa_bitmask_setall = (fun6)my_dlsym(handle, "numa_bitmask_setall");
146
147
148 return 0;
149 fail:
150 dlclose(handle);
151 handle = 0;
152 return -1;
153 }
154
155 static
156 bool
bit_is_set(unsigned long * mask,int bit)157 bit_is_set(unsigned long * mask, int bit)
158 {
159 int n = bit / (8 * sizeof(unsigned long));
160 int b = bit % (8 * sizeof(unsigned long));
161 return (mask[n] & (1UL << b)) != 0;
162 }
163
164 int
build_cputonodemap()165 NdbNuma::build_cputonodemap()
166 {
167 int len = 512;
168 unsigned long * buf = (unsigned long*)malloc(len);
169 if (buf == 0)
170 return -1;
171
172 int m = (* numa_max_node)();
173 for (int i = 0; i <= m; i++)
174 {
175 retry:
176 int r = (* numa_node_to_cpus)(i, buf, len);
177 if (r == -1)
178 {
179 if (errno != ERANGE)
180 goto fail;
181
182 len = len + 512;
183 if (len > 4096)
184 goto fail;
185
186 void * p = realloc(buf, len);
187 if (p == 0)
188 goto fail;
189
190 buf = (unsigned long*)p;
191 goto retry;
192 }
193 printf("node %d cpu(s): ", i);
194 for (int j = 0; j<8*len;j++)
195 if (bit_is_set(buf, j))
196 printf("%d ", j);
197 printf("\n");
198 }
199 free(buf);
200 return 0;
201 fail:
202 free(buf);
203 return -1;
204 }
205
206 extern "C"
207 int
NdbNuma_setInterleaved()208 NdbNuma_setInterleaved()
209 {
210 NdbNuma numa;
211 if (numa.open() == -1)
212 return -1;
213
214 if (numa.numa_set_interleave_mask == 0)
215 return -1;
216
217 if (numa.numa_all_nodes_ptr != 0)
218 {
219 /**
220 * libnuma v2
221 */
222 if (numa.numa_allocate_nodemask != 0 &&
223 numa.numa_bitmask_setall != 0 &&
224 numa.numa_bitmask_free != 0)
225 {
226 struct bitmask * bm = (* numa.numa_allocate_nodemask)();
227 if (bm != 0)
228 {
229 (* numa.numa_bitmask_setall)(bm);
230 (* numa.numa_set_interleave_mask)(bm);
231 (* numa.numa_bitmask_free)(bm);
232 }
233 else
234 {
235 return -1;
236 }
237 }
238 else
239 {
240 return -1;
241 }
242 }
243 else if (numa.numa_all_nodes != 0)
244 {
245 /**
246 * libnuma v1
247 */
248 (* numa.numa_set_interleave_mask)(numa.numa_all_nodes);
249 }
250 else
251 {
252 return -1;
253 }
254
255 return 0;
256 }
257
258 #else
259 extern "C"
260 int
NdbNuma_setInterleaved()261 NdbNuma_setInterleaved()
262 {
263 return -1;
264 }
265
266 extern "C"
267 int
NdbNuma_setInterleavedOnCpus(unsigned cpu[],unsigned len)268 NdbNuma_setInterleavedOnCpus(unsigned cpu[], unsigned len)
269 {
270 return -1;
271 }
272 #endif
273
274 #ifdef TEST_NDBNUMA
275 #include <NdbTap.hpp>
276
TAPTEST(SetInterleaved)277 TAPTEST(SetInterleaved)
278 {
279 NDB_TRACE_NUMA = 1;
280 NdbNuma_setInterleaved();
281 return 1; // OK
282 }
283 #endif
284