1 /**
2 * Copyright (C) Mellanox Technologies Ltd. 2001-2014. ALL RIGHTS RESERVED.
3 * Copyright (C) UT-Battelle, LLC. 2015. ALL RIGHTS RESERVED.
4 * Copyright (C) ARM Ltd. 2016-2017. ALL RIGHTS RESERVED.
5 *
6 * See file LICENSE for terms.
7 */
8
9 #ifdef HAVE_CONFIG_H
10 # include "config.h"
11 #endif
12
13 #include "uct_md.h"
14 #include "uct_iface.h"
15
16 #include <uct/api/uct.h>
17 #include <ucs/debug/log.h>
18 #include <ucs/debug/memtrack.h>
19 #include <ucs/type/class.h>
20 #include <ucs/sys/module.h>
21 #include <ucs/sys/string.h>
22 #include <ucs/arch/cpu.h>
23
24
25 ucs_config_field_t uct_md_config_table[] = {
26
27 {NULL}
28 };
29
30 ucs_config_field_t uct_md_config_rcache_table[] = {
31 {"RCACHE_MEM_PRIO", "1000", "Registration cache memory event priority",
32 ucs_offsetof(uct_md_rcache_config_t, event_prio), UCS_CONFIG_TYPE_UINT},
33
34 {"RCACHE_OVERHEAD", "180ns", "Registration cache lookup overhead",
35 ucs_offsetof(uct_md_rcache_config_t, overhead), UCS_CONFIG_TYPE_TIME},
36
37 {"RCACHE_ADDR_ALIGN", UCS_PP_MAKE_STRING(UCS_SYS_CACHE_LINE_SIZE),
38 "Registration cache address alignment, must be power of 2\n"
39 "between "UCS_PP_MAKE_STRING(UCS_PGT_ADDR_ALIGN)"and system page size",
40 ucs_offsetof(uct_md_rcache_config_t, alignment), UCS_CONFIG_TYPE_UINT},
41
42 {NULL}
43 };
44
45
uct_md_open(uct_component_h component,const char * md_name,const uct_md_config_t * config,uct_md_h * md_p)46 ucs_status_t uct_md_open(uct_component_h component, const char *md_name,
47 const uct_md_config_t *config, uct_md_h *md_p)
48 {
49 ucs_status_t status;
50 uct_md_h md;
51
52 status = component->md_open(component, md_name, config, &md);
53 if (status != UCS_OK) {
54 return status;
55 }
56
57 *md_p = md;
58 ucs_assert_always(md->component == component);
59 return UCS_OK;
60 }
61
uct_md_close(uct_md_h md)62 void uct_md_close(uct_md_h md)
63 {
64 md->ops->close(md);
65 }
66
uct_md_query_tl_resources(uct_md_h md,uct_tl_resource_desc_t ** resources_p,unsigned * num_resources_p)67 ucs_status_t uct_md_query_tl_resources(uct_md_h md,
68 uct_tl_resource_desc_t **resources_p,
69 unsigned *num_resources_p)
70 {
71 uct_component_t *component = md->component;
72 uct_tl_resource_desc_t *resources, *tmp;
73 uct_tl_device_resource_t *tl_devices;
74 unsigned i, num_resources, num_tl_devices;
75 ucs_status_t status;
76 uct_tl_t *tl;
77
78 resources = NULL;
79 num_resources = 0;
80
81 ucs_list_for_each(tl, &component->tl_list, list) {
82 status = tl->query_devices(md, &tl_devices, &num_tl_devices);
83 if (status != UCS_OK) {
84 ucs_debug("failed to query %s resources: %s", tl->name,
85 ucs_status_string(status));
86 continue;
87 }
88
89 if (num_tl_devices == 0) {
90 ucs_free(tl_devices);
91 continue;
92 }
93
94 tmp = ucs_realloc(resources,
95 sizeof(*resources) * (num_resources + num_tl_devices),
96 "md_resources");
97 if (tmp == NULL) {
98 ucs_free(tl_devices);
99 status = UCS_ERR_NO_MEMORY;
100 goto err;
101 }
102
103 /* add tl devices to overall list of resources */
104 for (i = 0; i < num_tl_devices; ++i) {
105 ucs_strncpy_zero(tmp[num_resources + i].tl_name, tl->name,
106 sizeof(tmp[num_resources + i].tl_name));
107 ucs_strncpy_zero(tmp[num_resources + i].dev_name, tl_devices[i].name,
108 sizeof(tmp[num_resources + i].dev_name));
109 tmp[num_resources + i].dev_type = tl_devices[i].type;
110 tmp[num_resources + i].sys_device = tl_devices[i].sys_device;
111 }
112
113 resources = tmp;
114 num_resources += num_tl_devices;
115 ucs_free(tl_devices);
116 }
117
118 *resources_p = resources;
119 *num_resources_p = num_resources;
120 return UCS_OK;
121
122 err:
123 ucs_free(resources);
124 return status;
125 }
126
uct_release_tl_resource_list(uct_tl_resource_desc_t * resources)127 void uct_release_tl_resource_list(uct_tl_resource_desc_t *resources)
128 {
129 ucs_free(resources);
130 }
131
132 ucs_status_t
uct_md_query_single_md_resource(uct_component_t * component,uct_md_resource_desc_t ** resources_p,unsigned * num_resources_p)133 uct_md_query_single_md_resource(uct_component_t *component,
134 uct_md_resource_desc_t **resources_p,
135 unsigned *num_resources_p)
136 {
137 uct_md_resource_desc_t *resource;
138
139 resource = ucs_malloc(sizeof(*resource), "md resource");
140 if (resource == NULL) {
141 return UCS_ERR_NO_MEMORY;
142 }
143
144 ucs_snprintf_zero(resource->md_name, UCT_MD_NAME_MAX, "%s",
145 component->name);
146
147 *resources_p = resource;
148 *num_resources_p = 1;
149 return UCS_OK;
150 }
151
152 ucs_status_t
uct_md_query_empty_md_resource(uct_md_resource_desc_t ** resources_p,unsigned * num_resources_p)153 uct_md_query_empty_md_resource(uct_md_resource_desc_t **resources_p,
154 unsigned *num_resources_p)
155 {
156 *resources_p = NULL;
157 *num_resources_p = 0;
158 return UCS_OK;
159 }
160
uct_md_stub_rkey_unpack(uct_component_t * component,const void * rkey_buffer,uct_rkey_t * rkey_p,void ** handle_p)161 ucs_status_t uct_md_stub_rkey_unpack(uct_component_t *component,
162 const void *rkey_buffer, uct_rkey_t *rkey_p,
163 void **handle_p)
164 {
165 *rkey_p = 0xdeadbeef;
166 *handle_p = NULL;
167 return UCS_OK;
168 }
169
uct_find_tl(uct_component_h component,uint64_t md_flags,const char * tl_name)170 static uct_tl_t *uct_find_tl(uct_component_h component, uint64_t md_flags,
171 const char *tl_name)
172 {
173 uct_tl_t *tl;
174
175 ucs_list_for_each(tl, &component->tl_list, list) {
176 if (((tl_name != NULL) && !strcmp(tl_name, tl->name)) ||
177 ((tl_name == NULL) && (md_flags & UCT_MD_FLAG_SOCKADDR))) {
178 return tl;
179 }
180 }
181 return NULL;
182 }
183
uct_md_iface_config_read(uct_md_h md,const char * tl_name,const char * env_prefix,const char * filename,uct_iface_config_t ** config_p)184 ucs_status_t uct_md_iface_config_read(uct_md_h md, const char *tl_name,
185 const char *env_prefix, const char *filename,
186 uct_iface_config_t **config_p)
187 {
188 uct_config_bundle_t *bundle = NULL;
189 uct_md_attr_t md_attr;
190 ucs_status_t status;
191 uct_tl_t *tl;
192
193 status = uct_md_query(md, &md_attr);
194 if (status != UCS_OK) {
195 ucs_error("Failed to query MD");
196 return status;
197 }
198
199 tl = uct_find_tl(md->component, md_attr.cap.flags, tl_name);
200 if (tl == NULL) {
201 if (tl_name == NULL) {
202 ucs_error("There is no sockaddr transport registered on the md");
203 } else {
204 ucs_error("Transport '%s' does not exist", tl_name);
205 }
206 status = UCS_ERR_NO_DEVICE; /* Non-existing transport */
207 return status;
208 }
209
210 status = uct_config_read(&bundle, tl->config.table, tl->config.size,
211 env_prefix, tl->config.prefix);
212 if (status != UCS_OK) {
213 ucs_error("Failed to read iface config");
214 return status;
215 }
216
217 *config_p = (uct_iface_config_t*) bundle->data;
218 /* coverity[leaked_storage] */
219 return UCS_OK;
220 }
221
uct_iface_open(uct_md_h md,uct_worker_h worker,const uct_iface_params_t * params,const uct_iface_config_t * config,uct_iface_h * iface_p)222 ucs_status_t uct_iface_open(uct_md_h md, uct_worker_h worker,
223 const uct_iface_params_t *params,
224 const uct_iface_config_t *config,
225 uct_iface_h *iface_p)
226 {
227 uct_md_attr_t md_attr;
228 ucs_status_t status;
229 uct_tl_t *tl;
230
231 status = uct_md_query(md, &md_attr);
232 if (status != UCS_OK) {
233 ucs_error("Failed to query MD");
234 return status;
235 }
236
237 UCT_CHECK_PARAM(params->field_mask & UCT_IFACE_PARAM_FIELD_OPEN_MODE,
238 "UCT_IFACE_PARAM_FIELD_OPEN_MODE is not defined");
239
240 if (params->open_mode & UCT_IFACE_OPEN_MODE_DEVICE) {
241 tl = uct_find_tl(md->component, md_attr.cap.flags,
242 params->mode.device.tl_name);
243 } else if ((params->open_mode & UCT_IFACE_OPEN_MODE_SOCKADDR_CLIENT) ||
244 (params->open_mode & UCT_IFACE_OPEN_MODE_SOCKADDR_SERVER)) {
245 tl = uct_find_tl(md->component, md_attr.cap.flags, NULL);
246 } else {
247 ucs_error("Invalid open mode %zu", params->open_mode);
248 return status;
249 }
250
251 if (tl == NULL) {
252 /* Non-existing transport */
253 return UCS_ERR_NO_DEVICE;
254 }
255
256 return tl->iface_open(md, worker, params, config, iface_p);
257 }
258
uct_md_config_read(uct_component_h component,const char * env_prefix,const char * filename,uct_md_config_t ** config_p)259 ucs_status_t uct_md_config_read(uct_component_h component,
260 const char *env_prefix, const char *filename,
261 uct_md_config_t **config_p)
262 {
263 uct_config_bundle_t *bundle = NULL;
264 ucs_status_t status;
265
266 status = uct_config_read(&bundle, component->md_config.table,
267 component->md_config.size, env_prefix,
268 component->md_config.prefix);
269 if (status != UCS_OK) {
270 ucs_error("Failed to read MD config");
271 return status;
272 }
273
274 *config_p = (uct_md_config_t*) bundle->data;
275 /* coverity[leaked_storage] */
276 return UCS_OK;
277 }
278
uct_config_release(void * config)279 void uct_config_release(void *config)
280 {
281 uct_config_bundle_t *bundle = (uct_config_bundle_t *)config - 1;
282
283 ucs_config_parser_release_opts(config, bundle->table);
284 ucs_free((void*)(bundle->table_prefix));
285 ucs_free(bundle);
286 }
287
uct_config_get(void * config,const char * name,char * value,size_t max)288 ucs_status_t uct_config_get(void *config, const char *name, char *value,
289 size_t max)
290 {
291 uct_config_bundle_t *bundle = (uct_config_bundle_t *)config - 1;
292 return ucs_config_parser_get_value(bundle->data, bundle->table, name, value,
293 max);
294 }
295
uct_config_modify(void * config,const char * name,const char * value)296 ucs_status_t uct_config_modify(void *config, const char *name, const char *value)
297 {
298 uct_config_bundle_t *bundle = (uct_config_bundle_t *)config - 1;
299 return ucs_config_parser_set_value(bundle->data, bundle->table, name, value);
300 }
301
uct_md_mkey_pack(uct_md_h md,uct_mem_h memh,void * rkey_buffer)302 ucs_status_t uct_md_mkey_pack(uct_md_h md, uct_mem_h memh, void *rkey_buffer)
303 {
304 void *rbuf = uct_md_fill_md_name(md, rkey_buffer);
305 return md->ops->mkey_pack(md, memh, rbuf);
306 }
307
uct_rkey_unpack(uct_component_h component,const void * rkey_buffer,uct_rkey_bundle_t * rkey_ob)308 ucs_status_t uct_rkey_unpack(uct_component_h component, const void *rkey_buffer,
309 uct_rkey_bundle_t *rkey_ob)
310 {
311 char component_name[UCT_COMPONENT_NAME_MAX + 1];
312
313 if (ENABLE_DEBUG_DATA) {
314 if (ENABLE_PARAMS_CHECK &&
315 strncmp(rkey_buffer, component->name, UCT_COMPONENT_NAME_MAX)) {
316 ucs_snprintf_zero(component_name, sizeof(component_name), "%s",
317 (const char*)rkey_buffer);
318 ucs_error("invalid component for rkey unpack; "
319 "expected: %s, actual: %s", component_name, component->name);
320 return UCS_ERR_INVALID_PARAM;
321 }
322
323 rkey_buffer = UCS_PTR_BYTE_OFFSET(rkey_buffer, UCT_COMPONENT_NAME_MAX);
324 }
325
326 return component->rkey_unpack(component, rkey_buffer, &rkey_ob->rkey,
327 &rkey_ob->handle);
328 }
329
uct_rkey_ptr(uct_component_h component,uct_rkey_bundle_t * rkey_ob,uint64_t remote_addr,void ** local_addr_p)330 ucs_status_t uct_rkey_ptr(uct_component_h component, uct_rkey_bundle_t *rkey_ob,
331 uint64_t remote_addr, void **local_addr_p)
332 {
333 return component->rkey_ptr(component, rkey_ob->rkey, rkey_ob->handle,
334 remote_addr, local_addr_p);
335 }
336
uct_rkey_release(uct_component_h component,const uct_rkey_bundle_t * rkey_ob)337 ucs_status_t uct_rkey_release(uct_component_h component,
338 const uct_rkey_bundle_t *rkey_ob)
339 {
340 return component->rkey_release(component, rkey_ob->rkey, rkey_ob->handle);
341 }
342
uct_md_query(uct_md_h md,uct_md_attr_t * md_attr)343 ucs_status_t uct_md_query(uct_md_h md, uct_md_attr_t *md_attr)
344 {
345 ucs_status_t status;
346
347 status = md->ops->query(md, md_attr);
348 if (status != UCS_OK) {
349 return status;
350 }
351
352 /* Component name + data */
353 memcpy(md_attr->component_name, md->component->name, UCT_COMPONENT_NAME_MAX);
354
355 #if ENABLE_DEBUG_DATA
356 /* MD name is packed into rkey in DEBUG mode only */
357 md_attr->rkey_packed_size += UCT_COMPONENT_NAME_MAX;
358 #endif
359
360 return UCS_OK;
361 }
362
uct_mem_check_flags(unsigned flags)363 static ucs_status_t uct_mem_check_flags(unsigned flags)
364 {
365 if (!(flags & UCT_MD_MEM_ACCESS_ALL)) {
366 return UCS_ERR_INVALID_PARAM;
367 }
368 return UCS_OK;
369 }
370
uct_md_mem_alloc(uct_md_h md,size_t * length_p,void ** address_p,unsigned flags,const char * alloc_name,uct_mem_h * memh_p)371 ucs_status_t uct_md_mem_alloc(uct_md_h md, size_t *length_p, void **address_p,
372 unsigned flags, const char *alloc_name, uct_mem_h *memh_p)
373 {
374 ucs_status_t status;
375
376 status = uct_mem_check_flags(flags);
377 if (status != UCS_OK) {
378 return status;
379 }
380
381 return md->ops->mem_alloc(md, length_p, address_p, flags, alloc_name, memh_p);
382 }
383
uct_md_mem_free(uct_md_h md,uct_mem_h memh)384 ucs_status_t uct_md_mem_free(uct_md_h md, uct_mem_h memh)
385 {
386 return md->ops->mem_free(md, memh);
387 }
388
389 ucs_status_t
uct_md_mem_advise(uct_md_h md,uct_mem_h memh,void * addr,size_t length,unsigned advice)390 uct_md_mem_advise(uct_md_h md, uct_mem_h memh, void *addr, size_t length,
391 unsigned advice)
392 {
393 if ((length == 0) || (addr == NULL)) {
394 return UCS_ERR_INVALID_PARAM;
395 }
396
397 return md->ops->mem_advise(md, memh, addr, length, advice);
398 }
399
uct_md_mem_reg(uct_md_h md,void * address,size_t length,unsigned flags,uct_mem_h * memh_p)400 ucs_status_t uct_md_mem_reg(uct_md_h md, void *address, size_t length,
401 unsigned flags, uct_mem_h *memh_p)
402 {
403 ucs_status_t status;
404
405 if ((length == 0) || (address == NULL)) {
406 return UCS_ERR_INVALID_PARAM;
407 }
408
409 status = uct_mem_check_flags(flags);
410 if (status != UCS_OK) {
411 return status;
412 }
413
414 return md->ops->mem_reg(md, address, length, flags, memh_p);
415 }
416
uct_md_mem_dereg(uct_md_h md,uct_mem_h memh)417 ucs_status_t uct_md_mem_dereg(uct_md_h md, uct_mem_h memh)
418 {
419 return md->ops->mem_dereg(md, memh);
420 }
421
uct_md_mem_query(uct_md_h md,const void * addr,const size_t length,uct_md_mem_attr_t * mem_attr_p)422 ucs_status_t uct_md_mem_query(uct_md_h md, const void *addr, const size_t length,
423 uct_md_mem_attr_t *mem_attr_p)
424 {
425 return md->ops->mem_query(md, addr, length, mem_attr_p);
426 }
427
uct_md_is_sockaddr_accessible(uct_md_h md,const ucs_sock_addr_t * sockaddr,uct_sockaddr_accessibility_t mode)428 int uct_md_is_sockaddr_accessible(uct_md_h md, const ucs_sock_addr_t *sockaddr,
429 uct_sockaddr_accessibility_t mode)
430 {
431 return md->ops->is_sockaddr_accessible(md, sockaddr, mode);
432 }
433
uct_md_detect_memory_type(uct_md_h md,const void * addr,size_t length,ucs_memory_type_t * mem_type_p)434 ucs_status_t uct_md_detect_memory_type(uct_md_h md, const void *addr, size_t length,
435 ucs_memory_type_t *mem_type_p)
436 {
437 return md->ops->detect_memory_type(md, addr, length, mem_type_p);
438 }
439