1 /**
2 * Copyright (c) UT-Battelle, LLC. 2014-2017. ALL RIGHTS RESERVED.
3 * Copyright (C) Mellanox Technologies Ltd. 2001-2019. ALL RIGHTS RESERVED.
4 * See file LICENSE for terms.
5 */
6
7 #ifdef HAVE_CONFIG_H
8 # include "config.h"
9 #endif
10
11 #include "ugni_device.h"
12 #include "ugni_iface.h"
13 #include "ugni_md.h"
14
15 /* Forward declarations */
16
17 UCS_CONFIG_DEFINE_ARRAY(ugni_alloc_methods, sizeof(uct_alloc_method_t),
18 UCS_CONFIG_TYPE_ENUM(uct_alloc_method_names));
19
20 pthread_mutex_t uct_ugni_global_lock = PTHREAD_MUTEX_INITIALIZER;
21
22 /* For Cray devices we have only one MD */
23 static ucs_status_t
uct_ugni_query_md_resources(uct_component_h component,uct_md_resource_desc_t ** resources_p,unsigned * num_resources_p)24 uct_ugni_query_md_resources(uct_component_h component,
25 uct_md_resource_desc_t **resources_p,
26 unsigned *num_resources_p)
27 {
28 if (getenv("PMI_GNI_PTAG") == NULL) {
29 return uct_md_query_empty_md_resource(resources_p, num_resources_p);
30 }
31
32 return uct_md_query_single_md_resource(component, resources_p,
33 num_resources_p);
34 }
35
uct_ugni_md_query(uct_md_h md,uct_md_attr_t * md_attr)36 static ucs_status_t uct_ugni_md_query(uct_md_h md, uct_md_attr_t *md_attr)
37 {
38 md_attr->rkey_packed_size = 3 * sizeof(uint64_t);
39 md_attr->cap.flags = UCT_MD_FLAG_REG |
40 UCT_MD_FLAG_NEED_MEMH |
41 UCT_MD_FLAG_NEED_RKEY;
42 md_attr->cap.reg_mem_types = UCS_MEMORY_TYPES_CPU_ACCESSIBLE;
43 md_attr->cap.access_mem_type = UCS_MEMORY_TYPE_HOST;
44 md_attr->cap.detect_mem_types = 0;
45 md_attr->cap.max_alloc = 0;
46 md_attr->cap.max_reg = ULONG_MAX;
47 md_attr->reg_cost = ucs_linear_func_make(1000.0e-9, 0.007e-9);
48 memset(&md_attr->local_cpus, 0xff, sizeof(md_attr->local_cpus));
49 return UCS_OK;
50 }
51
uct_ugni_mem_reg(uct_md_h md,void * address,size_t length,unsigned flags,uct_mem_h * memh_p)52 static ucs_status_t uct_ugni_mem_reg(uct_md_h md, void *address, size_t length,
53 unsigned flags, uct_mem_h *memh_p)
54 {
55 ucs_status_t status;
56 gni_return_t ugni_rc;
57 uct_ugni_md_t *ugni_md = ucs_derived_of(md, uct_ugni_md_t);
58 gni_mem_handle_t * mem_hndl = NULL;
59
60 if (0 == length) {
61 ucs_error("Unexpected length %zu", length);
62 return UCS_ERR_INVALID_PARAM;
63 }
64
65 mem_hndl = ucs_malloc(sizeof(gni_mem_handle_t), "gni_mem_handle_t");
66 if (NULL == mem_hndl) {
67 ucs_error("Failed to allocate memory for gni_mem_handle_t");
68 status = UCS_ERR_NO_MEMORY;
69 goto mem_err;
70 }
71
72 uct_ugni_cdm_lock(&ugni_md->cdm);
73 ugni_rc = GNI_MemRegister(ugni_md->cdm.nic_handle, (uint64_t)address,
74 length, NULL,
75 GNI_MEM_READWRITE | GNI_MEM_RELAXED_PI_ORDERING,
76 -1, mem_hndl);
77 uct_ugni_cdm_unlock(&ugni_md->cdm);
78 if (GNI_RC_SUCCESS != ugni_rc) {
79 ucs_error("GNI_MemRegister failed (addr %p, size %zu), Error status: %s %d",
80 address, length, gni_err_str[ugni_rc], ugni_rc);
81 status = UCS_ERR_IO_ERROR;
82 goto mem_err;
83 }
84
85 ucs_debug("Memory registration address %p, len %lu, keys [%"PRIx64" %"PRIx64"]",
86 address, length, mem_hndl->qword1, mem_hndl->qword2);
87 *memh_p = mem_hndl;
88 return UCS_OK;
89
90 mem_err:
91 free(mem_hndl);
92 return status;
93 }
94
uct_ugni_mem_dereg(uct_md_h md,uct_mem_h memh)95 static ucs_status_t uct_ugni_mem_dereg(uct_md_h md, uct_mem_h memh)
96 {
97 uct_ugni_md_t *ugni_md = ucs_derived_of(md, uct_ugni_md_t);
98 gni_mem_handle_t *mem_hndl = (gni_mem_handle_t *) memh;
99 gni_return_t ugni_rc;
100 ucs_status_t status = UCS_OK;
101
102 uct_ugni_cdm_lock(&ugni_md->cdm);
103 ugni_rc = GNI_MemDeregister(ugni_md->cdm.nic_handle, mem_hndl);
104 uct_ugni_cdm_unlock(&ugni_md->cdm);
105 if (GNI_RC_SUCCESS != ugni_rc) {
106 ucs_error("GNI_MemDeregister failed, Error status: %s %d",
107 gni_err_str[ugni_rc], ugni_rc);
108 status = UCS_ERR_IO_ERROR;
109 }
110 ucs_free(mem_hndl);
111
112 return status;
113 }
114
uct_ugni_rkey_pack(uct_md_h md,uct_mem_h memh,void * rkey_buffer)115 static ucs_status_t uct_ugni_rkey_pack(uct_md_h md, uct_mem_h memh,
116 void *rkey_buffer)
117 {
118 gni_mem_handle_t *mem_hndl = (gni_mem_handle_t *) memh;
119 uint64_t *ptr = rkey_buffer;
120
121 ptr[0] = UCT_UGNI_RKEY_MAGIC;
122 ptr[1] = mem_hndl->qword1;
123 ptr[2] = mem_hndl->qword2;
124 ucs_debug("Packed [ %"PRIx64" %"PRIx64" %"PRIx64"]", ptr[0], ptr[1], ptr[2]);
125 return UCS_OK;
126 }
127
uct_ugni_rkey_release(uct_component_t * component,uct_rkey_t rkey,void * handle)128 static ucs_status_t uct_ugni_rkey_release(uct_component_t *component,
129 uct_rkey_t rkey, void *handle)
130 {
131 ucs_assert(NULL == handle);
132 ucs_free((void *)rkey);
133 return UCS_OK;
134 }
135
uct_ugni_rkey_unpack(uct_component_t * component,const void * rkey_buffer,uct_rkey_t * rkey_p,void ** handle_p)136 static ucs_status_t uct_ugni_rkey_unpack(uct_component_t *component,
137 const void *rkey_buffer,
138 uct_rkey_t *rkey_p, void **handle_p)
139 {
140 const uint64_t *ptr = rkey_buffer;
141 gni_mem_handle_t *mem_hndl = NULL;
142 uint64_t magic = 0;
143
144 ucs_debug("Unpacking [ %"PRIx64" %"PRIx64" %"PRIx64"]", ptr[0], ptr[1], ptr[2]);
145 magic = ptr[0];
146 if (magic != UCT_UGNI_RKEY_MAGIC) {
147 ucs_error("Failed to identify key. Expected %llx but received %"PRIx64"",
148 UCT_UGNI_RKEY_MAGIC, magic);
149 return UCS_ERR_UNSUPPORTED;
150 }
151
152 mem_hndl = ucs_malloc(sizeof(gni_mem_handle_t), "gni_mem_handle_t");
153 if (NULL == mem_hndl) {
154 ucs_error("Failed to allocate memory for gni_mem_handle_t");
155 return UCS_ERR_NO_MEMORY;
156 }
157
158 mem_hndl->qword1 = ptr[1];
159 mem_hndl->qword2 = ptr[2];
160 *rkey_p = (uintptr_t)mem_hndl;
161 *handle_p = NULL;
162 return UCS_OK;
163 }
164
uct_ugni_md_close(uct_md_h md)165 static void uct_ugni_md_close(uct_md_h md)
166 {
167 uct_ugni_md_t *ugni_md = ucs_derived_of(md, uct_ugni_md_t);
168
169 pthread_mutex_lock(&uct_ugni_global_lock);
170 ugni_md->ref_count--;
171 if (!ugni_md->ref_count) {
172 ucs_debug("Tearing down MD CDM");
173 uct_ugni_destroy_cdm(&ugni_md->cdm);
174 }
175 pthread_mutex_unlock(&uct_ugni_global_lock);
176 }
177
178 static ucs_status_t
uct_ugni_md_open(uct_component_h component,const char * md_name,const uct_md_config_t * md_config,uct_md_h * md_p)179 uct_ugni_md_open(uct_component_h component,const char *md_name,
180 const uct_md_config_t *md_config, uct_md_h *md_p)
181 {
182 ucs_status_t status = UCS_OK;
183
184 pthread_mutex_lock(&uct_ugni_global_lock);
185 static uct_md_ops_t md_ops = {
186 .close = uct_ugni_md_close,
187 .query = uct_ugni_md_query,
188 .mem_alloc = (void*)ucs_empty_function,
189 .mem_free = (void*)ucs_empty_function,
190 .mem_reg = uct_ugni_mem_reg,
191 .mem_dereg = uct_ugni_mem_dereg,
192 .mkey_pack = uct_ugni_rkey_pack,
193 .detect_memory_type = ucs_empty_function_return_unsupported,
194 };
195
196 static uct_ugni_md_t md = {
197 .super.ops = &md_ops,
198 .super.component = &uct_ugni_component,
199 .ref_count = 0
200 };
201
202 *md_p = &md.super;
203
204 if (!md.ref_count) {
205 status = init_device_list();
206 if (UCS_OK != status) {
207 ucs_error("Failed to init device list, Error status: %d", status);
208 goto error;
209 }
210 status = uct_ugni_create_md_cdm(&md.cdm);
211 if (UCS_OK != status) {
212 ucs_error("Failed to UGNI NIC, Error status: %d", status);
213 goto error;
214 }
215 }
216
217 md.ref_count++;
218
219 error:
220 pthread_mutex_unlock(&uct_ugni_global_lock);
221 return status;
222 }
223
224 uct_component_t uct_ugni_component = {
225 .query_md_resources = uct_ugni_query_md_resources,
226 .md_open = uct_ugni_md_open,
227 .cm_open = ucs_empty_function_return_unsupported,
228 .rkey_unpack = uct_ugni_rkey_unpack,
229 .rkey_ptr = ucs_empty_function_return_unsupported,
230 .rkey_release = uct_ugni_rkey_release,
231 .name = UCT_UGNI_MD_NAME,
232 .md_config = {
233 .name = "UGNI memory domain",
234 .prefix = "UGNI_",
235 .table = uct_md_config_table,
236 .size = sizeof(uct_md_config_t),
237 },
238 .cm_config = UCS_CONFIG_EMPTY_GLOBAL_LIST_ENTRY,
239 .tl_list = UCT_COMPONENT_TL_LIST_INITIALIZER(&uct_ugni_component),
240 .flags = 0
241 };
242 UCT_COMPONENT_REGISTER(&uct_ugni_component);
243