1 /*
2 * Copyright (C) Advanced Micro Devices, Inc. 2019. ALL RIGHTS RESERVED.
3 * See file LICENSE for terms.
4 */
5
6 #ifdef HAVE_CONFIG_H
7 # include "config.h"
8 #endif
9
10 #include "rocm_gdr_iface.h"
11 #include "rocm_gdr_md.h"
12 #include "rocm_gdr_ep.h"
13
14 #include <uct/rocm/base/rocm_base.h>
15 #include <ucs/type/class.h>
16 #include <ucs/sys/string.h>
17
18
19 static ucs_config_field_t uct_rocm_gdr_iface_config_table[] = {
20
21 {"", "", NULL,
22 ucs_offsetof(uct_rocm_gdr_iface_config_t, super),
23 UCS_CONFIG_TYPE_TABLE(uct_iface_config_table)},
24
25 {NULL}
26 };
27
28 /* Forward declaration for the delete function */
29 static void UCS_CLASS_DELETE_FUNC_NAME(uct_rocm_gdr_iface_t)(uct_iface_t*);
30
31
uct_rocm_gdr_iface_get_address(uct_iface_h tl_iface,uct_iface_addr_t * iface_addr)32 static ucs_status_t uct_rocm_gdr_iface_get_address(uct_iface_h tl_iface,
33 uct_iface_addr_t *iface_addr)
34 {
35 uct_rocm_gdr_iface_t *iface = ucs_derived_of(tl_iface, uct_rocm_gdr_iface_t);
36
37 *(uct_rocm_gdr_iface_addr_t*)iface_addr = iface->id;
38 return UCS_OK;
39 }
40
uct_rocm_gdr_iface_is_reachable(const uct_iface_h tl_iface,const uct_device_addr_t * dev_addr,const uct_iface_addr_t * iface_addr)41 static int uct_rocm_gdr_iface_is_reachable(const uct_iface_h tl_iface,
42 const uct_device_addr_t *dev_addr,
43 const uct_iface_addr_t *iface_addr)
44 {
45 uct_rocm_gdr_iface_t *iface = ucs_derived_of(tl_iface, uct_rocm_gdr_iface_t);
46 uct_rocm_gdr_iface_addr_t *addr = (uct_rocm_gdr_iface_addr_t*)iface_addr;
47
48 return (addr != NULL) && (iface->id == *addr);
49 }
50
uct_rocm_gdr_iface_query(uct_iface_h tl_iface,uct_iface_attr_t * iface_attr)51 static ucs_status_t uct_rocm_gdr_iface_query(uct_iface_h tl_iface,
52 uct_iface_attr_t *iface_attr)
53 {
54 uct_rocm_gdr_iface_t *iface = ucs_derived_of(tl_iface, uct_rocm_gdr_iface_t);
55
56 uct_base_iface_query(&iface->super, iface_attr);
57
58 iface_attr->iface_addr_len = sizeof(uct_rocm_gdr_iface_addr_t);
59 iface_attr->device_addr_len = 0;
60 iface_attr->ep_addr_len = 0;
61 iface_attr->cap.flags = UCT_IFACE_FLAG_CONNECT_TO_IFACE |
62 UCT_IFACE_FLAG_GET_SHORT |
63 UCT_IFACE_FLAG_PUT_SHORT;
64
65 iface_attr->cap.put.max_short = UINT_MAX;
66 iface_attr->cap.put.max_bcopy = 0;
67 iface_attr->cap.put.min_zcopy = 0;
68 iface_attr->cap.put.max_zcopy = 0;
69 iface_attr->cap.put.opt_zcopy_align = 1;
70 iface_attr->cap.put.align_mtu = iface_attr->cap.put.opt_zcopy_align;
71 iface_attr->cap.put.max_iov = 1;
72
73 iface_attr->cap.get.max_short = UINT_MAX;
74 iface_attr->cap.get.max_bcopy = 0;
75 iface_attr->cap.get.min_zcopy = 0;
76 iface_attr->cap.get.max_zcopy = 0;
77 iface_attr->cap.get.opt_zcopy_align = 1;
78 iface_attr->cap.get.align_mtu = iface_attr->cap.get.opt_zcopy_align;
79 iface_attr->cap.get.max_iov = 1;
80
81 iface_attr->cap.am.max_short = 0;
82 iface_attr->cap.am.max_bcopy = 0;
83 iface_attr->cap.am.min_zcopy = 0;
84 iface_attr->cap.am.max_zcopy = 0;
85 iface_attr->cap.am.opt_zcopy_align = 1;
86 iface_attr->cap.am.align_mtu = iface_attr->cap.am.opt_zcopy_align;
87 iface_attr->cap.am.max_hdr = 0;
88 iface_attr->cap.am.max_iov = 1;
89
90 iface_attr->latency = ucs_linear_func_make(1e-6, 0);
91 iface_attr->bandwidth.dedicated = 0;
92 iface_attr->bandwidth.shared = 6911.0 * UCS_MBYTE;
93 iface_attr->overhead = 0;
94 iface_attr->priority = 0;
95
96 return UCS_OK;
97 }
98
99 static uct_iface_ops_t uct_rocm_gdr_iface_ops = {
100 .ep_get_short = uct_rocm_gdr_ep_get_short,
101 .ep_put_short = uct_rocm_gdr_ep_put_short,
102 .ep_pending_add = ucs_empty_function_return_busy,
103 .ep_pending_purge = ucs_empty_function,
104 .ep_flush = uct_base_ep_flush,
105 .ep_fence = uct_base_ep_fence,
106 .ep_create = UCS_CLASS_NEW_FUNC_NAME(uct_rocm_gdr_ep_t),
107 .ep_destroy = UCS_CLASS_DELETE_FUNC_NAME(uct_rocm_gdr_ep_t),
108 .iface_flush = uct_base_iface_flush,
109 .iface_fence = uct_base_iface_fence,
110 .iface_progress_enable = ucs_empty_function,
111 .iface_progress_disable = ucs_empty_function,
112 .iface_progress = ucs_empty_function_return_zero,
113 .iface_close = UCS_CLASS_DELETE_FUNC_NAME(uct_rocm_gdr_iface_t),
114 .iface_query = uct_rocm_gdr_iface_query,
115 .iface_get_device_address = (uct_iface_get_device_address_func_t)ucs_empty_function_return_success,
116 .iface_get_address = uct_rocm_gdr_iface_get_address,
117 .iface_is_reachable = uct_rocm_gdr_iface_is_reachable,
118 };
119
UCS_CLASS_INIT_FUNC(uct_rocm_gdr_iface_t,uct_md_h md,uct_worker_h worker,const uct_iface_params_t * params,const uct_iface_config_t * tl_config)120 static UCS_CLASS_INIT_FUNC(uct_rocm_gdr_iface_t, uct_md_h md, uct_worker_h worker,
121 const uct_iface_params_t *params,
122 const uct_iface_config_t *tl_config)
123 {
124 UCS_CLASS_CALL_SUPER_INIT(uct_base_iface_t, &uct_rocm_gdr_iface_ops, md, worker,
125 params, tl_config UCS_STATS_ARG(params->stats_root)
126 UCS_STATS_ARG(UCT_ROCM_GDR_TL_NAME));
127
128 self->id = ucs_generate_uuid((uintptr_t)self);
129
130 return UCS_OK;
131 }
132
UCS_CLASS_CLEANUP_FUNC(uct_rocm_gdr_iface_t)133 static UCS_CLASS_CLEANUP_FUNC(uct_rocm_gdr_iface_t)
134 {
135
136 }
137
138 UCS_CLASS_DEFINE(uct_rocm_gdr_iface_t, uct_base_iface_t);
139 UCS_CLASS_DEFINE_NEW_FUNC(uct_rocm_gdr_iface_t, uct_iface_t, uct_md_h, uct_worker_h,
140 const uct_iface_params_t*, const uct_iface_config_t*);
141 static UCS_CLASS_DEFINE_DELETE_FUNC(uct_rocm_gdr_iface_t, uct_iface_t);
142
143 UCT_TL_DEFINE(&uct_rocm_gdr_component, rocm_gdr, uct_rocm_base_query_devices,
144 uct_rocm_gdr_iface_t, "ROCM_GDR_",
145 uct_rocm_gdr_iface_config_table, uct_rocm_gdr_iface_config_t);
146