1 /* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */
2 /*
3 * Copyright (c) 2011-2018 Los Alamos National Security, LLC. All rights
4 * reserved.
5 * Copyright (c) 2011 UT-Battelle, LLC. All rights reserved.
6 * $COPYRIGHT$
7 *
8 * Additional copyrights may follow
9 *
10 * $HEADER$
11 */
12
13 #ifndef MCA_BTL_UGNI_ENDPOINT_H
14 #define MCA_BTL_UGNI_ENDPOINT_H
15
16 #include "btl_ugni.h"
17
18 enum mca_btl_ugni_endpoint_state_t {
19 MCA_BTL_UGNI_EP_STATE_INIT = 0,
20 MCA_BTL_UGNI_EP_STATE_CONNECTING,
21 MCA_BTL_UGNI_EP_STATE_CONNECTED,
22 };
23 typedef enum mca_btl_ugni_endpoint_state_t mca_btl_ugni_endpoint_state_t;
24
25 struct mca_btl_ugni_smsg_mbox_t;
26
27 struct mca_btl_ugni_endpoint_handle_t {
28 mca_btl_ugni_device_t *device;
29 gni_ep_handle_t gni_handle;
30 };
31
32 typedef struct mca_btl_ugni_endpoint_handle_t mca_btl_ugni_endpoint_handle_t;
33
34 typedef struct mca_btl_base_endpoint_t {
35 opal_list_item_t super;
36
37 opal_proc_t *peer_proc;
38
39 /** may need to lock recursively as the modex lookup could call opal_progress
40 * and hence our progress function. if this changes modify this mutex to not
41 * be recursive. also need to update the constructor function. */
42 opal_recursive_mutex_t lock;
43 mca_btl_ugni_endpoint_state_t state;
44
45 /** Remote NIC address */
46 uint32_t ep_rem_addr;
47
48 /** Remote CDM identifier (base) */
49 uint32_t ep_rem_id;
50
51 /** endpoint to use for SMSG messages */
52 mca_btl_ugni_endpoint_handle_t smsg_ep_handle;
53
54 /** temporary space to store the remote SMSG attributes */
55 mca_btl_ugni_endpoint_attr_t *remote_attr;
56
57 /** SMSG mailbox assigned to this endpoint */
58 struct mca_btl_ugni_smsg_mbox_t *mailbox;
59
60 /** Remote IRQ handle (for async completion) */
61 gni_mem_handle_t rmt_irq_mem_hndl;
62
63 /** frags waiting for SMSG credits */
64 opal_list_t frag_wait_list;
65
66 /** endpoint is currently wait-listed for SMSG progress */
67 bool wait_listed;
68
69 /** protect against race on connection */
70 bool dg_posted;
71
72 /** protect against re-entry to SMSG */
73 int32_t smsg_progressing;
74
75 int index;
76 } mca_btl_base_endpoint_t;
77
78 typedef mca_btl_base_endpoint_t mca_btl_ugni_endpoint_t;
79 OBJ_CLASS_DECLARATION(mca_btl_ugni_endpoint_t);
80
81 int mca_btl_ugni_ep_connect_progress (mca_btl_ugni_endpoint_t *ep);
82 int mca_btl_ugni_ep_disconnect (mca_btl_ugni_endpoint_t *ep, bool send_disconnect);
83 int mca_btl_ugni_wildcard_ep_post (mca_btl_ugni_module_t *ugni_module);
84 void mca_btl_ugni_release_ep (mca_btl_ugni_endpoint_t *ep);
85 int mca_btl_ugni_init_ep (mca_btl_ugni_module_t *ugni_module, mca_btl_ugni_endpoint_t **ep,
86 mca_btl_ugni_module_t *btl, opal_proc_t *peer_proc);
87
mca_btl_ugni_check_endpoint_state(mca_btl_ugni_endpoint_t * ep)88 static inline int mca_btl_ugni_check_endpoint_state (mca_btl_ugni_endpoint_t *ep) {
89 int rc;
90
91 if (OPAL_LIKELY(MCA_BTL_UGNI_EP_STATE_CONNECTED == ep->state)) {
92 return OPAL_SUCCESS;
93 }
94
95 opal_mutex_lock (&ep->lock);
96
97 switch (ep->state) {
98 case MCA_BTL_UGNI_EP_STATE_INIT:
99 rc = mca_btl_ugni_ep_connect_progress (ep);
100 if (OPAL_SUCCESS != rc) {
101 break;
102 }
103 case MCA_BTL_UGNI_EP_STATE_CONNECTING:
104 rc = OPAL_ERR_RESOURCE_BUSY;
105 break;
106 default:
107 rc = OPAL_SUCCESS;
108 }
109
110 opal_mutex_unlock (&ep->lock);
111
112 return rc;
113 }
114
115 /**
116 * Accessor function for endpoint btl
117 *
118 * @param[in] ep endpoint to query
119 *
120 * This helper function exists to make it easy to switch between using a single
121 * and multiple ugni modules. Currently there is only one so we just use the
122 * pointer in the component structure. This saves 4-8 bytes in the endpoint
123 * structure.
124 */
mca_btl_ugni_ep_btl(mca_btl_ugni_endpoint_t * ep)125 static inline mca_btl_ugni_module_t *mca_btl_ugni_ep_btl (mca_btl_ugni_endpoint_t *ep)
126 {
127 /* there is only one ugni module at this time. if that changes add a btl pointer back
128 * to the endpoint structure. */
129 return mca_btl_ugni_component.modules;
130 }
131
132 /**
133 * Initialize and bind an endpoint handle
134 *
135 * @param[in] ep BTL endpoint
136 * @param[in] cq completion queue
137 * @param[in] device device to bind with
138 * @param[in] ep_handle endpoint handle to initialize and bind
139 */
140 int mca_btl_ugni_ep_handle_init (mca_btl_ugni_endpoint_t *ep, gni_cq_handle_t cq,
141 mca_btl_ugni_device_t *device, mca_btl_ugni_endpoint_handle_t *ep_handle);
142
143 int mca_btl_ugni_ep_handle_cleanup (mca_btl_ugni_endpoint_handle_t *ep_handle);
144
145 #endif /* MCA_BTL_UGNI_ENDPOINT_H */
146