1 /**
2 * Copyright (C) Mellanox Technologies Ltd. 2001-2018. ALL RIGHTS RESERVED.
3 * Copyright (c) UT-Battelle, LLC. 2015. ALL RIGHTS RESERVED.
4 * Copyright (C) Los Alamos National Security, LLC. 2018. ALL RIGHTS RESERVED.
5 *
6 * See file LICENSE for terms.
7 */
8
9 #ifdef HAVE_CONFIG_H
10 # include "config.h"
11 #endif
12
13 #include "rma.h"
14
15 #include <ucp/proto/proto_am.inl>
16
17
ucp_rma_basic_progress_put(uct_pending_req_t * self)18 static ucs_status_t ucp_rma_basic_progress_put(uct_pending_req_t *self)
19 {
20 ucp_request_t *req = ucs_container_of(self, ucp_request_t, send.uct);
21 ucp_ep_t *ep = req->send.ep;
22 ucp_rkey_h rkey = req->send.rma.rkey;
23 ucp_lane_index_t lane = req->send.lane;
24 ucp_ep_rma_config_t *rma_config = &ucp_ep_config(ep)->rma[lane];
25 ucs_status_t status;
26 ssize_t packed_len;
27
28 ucs_assert(rkey->cache.ep_cfg_index == ep->cfg_index);
29 ucs_assert(rkey->cache.rma_lane == lane);
30
31 if ((req->send.length <= rma_config->max_put_short) ||
32 (req->send.length <= ucp_ep_config(ep)->bcopy_thresh))
33 {
34 packed_len = ucs_min(req->send.length, rma_config->max_put_short);
35 status = UCS_PROFILE_CALL(uct_ep_put_short,
36 ep->uct_eps[lane],
37 req->send.buffer,
38 packed_len,
39 req->send.rma.remote_addr,
40 rkey->cache.rma_rkey);
41 } else if (ucs_likely(req->send.length < rma_config->put_zcopy_thresh)) {
42 ucp_memcpy_pack_context_t pack_ctx;
43 pack_ctx.src = req->send.buffer;
44 pack_ctx.length = ucs_min(req->send.length, rma_config->max_put_bcopy);
45 packed_len = UCS_PROFILE_CALL(uct_ep_put_bcopy,
46 ep->uct_eps[lane],
47 ucp_memcpy_pack,
48 &pack_ctx,
49 req->send.rma.remote_addr,
50 rkey->cache.rma_rkey);
51 status = (packed_len > 0) ? UCS_OK : (ucs_status_t)packed_len;
52 } else {
53 uct_iov_t iov;
54
55 /* TODO: leave last fragment for bcopy */
56 packed_len = ucs_min(req->send.length, rma_config->max_put_zcopy);
57 /* TODO: use ucp_dt_iov_copy_uct */
58 iov.buffer = (void *)req->send.buffer;
59 iov.length = packed_len;
60 iov.count = 1;
61 iov.memh = req->send.state.dt.dt.contig.memh[0];
62
63 status = UCS_PROFILE_CALL(uct_ep_put_zcopy,
64 ep->uct_eps[lane],
65 &iov, 1,
66 req->send.rma.remote_addr,
67 rkey->cache.rma_rkey,
68 &req->send.state.uct_comp);
69 ucp_request_send_state_advance(req, NULL, UCP_REQUEST_SEND_PROTO_RMA,
70 status);
71 }
72
73 return ucp_rma_request_advance(req, packed_len, status);
74 }
75
ucp_rma_basic_progress_get(uct_pending_req_t * self)76 static ucs_status_t ucp_rma_basic_progress_get(uct_pending_req_t *self)
77 {
78 ucp_request_t *req = ucs_container_of(self, ucp_request_t, send.uct);
79 ucp_ep_t *ep = req->send.ep;
80 ucp_rkey_h rkey = req->send.rma.rkey;
81 ucp_lane_index_t lane = req->send.lane;
82 ucp_ep_rma_config_t *rma_config = &ucp_ep_config(ep)->rma[lane];
83 ucs_status_t status;
84 size_t frag_length;
85
86 ucs_assert(rkey->cache.ep_cfg_index == ep->cfg_index);
87 ucs_assert(rkey->cache.rma_lane == lane);
88
89 if (ucs_likely(req->send.length < rma_config->get_zcopy_thresh)) {
90 frag_length = ucs_min(rma_config->max_get_bcopy, req->send.length);
91 status = UCS_PROFILE_CALL(uct_ep_get_bcopy,
92 ep->uct_eps[lane],
93 (uct_unpack_callback_t)memcpy,
94 (void*)req->send.buffer,
95 frag_length,
96 req->send.rma.remote_addr,
97 rkey->cache.rma_rkey,
98 &req->send.state.uct_comp);
99 } else {
100 uct_iov_t iov;
101 frag_length = ucs_min(req->send.length, rma_config->max_get_zcopy);
102 iov.buffer = (void *)req->send.buffer;
103 iov.length = frag_length;
104 iov.count = 1;
105 iov.memh = req->send.state.dt.dt.contig.memh[0];
106
107 status = UCS_PROFILE_CALL(uct_ep_get_zcopy,
108 ep->uct_eps[lane],
109 &iov, 1,
110 req->send.rma.remote_addr,
111 rkey->cache.rma_rkey,
112 &req->send.state.uct_comp);
113 }
114
115 if (status == UCS_INPROGRESS) {
116 ucp_request_send_state_advance(req, 0, UCP_REQUEST_SEND_PROTO_RMA,
117 UCS_INPROGRESS);
118 }
119
120 return ucp_rma_request_advance(req, frag_length, status);
121 }
122
123 ucp_rma_proto_t ucp_rma_basic_proto = {
124 .name = "basic_rma",
125 .progress_put = ucp_rma_basic_progress_put,
126 .progress_get = ucp_rma_basic_progress_get
127 };
128