1 /*
2 * Copyright (c) 2013-2018 Intel Corporation. All rights reserved.
3 * Copyright (c) 2019 Amazon.com, Inc. or its affiliates.
4 * All rights reserved.
5 *
6 * This software is available to you under a choice of one of two
7 * licenses. You may choose to be licensed under the terms of the GNU
8 * General Public License (GPL) Version 2, available from the file
9 * COPYING in the main directory of this source tree, or the
10 * BSD license below:
11 *
12 * Redistribution and use in source and binary forms, with or
13 * without modification, are permitted provided that the following
14 * conditions are met:
15 *
16 * - Redistributions of source code must retain the above
17 * copyright notice, this list of conditions and the following
18 * disclaimer.
19 *
20 * - Redistributions in binary form must reproduce the above
21 * copyright notice, this list of conditions and the following
22 * disclaimer in the documentation and/or other materials
23 * provided with the distribution.
24 *
25 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
26 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
27 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
28 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
29 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
30 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
31 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
32 * SOFTWARE.
33 */
34
35 #include "ofi_util.h"
36 #include "rxr.h"
37 #include "rxr_cntr.h"
38
efa_cntr_wait(struct fid_cntr * cntr_fid,uint64_t threshold,int timeout)39 static int efa_cntr_wait(struct fid_cntr *cntr_fid, uint64_t threshold, int timeout)
40 {
41 struct util_cntr *cntr;
42 uint64_t start, errcnt;
43 int ret;
44 int numtry = 5;
45 int tryid = 0;
46 int waitim = 1;
47
48 cntr = container_of(cntr_fid, struct util_cntr, cntr_fid);
49 assert(cntr->wait);
50 errcnt = ofi_atomic_get64(&cntr->err);
51 start = (timeout >= 0) ? ofi_gettime_ms() : 0;
52
53 for (tryid = 0; tryid < numtry; ++tryid) {
54 cntr->progress(cntr);
55 if (threshold <= ofi_atomic_get64(&cntr->cnt))
56 return FI_SUCCESS;
57
58 if (errcnt != ofi_atomic_get64(&cntr->err))
59 return -FI_EAVAIL;
60
61 if (timeout >= 0) {
62 timeout -= (int)(ofi_gettime_ms() - start);
63 if (timeout <= 0)
64 return -FI_ETIMEDOUT;
65 }
66
67 ret = fi_wait(&cntr->wait->wait_fid, waitim);
68 if (ret == -FI_ETIMEDOUT)
69 ret = 0;
70
71 waitim *= 2;
72 }
73
74 return ret;
75 }
76
efa_cntr_open(struct fid_domain * domain,struct fi_cntr_attr * attr,struct fid_cntr ** cntr_fid,void * context)77 int efa_cntr_open(struct fid_domain *domain, struct fi_cntr_attr *attr,
78 struct fid_cntr **cntr_fid, void *context)
79 {
80 int ret;
81 struct util_cntr *cntr;
82
83 cntr = calloc(1, sizeof(*cntr));
84 if (!cntr)
85 return -FI_ENOMEM;
86
87 ret = ofi_cntr_init(&rxr_prov, domain, attr, cntr,
88 &ofi_cntr_progress, context);
89 if (ret)
90 goto free;
91
92 *cntr_fid = &cntr->cntr_fid;
93 cntr->cntr_fid.ops->wait = efa_cntr_wait;
94 return FI_SUCCESS;
95
96 free:
97 free(cntr);
98 return ret;
99 }
100
efa_cntr_report_tx_completion(struct util_ep * ep,uint64_t flags)101 void efa_cntr_report_tx_completion(struct util_ep *ep, uint64_t flags)
102 {
103 struct util_cntr *cntr;
104
105 flags &= (FI_SEND | FI_WRITE | FI_READ);
106 assert(flags == FI_SEND || flags == FI_WRITE || flags == FI_READ);
107
108 if (flags == FI_SEND)
109 cntr = ep->tx_cntr;
110 else if (flags == FI_WRITE)
111 cntr = ep->wr_cntr;
112 else if (flags == FI_READ)
113 cntr = ep->rd_cntr;
114 else
115 cntr = NULL;
116
117 if (cntr)
118 cntr->cntr_fid.ops->add(&cntr->cntr_fid, 1);
119 }
120
efa_cntr_report_rx_completion(struct util_ep * ep,uint64_t flags)121 void efa_cntr_report_rx_completion(struct util_ep *ep, uint64_t flags)
122 {
123 struct util_cntr *cntr;
124
125 flags &= (FI_RECV | FI_REMOTE_WRITE | FI_REMOTE_READ);
126 assert(flags == FI_RECV || flags == FI_REMOTE_WRITE || flags == FI_REMOTE_READ);
127
128 if (flags == FI_RECV)
129 cntr = ep->rx_cntr;
130 else if (flags == FI_REMOTE_READ)
131 cntr = ep->rem_rd_cntr;
132 else if (flags == FI_REMOTE_WRITE)
133 cntr = ep->rem_wr_cntr;
134 else
135 cntr = NULL;
136
137 if (cntr)
138 cntr->cntr_fid.ops->add(&cntr->cntr_fid, 1);
139 }
140
efa_cntr_report_error(struct util_ep * ep,uint64_t flags)141 void efa_cntr_report_error(struct util_ep *ep, uint64_t flags)
142 {
143 flags = flags & (FI_SEND | FI_READ | FI_WRITE | FI_ATOMIC |
144 FI_RECV | FI_REMOTE_READ | FI_REMOTE_WRITE);
145
146 struct util_cntr *cntr;
147
148 if (flags == FI_WRITE || flags == FI_ATOMIC)
149 cntr = ep->wr_cntr;
150 else if (flags == FI_READ)
151 cntr = ep->rd_cntr;
152 else if (flags == FI_SEND)
153 cntr = ep->tx_cntr;
154 else if (flags == FI_RECV)
155 cntr = ep->rx_cntr;
156 else if (flags == FI_REMOTE_READ)
157 cntr = ep->rem_rd_cntr;
158 else if (flags == FI_REMOTE_WRITE)
159 cntr = ep->rem_wr_cntr;
160 else
161 cntr = NULL;
162
163 if (cntr)
164 cntr->cntr_fid.ops->adderr(&cntr->cntr_fid, 1);
165 }
166
167