1 /*
2 * Copyright (C) 2016 by Argonne National Laboratory.
3 *
4 * This software is available to you under a choice of one of two
5 * licenses. You may choose to be licensed under the terms of the GNU
6 * General Public License (GPL) Version 2, available from the file
7 * COPYING in the main directory of this source tree, or the
8 * BSD license below:
9 *
10 * Redistribution and use in source and binary forms, with or
11 * without modification, are permitted provided that the following
12 * conditions are met:
13 *
14 * - Redistributions of source code must retain the above
15 * copyright notice, this list of conditions and the following
16 * disclaimer.
17 *
18 * - Redistributions in binary form must reproduce the above
19 * copyright notice, this list of conditions and the following
20 * disclaimer in the documentation and/or other materials
21 * provided with the distribution.
22 *
23 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
24 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
25 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
26 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
27 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
28 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
29 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
30 * SOFTWARE.
31 */
32 #include "rdma/bgq/fi_bgq.h"
33
fi_bgq_set_default_info()34 int fi_bgq_set_default_info()
35 {
36 struct fi_info *fi;
37 uint32_t ppn = Kernel_ProcessCount();
38
39 /*
40 * See: fi_bgq_stx_init() for the number of mu injection fifos
41 * allocated for each tx context. Each rx context uses one
42 * mu injection fifo and one mu reception fifo.
43 */
44 const unsigned tx_ctx_cnt = (((BGQ_MU_NUM_INJ_FIFO_GROUPS-1) * BGQ_MU_NUM_INJ_FIFOS_PER_GROUP) / 3) / ppn;
45
46 /*
47 * The number of rx contexts on a node is the minimum of:
48 * 1. number of mu injection fifos on the node not used by tx contexts
49 * 2. total number mu reception fifos on the node
50 */
51 const unsigned rx_ctx_cnt = MIN((((BGQ_MU_NUM_INJ_FIFO_GROUPS-1) * BGQ_MU_NUM_INJ_FIFOS_PER_GROUP) - (tx_ctx_cnt * ppn)), ((BGQ_MU_NUM_REC_FIFO_GROUPS-1) * BGQ_MU_NUM_REC_FIFOS_PER_GROUP)) / ppn;
52
53 fi = fi_dupinfo(NULL);
54 if (!fi) {
55 errno = FI_ENOMEM;
56 return -errno;
57 }
58
59 fi_bgq_global.info = fi;
60
61 *fi->tx_attr = (struct fi_tx_attr) {
62 .caps = FI_RMA | FI_ATOMIC | FI_TRANSMIT_COMPLETE,
63 .mode = FI_ASYNC_IOV,
64 .op_flags = FI_TRANSMIT_COMPLETE,
65 .msg_order = FI_ORDER_SAS | OFI_ORDER_WAW_SET |
66 OFI_ORDER_RAW_SET | OFI_ORDER_RAR_SET,
67 .comp_order = FI_ORDER_NONE,
68 .inject_size = FI_BGQ_INJECT_SIZE,
69 .size = FI_BGQ_TX_SIZE,
70 .iov_limit = SIZE_MAX,
71 .rma_iov_limit = 0
72 };
73
74 *fi->rx_attr = (struct fi_rx_attr) {
75 .caps = FI_RMA | FI_ATOMIC | FI_NAMED_RX_CTX,
76 .mode = FI_ASYNC_IOV,
77 .op_flags = 0,
78 .msg_order = 0,
79 .comp_order = FI_ORDER_NONE,
80 .total_buffered_recv = FI_BGQ_TOTAL_BUFFERED_RECV,
81 .size = FI_BGQ_RX_SIZE,
82 .iov_limit = SIZE_MAX
83 };
84
85 *fi->ep_attr = (struct fi_ep_attr) {
86 .type = FI_EP_RDM,
87 .protocol = FI_BGQ_PROTOCOL,
88 .protocol_version = FI_BGQ_PROTOCOL_VERSION,
89 .max_msg_size = FI_BGQ_MAX_MSG_SIZE,
90 .msg_prefix_size = FI_BGQ_MAX_PREFIX_SIZE,
91 .max_order_raw_size = FI_BGQ_MAX_ORDER_RAW_SIZE,
92 .max_order_war_size = FI_BGQ_MAX_ORDER_WAR_SIZE,
93 .max_order_waw_size = FI_BGQ_MAX_ORDER_WAW_SIZE,
94 .mem_tag_format = FI_BGQ_MEM_TAG_FORMAT,
95 .tx_ctx_cnt = tx_ctx_cnt,
96 .rx_ctx_cnt = rx_ctx_cnt,
97 };
98
99 *fi->domain_attr = (struct fi_domain_attr) {
100 .domain = NULL,
101 .name = NULL, /* TODO: runtime query for name? */
102 .threading = FI_THREAD_FID,
103 .control_progress = FI_PROGRESS_MANUAL,
104 .data_progress = FI_BGQ_FABRIC_DIRECT_PROGRESS,
105 .resource_mgmt = FI_RM_DISABLED,
106 .av_type = FI_AV_MAP,
107 .mr_mode = FI_BGQ_FABRIC_DIRECT_MR,
108 .mr_key_size = 2,
109 .cq_data_size = FI_BGQ_REMOTE_CQ_DATA_SIZE,
110 .cq_cnt = 128 / ppn,
111 .ep_cnt = SIZE_MAX,
112 .tx_ctx_cnt = tx_ctx_cnt,
113 .rx_ctx_cnt = rx_ctx_cnt,
114
115 .max_ep_tx_ctx = ((BGQ_MU_NUM_INJ_FIFO_GROUPS-1) * BGQ_MU_NUM_INJ_FIFOS_PER_GROUP) / ppn / 2,
116 .max_ep_rx_ctx = ((BGQ_MU_NUM_REC_FIFO_GROUPS-1) * BGQ_MU_NUM_REC_FIFOS_PER_GROUP) / ppn,
117 .max_ep_stx_ctx = ((BGQ_MU_NUM_INJ_FIFO_GROUPS-1) * BGQ_MU_NUM_INJ_FIFOS_PER_GROUP) / ppn / 2,
118 .max_ep_srx_ctx = 0
119 };
120
121 *fi->fabric_attr = (struct fi_fabric_attr) {
122 .fabric = NULL,
123 .name = strdup(FI_BGQ_FABRIC_NAME),
124 .prov_name = strdup(FI_BGQ_PROVIDER_NAME),
125 .prov_version = FI_BGQ_PROVIDER_VERSION
126 };
127
128 fi->caps = FI_BGQ_DEFAULT_CAPS;
129 fi->mode = FI_ASYNC_IOV;
130 fi->mode |= (FI_CONTEXT);
131 fi->mode &= (~FI_LOCAL_MR);
132 fi->mode &= (~FI_MSG_PREFIX);
133
134 fi->addr_format = FI_ADDR_BGQ;
135 fi->src_addrlen = 24; // includes null
136 fi->dest_addrlen = 24; // includes null
137 fi->dest_addr = NULL;
138 fi->next = NULL;
139
140 return 0;
141 }
142