1 /* 2 * Copyright (C) 1999-2001 The Regents of the University of California 3 * (through E.O. Lawrence Berkeley National Laboratory), subject to 4 * approval by the U.S. Department of Energy. 5 * 6 * Use of this software is under license. The license agreement is included 7 * in the file MVICH_LICENSE.TXT. 8 * 9 * Developed at Berkeley Lab as part of MVICH. 10 * 11 * Authors: Bill Saphir <wcsaphir@lbl.gov> 12 * Michael Welcome <mlwelcome@lbl.gov> 13 */ 14 15 /* Copyright (c) 2002-2008, The Ohio State University. All rights 16 * reserved. 17 * 18 * This file is part of the MVAPICH software package developed by the 19 * team members of The Ohio State University's Network-Based Computing 20 * Laboratory (NBCL), headed by Professor Dhabaleswar K. (DK) Panda. 21 * 22 * For detailed copyright and licensing information, please refer to the 23 * copyright file COPYRIGHT_MVAPICH in the top level MPICH directory. 24 * 25 */ 26 27 28 #ifndef _CBUF_H 29 #define _CBUF_H 30 31 #include <stdio.h> 32 #include <pthread.h> 33 #include <infiniband/verbs.h> 34 #include <malloc.h> 35 #include <string.h> 36 37 #ifdef _IA64_ 38 #define CBUF_FLAG_TYPE uint64_t 39 #else 40 #define CBUF_FLAG_TYPE uint32_t 41 #endif 42 43 #if (defined(RDMA_FAST_PATH) || defined(ADAPTIVE_RDMA_FAST_PATH)) 44 45 #define CBUF_BUFFER_SIZE (viadev_cbuf_total_size - \ 46 2*sizeof(CBUF_FLAG_TYPE)) 47 48 #else 49 50 #define CBUF_BUFFER_SIZE (viadev_cbuf_total_size) 51 52 #endif 53 54 /* 55 * brief justification for cbuf format: 56 * descriptor must be aligned (64 bytes). 57 * cbuf size must be multiple of this alignment to allow contiguous allocation 58 * descriptor and buffer should be contiguous to allow via implementations that 59 * optimize contiguous descriptor/data (? how likely ?) 60 * need to be able to store send handle in cbuf so that we can mark sends 61 * complete when communication completes. don't want to store 62 * it in packet header because we don't always need to send over the network. 63 * don't want to store at beginning or between desc and buffer (see above) so 64 * store at end. 65 */ 66 67 #define QP_CON_REQ 1 68 #define QP_CON_ACK 2 69 #define QP_CON_BREAK_FROM_CLIENT 3 70 #define QP_CON_BREAK_FROM_SERVER 4 71 72 typedef struct { 73 uint32_t src_rank; 74 uint16_t lid; 75 uint32_t qpnum; 76 int msg_type; 77 } armci_ud_rank; 78 79 struct ibv_descriptor { 80 union { 81 struct ibv_recv_wr rr; 82 struct ibv_send_wr sr; 83 } u; 84 struct ibv_sge sg_entry; 85 void *next; 86 }; 87 88 typedef struct ibv_descriptor IBV_DESCRIPTOR; 89 90 typedef struct cbuf { 91 92 #if (defined(RDMA_FAST_PATH) || defined(ADAPTIVE_RDMA_FAST_PATH)) 93 CBUF_FLAG_TYPE *head_flag; 94 #endif 95 96 unsigned char *buffer; 97 98 #if (defined(RDMA_FAST_PATH) || defined(ADAPTIVE_RDMA_FAST_PATH)) 99 int padding; 100 #endif 101 102 IBV_DESCRIPTOR desc; 103 104 /* NULL shandle means not send or not complete. Non-null 105 * means pointer to send handle that is now complete. Used 106 * by viadev_process_send 107 */ 108 void *shandle; 109 struct cbuf_region *region; 110 int grank; 111 112 uint16_t bytes_remaining; 113 uint16_t len; 114 unsigned char *data_start; 115 uint16_t ref_count; 116 117 } cbuf; 118 /* one for head and one for tail */ 119 #define CBUF_FAST_RDMA_EXTRA_BYTES (2*sizeof(CBUF_FLAG_TYPE)) 120 121 #define FAST_RDMA_ALT_TAG 0x8000 122 #define FAST_RDMA_SIZE_MASK 0x7fff 123 124 /* 125 * Vbufs are allocated in blocks and threaded on a single free list. 126 * 127 * These data structures record information on all the cbuf 128 * regions that have been allocated. They can be used for 129 * error checking and to un-register and deallocate the regions 130 * at program termination. 131 * 132 */ 133 typedef struct cbuf_region { 134 struct ibv_mr *mem_handle; /* memory handle for entire region */ 135 136 void *malloc_start; /* used to free region later */ 137 void *malloc_end; /* to bracket mem region */ 138 void *malloc_buf_start; /* used to free DMA region later */ 139 void *malloc_buf_end; /* bracket DMA region */ 140 int count; /* number of cbufs in region */ 141 struct cbuf *cbuf_head; /* first cbuf in region */ 142 struct cbuf_region *next; /* thread cbuf regions */ 143 } cbuf_region; 144 145 /* ack. needed here after cbuf is defined. need to clean up header files 146 * and dependencies. 147 */ 148 149 typedef unsigned long aint_t; 150 151 extern int viadev_cbuf_max; 152 extern int viadev_cbuf_total_size; 153 extern int viadev_cbuf_secondary_pool_size; 154 155 156 void dump_cbuf_regions(void); 157 void dump_cbuf_region(cbuf_region * r); 158 159 160 void allocate_cbufs(int ncbufs); 161 void deallocate_cbufs(void); 162 163 cbuf *get_cbuf(void); 164 void release_cbuf(cbuf * v); 165 void cbuf_init_send(cbuf * v, unsigned long len); 166 void cbuf_init_recv(cbuf * v, unsigned long len); 167 void cbuf_init_sendrecv(cbuf * v, unsigned long len); 168 169 void cbuf_init_rput(cbuf * v, 170 void *local_address, 171 uint32_t local_memhandle, 172 void *remote_address, 173 uint32_t remote_memhandle_rkey, int nbytes); 174 void cbuf_init_rget(cbuf * v, 175 void *local_address, 176 uint32_t local_memhandle, 177 void *remote_address, 178 uint32_t remote_memhandle_rkey, int nbytes); 179 180 void init_cbuf_lock(); 181 182 void dump_cbuf(char *msg, cbuf * v); 183 184 struct ibv_mr * armci_register_memory(void *, int); 185 186 /* 187 * Macros for working with cbufs 188 */ 189 190 #define CBUF_BUFFER_START(v) (v->buffer) 191 192 #define CBUF_DATA_SIZE(type) (CBUF_BUFFER_SIZE - sizeof(type)) 193 194 #endif /* _CBUF_H */ 195