1 /* SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB */ 2 /* 3 * Copyright (c) 2016 Mellanox Technologies Ltd. All rights reserved. 4 * Copyright (c) 2015 System Fabric Works, Inc. All rights reserved. 5 */ 6 7 #ifndef RXE_QUEUE_H 8 #define RXE_QUEUE_H 9 10 /* Implements a simple circular buffer that is shared between user 11 * and the driver and can be resized. The requested element size is 12 * rounded up to a power of 2 and the number of elements in the buffer 13 * is also rounded up to a power of 2. Since the queue is empty when 14 * the producer and consumer indices match the maximum capacity of the 15 * queue is one less than the number of element slots. 16 * 17 * Notes: 18 * - The driver indices are always masked off to q->index_mask 19 * before storing so do not need to be checked on reads. 20 * - The user whether user space or kernel is generally 21 * not trusted so its parameters are masked to make sure 22 * they do not access the queue out of bounds on reads. 23 * - The driver indices for queues must not be written 24 * by user so a local copy is used and a shared copy is 25 * stored when the local copy is changed. 26 * - By passing the type in the parameter list separate from q 27 * the compiler can eliminate the switch statement when the 28 * actual queue type is known when the function is called at 29 * compile time. 30 * - These queues are lock free. The user and driver must protect 31 * changes to their end of the queues with locks if more than one 32 * CPU can be accessing it at the same time. 33 */ 34 35 /** 36 * enum queue_type - type of queue 37 * @QUEUE_TYPE_TO_CLIENT: Queue is written by rxe driver and 38 * read by client. Used by rxe driver only. 39 * @QUEUE_TYPE_FROM_CLIENT: Queue is written by client and 40 * read by rxe driver. Used by rxe driver only. 41 * @QUEUE_TYPE_TO_DRIVER: Queue is written by client and 42 * read by rxe driver. Used by kernel client only. 43 * @QUEUE_TYPE_FROM_DRIVER: Queue is written by rxe driver and 44 * read by client. Used by kernel client only. 45 */ 46 enum queue_type { 47 QUEUE_TYPE_TO_CLIENT, 48 QUEUE_TYPE_FROM_CLIENT, 49 QUEUE_TYPE_TO_DRIVER, 50 QUEUE_TYPE_FROM_DRIVER, 51 }; 52 53 struct rxe_queue_buf; 54 55 struct rxe_queue { 56 struct rxe_dev *rxe; 57 struct rxe_queue_buf *buf; 58 struct rxe_mmap_info *ip; 59 size_t buf_size; 60 size_t elem_size; 61 unsigned int log2_elem_size; 62 u32 index_mask; 63 enum queue_type type; 64 /* private copy of index for shared queues between 65 * kernel space and user space. Kernel reads and writes 66 * this copy and then replicates to rxe_queue_buf 67 * for read access by user space. 68 */ 69 u32 index; 70 }; 71 72 int do_mmap_info(struct rxe_dev *rxe, struct mminfo __user *outbuf, 73 struct ib_udata *udata, struct rxe_queue_buf *buf, 74 size_t buf_size, struct rxe_mmap_info **ip_p); 75 76 void rxe_queue_reset(struct rxe_queue *q); 77 78 struct rxe_queue *rxe_queue_init(struct rxe_dev *rxe, int *num_elem, 79 unsigned int elem_size, enum queue_type type); 80 81 int rxe_queue_resize(struct rxe_queue *q, unsigned int *num_elem_p, 82 unsigned int elem_size, struct ib_udata *udata, 83 struct mminfo __user *outbuf, 84 spinlock_t *producer_lock, spinlock_t *consumer_lock); 85 86 void rxe_queue_cleanup(struct rxe_queue *queue); 87 88 static inline u32 queue_next_index(struct rxe_queue *q, int index) 89 { 90 return (index + 1) & q->index_mask; 91 } 92 93 static inline u32 queue_get_producer(const struct rxe_queue *q, 94 enum queue_type type) 95 { 96 u32 prod; 97 98 switch (type) { 99 case QUEUE_TYPE_FROM_CLIENT: 100 /* protect user index */ 101 prod = smp_load_acquire(&q->buf->producer_index); 102 break; 103 case QUEUE_TYPE_TO_CLIENT: 104 prod = q->index; 105 break; 106 case QUEUE_TYPE_FROM_DRIVER: 107 /* protect driver index */ 108 prod = smp_load_acquire(&q->buf->producer_index); 109 break; 110 case QUEUE_TYPE_TO_DRIVER: 111 prod = q->buf->producer_index; 112 break; 113 } 114 115 return prod; 116 } 117 118 static inline u32 queue_get_consumer(const struct rxe_queue *q, 119 enum queue_type type) 120 { 121 u32 cons; 122 123 switch (type) { 124 case QUEUE_TYPE_FROM_CLIENT: 125 cons = q->index; 126 break; 127 case QUEUE_TYPE_TO_CLIENT: 128 /* protect user index */ 129 cons = smp_load_acquire(&q->buf->consumer_index); 130 break; 131 case QUEUE_TYPE_FROM_DRIVER: 132 cons = q->buf->consumer_index; 133 break; 134 case QUEUE_TYPE_TO_DRIVER: 135 /* protect driver index */ 136 cons = smp_load_acquire(&q->buf->consumer_index); 137 break; 138 } 139 140 return cons; 141 } 142 143 static inline int queue_empty(struct rxe_queue *q, enum queue_type type) 144 { 145 u32 prod = queue_get_producer(q, type); 146 u32 cons = queue_get_consumer(q, type); 147 148 return ((prod - cons) & q->index_mask) == 0; 149 } 150 151 static inline int queue_full(struct rxe_queue *q, enum queue_type type) 152 { 153 u32 prod = queue_get_producer(q, type); 154 u32 cons = queue_get_consumer(q, type); 155 156 return ((prod + 1 - cons) & q->index_mask) == 0; 157 } 158 159 static inline u32 queue_count(const struct rxe_queue *q, 160 enum queue_type type) 161 { 162 u32 prod = queue_get_producer(q, type); 163 u32 cons = queue_get_consumer(q, type); 164 165 return (prod - cons) & q->index_mask; 166 } 167 168 static inline void queue_advance_producer(struct rxe_queue *q, 169 enum queue_type type) 170 { 171 u32 prod; 172 173 switch (type) { 174 case QUEUE_TYPE_FROM_CLIENT: 175 pr_warn("%s: attempt to advance client index\n", 176 __func__); 177 break; 178 case QUEUE_TYPE_TO_CLIENT: 179 prod = q->index; 180 prod = (prod + 1) & q->index_mask; 181 q->index = prod; 182 /* protect user index */ 183 smp_store_release(&q->buf->producer_index, prod); 184 break; 185 case QUEUE_TYPE_FROM_DRIVER: 186 pr_warn("%s: attempt to advance driver index\n", 187 __func__); 188 break; 189 case QUEUE_TYPE_TO_DRIVER: 190 prod = q->buf->producer_index; 191 prod = (prod + 1) & q->index_mask; 192 q->buf->producer_index = prod; 193 break; 194 } 195 } 196 197 static inline void queue_advance_consumer(struct rxe_queue *q, 198 enum queue_type type) 199 { 200 u32 cons; 201 202 switch (type) { 203 case QUEUE_TYPE_FROM_CLIENT: 204 cons = q->index; 205 cons = (cons + 1) & q->index_mask; 206 q->index = cons; 207 /* protect user index */ 208 smp_store_release(&q->buf->consumer_index, cons); 209 break; 210 case QUEUE_TYPE_TO_CLIENT: 211 pr_warn("%s: attempt to advance client index\n", 212 __func__); 213 break; 214 case QUEUE_TYPE_FROM_DRIVER: 215 cons = q->buf->consumer_index; 216 cons = (cons + 1) & q->index_mask; 217 q->buf->consumer_index = cons; 218 break; 219 case QUEUE_TYPE_TO_DRIVER: 220 pr_warn("%s: attempt to advance driver index\n", 221 __func__); 222 break; 223 } 224 } 225 226 static inline void *queue_producer_addr(struct rxe_queue *q, 227 enum queue_type type) 228 { 229 u32 prod = queue_get_producer(q, type); 230 231 return q->buf->data + (prod << q->log2_elem_size); 232 } 233 234 static inline void *queue_consumer_addr(struct rxe_queue *q, 235 enum queue_type type) 236 { 237 u32 cons = queue_get_consumer(q, type); 238 239 return q->buf->data + (cons << q->log2_elem_size); 240 } 241 242 static inline void *queue_addr_from_index(struct rxe_queue *q, u32 index) 243 { 244 return q->buf->data + ((index & q->index_mask) 245 << q->log2_elem_size); 246 } 247 248 static inline u32 queue_index_from_addr(const struct rxe_queue *q, 249 const void *addr) 250 { 251 return (((u8 *)addr - q->buf->data) >> q->log2_elem_size) 252 & q->index_mask; 253 } 254 255 static inline void *queue_head(struct rxe_queue *q, enum queue_type type) 256 { 257 return queue_empty(q, type) ? NULL : queue_consumer_addr(q, type); 258 } 259 260 #endif /* RXE_QUEUE_H */ 261