1 /* 2 * Copyright (c) 2013-2015, Mellanox Technologies, Ltd. All rights reserved. 3 * 4 * This software is available to you under a choice of one of two 5 * licenses. You may choose to be licensed under the terms of the GNU 6 * General Public License (GPL) Version 2, available from the file 7 * COPYING in the main directory of this source tree, or the 8 * OpenIB.org BSD license below: 9 * 10 * Redistribution and use in source and binary forms, with or 11 * without modification, are permitted provided that the following 12 * conditions are met: 13 * 14 * - Redistributions of source code must retain the above 15 * copyright notice, this list of conditions and the following 16 * disclaimer. 17 * 18 * - Redistributions in binary form must reproduce the above 19 * copyright notice, this list of conditions and the following 20 * disclaimer in the documentation and/or other materials 21 * provided with the distribution. 22 * 23 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, 24 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF 25 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND 26 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS 27 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN 28 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN 29 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 30 * SOFTWARE. 31 */ 32 33 #ifndef __MLX5_WQ_H__ 34 #define __MLX5_WQ_H__ 35 36 #include <linux/mlx5/mlx5_ifc.h> 37 #include <linux/mlx5/cq.h> 38 #include <linux/mlx5/qp.h> 39 40 struct mlx5_wq_param { 41 int buf_numa_node; 42 int db_numa_node; 43 }; 44 45 struct mlx5_wq_ctrl { 46 struct mlx5_core_dev *mdev; 47 struct mlx5_frag_buf buf; 48 struct mlx5_db db; 49 }; 50 51 struct mlx5_wq_cyc { 52 struct mlx5_frag_buf_ctrl fbc; 53 __be32 *db; 54 u16 sz; 55 u16 wqe_ctr; 56 u16 cur_sz; 57 }; 58 59 struct mlx5_wq_qp { 60 struct mlx5_wq_cyc rq; 61 struct mlx5_wq_cyc sq; 62 }; 63 64 struct mlx5_cqwq { 65 struct mlx5_frag_buf_ctrl fbc; 66 __be32 *db; 67 u32 cc; /* consumer counter */ 68 }; 69 70 struct mlx5_wq_ll { 71 struct mlx5_frag_buf_ctrl fbc; 72 __be32 *db; 73 __be16 *tail_next; 74 u16 head; 75 u16 wqe_ctr; 76 u16 cur_sz; 77 }; 78 79 int mlx5_wq_cyc_create(struct mlx5_core_dev *mdev, struct mlx5_wq_param *param, 80 void *wqc, struct mlx5_wq_cyc *wq, 81 struct mlx5_wq_ctrl *wq_ctrl); 82 u32 mlx5_wq_cyc_get_size(struct mlx5_wq_cyc *wq); 83 84 int mlx5_wq_qp_create(struct mlx5_core_dev *mdev, struct mlx5_wq_param *param, 85 void *qpc, struct mlx5_wq_qp *wq, 86 struct mlx5_wq_ctrl *wq_ctrl); 87 88 int mlx5_cqwq_create(struct mlx5_core_dev *mdev, struct mlx5_wq_param *param, 89 void *cqc, struct mlx5_cqwq *wq, 90 struct mlx5_wq_ctrl *wq_ctrl); 91 u32 mlx5_cqwq_get_size(struct mlx5_cqwq *wq); 92 u8 mlx5_cqwq_get_log_stride_size(struct mlx5_cqwq *wq); 93 94 int mlx5_wq_ll_create(struct mlx5_core_dev *mdev, struct mlx5_wq_param *param, 95 void *wqc, struct mlx5_wq_ll *wq, 96 struct mlx5_wq_ctrl *wq_ctrl); 97 u32 mlx5_wq_ll_get_size(struct mlx5_wq_ll *wq); 98 99 void mlx5_wq_destroy(struct mlx5_wq_ctrl *wq_ctrl); 100 101 static inline int mlx5_wq_cyc_is_full(struct mlx5_wq_cyc *wq) 102 { 103 return wq->cur_sz == wq->sz; 104 } 105 106 static inline int mlx5_wq_cyc_missing(struct mlx5_wq_cyc *wq) 107 { 108 return wq->sz - wq->cur_sz; 109 } 110 111 static inline int mlx5_wq_cyc_is_empty(struct mlx5_wq_cyc *wq) 112 { 113 return !wq->cur_sz; 114 } 115 116 static inline void mlx5_wq_cyc_push(struct mlx5_wq_cyc *wq) 117 { 118 wq->wqe_ctr++; 119 wq->cur_sz++; 120 } 121 122 static inline void mlx5_wq_cyc_push_n(struct mlx5_wq_cyc *wq, u8 n) 123 { 124 wq->wqe_ctr += n; 125 wq->cur_sz += n; 126 } 127 128 static inline void mlx5_wq_cyc_pop(struct mlx5_wq_cyc *wq) 129 { 130 wq->cur_sz--; 131 } 132 133 static inline void mlx5_wq_cyc_update_db_record(struct mlx5_wq_cyc *wq) 134 { 135 *wq->db = cpu_to_be32(wq->wqe_ctr); 136 } 137 138 static inline u16 mlx5_wq_cyc_ctr2ix(struct mlx5_wq_cyc *wq, u16 ctr) 139 { 140 return ctr & wq->fbc.sz_m1; 141 } 142 143 static inline u16 mlx5_wq_cyc_get_head(struct mlx5_wq_cyc *wq) 144 { 145 return mlx5_wq_cyc_ctr2ix(wq, wq->wqe_ctr); 146 } 147 148 static inline u16 mlx5_wq_cyc_get_tail(struct mlx5_wq_cyc *wq) 149 { 150 return mlx5_wq_cyc_ctr2ix(wq, wq->wqe_ctr - wq->cur_sz); 151 } 152 153 static inline void *mlx5_wq_cyc_get_wqe(struct mlx5_wq_cyc *wq, u16 ix) 154 { 155 return mlx5_frag_buf_get_wqe(&wq->fbc, ix); 156 } 157 158 static inline u16 mlx5_wq_cyc_get_contig_wqebbs(struct mlx5_wq_cyc *wq, u16 ix) 159 { 160 return mlx5_frag_buf_get_idx_last_contig_stride(&wq->fbc, ix) - ix + 1; 161 } 162 163 static inline int mlx5_wq_cyc_cc_bigger(u16 cc1, u16 cc2) 164 { 165 int equal = (cc1 == cc2); 166 int smaller = 0x8000 & (cc1 - cc2); 167 168 return !equal && !smaller; 169 } 170 171 static inline u32 mlx5_cqwq_ctr2ix(struct mlx5_cqwq *wq, u32 ctr) 172 { 173 return ctr & wq->fbc.sz_m1; 174 } 175 176 static inline u32 mlx5_cqwq_get_ci(struct mlx5_cqwq *wq) 177 { 178 return mlx5_cqwq_ctr2ix(wq, wq->cc); 179 } 180 181 static inline struct mlx5_cqe64 *mlx5_cqwq_get_wqe(struct mlx5_cqwq *wq, u32 ix) 182 { 183 struct mlx5_cqe64 *cqe = mlx5_frag_buf_get_wqe(&wq->fbc, ix); 184 185 /* For 128B CQEs the data is in the last 64B */ 186 cqe += wq->fbc.log_stride == 7; 187 188 return cqe; 189 } 190 191 static inline u32 mlx5_cqwq_get_ctr_wrap_cnt(struct mlx5_cqwq *wq, u32 ctr) 192 { 193 return ctr >> wq->fbc.log_sz; 194 } 195 196 static inline u32 mlx5_cqwq_get_wrap_cnt(struct mlx5_cqwq *wq) 197 { 198 return mlx5_cqwq_get_ctr_wrap_cnt(wq, wq->cc); 199 } 200 201 static inline void mlx5_cqwq_pop(struct mlx5_cqwq *wq) 202 { 203 wq->cc++; 204 } 205 206 static inline void mlx5_cqwq_update_db_record(struct mlx5_cqwq *wq) 207 { 208 *wq->db = cpu_to_be32(wq->cc & 0xffffff); 209 } 210 211 static inline struct mlx5_cqe64 *mlx5_cqwq_get_cqe(struct mlx5_cqwq *wq) 212 { 213 u32 ci = mlx5_cqwq_get_ci(wq); 214 struct mlx5_cqe64 *cqe = mlx5_cqwq_get_wqe(wq, ci); 215 u8 cqe_ownership_bit = cqe->op_own & MLX5_CQE_OWNER_MASK; 216 u8 sw_ownership_val = mlx5_cqwq_get_wrap_cnt(wq) & 1; 217 218 if (cqe_ownership_bit != sw_ownership_val) 219 return NULL; 220 221 /* ensure cqe content is read after cqe ownership bit */ 222 dma_rmb(); 223 224 return cqe; 225 } 226 227 static inline int mlx5_wq_ll_is_full(struct mlx5_wq_ll *wq) 228 { 229 return wq->cur_sz == wq->fbc.sz_m1; 230 } 231 232 static inline int mlx5_wq_ll_is_empty(struct mlx5_wq_ll *wq) 233 { 234 return !wq->cur_sz; 235 } 236 237 static inline int mlx5_wq_ll_missing(struct mlx5_wq_ll *wq) 238 { 239 return wq->fbc.sz_m1 - wq->cur_sz; 240 } 241 242 static inline void *mlx5_wq_ll_get_wqe(struct mlx5_wq_ll *wq, u16 ix) 243 { 244 return mlx5_frag_buf_get_wqe(&wq->fbc, ix); 245 } 246 247 static inline u16 mlx5_wq_ll_get_wqe_next_ix(struct mlx5_wq_ll *wq, u16 ix) 248 { 249 struct mlx5_wqe_srq_next_seg *wqe = mlx5_wq_ll_get_wqe(wq, ix); 250 251 return be16_to_cpu(wqe->next_wqe_index); 252 } 253 254 static inline void mlx5_wq_ll_push(struct mlx5_wq_ll *wq, u16 head_next) 255 { 256 wq->head = head_next; 257 wq->wqe_ctr++; 258 wq->cur_sz++; 259 } 260 261 static inline void mlx5_wq_ll_pop(struct mlx5_wq_ll *wq, __be16 ix, 262 __be16 *next_tail_next) 263 { 264 *wq->tail_next = ix; 265 wq->tail_next = next_tail_next; 266 wq->cur_sz--; 267 } 268 269 static inline void mlx5_wq_ll_update_db_record(struct mlx5_wq_ll *wq) 270 { 271 *wq->db = cpu_to_be32(wq->wqe_ctr); 272 } 273 274 #endif /* __MLX5_WQ_H__ */ 275