1 /*- 2 * Copyright (c) 2013-2017, Mellanox Technologies, Ltd. All rights reserved. 3 * 4 * Redistribution and use in source and binary forms, with or without 5 * modification, are permitted provided that the following conditions 6 * are met: 7 * 1. Redistributions of source code must retain the above copyright 8 * notice, this list of conditions and the following disclaimer. 9 * 2. Redistributions in binary form must reproduce the above copyright 10 * notice, this list of conditions and the following disclaimer in the 11 * documentation and/or other materials provided with the distribution. 12 * 13 * THIS SOFTWARE IS PROVIDED BY AUTHOR AND CONTRIBUTORS `AS IS' AND 14 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 15 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 16 * ARE DISCLAIMED. IN NO EVENT SHALL AUTHOR OR CONTRIBUTORS BE LIABLE 17 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 18 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 19 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 20 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 21 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 22 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 23 * SUCH DAMAGE. 24 * 25 * $FreeBSD$ 26 */ 27 28 #include <linux/kernel.h> 29 #include <linux/module.h> 30 #include <linux/hardirq.h> 31 #include <dev/mlx5/driver.h> 32 #include <rdma/ib_verbs.h> 33 #include <dev/mlx5/cq.h> 34 #include "mlx5_core.h" 35 36 #include <sys/epoch.h> 37 38 static void 39 mlx5_cq_table_write_lock(struct mlx5_cq_table *table) 40 { 41 42 atomic_inc(&table->writercount); 43 /* make sure all see the updated writercount */ 44 NET_EPOCH_WAIT(); 45 spin_lock(&table->writerlock); 46 } 47 48 static void 49 mlx5_cq_table_write_unlock(struct mlx5_cq_table *table) 50 { 51 52 spin_unlock(&table->writerlock); 53 atomic_dec(&table->writercount); 54 /* drain all pending CQ callers */ 55 NET_EPOCH_WAIT(); 56 } 57 58 void mlx5_cq_completion(struct mlx5_core_dev *dev, u32 cqn) 59 { 60 struct mlx5_cq_table *table = &dev->priv.cq_table; 61 struct mlx5_core_cq *cq; 62 struct epoch_tracker et; 63 bool do_lock; 64 65 NET_EPOCH_ENTER(et); 66 67 do_lock = atomic_read(&table->writercount) != 0; 68 if (unlikely(do_lock)) 69 spin_lock(&table->writerlock); 70 71 if (likely(cqn < MLX5_CQ_LINEAR_ARRAY_SIZE)) 72 cq = table->linear_array[cqn].cq; 73 else 74 cq = radix_tree_lookup(&table->tree, cqn); 75 76 if (unlikely(do_lock)) 77 spin_unlock(&table->writerlock); 78 79 if (likely(cq != NULL)) { 80 ++cq->arm_sn; 81 cq->comp(cq); 82 } else { 83 mlx5_core_warn(dev, 84 "Completion event for bogus CQ 0x%x\n", cqn); 85 } 86 87 NET_EPOCH_EXIT(et); 88 } 89 90 void mlx5_cq_event(struct mlx5_core_dev *dev, u32 cqn, int event_type) 91 { 92 struct mlx5_cq_table *table = &dev->priv.cq_table; 93 struct mlx5_core_cq *cq; 94 struct epoch_tracker et; 95 bool do_lock; 96 97 NET_EPOCH_ENTER(et); 98 99 do_lock = atomic_read(&table->writercount) != 0; 100 if (unlikely(do_lock)) 101 spin_lock(&table->writerlock); 102 103 if (likely(cqn < MLX5_CQ_LINEAR_ARRAY_SIZE)) 104 cq = table->linear_array[cqn].cq; 105 else 106 cq = radix_tree_lookup(&table->tree, cqn); 107 108 if (unlikely(do_lock)) 109 spin_unlock(&table->writerlock); 110 111 if (likely(cq != NULL)) { 112 cq->event(cq, event_type); 113 } else { 114 mlx5_core_warn(dev, 115 "Asynchronous event for bogus CQ 0x%x\n", cqn); 116 } 117 118 NET_EPOCH_EXIT(et); 119 } 120 121 int mlx5_core_create_cq(struct mlx5_core_dev *dev, struct mlx5_core_cq *cq, 122 u32 *in, int inlen) 123 { 124 struct mlx5_cq_table *table = &dev->priv.cq_table; 125 u32 out[MLX5_ST_SZ_DW(create_cq_out)] = {0}; 126 u32 din[MLX5_ST_SZ_DW(destroy_cq_in)] = {0}; 127 u32 dout[MLX5_ST_SZ_DW(destroy_cq_out)] = {0}; 128 int err; 129 130 MLX5_SET(create_cq_in, in, opcode, MLX5_CMD_OP_CREATE_CQ); 131 err = mlx5_cmd_exec(dev, in, inlen, out, sizeof(out)); 132 if (err) 133 return err; 134 135 cq->cqn = MLX5_GET(create_cq_out, out, cqn); 136 cq->cons_index = 0; 137 cq->arm_sn = 0; 138 139 mlx5_cq_table_write_lock(table); 140 err = radix_tree_insert(&table->tree, cq->cqn, cq); 141 if (likely(err == 0 && cq->cqn < MLX5_CQ_LINEAR_ARRAY_SIZE)) 142 table->linear_array[cq->cqn].cq = cq; 143 mlx5_cq_table_write_unlock(table); 144 145 if (err) 146 goto err_cmd; 147 148 cq->pid = curthread->td_proc->p_pid; 149 150 return 0; 151 152 err_cmd: 153 MLX5_SET(destroy_cq_in, din, opcode, MLX5_CMD_OP_DESTROY_CQ); 154 MLX5_SET(destroy_cq_in, din, cqn, cq->cqn); 155 mlx5_cmd_exec(dev, din, sizeof(din), dout, sizeof(dout)); 156 return err; 157 } 158 EXPORT_SYMBOL(mlx5_core_create_cq); 159 160 int mlx5_core_destroy_cq(struct mlx5_core_dev *dev, struct mlx5_core_cq *cq) 161 { 162 struct mlx5_cq_table *table = &dev->priv.cq_table; 163 u32 out[MLX5_ST_SZ_DW(destroy_cq_out)] = {0}; 164 u32 in[MLX5_ST_SZ_DW(destroy_cq_in)] = {0}; 165 struct mlx5_core_cq *tmp; 166 167 mlx5_cq_table_write_lock(table); 168 if (likely(cq->cqn < MLX5_CQ_LINEAR_ARRAY_SIZE)) 169 table->linear_array[cq->cqn].cq = NULL; 170 tmp = radix_tree_delete(&table->tree, cq->cqn); 171 mlx5_cq_table_write_unlock(table); 172 173 if (unlikely(tmp == NULL)) { 174 mlx5_core_warn(dev, "cq 0x%x not found in tree\n", cq->cqn); 175 return -EINVAL; 176 } else if (unlikely(tmp != cq)) { 177 mlx5_core_warn(dev, "corrupted cqn 0x%x\n", cq->cqn); 178 return -EINVAL; 179 } 180 181 MLX5_SET(destroy_cq_in, in, opcode, MLX5_CMD_OP_DESTROY_CQ); 182 MLX5_SET(destroy_cq_in, in, cqn, cq->cqn); 183 return mlx5_cmd_exec(dev, in, sizeof(in), out, sizeof(out)); 184 } 185 EXPORT_SYMBOL(mlx5_core_destroy_cq); 186 187 int mlx5_core_query_cq(struct mlx5_core_dev *dev, struct mlx5_core_cq *cq, 188 u32 *out, int outlen) 189 { 190 u32 in[MLX5_ST_SZ_DW(query_cq_in)] = {0}; 191 192 MLX5_SET(query_cq_in, in, opcode, MLX5_CMD_OP_QUERY_CQ); 193 MLX5_SET(query_cq_in, in, cqn, cq->cqn); 194 195 return mlx5_cmd_exec(dev, in, sizeof(in), out, outlen); 196 } 197 EXPORT_SYMBOL(mlx5_core_query_cq); 198 199 200 int mlx5_core_modify_cq(struct mlx5_core_dev *dev, struct mlx5_core_cq *cq, 201 u32 *in, int inlen) 202 { 203 u32 out[MLX5_ST_SZ_DW(modify_cq_out)] = {0}; 204 205 MLX5_SET(modify_cq_in, in, opcode, MLX5_CMD_OP_MODIFY_CQ); 206 return mlx5_cmd_exec(dev, in, inlen, out, sizeof(out)); 207 } 208 EXPORT_SYMBOL(mlx5_core_modify_cq); 209 210 int mlx5_core_modify_cq_moderation(struct mlx5_core_dev *dev, 211 struct mlx5_core_cq *cq, 212 u16 cq_period, 213 u16 cq_max_count) 214 { 215 u32 in[MLX5_ST_SZ_DW(modify_cq_in)] = {0}; 216 void *cqc; 217 218 MLX5_SET(modify_cq_in, in, cqn, cq->cqn); 219 cqc = MLX5_ADDR_OF(modify_cq_in, in, cq_context); 220 MLX5_SET(cqc, cqc, cq_period, cq_period); 221 MLX5_SET(cqc, cqc, cq_max_count, cq_max_count); 222 MLX5_SET(modify_cq_in, in, 223 modify_field_select_resize_field_select.modify_field_select.modify_field_select, 224 MLX5_CQ_MODIFY_PERIOD | MLX5_CQ_MODIFY_COUNT); 225 226 return mlx5_core_modify_cq(dev, cq, in, sizeof(in)); 227 } 228 229 int mlx5_core_modify_cq_moderation_mode(struct mlx5_core_dev *dev, 230 struct mlx5_core_cq *cq, 231 u16 cq_period, 232 u16 cq_max_count, 233 u8 cq_mode) 234 { 235 u32 in[MLX5_ST_SZ_DW(modify_cq_in)] = {0}; 236 void *cqc; 237 238 MLX5_SET(modify_cq_in, in, cqn, cq->cqn); 239 cqc = MLX5_ADDR_OF(modify_cq_in, in, cq_context); 240 MLX5_SET(cqc, cqc, cq_period, cq_period); 241 MLX5_SET(cqc, cqc, cq_max_count, cq_max_count); 242 MLX5_SET(cqc, cqc, cq_period_mode, cq_mode); 243 MLX5_SET(modify_cq_in, in, 244 modify_field_select_resize_field_select.modify_field_select.modify_field_select, 245 MLX5_CQ_MODIFY_PERIOD | MLX5_CQ_MODIFY_COUNT | MLX5_CQ_MODIFY_PERIOD_MODE); 246 247 return mlx5_core_modify_cq(dev, cq, in, sizeof(in)); 248 } 249 250 int mlx5_init_cq_table(struct mlx5_core_dev *dev) 251 { 252 struct mlx5_cq_table *table = &dev->priv.cq_table; 253 254 memset(table, 0, sizeof(*table)); 255 spin_lock_init(&table->writerlock); 256 INIT_RADIX_TREE(&table->tree, GFP_ATOMIC); 257 258 return 0; 259 } 260 261 void mlx5_cleanup_cq_table(struct mlx5_core_dev *dev) 262 { 263 } 264