1 /*- 2 * Copyright (c) 2013-2018, Mellanox Technologies, Ltd. All rights reserved. 3 * 4 * Redistribution and use in source and binary forms, with or without 5 * modification, are permitted provided that the following conditions 6 * are met: 7 * 1. Redistributions of source code must retain the above copyright 8 * notice, this list of conditions and the following disclaimer. 9 * 2. Redistributions in binary form must reproduce the above copyright 10 * notice, this list of conditions and the following disclaimer in the 11 * documentation and/or other materials provided with the distribution. 12 * 13 * THIS SOFTWARE IS PROVIDED BY AUTHOR AND CONTRIBUTORS `AS IS' AND 14 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 15 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 16 * ARE DISCLAIMED. IN NO EVENT SHALL AUTHOR OR CONTRIBUTORS BE LIABLE 17 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 18 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 19 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 20 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 21 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 22 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 23 * SUCH DAMAGE. 24 */ 25 26 #include "opt_rss.h" 27 #include "opt_ratelimit.h" 28 29 #include <linux/kernel.h> 30 #include <linux/module.h> 31 #include <dev/mlx5/driver.h> 32 #include <dev/mlx5/mlx5_core/mlx5_core.h> 33 34 static int mlx5_relaxed_ordering_write; 35 SYSCTL_INT(_hw_mlx5, OID_AUTO, relaxed_ordering_write, CTLFLAG_RWTUN, 36 &mlx5_relaxed_ordering_write, 0, 37 "Set to enable relaxed ordering for PCIe writes"); 38 39 void mlx5_init_mr_table(struct mlx5_core_dev *dev) 40 { 41 struct mlx5_mr_table *table = &dev->priv.mr_table; 42 43 memset(table, 0, sizeof(*table)); 44 spin_lock_init(&table->lock); 45 INIT_RADIX_TREE(&table->tree, GFP_ATOMIC); 46 } 47 48 void mlx5_cleanup_mr_table(struct mlx5_core_dev *dev) 49 { 50 } 51 52 int mlx5_core_create_mkey_cb(struct mlx5_core_dev *dev, 53 struct mlx5_core_mkey *mkey, 54 struct mlx5_async_ctx *async_ctx, u32 *in, 55 int inlen, u32 *out, int outlen, 56 mlx5_async_cbk_t callback, 57 struct mlx5_async_work *context) 58 { 59 struct mlx5_mr_table *table = &dev->priv.mr_table; 60 u32 lout[MLX5_ST_SZ_DW(create_mkey_out)] = {0}; 61 u32 mkey_index; 62 void *mkc; 63 unsigned long flags; 64 int err; 65 u8 key; 66 67 spin_lock_irq(&dev->priv.mkey_lock); 68 key = dev->priv.mkey_key++; 69 spin_unlock_irq(&dev->priv.mkey_lock); 70 mkc = MLX5_ADDR_OF(create_mkey_in, in, memory_key_mkey_entry); 71 MLX5_SET(create_mkey_in, in, opcode, MLX5_CMD_OP_CREATE_MKEY); 72 MLX5_SET(mkc, mkc, mkey_7_0, key); 73 74 if (mlx5_relaxed_ordering_write != 0) { 75 if (MLX5_CAP_GEN(dev, relaxed_ordering_write)) 76 MLX5_SET(mkc, mkc, relaxed_ordering_write, 1); 77 else 78 return (-EPROTONOSUPPORT); 79 } 80 81 if (callback) 82 return mlx5_cmd_exec_cb(async_ctx, in, inlen, out, outlen, 83 callback, context); 84 85 err = mlx5_cmd_exec(dev, in, inlen, lout, sizeof(lout)); 86 if (err) { 87 mlx5_core_dbg(dev, "cmd exec failed %d\n", err); 88 return err; 89 } 90 91 mkey_index = MLX5_GET(create_mkey_out, lout, mkey_index); 92 mkey->iova = MLX5_GET64(mkc, mkc, start_addr); 93 mkey->size = MLX5_GET64(mkc, mkc, len); 94 mkey->key = mlx5_idx_to_mkey(mkey_index) | key; 95 mkey->pd = MLX5_GET(mkc, mkc, pd); 96 97 mlx5_core_dbg(dev, "out 0x%x, key 0x%x, mkey 0x%x\n", 98 mkey_index, key, mkey->key); 99 100 /* connect to MR tree */ 101 spin_lock_irqsave(&table->lock, flags); 102 err = radix_tree_insert(&table->tree, mlx5_mkey_to_idx(mkey->key), mkey); 103 spin_unlock_irqrestore(&table->lock, flags); 104 if (err) { 105 mlx5_core_warn(dev, "failed radix tree insert of mr 0x%x, %d\n", 106 mkey->key, err); 107 mlx5_core_destroy_mkey(dev, mkey); 108 } 109 110 return err; 111 } 112 EXPORT_SYMBOL(mlx5_core_create_mkey_cb); 113 114 int mlx5_core_create_mkey(struct mlx5_core_dev *dev, 115 struct mlx5_core_mkey *mkey, 116 u32 *in, int inlen) 117 { 118 return mlx5_core_create_mkey_cb(dev, mkey, NULL, in, inlen, 119 NULL, 0, NULL, NULL); 120 } 121 EXPORT_SYMBOL(mlx5_core_create_mkey); 122 123 int mlx5_core_destroy_mkey(struct mlx5_core_dev *dev, struct mlx5_core_mkey *mkey) 124 { 125 struct mlx5_mr_table *table = &dev->priv.mr_table; 126 u32 out[MLX5_ST_SZ_DW(destroy_mkey_out)] = {0}; 127 u32 in[MLX5_ST_SZ_DW(destroy_mkey_in)] = {0}; 128 struct mlx5_core_mkey *deleted_mr; 129 unsigned long flags; 130 131 spin_lock_irqsave(&table->lock, flags); 132 deleted_mr = radix_tree_delete(&table->tree, mlx5_mkey_to_idx(mkey->key)); 133 spin_unlock_irqrestore(&table->lock, flags); 134 if (!deleted_mr) { 135 mlx5_core_warn(dev, "failed radix tree delete of mr 0x%x\n", mkey->key); 136 return -ENOENT; 137 } 138 139 MLX5_SET(destroy_mkey_in, in, opcode, MLX5_CMD_OP_DESTROY_MKEY); 140 MLX5_SET(destroy_mkey_in, in, mkey_index, mlx5_mkey_to_idx(mkey->key)); 141 142 return mlx5_cmd_exec(dev, in, sizeof(in), out, sizeof(out)); 143 } 144 EXPORT_SYMBOL(mlx5_core_destroy_mkey); 145 146 int mlx5_core_query_mkey(struct mlx5_core_dev *dev, struct mlx5_core_mkey *mkey, 147 u32 *out, int outlen) 148 { 149 u32 in[MLX5_ST_SZ_DW(query_mkey_in)] = {0}; 150 151 memset(out, 0, outlen); 152 MLX5_SET(query_mkey_in, in, opcode, MLX5_CMD_OP_QUERY_MKEY); 153 MLX5_SET(query_mkey_in, in, mkey_index, mlx5_mkey_to_idx(mkey->key)); 154 155 return mlx5_cmd_exec(dev, in, sizeof(in), out, outlen); 156 } 157 EXPORT_SYMBOL(mlx5_core_query_mkey); 158 159 int mlx5_core_dump_fill_mkey(struct mlx5_core_dev *dev, struct mlx5_core_mkey *_mkey, 160 u32 *mkey) 161 { 162 u32 out[MLX5_ST_SZ_DW(query_special_contexts_out)] = {0}; 163 u32 in[MLX5_ST_SZ_DW(query_special_contexts_in)] = {0}; 164 int err; 165 166 MLX5_SET(query_special_contexts_in, in, opcode, 167 MLX5_CMD_OP_QUERY_SPECIAL_CONTEXTS); 168 err = mlx5_cmd_exec(dev, in, sizeof(in), out, sizeof(out)); 169 if (!err) 170 *mkey = MLX5_GET(query_special_contexts_out, out, dump_fill_mkey); 171 172 return err; 173 } 174 EXPORT_SYMBOL(mlx5_core_dump_fill_mkey); 175 176 static inline u32 mlx5_get_psv(u32 *out, int psv_index) 177 { 178 switch (psv_index) { 179 case 1: return MLX5_GET(create_psv_out, out, psv1_index); 180 case 2: return MLX5_GET(create_psv_out, out, psv2_index); 181 case 3: return MLX5_GET(create_psv_out, out, psv3_index); 182 default: return MLX5_GET(create_psv_out, out, psv0_index); 183 } 184 } 185 186 int mlx5_core_create_psv(struct mlx5_core_dev *dev, u32 pdn, 187 int npsvs, u32 *sig_index) 188 { 189 u32 out[MLX5_ST_SZ_DW(create_psv_out)] = {0}; 190 u32 in[MLX5_ST_SZ_DW(create_psv_in)] = {0}; 191 int i, err; 192 193 if (npsvs > MLX5_MAX_PSVS) 194 return -EINVAL; 195 196 MLX5_SET(create_psv_in, in, opcode, MLX5_CMD_OP_CREATE_PSV); 197 MLX5_SET(create_psv_in, in, pd, pdn); 198 MLX5_SET(create_psv_in, in, num_psv, npsvs); 199 err = mlx5_cmd_exec(dev, in, sizeof(in), out, sizeof(out)); 200 if (err) { 201 mlx5_core_err(dev, "create_psv cmd exec failed %d\n", err); 202 return err; 203 } 204 205 for (i = 0; i < npsvs; i++) 206 sig_index[i] = mlx5_get_psv(out, i); 207 208 return err; 209 } 210 EXPORT_SYMBOL(mlx5_core_create_psv); 211 212 int mlx5_core_destroy_psv(struct mlx5_core_dev *dev, int psv_num) 213 { 214 u32 out[MLX5_ST_SZ_DW(destroy_psv_out)] = {0}; 215 u32 in[MLX5_ST_SZ_DW(destroy_psv_in)] = {0}; 216 217 MLX5_SET(destroy_psv_in, in, opcode, MLX5_CMD_OP_DESTROY_PSV); 218 MLX5_SET(destroy_psv_in, in, psvn, psv_num); 219 return mlx5_cmd_exec(dev, in, sizeof(in), out, sizeof(out)); 220 } 221 EXPORT_SYMBOL(mlx5_core_destroy_psv); 222