112515907SHans Petter Selasky /*-
2e47a6525SHans Petter Selasky * Copyright (c) 2013-2021, Mellanox Technologies, Ltd. All rights reserved.
312515907SHans Petter Selasky *
412515907SHans Petter Selasky * Redistribution and use in source and binary forms, with or without
512515907SHans Petter Selasky * modification, are permitted provided that the following conditions
612515907SHans Petter Selasky * are met:
712515907SHans Petter Selasky * 1. Redistributions of source code must retain the above copyright
812515907SHans Petter Selasky * notice, this list of conditions and the following disclaimer.
912515907SHans Petter Selasky * 2. Redistributions in binary form must reproduce the above copyright
1012515907SHans Petter Selasky * notice, this list of conditions and the following disclaimer in the
1112515907SHans Petter Selasky * documentation and/or other materials provided with the distribution.
1212515907SHans Petter Selasky *
1312515907SHans Petter Selasky * THIS SOFTWARE IS PROVIDED BY AUTHOR AND CONTRIBUTORS `AS IS' AND
1412515907SHans Petter Selasky * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
1512515907SHans Petter Selasky * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
1612515907SHans Petter Selasky * ARE DISCLAIMED. IN NO EVENT SHALL AUTHOR OR CONTRIBUTORS BE LIABLE
1712515907SHans Petter Selasky * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
1812515907SHans Petter Selasky * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
1912515907SHans Petter Selasky * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
2012515907SHans Petter Selasky * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
2112515907SHans Petter Selasky * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
2212515907SHans Petter Selasky * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
2312515907SHans Petter Selasky * SUCH DAMAGE.
2412515907SHans Petter Selasky */
2512515907SHans Petter Selasky
2670600979SKonstantin Belousov #include "opt_rss.h"
2770600979SKonstantin Belousov #include "opt_ratelimit.h"
2870600979SKonstantin Belousov
298e6e287fSHans Petter Selasky #include <linux/module.h>
3012515907SHans Petter Selasky #include <linux/errno.h>
3112515907SHans Petter Selasky #include <linux/pci.h>
3212515907SHans Petter Selasky #include <linux/dma-mapping.h>
3312515907SHans Petter Selasky #include <linux/slab.h>
348e6e287fSHans Petter Selasky #if defined(CONFIG_X86)
358e6e287fSHans Petter Selasky #include <asm/pat.h>
368e6e287fSHans Petter Selasky #endif
3712515907SHans Petter Selasky #include <linux/sched.h>
388e6e287fSHans Petter Selasky #include <linux/delay.h>
3912515907SHans Petter Selasky #include <linux/fs.h>
4012515907SHans Petter Selasky #undef inode
4112515907SHans Petter Selasky #include <rdma/ib_user_verbs.h>
428e6e287fSHans Petter Selasky #include <rdma/ib_addr.h>
438e6e287fSHans Petter Selasky #include <rdma/ib_cache.h>
44d9142151SHans Petter Selasky #include <dev/mlx5/port.h>
458e6e287fSHans Petter Selasky #include <dev/mlx5/vport.h>
468e6e287fSHans Petter Selasky #include <linux/list.h>
4712515907SHans Petter Selasky #include <rdma/ib_smi.h>
4812515907SHans Petter Selasky #include <rdma/ib_umem.h>
49b633e08cSHans Petter Selasky #include <rdma/uverbs_ioctl.h>
508e6e287fSHans Petter Selasky #include <linux/in.h>
518e6e287fSHans Petter Selasky #include <linux/etherdevice.h>
528e6e287fSHans Petter Selasky #include <dev/mlx5/fs.h>
53028130b8SKonstantin Belousov #include <dev/mlx5/mlx5_ib/mlx5_ib.h>
5412515907SHans Petter Selasky
5512515907SHans Petter Selasky MODULE_DESCRIPTION("Mellanox Connect-IB HCA IB driver");
5612515907SHans Petter Selasky MODULE_LICENSE("Dual BSD/GPL");
5712515907SHans Petter Selasky MODULE_DEPEND(mlx5ib, linuxkpi, 1, 1, 1);
5812515907SHans Petter Selasky MODULE_DEPEND(mlx5ib, mlx5, 1, 1, 1);
5912515907SHans Petter Selasky MODULE_DEPEND(mlx5ib, ibcore, 1, 1, 1);
6012515907SHans Petter Selasky MODULE_VERSION(mlx5ib, 1);
6112515907SHans Petter Selasky
628e6e287fSHans Petter Selasky enum {
638e6e287fSHans Petter Selasky MLX5_ATOMIC_SIZE_QP_8BYTES = 1 << 3,
648e6e287fSHans Petter Selasky };
6512515907SHans Petter Selasky
6612515907SHans Petter Selasky static enum rdma_link_layer
mlx5_port_type_cap_to_rdma_ll(int port_type_cap)678e6e287fSHans Petter Selasky mlx5_port_type_cap_to_rdma_ll(int port_type_cap)
6812515907SHans Petter Selasky {
698e6e287fSHans Petter Selasky switch (port_type_cap) {
7012515907SHans Petter Selasky case MLX5_CAP_PORT_TYPE_IB:
7112515907SHans Petter Selasky return IB_LINK_LAYER_INFINIBAND;
7212515907SHans Petter Selasky case MLX5_CAP_PORT_TYPE_ETH:
7312515907SHans Petter Selasky return IB_LINK_LAYER_ETHERNET;
7412515907SHans Petter Selasky default:
7512515907SHans Petter Selasky return IB_LINK_LAYER_UNSPECIFIED;
7612515907SHans Petter Selasky }
7712515907SHans Petter Selasky }
7812515907SHans Petter Selasky
798e6e287fSHans Petter Selasky static enum rdma_link_layer
mlx5_ib_port_link_layer(struct ib_device * device,u8 port_num)808e6e287fSHans Petter Selasky mlx5_ib_port_link_layer(struct ib_device *device, u8 port_num)
818e6e287fSHans Petter Selasky {
828e6e287fSHans Petter Selasky struct mlx5_ib_dev *dev = to_mdev(device);
838e6e287fSHans Petter Selasky int port_type_cap = MLX5_CAP_GEN(dev->mdev, port_type);
848e6e287fSHans Petter Selasky
858e6e287fSHans Petter Selasky return mlx5_port_type_cap_to_rdma_ll(port_type_cap);
868e6e287fSHans Petter Selasky }
878e6e287fSHans Petter Selasky
mlx5_netdev_match(if_t ndev,struct mlx5_core_dev * mdev,const char * dname)885dc00f00SJustin Hibbits static bool mlx5_netdev_match(if_t ndev,
898e6e287fSHans Petter Selasky struct mlx5_core_dev *mdev,
908e6e287fSHans Petter Selasky const char *dname)
918e6e287fSHans Petter Selasky {
925dc00f00SJustin Hibbits return if_gettype(ndev) == IFT_ETHER &&
935dc00f00SJustin Hibbits if_getdname(ndev) != NULL &&
945dc00f00SJustin Hibbits strcmp(if_getdname(ndev), dname) == 0 &&
955dc00f00SJustin Hibbits if_getsoftc(ndev) != NULL &&
965dc00f00SJustin Hibbits *(struct mlx5_core_dev **)if_getsoftc(ndev) == mdev;
978e6e287fSHans Petter Selasky }
988e6e287fSHans Petter Selasky
mlx5_netdev_event(struct notifier_block * this,unsigned long event,void * ptr)998e6e287fSHans Petter Selasky static int mlx5_netdev_event(struct notifier_block *this,
1008e6e287fSHans Petter Selasky unsigned long event, void *ptr)
1018e6e287fSHans Petter Selasky {
1025dc00f00SJustin Hibbits if_t ndev = netdev_notifier_info_to_ifp(ptr);
1038e6e287fSHans Petter Selasky struct mlx5_ib_dev *ibdev = container_of(this, struct mlx5_ib_dev,
1048e6e287fSHans Petter Selasky roce.nb);
1058e6e287fSHans Petter Selasky
1068e6e287fSHans Petter Selasky switch (event) {
1078e6e287fSHans Petter Selasky case NETDEV_REGISTER:
1088e6e287fSHans Petter Selasky case NETDEV_UNREGISTER:
1098e6e287fSHans Petter Selasky write_lock(&ibdev->roce.netdev_lock);
1108e6e287fSHans Petter Selasky /* check if network interface belongs to mlx5en */
1118e6e287fSHans Petter Selasky if (mlx5_netdev_match(ndev, ibdev->mdev, "mce"))
1128e6e287fSHans Petter Selasky ibdev->roce.netdev = (event == NETDEV_UNREGISTER) ?
1138e6e287fSHans Petter Selasky NULL : ndev;
1148e6e287fSHans Petter Selasky write_unlock(&ibdev->roce.netdev_lock);
1158e6e287fSHans Petter Selasky break;
1168e6e287fSHans Petter Selasky
1178e6e287fSHans Petter Selasky case NETDEV_UP:
1188e6e287fSHans Petter Selasky case NETDEV_DOWN: {
1195dc00f00SJustin Hibbits if_t upper = NULL;
1208e6e287fSHans Petter Selasky
1218e6e287fSHans Petter Selasky if ((upper == ndev || (!upper && ndev == ibdev->roce.netdev))
1228e6e287fSHans Petter Selasky && ibdev->ib_active) {
1238e6e287fSHans Petter Selasky struct ib_event ibev = {0};
1248e6e287fSHans Petter Selasky
1258e6e287fSHans Petter Selasky ibev.device = &ibdev->ib_dev;
1268e6e287fSHans Petter Selasky ibev.event = (event == NETDEV_UP) ?
1278e6e287fSHans Petter Selasky IB_EVENT_PORT_ACTIVE : IB_EVENT_PORT_ERR;
1288e6e287fSHans Petter Selasky ibev.element.port_num = 1;
1298e6e287fSHans Petter Selasky ib_dispatch_event(&ibev);
1308e6e287fSHans Petter Selasky }
1318e6e287fSHans Petter Selasky break;
1328e6e287fSHans Petter Selasky }
1338e6e287fSHans Petter Selasky
1348e6e287fSHans Petter Selasky default:
1358e6e287fSHans Petter Selasky break;
1368e6e287fSHans Petter Selasky }
1378e6e287fSHans Petter Selasky
1388e6e287fSHans Petter Selasky return NOTIFY_DONE;
1398e6e287fSHans Petter Selasky }
1408e6e287fSHans Petter Selasky
mlx5_ib_get_netdev(struct ib_device * device,u8 port_num)1415dc00f00SJustin Hibbits static if_t mlx5_ib_get_netdev(struct ib_device *device,
1428e6e287fSHans Petter Selasky u8 port_num)
1438e6e287fSHans Petter Selasky {
1448e6e287fSHans Petter Selasky struct mlx5_ib_dev *ibdev = to_mdev(device);
1455dc00f00SJustin Hibbits if_t ndev;
1468e6e287fSHans Petter Selasky
1474ded022dSBjoern A. Zeeb /* Ensure ndev does not disappear before we invoke if_ref()
1488e6e287fSHans Petter Selasky */
1498e6e287fSHans Petter Selasky read_lock(&ibdev->roce.netdev_lock);
1508e6e287fSHans Petter Selasky ndev = ibdev->roce.netdev;
1518e6e287fSHans Petter Selasky if (ndev)
1524ded022dSBjoern A. Zeeb if_ref(ndev);
1538e6e287fSHans Petter Selasky read_unlock(&ibdev->roce.netdev_lock);
1548e6e287fSHans Petter Selasky
1558e6e287fSHans Petter Selasky return ndev;
1568e6e287fSHans Petter Selasky }
1578e6e287fSHans Petter Selasky
translate_eth_proto_oper(u32 eth_proto_oper,u8 * active_speed,u8 * active_width)158d9142151SHans Petter Selasky static int translate_eth_proto_oper(u32 eth_proto_oper, u8 *active_speed,
159d9142151SHans Petter Selasky u8 *active_width)
160d9142151SHans Petter Selasky {
161d9142151SHans Petter Selasky switch (eth_proto_oper) {
162d9142151SHans Petter Selasky case MLX5E_PROT_MASK(MLX5E_1000BASE_CX_SGMII):
163d9142151SHans Petter Selasky case MLX5E_PROT_MASK(MLX5E_1000BASE_KX):
164d9142151SHans Petter Selasky case MLX5E_PROT_MASK(MLX5E_100BASE_TX):
165d9142151SHans Petter Selasky case MLX5E_PROT_MASK(MLX5E_1000BASE_T):
166d9142151SHans Petter Selasky *active_width = IB_WIDTH_1X;
167d9142151SHans Petter Selasky *active_speed = IB_SPEED_SDR;
168d9142151SHans Petter Selasky break;
169d9142151SHans Petter Selasky case MLX5E_PROT_MASK(MLX5E_10GBASE_T):
170d9142151SHans Petter Selasky case MLX5E_PROT_MASK(MLX5E_10GBASE_CX4):
171d9142151SHans Petter Selasky case MLX5E_PROT_MASK(MLX5E_10GBASE_KX4):
172d9142151SHans Petter Selasky case MLX5E_PROT_MASK(MLX5E_10GBASE_KR):
173d9142151SHans Petter Selasky case MLX5E_PROT_MASK(MLX5E_10GBASE_CR):
174d9142151SHans Petter Selasky case MLX5E_PROT_MASK(MLX5E_10GBASE_SR):
175ac87880aSHans Petter Selasky case MLX5E_PROT_MASK(MLX5E_10GBASE_ER_LR):
176d9142151SHans Petter Selasky *active_width = IB_WIDTH_1X;
177d9142151SHans Petter Selasky *active_speed = IB_SPEED_QDR;
178d9142151SHans Petter Selasky break;
179d9142151SHans Petter Selasky case MLX5E_PROT_MASK(MLX5E_25GBASE_CR):
180d9142151SHans Petter Selasky case MLX5E_PROT_MASK(MLX5E_25GBASE_KR):
181d9142151SHans Petter Selasky case MLX5E_PROT_MASK(MLX5E_25GBASE_SR):
182d9142151SHans Petter Selasky *active_width = IB_WIDTH_1X;
183d9142151SHans Petter Selasky *active_speed = IB_SPEED_EDR;
184d9142151SHans Petter Selasky break;
185d9142151SHans Petter Selasky case MLX5E_PROT_MASK(MLX5E_40GBASE_CR4):
186d9142151SHans Petter Selasky case MLX5E_PROT_MASK(MLX5E_40GBASE_KR4):
187d9142151SHans Petter Selasky case MLX5E_PROT_MASK(MLX5E_40GBASE_SR4):
188ac87880aSHans Petter Selasky case MLX5E_PROT_MASK(MLX5E_40GBASE_LR4_ER4):
189d9142151SHans Petter Selasky *active_width = IB_WIDTH_4X;
190d9142151SHans Petter Selasky *active_speed = IB_SPEED_QDR;
191d9142151SHans Petter Selasky break;
192d9142151SHans Petter Selasky case MLX5E_PROT_MASK(MLX5E_50GBASE_CR2):
193d9142151SHans Petter Selasky case MLX5E_PROT_MASK(MLX5E_50GBASE_KR2):
19496dad2b7SKonstantin Belousov case MLX5E_PROT_MASK(MLX5E_50GBASE_KR4):
195d9142151SHans Petter Selasky case MLX5E_PROT_MASK(MLX5E_50GBASE_SR2):
196d9142151SHans Petter Selasky *active_width = IB_WIDTH_1X;
197d9142151SHans Petter Selasky *active_speed = IB_SPEED_HDR;
198d9142151SHans Petter Selasky break;
199d9142151SHans Petter Selasky case MLX5E_PROT_MASK(MLX5E_56GBASE_R4):
200d9142151SHans Petter Selasky *active_width = IB_WIDTH_4X;
201d9142151SHans Petter Selasky *active_speed = IB_SPEED_FDR;
202d9142151SHans Petter Selasky break;
203d9142151SHans Petter Selasky case MLX5E_PROT_MASK(MLX5E_100GBASE_CR4):
204d9142151SHans Petter Selasky case MLX5E_PROT_MASK(MLX5E_100GBASE_SR4):
205d9142151SHans Petter Selasky case MLX5E_PROT_MASK(MLX5E_100GBASE_KR4):
206d9142151SHans Petter Selasky case MLX5E_PROT_MASK(MLX5E_100GBASE_LR4):
207d9142151SHans Petter Selasky *active_width = IB_WIDTH_4X;
208d9142151SHans Petter Selasky *active_speed = IB_SPEED_EDR;
209d9142151SHans Petter Selasky break;
210d9142151SHans Petter Selasky default:
21167db6873SSlava Shwartsman *active_width = IB_WIDTH_4X;
21267db6873SSlava Shwartsman *active_speed = IB_SPEED_QDR;
213d9142151SHans Petter Selasky return -EINVAL;
214d9142151SHans Petter Selasky }
215d9142151SHans Petter Selasky
216d9142151SHans Petter Selasky return 0;
217d9142151SHans Petter Selasky }
218d9142151SHans Petter Selasky
translate_eth_ext_proto_oper(u32 eth_proto_oper,u8 * active_speed,u8 * active_width)21986a39779SHans Petter Selasky static int translate_eth_ext_proto_oper(u32 eth_proto_oper, u8 *active_speed,
22086a39779SHans Petter Selasky u8 *active_width)
22186a39779SHans Petter Selasky {
22286a39779SHans Petter Selasky switch (eth_proto_oper) {
22386a39779SHans Petter Selasky case MLX5E_PROT_MASK(MLX5E_SGMII_100M):
22486a39779SHans Petter Selasky case MLX5E_PROT_MASK(MLX5E_1000BASE_X_SGMII):
22586a39779SHans Petter Selasky *active_width = IB_WIDTH_1X;
22686a39779SHans Petter Selasky *active_speed = IB_SPEED_SDR;
22786a39779SHans Petter Selasky break;
22886a39779SHans Petter Selasky case MLX5E_PROT_MASK(MLX5E_5GBASE_R):
22986a39779SHans Petter Selasky *active_width = IB_WIDTH_1X;
23086a39779SHans Petter Selasky *active_speed = IB_SPEED_DDR;
23186a39779SHans Petter Selasky break;
23286a39779SHans Petter Selasky case MLX5E_PROT_MASK(MLX5E_10GBASE_XFI_XAUI_1):
23386a39779SHans Petter Selasky *active_width = IB_WIDTH_1X;
23486a39779SHans Petter Selasky *active_speed = IB_SPEED_QDR;
23586a39779SHans Petter Selasky break;
23686a39779SHans Petter Selasky case MLX5E_PROT_MASK(MLX5E_40GBASE_XLAUI_4_XLPPI_4):
23786a39779SHans Petter Selasky *active_width = IB_WIDTH_4X;
23886a39779SHans Petter Selasky *active_speed = IB_SPEED_QDR;
23986a39779SHans Petter Selasky break;
24086a39779SHans Petter Selasky case MLX5E_PROT_MASK(MLX5E_25GAUI_1_25GBASE_CR_KR):
24186a39779SHans Petter Selasky *active_width = IB_WIDTH_1X;
24286a39779SHans Petter Selasky *active_speed = IB_SPEED_EDR;
24386a39779SHans Petter Selasky break;
24486a39779SHans Petter Selasky case MLX5E_PROT_MASK(MLX5E_50GAUI_2_LAUI_2_50GBASE_CR2_KR2):
24586a39779SHans Petter Selasky *active_width = IB_WIDTH_2X;
24686a39779SHans Petter Selasky *active_speed = IB_SPEED_EDR;
24786a39779SHans Petter Selasky break;
24886a39779SHans Petter Selasky case MLX5E_PROT_MASK(MLX5E_50GAUI_1_LAUI_1_50GBASE_CR_KR):
24986a39779SHans Petter Selasky *active_width = IB_WIDTH_1X;
25086a39779SHans Petter Selasky *active_speed = IB_SPEED_HDR;
25186a39779SHans Petter Selasky break;
25286a39779SHans Petter Selasky case MLX5E_PROT_MASK(MLX5E_CAUI_4_100GBASE_CR4_KR4):
25386a39779SHans Petter Selasky *active_width = IB_WIDTH_4X;
25486a39779SHans Petter Selasky *active_speed = IB_SPEED_EDR;
25586a39779SHans Petter Selasky break;
25686a39779SHans Petter Selasky case MLX5E_PROT_MASK(MLX5E_100GAUI_2_100GBASE_CR2_KR2):
25786a39779SHans Petter Selasky *active_width = IB_WIDTH_2X;
25886a39779SHans Petter Selasky *active_speed = IB_SPEED_HDR;
25986a39779SHans Petter Selasky break;
26091c8ffd7SHans Petter Selasky case MLX5E_PROT_MASK(MLX5E_100GAUI_1_100GBASE_CR_KR):
26191c8ffd7SHans Petter Selasky *active_width = IB_WIDTH_1X;
26291c8ffd7SHans Petter Selasky *active_speed = IB_SPEED_NDR;
26391c8ffd7SHans Petter Selasky break;
26486a39779SHans Petter Selasky case MLX5E_PROT_MASK(MLX5E_200GAUI_4_200GBASE_CR4_KR4):
26586a39779SHans Petter Selasky *active_width = IB_WIDTH_4X;
26686a39779SHans Petter Selasky *active_speed = IB_SPEED_HDR;
26786a39779SHans Petter Selasky break;
26891c8ffd7SHans Petter Selasky case MLX5E_PROT_MASK(MLX5E_200GAUI_2_200GBASE_CR2_KR2):
26991c8ffd7SHans Petter Selasky *active_width = IB_WIDTH_2X;
27091c8ffd7SHans Petter Selasky *active_speed = IB_SPEED_NDR;
27191c8ffd7SHans Petter Selasky break;
27291c8ffd7SHans Petter Selasky case MLX5E_PROT_MASK(MLX5E_400GAUI_4_400GBASE_CR4_KR4):
27391c8ffd7SHans Petter Selasky *active_width = IB_WIDTH_4X;
27491c8ffd7SHans Petter Selasky *active_speed = IB_SPEED_NDR;
27591c8ffd7SHans Petter Selasky break;
27686a39779SHans Petter Selasky default:
27786a39779SHans Petter Selasky *active_width = IB_WIDTH_4X;
27886a39779SHans Petter Selasky *active_speed = IB_SPEED_QDR;
27986a39779SHans Petter Selasky return -EINVAL;
28086a39779SHans Petter Selasky }
28186a39779SHans Petter Selasky
28286a39779SHans Petter Selasky return 0;
28386a39779SHans Petter Selasky }
28486a39779SHans Petter Selasky
mlx5_query_port_roce(struct ib_device * device,u8 port_num,struct ib_port_attr * props)2858e6e287fSHans Petter Selasky static int mlx5_query_port_roce(struct ib_device *device, u8 port_num,
2868e6e287fSHans Petter Selasky struct ib_port_attr *props)
2878e6e287fSHans Petter Selasky {
2888e6e287fSHans Petter Selasky struct mlx5_ib_dev *dev = to_mdev(device);
28986a39779SHans Petter Selasky u32 out[MLX5_ST_SZ_DW(ptys_reg)] = {};
2905dc00f00SJustin Hibbits if_t ndev;
2918e6e287fSHans Petter Selasky enum ib_mtu ndev_ib_mtu;
2928e6e287fSHans Petter Selasky u16 qkey_viol_cntr;
293d9142151SHans Petter Selasky u32 eth_prot_oper;
29486a39779SHans Petter Selasky bool ext;
295d9142151SHans Petter Selasky int err;
2968e6e287fSHans Petter Selasky
2978e6e287fSHans Petter Selasky memset(props, 0, sizeof(*props));
2988e6e287fSHans Petter Selasky
299d9142151SHans Petter Selasky /* Possible bad flows are checked before filling out props so in case
300d9142151SHans Petter Selasky * of an error it will still be zeroed out.
301d9142151SHans Petter Selasky */
30286a39779SHans Petter Selasky err = mlx5_query_port_ptys(dev->mdev, out, sizeof(out), MLX5_PTYS_EN,
30386a39779SHans Petter Selasky port_num);
304d9142151SHans Petter Selasky if (err)
305d9142151SHans Petter Selasky return err;
306d9142151SHans Petter Selasky
30786a39779SHans Petter Selasky ext = MLX5_CAP_PCAM_FEATURE(dev->mdev, ptys_extended_ethernet);
30886a39779SHans Petter Selasky eth_prot_oper = MLX5_GET_ETH_PROTO(ptys_reg, out, ext, eth_proto_oper);
30986a39779SHans Petter Selasky
31086a39779SHans Petter Selasky if (ext)
31186a39779SHans Petter Selasky translate_eth_ext_proto_oper(eth_prot_oper, &props->active_speed,
31286a39779SHans Petter Selasky &props->active_width);
31386a39779SHans Petter Selasky else
314d9142151SHans Petter Selasky translate_eth_proto_oper(eth_prot_oper, &props->active_speed,
315d9142151SHans Petter Selasky &props->active_width);
316d9142151SHans Petter Selasky
3178e6e287fSHans Petter Selasky props->port_cap_flags |= IB_PORT_CM_SUP;
3188e6e287fSHans Petter Selasky props->port_cap_flags |= IB_PORT_IP_BASED_GIDS;
3198e6e287fSHans Petter Selasky
3208e6e287fSHans Petter Selasky props->gid_tbl_len = MLX5_CAP_ROCE(dev->mdev,
3218e6e287fSHans Petter Selasky roce_address_table_size);
3228e6e287fSHans Petter Selasky props->max_mtu = IB_MTU_4096;
3238e6e287fSHans Petter Selasky props->max_msg_sz = 1 << MLX5_CAP_GEN(dev->mdev, log_max_msg);
3248e6e287fSHans Petter Selasky props->pkey_tbl_len = 1;
3258e6e287fSHans Petter Selasky props->state = IB_PORT_DOWN;
3264238b4a7SHans Petter Selasky props->phys_state = IB_PORT_PHYS_STATE_DISABLED;
3278e6e287fSHans Petter Selasky
3288e6e287fSHans Petter Selasky mlx5_query_nic_vport_qkey_viol_cntr(dev->mdev, &qkey_viol_cntr);
3298e6e287fSHans Petter Selasky props->qkey_viol_cntr = qkey_viol_cntr;
3308e6e287fSHans Petter Selasky
3318e6e287fSHans Petter Selasky ndev = mlx5_ib_get_netdev(device, port_num);
3328e6e287fSHans Petter Selasky if (!ndev)
3338e6e287fSHans Petter Selasky return 0;
3348e6e287fSHans Petter Selasky
3355dc00f00SJustin Hibbits if (if_getdrvflags(ndev) & IFF_DRV_RUNNING &&
3365dc00f00SJustin Hibbits if_getlinkstate(ndev) == LINK_STATE_UP) {
3378e6e287fSHans Petter Selasky props->state = IB_PORT_ACTIVE;
3384238b4a7SHans Petter Selasky props->phys_state = IB_PORT_PHYS_STATE_LINK_UP;
3398e6e287fSHans Petter Selasky }
3408e6e287fSHans Petter Selasky
3415dc00f00SJustin Hibbits ndev_ib_mtu = iboe_get_mtu(if_getmtu(ndev));
3428e6e287fSHans Petter Selasky
3434ded022dSBjoern A. Zeeb if_rele(ndev);
3448e6e287fSHans Petter Selasky
3458e6e287fSHans Petter Selasky props->active_mtu = min(props->max_mtu, ndev_ib_mtu);
3468e6e287fSHans Petter Selasky return 0;
3478e6e287fSHans Petter Selasky }
3488e6e287fSHans Petter Selasky
ib_gid_to_mlx5_roce_addr(const union ib_gid * gid,const struct ib_gid_attr * attr,void * mlx5_addr)3498e6e287fSHans Petter Selasky static void ib_gid_to_mlx5_roce_addr(const union ib_gid *gid,
3508e6e287fSHans Petter Selasky const struct ib_gid_attr *attr,
3518e6e287fSHans Petter Selasky void *mlx5_addr)
3528e6e287fSHans Petter Selasky {
3538e6e287fSHans Petter Selasky #define MLX5_SET_RA(p, f, v) MLX5_SET(roce_addr_layout, p, f, v)
3548e6e287fSHans Petter Selasky char *mlx5_addr_l3_addr = MLX5_ADDR_OF(roce_addr_layout, mlx5_addr,
3558e6e287fSHans Petter Selasky source_l3_address);
3568e6e287fSHans Petter Selasky void *mlx5_addr_mac = MLX5_ADDR_OF(roce_addr_layout, mlx5_addr,
3578e6e287fSHans Petter Selasky source_mac_47_32);
358f4546fa3SHans Petter Selasky u16 vlan_id;
3598e6e287fSHans Petter Selasky
3608e6e287fSHans Petter Selasky if (!gid)
3618e6e287fSHans Petter Selasky return;
3625dc00f00SJustin Hibbits ether_addr_copy(mlx5_addr_mac, if_getlladdr(attr->ndev));
3638e6e287fSHans Petter Selasky
364f4546fa3SHans Petter Selasky vlan_id = rdma_vlan_dev_vlan_id(attr->ndev);
365f4546fa3SHans Petter Selasky if (vlan_id != 0xffff) {
3668e6e287fSHans Petter Selasky MLX5_SET_RA(mlx5_addr, vlan_valid, 1);
367f4546fa3SHans Petter Selasky MLX5_SET_RA(mlx5_addr, vlan_id, vlan_id);
3688e6e287fSHans Petter Selasky }
3698e6e287fSHans Petter Selasky
3708e6e287fSHans Petter Selasky switch (attr->gid_type) {
3718e6e287fSHans Petter Selasky case IB_GID_TYPE_IB:
3728e6e287fSHans Petter Selasky MLX5_SET_RA(mlx5_addr, roce_version, MLX5_ROCE_VERSION_1);
3738e6e287fSHans Petter Selasky break;
3748e6e287fSHans Petter Selasky case IB_GID_TYPE_ROCE_UDP_ENCAP:
3758e6e287fSHans Petter Selasky MLX5_SET_RA(mlx5_addr, roce_version, MLX5_ROCE_VERSION_2);
3768e6e287fSHans Petter Selasky break;
3778e6e287fSHans Petter Selasky
3788e6e287fSHans Petter Selasky default:
3798e6e287fSHans Petter Selasky WARN_ON(true);
3808e6e287fSHans Petter Selasky }
3818e6e287fSHans Petter Selasky
3828e6e287fSHans Petter Selasky if (attr->gid_type != IB_GID_TYPE_IB) {
3838e6e287fSHans Petter Selasky if (ipv6_addr_v4mapped((void *)gid))
3848e6e287fSHans Petter Selasky MLX5_SET_RA(mlx5_addr, roce_l3_type,
3858e6e287fSHans Petter Selasky MLX5_ROCE_L3_TYPE_IPV4);
3868e6e287fSHans Petter Selasky else
3878e6e287fSHans Petter Selasky MLX5_SET_RA(mlx5_addr, roce_l3_type,
3888e6e287fSHans Petter Selasky MLX5_ROCE_L3_TYPE_IPV6);
3898e6e287fSHans Petter Selasky }
3908e6e287fSHans Petter Selasky
3918e6e287fSHans Petter Selasky if ((attr->gid_type == IB_GID_TYPE_IB) ||
3928e6e287fSHans Petter Selasky !ipv6_addr_v4mapped((void *)gid))
3938e6e287fSHans Petter Selasky memcpy(mlx5_addr_l3_addr, gid, sizeof(*gid));
3948e6e287fSHans Petter Selasky else
3958e6e287fSHans Petter Selasky memcpy(&mlx5_addr_l3_addr[12], &gid->raw[12], 4);
3968e6e287fSHans Petter Selasky }
3978e6e287fSHans Petter Selasky
set_roce_addr(struct ib_device * device,u8 port_num,unsigned int index,const union ib_gid * gid,const struct ib_gid_attr * attr)3988e6e287fSHans Petter Selasky static int set_roce_addr(struct ib_device *device, u8 port_num,
3998e6e287fSHans Petter Selasky unsigned int index,
4008e6e287fSHans Petter Selasky const union ib_gid *gid,
4018e6e287fSHans Petter Selasky const struct ib_gid_attr *attr)
4028e6e287fSHans Petter Selasky {
4038e6e287fSHans Petter Selasky struct mlx5_ib_dev *dev = to_mdev(device);
4048e6e287fSHans Petter Selasky u32 in[MLX5_ST_SZ_DW(set_roce_address_in)] = {0};
4058e6e287fSHans Petter Selasky u32 out[MLX5_ST_SZ_DW(set_roce_address_out)] = {0};
4068e6e287fSHans Petter Selasky void *in_addr = MLX5_ADDR_OF(set_roce_address_in, in, roce_address);
4078e6e287fSHans Petter Selasky enum rdma_link_layer ll = mlx5_ib_port_link_layer(device, port_num);
4088e6e287fSHans Petter Selasky
4098e6e287fSHans Petter Selasky if (ll != IB_LINK_LAYER_ETHERNET)
4108e6e287fSHans Petter Selasky return -EINVAL;
4118e6e287fSHans Petter Selasky
4128e6e287fSHans Petter Selasky ib_gid_to_mlx5_roce_addr(gid, attr, in_addr);
4138e6e287fSHans Petter Selasky
4148e6e287fSHans Petter Selasky MLX5_SET(set_roce_address_in, in, roce_address_index, index);
4158e6e287fSHans Petter Selasky MLX5_SET(set_roce_address_in, in, opcode, MLX5_CMD_OP_SET_ROCE_ADDRESS);
4168e6e287fSHans Petter Selasky return mlx5_cmd_exec(dev->mdev, in, sizeof(in), out, sizeof(out));
4178e6e287fSHans Petter Selasky }
4188e6e287fSHans Petter Selasky
mlx5_ib_add_gid(struct ib_device * device,u8 port_num,unsigned int index,const union ib_gid * gid,const struct ib_gid_attr * attr,__always_unused void ** context)4198e6e287fSHans Petter Selasky static int mlx5_ib_add_gid(struct ib_device *device, u8 port_num,
4208e6e287fSHans Petter Selasky unsigned int index, const union ib_gid *gid,
4218e6e287fSHans Petter Selasky const struct ib_gid_attr *attr,
4228e6e287fSHans Petter Selasky __always_unused void **context)
4238e6e287fSHans Petter Selasky {
4248e6e287fSHans Petter Selasky return set_roce_addr(device, port_num, index, gid, attr);
4258e6e287fSHans Petter Selasky }
4268e6e287fSHans Petter Selasky
mlx5_ib_del_gid(struct ib_device * device,u8 port_num,unsigned int index,__always_unused void ** context)4278e6e287fSHans Petter Selasky static int mlx5_ib_del_gid(struct ib_device *device, u8 port_num,
4288e6e287fSHans Petter Selasky unsigned int index, __always_unused void **context)
4298e6e287fSHans Petter Selasky {
4308e6e287fSHans Petter Selasky return set_roce_addr(device, port_num, index, NULL, NULL);
4318e6e287fSHans Petter Selasky }
4328e6e287fSHans Petter Selasky
mlx5_get_roce_udp_sport(struct mlx5_ib_dev * dev,u8 port_num,int index)4338e6e287fSHans Petter Selasky __be16 mlx5_get_roce_udp_sport(struct mlx5_ib_dev *dev, u8 port_num,
4348e6e287fSHans Petter Selasky int index)
4358e6e287fSHans Petter Selasky {
4368e6e287fSHans Petter Selasky struct ib_gid_attr attr;
4378e6e287fSHans Petter Selasky union ib_gid gid;
4388e6e287fSHans Petter Selasky
4398e6e287fSHans Petter Selasky if (ib_get_cached_gid(&dev->ib_dev, port_num, index, &gid, &attr))
4408e6e287fSHans Petter Selasky return 0;
4418e6e287fSHans Petter Selasky
4428e6e287fSHans Petter Selasky if (!attr.ndev)
4438e6e287fSHans Petter Selasky return 0;
4448e6e287fSHans Petter Selasky
4454ded022dSBjoern A. Zeeb if_rele(attr.ndev);
4468e6e287fSHans Petter Selasky
4478e6e287fSHans Petter Selasky if (attr.gid_type != IB_GID_TYPE_ROCE_UDP_ENCAP)
4488e6e287fSHans Petter Selasky return 0;
4498e6e287fSHans Petter Selasky
4508e6e287fSHans Petter Selasky return cpu_to_be16(MLX5_CAP_ROCE(dev->mdev, r_roce_min_src_udp_port));
4518e6e287fSHans Petter Selasky }
4528e6e287fSHans Petter Selasky
mlx5_get_roce_gid_type(struct mlx5_ib_dev * dev,u8 port_num,int index,enum ib_gid_type * gid_type)453ed0cee0bSHans Petter Selasky int mlx5_get_roce_gid_type(struct mlx5_ib_dev *dev, u8 port_num,
454ed0cee0bSHans Petter Selasky int index, enum ib_gid_type *gid_type)
455ed0cee0bSHans Petter Selasky {
456ed0cee0bSHans Petter Selasky struct ib_gid_attr attr;
457ed0cee0bSHans Petter Selasky union ib_gid gid;
458ed0cee0bSHans Petter Selasky int ret;
459ed0cee0bSHans Petter Selasky
460ed0cee0bSHans Petter Selasky ret = ib_get_cached_gid(&dev->ib_dev, port_num, index, &gid, &attr);
461ed0cee0bSHans Petter Selasky if (ret)
462ed0cee0bSHans Petter Selasky return ret;
463ed0cee0bSHans Petter Selasky
464ed0cee0bSHans Petter Selasky if (!attr.ndev)
465ed0cee0bSHans Petter Selasky return -ENODEV;
466ed0cee0bSHans Petter Selasky
4674ded022dSBjoern A. Zeeb if_rele(attr.ndev);
468ed0cee0bSHans Petter Selasky
469ed0cee0bSHans Petter Selasky *gid_type = attr.gid_type;
470ed0cee0bSHans Petter Selasky
471ed0cee0bSHans Petter Selasky return 0;
472ed0cee0bSHans Petter Selasky }
473ed0cee0bSHans Petter Selasky
mlx5_use_mad_ifc(struct mlx5_ib_dev * dev)47412515907SHans Petter Selasky static int mlx5_use_mad_ifc(struct mlx5_ib_dev *dev)
47512515907SHans Petter Selasky {
4768e6e287fSHans Petter Selasky if (MLX5_CAP_GEN(dev->mdev, port_type) == MLX5_CAP_PORT_TYPE_IB)
4778e6e287fSHans Petter Selasky return !MLX5_CAP_GEN(dev->mdev, ib_virt);
4788e6e287fSHans Petter Selasky return 0;
47912515907SHans Petter Selasky }
48012515907SHans Petter Selasky
48112515907SHans Petter Selasky enum {
48212515907SHans Petter Selasky MLX5_VPORT_ACCESS_METHOD_MAD,
48312515907SHans Petter Selasky MLX5_VPORT_ACCESS_METHOD_HCA,
48412515907SHans Petter Selasky MLX5_VPORT_ACCESS_METHOD_NIC,
48512515907SHans Petter Selasky };
48612515907SHans Petter Selasky
mlx5_get_vport_access_method(struct ib_device * ibdev)48712515907SHans Petter Selasky static int mlx5_get_vport_access_method(struct ib_device *ibdev)
48812515907SHans Petter Selasky {
48912515907SHans Petter Selasky if (mlx5_use_mad_ifc(to_mdev(ibdev)))
49012515907SHans Petter Selasky return MLX5_VPORT_ACCESS_METHOD_MAD;
49112515907SHans Petter Selasky
49212515907SHans Petter Selasky if (mlx5_ib_port_link_layer(ibdev, 1) ==
49312515907SHans Petter Selasky IB_LINK_LAYER_ETHERNET)
49412515907SHans Petter Selasky return MLX5_VPORT_ACCESS_METHOD_NIC;
49512515907SHans Petter Selasky
49612515907SHans Petter Selasky return MLX5_VPORT_ACCESS_METHOD_HCA;
49712515907SHans Petter Selasky }
49812515907SHans Petter Selasky
get_atomic_caps(struct mlx5_ib_dev * dev,struct ib_device_attr * props)4998e6e287fSHans Petter Selasky static void get_atomic_caps(struct mlx5_ib_dev *dev,
5008e6e287fSHans Petter Selasky struct ib_device_attr *props)
5018e6e287fSHans Petter Selasky {
5028e6e287fSHans Petter Selasky u8 tmp;
5038e6e287fSHans Petter Selasky u8 atomic_operations = MLX5_CAP_ATOMIC(dev->mdev, atomic_operations);
5048e6e287fSHans Petter Selasky u8 atomic_size_qp = MLX5_CAP_ATOMIC(dev->mdev, atomic_size_qp);
5058e6e287fSHans Petter Selasky u8 atomic_req_8B_endianness_mode =
5068e6e287fSHans Petter Selasky MLX5_CAP_ATOMIC(dev->mdev, atomic_req_8B_endianess_mode);
5078e6e287fSHans Petter Selasky
5088e6e287fSHans Petter Selasky /* Check if HW supports 8 bytes standard atomic operations and capable
5098e6e287fSHans Petter Selasky * of host endianness respond
5108e6e287fSHans Petter Selasky */
5118e6e287fSHans Petter Selasky tmp = MLX5_ATOMIC_OPS_CMP_SWAP | MLX5_ATOMIC_OPS_FETCH_ADD;
5128e6e287fSHans Petter Selasky if (((atomic_operations & tmp) == tmp) &&
5138e6e287fSHans Petter Selasky (atomic_size_qp & MLX5_ATOMIC_SIZE_QP_8BYTES) &&
5148e6e287fSHans Petter Selasky (atomic_req_8B_endianness_mode)) {
5158e6e287fSHans Petter Selasky props->atomic_cap = IB_ATOMIC_HCA;
5168e6e287fSHans Petter Selasky } else {
5178e6e287fSHans Petter Selasky props->atomic_cap = IB_ATOMIC_NONE;
5188e6e287fSHans Petter Selasky }
5198e6e287fSHans Petter Selasky }
5208e6e287fSHans Petter Selasky
mlx5_query_system_image_guid(struct ib_device * ibdev,__be64 * sys_image_guid)52112515907SHans Petter Selasky static int mlx5_query_system_image_guid(struct ib_device *ibdev,
52212515907SHans Petter Selasky __be64 *sys_image_guid)
52312515907SHans Petter Selasky {
52412515907SHans Petter Selasky struct mlx5_ib_dev *dev = to_mdev(ibdev);
52512515907SHans Petter Selasky struct mlx5_core_dev *mdev = dev->mdev;
52612515907SHans Petter Selasky u64 tmp;
52712515907SHans Petter Selasky int err;
52812515907SHans Petter Selasky
52912515907SHans Petter Selasky switch (mlx5_get_vport_access_method(ibdev)) {
53012515907SHans Petter Selasky case MLX5_VPORT_ACCESS_METHOD_MAD:
5318e6e287fSHans Petter Selasky return mlx5_query_mad_ifc_system_image_guid(ibdev,
53212515907SHans Petter Selasky sys_image_guid);
53312515907SHans Petter Selasky
53412515907SHans Petter Selasky case MLX5_VPORT_ACCESS_METHOD_HCA:
53512515907SHans Petter Selasky err = mlx5_query_hca_vport_system_image_guid(mdev, &tmp);
5368e6e287fSHans Petter Selasky break;
53712515907SHans Petter Selasky
53812515907SHans Petter Selasky case MLX5_VPORT_ACCESS_METHOD_NIC:
53912515907SHans Petter Selasky err = mlx5_query_nic_vport_system_image_guid(mdev, &tmp);
5408e6e287fSHans Petter Selasky break;
54112515907SHans Petter Selasky
54212515907SHans Petter Selasky default:
54312515907SHans Petter Selasky return -EINVAL;
54412515907SHans Petter Selasky }
5458e6e287fSHans Petter Selasky
5468e6e287fSHans Petter Selasky if (!err)
5478e6e287fSHans Petter Selasky *sys_image_guid = cpu_to_be64(tmp);
5488e6e287fSHans Petter Selasky
5498e6e287fSHans Petter Selasky return err;
5508e6e287fSHans Petter Selasky
55112515907SHans Petter Selasky }
55212515907SHans Petter Selasky
mlx5_query_max_pkeys(struct ib_device * ibdev,u16 * max_pkeys)55312515907SHans Petter Selasky static int mlx5_query_max_pkeys(struct ib_device *ibdev,
55412515907SHans Petter Selasky u16 *max_pkeys)
55512515907SHans Petter Selasky {
55612515907SHans Petter Selasky struct mlx5_ib_dev *dev = to_mdev(ibdev);
55712515907SHans Petter Selasky struct mlx5_core_dev *mdev = dev->mdev;
55812515907SHans Petter Selasky
55912515907SHans Petter Selasky switch (mlx5_get_vport_access_method(ibdev)) {
56012515907SHans Petter Selasky case MLX5_VPORT_ACCESS_METHOD_MAD:
5618e6e287fSHans Petter Selasky return mlx5_query_mad_ifc_max_pkeys(ibdev, max_pkeys);
56212515907SHans Petter Selasky
56312515907SHans Petter Selasky case MLX5_VPORT_ACCESS_METHOD_HCA:
56412515907SHans Petter Selasky case MLX5_VPORT_ACCESS_METHOD_NIC:
56512515907SHans Petter Selasky *max_pkeys = mlx5_to_sw_pkey_sz(MLX5_CAP_GEN(mdev,
56612515907SHans Petter Selasky pkey_table_size));
56712515907SHans Petter Selasky return 0;
56812515907SHans Petter Selasky
56912515907SHans Petter Selasky default:
57012515907SHans Petter Selasky return -EINVAL;
57112515907SHans Petter Selasky }
57212515907SHans Petter Selasky }
57312515907SHans Petter Selasky
mlx5_query_vendor_id(struct ib_device * ibdev,u32 * vendor_id)57412515907SHans Petter Selasky static int mlx5_query_vendor_id(struct ib_device *ibdev,
57512515907SHans Petter Selasky u32 *vendor_id)
57612515907SHans Petter Selasky {
57712515907SHans Petter Selasky struct mlx5_ib_dev *dev = to_mdev(ibdev);
57812515907SHans Petter Selasky
57912515907SHans Petter Selasky switch (mlx5_get_vport_access_method(ibdev)) {
58012515907SHans Petter Selasky case MLX5_VPORT_ACCESS_METHOD_MAD:
5818e6e287fSHans Petter Selasky return mlx5_query_mad_ifc_vendor_id(ibdev, vendor_id);
58212515907SHans Petter Selasky
58312515907SHans Petter Selasky case MLX5_VPORT_ACCESS_METHOD_HCA:
58412515907SHans Petter Selasky case MLX5_VPORT_ACCESS_METHOD_NIC:
58512515907SHans Petter Selasky return mlx5_core_query_vendor_id(dev->mdev, vendor_id);
58612515907SHans Petter Selasky
58712515907SHans Petter Selasky default:
58812515907SHans Petter Selasky return -EINVAL;
58912515907SHans Petter Selasky }
59012515907SHans Petter Selasky }
59112515907SHans Petter Selasky
mlx5_query_node_guid(struct mlx5_ib_dev * dev,__be64 * node_guid)59212515907SHans Petter Selasky static int mlx5_query_node_guid(struct mlx5_ib_dev *dev,
59312515907SHans Petter Selasky __be64 *node_guid)
59412515907SHans Petter Selasky {
59512515907SHans Petter Selasky u64 tmp;
59612515907SHans Petter Selasky int err;
59712515907SHans Petter Selasky
59812515907SHans Petter Selasky switch (mlx5_get_vport_access_method(&dev->ib_dev)) {
59912515907SHans Petter Selasky case MLX5_VPORT_ACCESS_METHOD_MAD:
6008e6e287fSHans Petter Selasky return mlx5_query_mad_ifc_node_guid(dev, node_guid);
60112515907SHans Petter Selasky
60212515907SHans Petter Selasky case MLX5_VPORT_ACCESS_METHOD_HCA:
60312515907SHans Petter Selasky err = mlx5_query_hca_vport_node_guid(dev->mdev, &tmp);
6048e6e287fSHans Petter Selasky break;
60512515907SHans Petter Selasky
60612515907SHans Petter Selasky case MLX5_VPORT_ACCESS_METHOD_NIC:
60712515907SHans Petter Selasky err = mlx5_query_nic_vport_node_guid(dev->mdev, &tmp);
6088e6e287fSHans Petter Selasky break;
60912515907SHans Petter Selasky
61012515907SHans Petter Selasky default:
61112515907SHans Petter Selasky return -EINVAL;
61212515907SHans Petter Selasky }
6138e6e287fSHans Petter Selasky
6148e6e287fSHans Petter Selasky if (!err)
6158e6e287fSHans Petter Selasky *node_guid = cpu_to_be64(tmp);
6168e6e287fSHans Petter Selasky
6178e6e287fSHans Petter Selasky return err;
61812515907SHans Petter Selasky }
61912515907SHans Petter Selasky
62012515907SHans Petter Selasky struct mlx5_reg_node_desc {
6218e6e287fSHans Petter Selasky u8 desc[IB_DEVICE_NODE_DESC_MAX];
62212515907SHans Petter Selasky };
62312515907SHans Petter Selasky
mlx5_query_node_desc(struct mlx5_ib_dev * dev,char * node_desc)62412515907SHans Petter Selasky static int mlx5_query_node_desc(struct mlx5_ib_dev *dev, char *node_desc)
62512515907SHans Petter Selasky {
62612515907SHans Petter Selasky struct mlx5_reg_node_desc in;
62712515907SHans Petter Selasky
62812515907SHans Petter Selasky if (mlx5_use_mad_ifc(dev))
6298e6e287fSHans Petter Selasky return mlx5_query_mad_ifc_node_desc(dev, node_desc);
63012515907SHans Petter Selasky
63112515907SHans Petter Selasky memset(&in, 0, sizeof(in));
63212515907SHans Petter Selasky
63312515907SHans Petter Selasky return mlx5_core_access_reg(dev->mdev, &in, sizeof(in), node_desc,
63412515907SHans Petter Selasky sizeof(struct mlx5_reg_node_desc),
63512515907SHans Petter Selasky MLX5_REG_NODE_DESC, 0, 0);
63612515907SHans Petter Selasky }
63712515907SHans Petter Selasky
mlx5_ib_query_device(struct ib_device * ibdev,struct ib_device_attr * props,struct ib_udata * uhw)63812515907SHans Petter Selasky static int mlx5_ib_query_device(struct ib_device *ibdev,
6398e6e287fSHans Petter Selasky struct ib_device_attr *props,
6408e6e287fSHans Petter Selasky struct ib_udata *uhw)
64112515907SHans Petter Selasky {
64212515907SHans Petter Selasky struct mlx5_ib_dev *dev = to_mdev(ibdev);
64312515907SHans Petter Selasky struct mlx5_core_dev *mdev = dev->mdev;
6448e6e287fSHans Petter Selasky int err = -ENOMEM;
645c788dceaSHans Petter Selasky int max_sq_desc;
64612515907SHans Petter Selasky int max_rq_sg;
64712515907SHans Petter Selasky int max_sq_sg;
6488e6e287fSHans Petter Selasky u64 min_page_size = 1ull << MLX5_CAP_GEN(mdev, log_pg_sz);
6498e6e287fSHans Petter Selasky struct mlx5_ib_query_device_resp resp = {};
6508e6e287fSHans Petter Selasky size_t resp_len;
6518e6e287fSHans Petter Selasky u64 max_tso;
65212515907SHans Petter Selasky
6538e6e287fSHans Petter Selasky resp_len = sizeof(resp.comp_mask) + sizeof(resp.response_length);
6548e6e287fSHans Petter Selasky if (uhw->outlen && uhw->outlen < resp_len)
6558e6e287fSHans Petter Selasky return -EINVAL;
6568e6e287fSHans Petter Selasky else
6578e6e287fSHans Petter Selasky resp.response_length = resp_len;
6588e6e287fSHans Petter Selasky
6598e6e287fSHans Petter Selasky if (uhw->inlen && !ib_is_udata_cleared(uhw, 0, uhw->inlen))
6608e6e287fSHans Petter Selasky return -EINVAL;
66112515907SHans Petter Selasky
66212515907SHans Petter Selasky memset(props, 0, sizeof(*props));
66312515907SHans Petter Selasky err = mlx5_query_system_image_guid(ibdev,
66412515907SHans Petter Selasky &props->sys_image_guid);
66512515907SHans Petter Selasky if (err)
66612515907SHans Petter Selasky return err;
66712515907SHans Petter Selasky
66812515907SHans Petter Selasky err = mlx5_query_max_pkeys(ibdev, &props->max_pkeys);
66912515907SHans Petter Selasky if (err)
67012515907SHans Petter Selasky return err;
67112515907SHans Petter Selasky
67212515907SHans Petter Selasky err = mlx5_query_vendor_id(ibdev, &props->vendor_id);
67312515907SHans Petter Selasky if (err)
67412515907SHans Petter Selasky return err;
67512515907SHans Petter Selasky
67612515907SHans Petter Selasky props->fw_ver = ((u64)fw_rev_maj(dev->mdev) << 32) |
677c9e9b5c1SSlava Shwartsman ((u32)fw_rev_min(dev->mdev) << 16) |
67812515907SHans Petter Selasky fw_rev_sub(dev->mdev);
67912515907SHans Petter Selasky props->device_cap_flags = IB_DEVICE_CHANGE_PHY_PORT |
68012515907SHans Petter Selasky IB_DEVICE_PORT_ACTIVE_EVENT |
68112515907SHans Petter Selasky IB_DEVICE_SYS_IMAGE_GUID |
682da4068c4SZhenlei Huang IB_DEVICE_RC_RNR_NAK_GEN |
683da4068c4SZhenlei Huang IB_DEVICE_KNOWSEPOCH;
68412515907SHans Petter Selasky
68512515907SHans Petter Selasky if (MLX5_CAP_GEN(mdev, pkv))
68612515907SHans Petter Selasky props->device_cap_flags |= IB_DEVICE_BAD_PKEY_CNTR;
68712515907SHans Petter Selasky if (MLX5_CAP_GEN(mdev, qkv))
68812515907SHans Petter Selasky props->device_cap_flags |= IB_DEVICE_BAD_QKEY_CNTR;
68912515907SHans Petter Selasky if (MLX5_CAP_GEN(mdev, apm))
69012515907SHans Petter Selasky props->device_cap_flags |= IB_DEVICE_AUTO_PATH_MIG;
69112515907SHans Petter Selasky if (MLX5_CAP_GEN(mdev, xrc))
69212515907SHans Petter Selasky props->device_cap_flags |= IB_DEVICE_XRC;
6938e6e287fSHans Petter Selasky if (MLX5_CAP_GEN(mdev, imaicl)) {
6948e6e287fSHans Petter Selasky props->device_cap_flags |= IB_DEVICE_MEM_WINDOW |
6958e6e287fSHans Petter Selasky IB_DEVICE_MEM_WINDOW_TYPE_2B;
6968e6e287fSHans Petter Selasky props->max_mw = 1 << MLX5_CAP_GEN(mdev, log_max_mkey);
6978e6e287fSHans Petter Selasky /* We support 'Gappy' memory registration too */
6988e6e287fSHans Petter Selasky props->device_cap_flags |= IB_DEVICE_SG_GAPS_REG;
6998e6e287fSHans Petter Selasky }
70012515907SHans Petter Selasky props->device_cap_flags |= IB_DEVICE_MEM_MGT_EXTENSIONS;
7018e6e287fSHans Petter Selasky if (MLX5_CAP_GEN(mdev, sho)) {
7028e6e287fSHans Petter Selasky props->device_cap_flags |= IB_DEVICE_SIGNATURE_HANDOVER;
7038e6e287fSHans Petter Selasky /* At this stage no support for signature handover */
7048e6e287fSHans Petter Selasky props->sig_prot_cap = IB_PROT_T10DIF_TYPE_1 |
7058e6e287fSHans Petter Selasky IB_PROT_T10DIF_TYPE_2 |
7068e6e287fSHans Petter Selasky IB_PROT_T10DIF_TYPE_3;
7078e6e287fSHans Petter Selasky props->sig_guard_cap = IB_GUARD_T10DIF_CRC |
7088e6e287fSHans Petter Selasky IB_GUARD_T10DIF_CSUM;
7098e6e287fSHans Petter Selasky }
71012515907SHans Petter Selasky if (MLX5_CAP_GEN(mdev, block_lb_mc))
71112515907SHans Petter Selasky props->device_cap_flags |= IB_DEVICE_BLOCK_MULTICAST_LOOPBACK;
71212515907SHans Petter Selasky
7138e6e287fSHans Petter Selasky if (MLX5_CAP_GEN(dev->mdev, eth_net_offloads)) {
7148e6e287fSHans Petter Selasky if (MLX5_CAP_ETH(mdev, csum_cap))
7158e6e287fSHans Petter Selasky props->device_cap_flags |= IB_DEVICE_RAW_IP_CSUM;
7168e6e287fSHans Petter Selasky
7178e6e287fSHans Petter Selasky if (field_avail(typeof(resp), tso_caps, uhw->outlen)) {
7188e6e287fSHans Petter Selasky max_tso = MLX5_CAP_ETH(mdev, max_lso_cap);
7198e6e287fSHans Petter Selasky if (max_tso) {
7208e6e287fSHans Petter Selasky resp.tso_caps.max_tso = 1 << max_tso;
7218e6e287fSHans Petter Selasky resp.tso_caps.supported_qpts |=
7228e6e287fSHans Petter Selasky 1 << IB_QPT_RAW_PACKET;
7238e6e287fSHans Petter Selasky resp.response_length += sizeof(resp.tso_caps);
7248e6e287fSHans Petter Selasky }
7258e6e287fSHans Petter Selasky }
7268e6e287fSHans Petter Selasky
7278e6e287fSHans Petter Selasky if (field_avail(typeof(resp), rss_caps, uhw->outlen)) {
7288e6e287fSHans Petter Selasky resp.rss_caps.rx_hash_function =
7298e6e287fSHans Petter Selasky MLX5_RX_HASH_FUNC_TOEPLITZ;
7308e6e287fSHans Petter Selasky resp.rss_caps.rx_hash_fields_mask =
7318e6e287fSHans Petter Selasky MLX5_RX_HASH_SRC_IPV4 |
7328e6e287fSHans Petter Selasky MLX5_RX_HASH_DST_IPV4 |
7338e6e287fSHans Petter Selasky MLX5_RX_HASH_SRC_IPV6 |
7348e6e287fSHans Petter Selasky MLX5_RX_HASH_DST_IPV6 |
7358e6e287fSHans Petter Selasky MLX5_RX_HASH_SRC_PORT_TCP |
7368e6e287fSHans Petter Selasky MLX5_RX_HASH_DST_PORT_TCP |
7378e6e287fSHans Petter Selasky MLX5_RX_HASH_SRC_PORT_UDP |
7388e6e287fSHans Petter Selasky MLX5_RX_HASH_DST_PORT_UDP;
7398e6e287fSHans Petter Selasky resp.response_length += sizeof(resp.rss_caps);
7408e6e287fSHans Petter Selasky }
7418e6e287fSHans Petter Selasky } else {
7428e6e287fSHans Petter Selasky if (field_avail(typeof(resp), tso_caps, uhw->outlen))
7438e6e287fSHans Petter Selasky resp.response_length += sizeof(resp.tso_caps);
7448e6e287fSHans Petter Selasky if (field_avail(typeof(resp), rss_caps, uhw->outlen))
7458e6e287fSHans Petter Selasky resp.response_length += sizeof(resp.rss_caps);
7468e6e287fSHans Petter Selasky }
7478e6e287fSHans Petter Selasky
7488e6e287fSHans Petter Selasky if (MLX5_CAP_GEN(mdev, ipoib_ipoib_offloads)) {
7498e6e287fSHans Petter Selasky props->device_cap_flags |= IB_DEVICE_UD_IP_CSUM;
7508e6e287fSHans Petter Selasky props->device_cap_flags |= IB_DEVICE_UD_TSO;
7518e6e287fSHans Petter Selasky }
7528e6e287fSHans Petter Selasky
7538e6e287fSHans Petter Selasky if (MLX5_CAP_GEN(dev->mdev, eth_net_offloads) &&
7548e6e287fSHans Petter Selasky MLX5_CAP_ETH(dev->mdev, scatter_fcs))
7558e6e287fSHans Petter Selasky props->device_cap_flags |= IB_DEVICE_RAW_SCATTER_FCS;
7568e6e287fSHans Petter Selasky
7578e6e287fSHans Petter Selasky if (mlx5_get_flow_namespace(dev->mdev, MLX5_FLOW_NAMESPACE_BYPASS))
7588e6e287fSHans Petter Selasky props->device_cap_flags |= IB_DEVICE_MANAGED_FLOW_STEERING;
7598e6e287fSHans Petter Selasky
76012515907SHans Petter Selasky props->vendor_part_id = mdev->pdev->device;
76112515907SHans Petter Selasky props->hw_ver = mdev->pdev->revision;
76212515907SHans Petter Selasky
76312515907SHans Petter Selasky props->max_mr_size = ~0ull;
7648e6e287fSHans Petter Selasky props->page_size_cap = ~(min_page_size - 1);
76512515907SHans Petter Selasky props->max_qp = 1 << MLX5_CAP_GEN(mdev, log_max_qp);
76612515907SHans Petter Selasky props->max_qp_wr = 1 << MLX5_CAP_GEN(mdev, log_max_qp_sz);
76712515907SHans Petter Selasky max_rq_sg = MLX5_CAP_GEN(mdev, max_wqe_sz_rq) /
76812515907SHans Petter Selasky sizeof(struct mlx5_wqe_data_seg);
769c788dceaSHans Petter Selasky max_sq_desc = min_t(int, MLX5_CAP_GEN(mdev, max_wqe_sz_sq), 512);
770c788dceaSHans Petter Selasky max_sq_sg = (max_sq_desc - sizeof(struct mlx5_wqe_ctrl_seg) -
771c788dceaSHans Petter Selasky sizeof(struct mlx5_wqe_raddr_seg)) /
7728e6e287fSHans Petter Selasky sizeof(struct mlx5_wqe_data_seg);
77312515907SHans Petter Selasky props->max_sge = min(max_rq_sg, max_sq_sg);
7748e6e287fSHans Petter Selasky props->max_sge_rd = MLX5_MAX_SGE_RD;
77512515907SHans Petter Selasky props->max_cq = 1 << MLX5_CAP_GEN(mdev, log_max_cq);
77612515907SHans Petter Selasky props->max_cqe = (1 << MLX5_CAP_GEN(mdev, log_max_cq_sz)) - 1;
77712515907SHans Petter Selasky props->max_mr = 1 << MLX5_CAP_GEN(mdev, log_max_mkey);
77812515907SHans Petter Selasky props->max_pd = 1 << MLX5_CAP_GEN(mdev, log_max_pd);
77912515907SHans Petter Selasky props->max_qp_rd_atom = 1 << MLX5_CAP_GEN(mdev, log_max_ra_req_qp);
78012515907SHans Petter Selasky props->max_qp_init_rd_atom = 1 << MLX5_CAP_GEN(mdev, log_max_ra_res_qp);
78112515907SHans Petter Selasky props->max_srq = 1 << MLX5_CAP_GEN(mdev, log_max_srq);
78212515907SHans Petter Selasky props->max_srq_wr = (1 << MLX5_CAP_GEN(mdev, log_max_srq_sz)) - 1;
78312515907SHans Petter Selasky props->local_ca_ack_delay = MLX5_CAP_GEN(mdev, local_ca_ack_delay);
78412515907SHans Petter Selasky props->max_res_rd_atom = props->max_qp_rd_atom * props->max_qp;
78512515907SHans Petter Selasky props->max_srq_sge = max_rq_sg - 1;
7868e6e287fSHans Petter Selasky props->max_fast_reg_page_list_len =
7878e6e287fSHans Petter Selasky 1 << MLX5_CAP_GEN(mdev, log_max_klm_list_size);
78812515907SHans Petter Selasky get_atomic_caps(dev, props);
7898e6e287fSHans Petter Selasky props->masked_atomic_cap = IB_ATOMIC_NONE;
79012515907SHans Petter Selasky props->max_mcast_grp = 1 << MLX5_CAP_GEN(mdev, log_max_mcg);
79112515907SHans Petter Selasky props->max_mcast_qp_attach = MLX5_CAP_GEN(mdev, max_qp_mcg);
79212515907SHans Petter Selasky props->max_total_mcast_qp_attach = props->max_mcast_qp_attach *
79312515907SHans Petter Selasky props->max_mcast_grp;
79412515907SHans Petter Selasky props->max_map_per_fmr = INT_MAX; /* no limit in ConnectIB */
7958e6e287fSHans Petter Selasky props->hca_core_clock = MLX5_CAP_GEN(mdev, device_frequency_khz);
7968e6e287fSHans Petter Selasky props->timestamp_mask = 0x7FFFFFFFFFFFFFFFULL;
7978e6e287fSHans Petter Selasky
7988e6e287fSHans Petter Selasky #ifdef CONFIG_INFINIBAND_ON_DEMAND_PAGING
7998e6e287fSHans Petter Selasky if (MLX5_CAP_GEN(mdev, pg))
8008e6e287fSHans Petter Selasky props->device_cap_flags |= IB_DEVICE_ON_DEMAND_PAGING;
8018e6e287fSHans Petter Selasky props->odp_caps = dev->odp_caps;
8028e6e287fSHans Petter Selasky #endif
8038e6e287fSHans Petter Selasky
8048e6e287fSHans Petter Selasky if (MLX5_CAP_GEN(mdev, cd))
8058e6e287fSHans Petter Selasky props->device_cap_flags |= IB_DEVICE_CROSS_CHANNEL;
8068e6e287fSHans Petter Selasky
8078e6e287fSHans Petter Selasky if (!mlx5_core_is_pf(mdev))
8088e6e287fSHans Petter Selasky props->device_cap_flags |= IB_DEVICE_VIRTUAL_FUNCTION;
8098e6e287fSHans Petter Selasky
8108e6e287fSHans Petter Selasky if (mlx5_ib_port_link_layer(ibdev, 1) ==
8118e6e287fSHans Petter Selasky IB_LINK_LAYER_ETHERNET) {
8128e6e287fSHans Petter Selasky props->rss_caps.max_rwq_indirection_tables =
8138e6e287fSHans Petter Selasky 1 << MLX5_CAP_GEN(dev->mdev, log_max_rqt);
8148e6e287fSHans Petter Selasky props->rss_caps.max_rwq_indirection_table_size =
8158e6e287fSHans Petter Selasky 1 << MLX5_CAP_GEN(dev->mdev, log_max_rqt_size);
8168e6e287fSHans Petter Selasky props->rss_caps.supported_qpts = 1 << IB_QPT_RAW_PACKET;
8178e6e287fSHans Petter Selasky props->max_wq_type_rq =
8188e6e287fSHans Petter Selasky 1 << MLX5_CAP_GEN(dev->mdev, log_max_rq);
8198e6e287fSHans Petter Selasky }
8208e6e287fSHans Petter Selasky
8218e6e287fSHans Petter Selasky if (uhw->outlen) {
8228e6e287fSHans Petter Selasky err = ib_copy_to_udata(uhw, &resp, resp.response_length);
8238e6e287fSHans Petter Selasky
8248e6e287fSHans Petter Selasky if (err)
8258e6e287fSHans Petter Selasky return err;
8268e6e287fSHans Petter Selasky }
82712515907SHans Petter Selasky
82812515907SHans Petter Selasky return 0;
82912515907SHans Petter Selasky }
83012515907SHans Petter Selasky
83112515907SHans Petter Selasky enum mlx5_ib_width {
83212515907SHans Petter Selasky MLX5_IB_WIDTH_1X = 1 << 0,
83312515907SHans Petter Selasky MLX5_IB_WIDTH_2X = 1 << 1,
83412515907SHans Petter Selasky MLX5_IB_WIDTH_4X = 1 << 2,
83512515907SHans Petter Selasky MLX5_IB_WIDTH_8X = 1 << 3,
83612515907SHans Petter Selasky MLX5_IB_WIDTH_12X = 1 << 4
83712515907SHans Petter Selasky };
83812515907SHans Petter Selasky
translate_active_width(struct ib_device * ibdev,u8 active_width,u8 * ib_width)83912515907SHans Petter Selasky static int translate_active_width(struct ib_device *ibdev, u8 active_width,
84012515907SHans Petter Selasky u8 *ib_width)
84112515907SHans Petter Selasky {
84212515907SHans Petter Selasky struct mlx5_ib_dev *dev = to_mdev(ibdev);
84312515907SHans Petter Selasky int err = 0;
84412515907SHans Petter Selasky
84512515907SHans Petter Selasky if (active_width & MLX5_IB_WIDTH_1X) {
84612515907SHans Petter Selasky *ib_width = IB_WIDTH_1X;
84712515907SHans Petter Selasky } else if (active_width & MLX5_IB_WIDTH_2X) {
84886a39779SHans Petter Selasky *ib_width = IB_WIDTH_2X;
84912515907SHans Petter Selasky } else if (active_width & MLX5_IB_WIDTH_4X) {
85012515907SHans Petter Selasky *ib_width = IB_WIDTH_4X;
85112515907SHans Petter Selasky } else if (active_width & MLX5_IB_WIDTH_8X) {
85212515907SHans Petter Selasky *ib_width = IB_WIDTH_8X;
85312515907SHans Petter Selasky } else if (active_width & MLX5_IB_WIDTH_12X) {
85412515907SHans Petter Selasky *ib_width = IB_WIDTH_12X;
85512515907SHans Petter Selasky } else {
85612515907SHans Petter Selasky mlx5_ib_dbg(dev, "Invalid active_width %d\n",
85712515907SHans Petter Selasky (int)active_width);
85812515907SHans Petter Selasky err = -EINVAL;
85912515907SHans Petter Selasky }
86012515907SHans Petter Selasky
86112515907SHans Petter Selasky return err;
86212515907SHans Petter Selasky }
86312515907SHans Petter Selasky
86412515907SHans Petter Selasky enum ib_max_vl_num {
86512515907SHans Petter Selasky __IB_MAX_VL_0 = 1,
86612515907SHans Petter Selasky __IB_MAX_VL_0_1 = 2,
86712515907SHans Petter Selasky __IB_MAX_VL_0_3 = 3,
86812515907SHans Petter Selasky __IB_MAX_VL_0_7 = 4,
86912515907SHans Petter Selasky __IB_MAX_VL_0_14 = 5,
87012515907SHans Petter Selasky };
87112515907SHans Petter Selasky
87212515907SHans Petter Selasky enum mlx5_vl_hw_cap {
87312515907SHans Petter Selasky MLX5_VL_HW_0 = 1,
87412515907SHans Petter Selasky MLX5_VL_HW_0_1 = 2,
87512515907SHans Petter Selasky MLX5_VL_HW_0_2 = 3,
87612515907SHans Petter Selasky MLX5_VL_HW_0_3 = 4,
87712515907SHans Petter Selasky MLX5_VL_HW_0_4 = 5,
87812515907SHans Petter Selasky MLX5_VL_HW_0_5 = 6,
87912515907SHans Petter Selasky MLX5_VL_HW_0_6 = 7,
88012515907SHans Petter Selasky MLX5_VL_HW_0_7 = 8,
88112515907SHans Petter Selasky MLX5_VL_HW_0_14 = 15
88212515907SHans Petter Selasky };
88312515907SHans Petter Selasky
translate_max_vl_num(struct ib_device * ibdev,u8 vl_hw_cap,u8 * max_vl_num)88412515907SHans Petter Selasky static int translate_max_vl_num(struct ib_device *ibdev, u8 vl_hw_cap,
88512515907SHans Petter Selasky u8 *max_vl_num)
88612515907SHans Petter Selasky {
88712515907SHans Petter Selasky switch (vl_hw_cap) {
88812515907SHans Petter Selasky case MLX5_VL_HW_0:
88912515907SHans Petter Selasky *max_vl_num = __IB_MAX_VL_0;
89012515907SHans Petter Selasky break;
89112515907SHans Petter Selasky case MLX5_VL_HW_0_1:
89212515907SHans Petter Selasky *max_vl_num = __IB_MAX_VL_0_1;
89312515907SHans Petter Selasky break;
89412515907SHans Petter Selasky case MLX5_VL_HW_0_3:
89512515907SHans Petter Selasky *max_vl_num = __IB_MAX_VL_0_3;
89612515907SHans Petter Selasky break;
89712515907SHans Petter Selasky case MLX5_VL_HW_0_7:
89812515907SHans Petter Selasky *max_vl_num = __IB_MAX_VL_0_7;
89912515907SHans Petter Selasky break;
90012515907SHans Petter Selasky case MLX5_VL_HW_0_14:
90112515907SHans Petter Selasky *max_vl_num = __IB_MAX_VL_0_14;
90212515907SHans Petter Selasky break;
90312515907SHans Petter Selasky
90412515907SHans Petter Selasky default:
90512515907SHans Petter Selasky return -EINVAL;
90612515907SHans Petter Selasky }
90712515907SHans Petter Selasky
90812515907SHans Petter Selasky return 0;
90912515907SHans Petter Selasky }
91012515907SHans Petter Selasky
mlx5_query_hca_port(struct ib_device * ibdev,u8 port,struct ib_port_attr * props)9118e6e287fSHans Petter Selasky static int mlx5_query_hca_port(struct ib_device *ibdev, u8 port,
91212515907SHans Petter Selasky struct ib_port_attr *props)
91312515907SHans Petter Selasky {
91412515907SHans Petter Selasky struct mlx5_ib_dev *dev = to_mdev(ibdev);
91512515907SHans Petter Selasky struct mlx5_core_dev *mdev = dev->mdev;
91612515907SHans Petter Selasky u32 *rep;
9178e6e287fSHans Petter Selasky int replen = MLX5_ST_SZ_BYTES(query_hca_vport_context_out);
91812515907SHans Petter Selasky struct mlx5_ptys_reg *ptys;
91912515907SHans Petter Selasky struct mlx5_pmtu_reg *pmtu;
92012515907SHans Petter Selasky struct mlx5_pvlc_reg pvlc;
92112515907SHans Petter Selasky void *ctx;
92212515907SHans Petter Selasky int err;
92312515907SHans Petter Selasky
9248e6e287fSHans Petter Selasky rep = mlx5_vzalloc(replen);
92512515907SHans Petter Selasky ptys = kzalloc(sizeof(*ptys), GFP_KERNEL);
92612515907SHans Petter Selasky pmtu = kzalloc(sizeof(*pmtu), GFP_KERNEL);
92712515907SHans Petter Selasky if (!rep || !ptys || !pmtu) {
92812515907SHans Petter Selasky err = -ENOMEM;
92912515907SHans Petter Selasky goto out;
93012515907SHans Petter Selasky }
93112515907SHans Petter Selasky
93212515907SHans Petter Selasky memset(props, 0, sizeof(*props));
93312515907SHans Petter Selasky
9348e6e287fSHans Petter Selasky err = mlx5_query_hca_vport_context(mdev, port, 0, rep, replen);
93512515907SHans Petter Selasky if (err)
93612515907SHans Petter Selasky goto out;
93712515907SHans Petter Selasky
93812515907SHans Petter Selasky ctx = MLX5_ADDR_OF(query_hca_vport_context_out, rep, hca_vport_context);
93912515907SHans Petter Selasky
94012515907SHans Petter Selasky props->lid = MLX5_GET(hca_vport_context, ctx, lid);
94112515907SHans Petter Selasky props->lmc = MLX5_GET(hca_vport_context, ctx, lmc);
94212515907SHans Petter Selasky props->sm_lid = MLX5_GET(hca_vport_context, ctx, sm_lid);
94312515907SHans Petter Selasky props->sm_sl = MLX5_GET(hca_vport_context, ctx, sm_sl);
94412515907SHans Petter Selasky props->state = MLX5_GET(hca_vport_context, ctx, vport_state);
94512515907SHans Petter Selasky props->phys_state = MLX5_GET(hca_vport_context, ctx,
94612515907SHans Petter Selasky port_physical_state);
94712515907SHans Petter Selasky props->port_cap_flags = MLX5_GET(hca_vport_context, ctx, cap_mask1);
94812515907SHans Petter Selasky props->gid_tbl_len = mlx5_get_gid_table_len(MLX5_CAP_GEN(mdev, gid_table_size));
94912515907SHans Petter Selasky props->max_msg_sz = 1 << MLX5_CAP_GEN(mdev, log_max_msg);
95012515907SHans Petter Selasky props->pkey_tbl_len = mlx5_to_sw_pkey_sz(MLX5_CAP_GEN(mdev, pkey_table_size));
95112515907SHans Petter Selasky props->bad_pkey_cntr = MLX5_GET(hca_vport_context, ctx,
95212515907SHans Petter Selasky pkey_violation_counter);
95312515907SHans Petter Selasky props->qkey_viol_cntr = MLX5_GET(hca_vport_context, ctx,
95412515907SHans Petter Selasky qkey_violation_counter);
95512515907SHans Petter Selasky props->subnet_timeout = MLX5_GET(hca_vport_context, ctx,
95612515907SHans Petter Selasky subnet_timeout);
95712515907SHans Petter Selasky props->init_type_reply = MLX5_GET(hca_vport_context, ctx,
95812515907SHans Petter Selasky init_type_reply);
9598e6e287fSHans Petter Selasky props->grh_required = MLX5_GET(hca_vport_context, ctx, grh_required);
96012515907SHans Petter Selasky
96112515907SHans Petter Selasky ptys->proto_mask |= MLX5_PTYS_IB;
96212515907SHans Petter Selasky ptys->local_port = port;
96312515907SHans Petter Selasky err = mlx5_core_access_ptys(mdev, ptys, 0);
96412515907SHans Petter Selasky if (err)
96512515907SHans Petter Selasky goto out;
96612515907SHans Petter Selasky
96712515907SHans Petter Selasky err = translate_active_width(ibdev, ptys->ib_link_width_oper,
96812515907SHans Petter Selasky &props->active_width);
96912515907SHans Petter Selasky if (err)
97012515907SHans Petter Selasky goto out;
97112515907SHans Petter Selasky
97212515907SHans Petter Selasky props->active_speed = (u8)ptys->ib_proto_oper;
97312515907SHans Petter Selasky
97412515907SHans Petter Selasky pmtu->local_port = port;
97512515907SHans Petter Selasky err = mlx5_core_access_pmtu(mdev, pmtu, 0);
97612515907SHans Petter Selasky if (err)
97712515907SHans Petter Selasky goto out;
97812515907SHans Petter Selasky
97912515907SHans Petter Selasky props->max_mtu = pmtu->max_mtu;
98012515907SHans Petter Selasky props->active_mtu = pmtu->oper_mtu;
98112515907SHans Petter Selasky
98212515907SHans Petter Selasky memset(&pvlc, 0, sizeof(pvlc));
98312515907SHans Petter Selasky pvlc.local_port = port;
98412515907SHans Petter Selasky err = mlx5_core_access_pvlc(mdev, &pvlc, 0);
98512515907SHans Petter Selasky if (err)
98612515907SHans Petter Selasky goto out;
98712515907SHans Petter Selasky
98812515907SHans Petter Selasky err = translate_max_vl_num(ibdev, pvlc.vl_hw_cap,
98912515907SHans Petter Selasky &props->max_vl_num);
99012515907SHans Petter Selasky out:
99112515907SHans Petter Selasky kvfree(rep);
99212515907SHans Petter Selasky kfree(ptys);
99312515907SHans Petter Selasky kfree(pmtu);
99412515907SHans Petter Selasky return err;
99512515907SHans Petter Selasky }
99612515907SHans Petter Selasky
mlx5_ib_query_port(struct ib_device * ibdev,u8 port,struct ib_port_attr * props)99712515907SHans Petter Selasky int mlx5_ib_query_port(struct ib_device *ibdev, u8 port,
99812515907SHans Petter Selasky struct ib_port_attr *props)
99912515907SHans Petter Selasky {
100012515907SHans Petter Selasky switch (mlx5_get_vport_access_method(ibdev)) {
100112515907SHans Petter Selasky case MLX5_VPORT_ACCESS_METHOD_MAD:
10028e6e287fSHans Petter Selasky return mlx5_query_mad_ifc_port(ibdev, port, props);
100312515907SHans Petter Selasky
100412515907SHans Petter Selasky case MLX5_VPORT_ACCESS_METHOD_HCA:
10058e6e287fSHans Petter Selasky return mlx5_query_hca_port(ibdev, port, props);
100612515907SHans Petter Selasky
100712515907SHans Petter Selasky case MLX5_VPORT_ACCESS_METHOD_NIC:
100812515907SHans Petter Selasky return mlx5_query_port_roce(ibdev, port, props);
100912515907SHans Petter Selasky
101012515907SHans Petter Selasky default:
101112515907SHans Petter Selasky return -EINVAL;
101212515907SHans Petter Selasky }
101312515907SHans Petter Selasky }
101412515907SHans Petter Selasky
mlx5_ib_query_gid(struct ib_device * ibdev,u8 port,int index,union ib_gid * gid)101512515907SHans Petter Selasky static int mlx5_ib_query_gid(struct ib_device *ibdev, u8 port, int index,
101612515907SHans Petter Selasky union ib_gid *gid)
101712515907SHans Petter Selasky {
101812515907SHans Petter Selasky struct mlx5_ib_dev *dev = to_mdev(ibdev);
101912515907SHans Petter Selasky struct mlx5_core_dev *mdev = dev->mdev;
102012515907SHans Petter Selasky
102112515907SHans Petter Selasky switch (mlx5_get_vport_access_method(ibdev)) {
102212515907SHans Petter Selasky case MLX5_VPORT_ACCESS_METHOD_MAD:
10238e6e287fSHans Petter Selasky return mlx5_query_mad_ifc_gids(ibdev, port, index, gid);
102412515907SHans Petter Selasky
102512515907SHans Petter Selasky case MLX5_VPORT_ACCESS_METHOD_HCA:
102612515907SHans Petter Selasky return mlx5_query_hca_vport_gid(mdev, port, 0, index, gid);
102712515907SHans Petter Selasky
102812515907SHans Petter Selasky default:
102912515907SHans Petter Selasky return -EINVAL;
103012515907SHans Petter Selasky }
10318e6e287fSHans Petter Selasky
103212515907SHans Petter Selasky }
103312515907SHans Petter Selasky
mlx5_ib_query_pkey(struct ib_device * ibdev,u8 port,u16 index,u16 * pkey)103412515907SHans Petter Selasky static int mlx5_ib_query_pkey(struct ib_device *ibdev, u8 port, u16 index,
103512515907SHans Petter Selasky u16 *pkey)
103612515907SHans Petter Selasky {
103712515907SHans Petter Selasky struct mlx5_ib_dev *dev = to_mdev(ibdev);
103812515907SHans Petter Selasky struct mlx5_core_dev *mdev = dev->mdev;
103912515907SHans Petter Selasky
104012515907SHans Petter Selasky switch (mlx5_get_vport_access_method(ibdev)) {
104112515907SHans Petter Selasky case MLX5_VPORT_ACCESS_METHOD_MAD:
10428e6e287fSHans Petter Selasky return mlx5_query_mad_ifc_pkey(ibdev, port, index, pkey);
104312515907SHans Petter Selasky
104412515907SHans Petter Selasky case MLX5_VPORT_ACCESS_METHOD_HCA:
104512515907SHans Petter Selasky case MLX5_VPORT_ACCESS_METHOD_NIC:
104612515907SHans Petter Selasky return mlx5_query_hca_vport_pkey(mdev, 0, port, 0, index,
104712515907SHans Petter Selasky pkey);
104812515907SHans Petter Selasky default:
104912515907SHans Petter Selasky return -EINVAL;
105012515907SHans Petter Selasky }
105112515907SHans Petter Selasky }
105212515907SHans Petter Selasky
mlx5_ib_modify_device(struct ib_device * ibdev,int mask,struct ib_device_modify * props)105312515907SHans Petter Selasky static int mlx5_ib_modify_device(struct ib_device *ibdev, int mask,
105412515907SHans Petter Selasky struct ib_device_modify *props)
105512515907SHans Petter Selasky {
105612515907SHans Petter Selasky struct mlx5_ib_dev *dev = to_mdev(ibdev);
105712515907SHans Petter Selasky struct mlx5_reg_node_desc in;
105812515907SHans Petter Selasky struct mlx5_reg_node_desc out;
105912515907SHans Petter Selasky int err;
106012515907SHans Petter Selasky
106112515907SHans Petter Selasky if (mask & ~IB_DEVICE_MODIFY_NODE_DESC)
106212515907SHans Petter Selasky return -EOPNOTSUPP;
106312515907SHans Petter Selasky
106412515907SHans Petter Selasky if (!(mask & IB_DEVICE_MODIFY_NODE_DESC))
106512515907SHans Petter Selasky return 0;
106612515907SHans Petter Selasky
106712515907SHans Petter Selasky /*
106812515907SHans Petter Selasky * If possible, pass node desc to FW, so it can generate
106912515907SHans Petter Selasky * a 144 trap. If cmd fails, just ignore.
107012515907SHans Petter Selasky */
10718e6e287fSHans Petter Selasky memcpy(&in, props->node_desc, IB_DEVICE_NODE_DESC_MAX);
107212515907SHans Petter Selasky err = mlx5_core_access_reg(dev->mdev, &in, sizeof(in), &out,
107312515907SHans Petter Selasky sizeof(out), MLX5_REG_NODE_DESC, 0, 1);
107412515907SHans Petter Selasky if (err)
107512515907SHans Petter Selasky return err;
107612515907SHans Petter Selasky
10778e6e287fSHans Petter Selasky memcpy(ibdev->node_desc, props->node_desc, IB_DEVICE_NODE_DESC_MAX);
107812515907SHans Petter Selasky
107912515907SHans Petter Selasky return err;
108012515907SHans Petter Selasky }
108112515907SHans Petter Selasky
mlx5_ib_modify_port(struct ib_device * ibdev,u8 port,int mask,struct ib_port_modify * props)108212515907SHans Petter Selasky static int mlx5_ib_modify_port(struct ib_device *ibdev, u8 port, int mask,
108312515907SHans Petter Selasky struct ib_port_modify *props)
108412515907SHans Petter Selasky {
108512515907SHans Petter Selasky struct mlx5_ib_dev *dev = to_mdev(ibdev);
108612515907SHans Petter Selasky struct ib_port_attr attr;
108712515907SHans Petter Selasky u32 tmp;
108812515907SHans Petter Selasky int err;
108912515907SHans Petter Selasky
1090ebf7e777SHans Petter Selasky /*
1091ebf7e777SHans Petter Selasky * CM layer calls ib_modify_port() regardless of the link
1092ebf7e777SHans Petter Selasky * layer. For Ethernet ports, qkey violation and Port
1093ebf7e777SHans Petter Selasky * capabilities are meaningless.
1094ebf7e777SHans Petter Selasky */
1095ebf7e777SHans Petter Selasky if (mlx5_ib_port_link_layer(ibdev, port) == IB_LINK_LAYER_ETHERNET)
1096ebf7e777SHans Petter Selasky return 0;
1097ebf7e777SHans Petter Selasky
109812515907SHans Petter Selasky mutex_lock(&dev->cap_mask_mutex);
109912515907SHans Petter Selasky
110012515907SHans Petter Selasky err = mlx5_ib_query_port(ibdev, port, &attr);
110112515907SHans Petter Selasky if (err)
110212515907SHans Petter Selasky goto out;
110312515907SHans Petter Selasky
110412515907SHans Petter Selasky tmp = (attr.port_cap_flags | props->set_port_cap_mask) &
110512515907SHans Petter Selasky ~props->clr_port_cap_mask;
110612515907SHans Petter Selasky
110712515907SHans Petter Selasky err = mlx5_set_port_caps(dev->mdev, port, tmp);
110812515907SHans Petter Selasky
110912515907SHans Petter Selasky out:
111012515907SHans Petter Selasky mutex_unlock(&dev->cap_mask_mutex);
111112515907SHans Petter Selasky return err;
111212515907SHans Petter Selasky }
111312515907SHans Petter Selasky
print_lib_caps(struct mlx5_ib_dev * dev,u64 caps)1114b633e08cSHans Petter Selasky static void print_lib_caps(struct mlx5_ib_dev *dev, u64 caps)
1115b633e08cSHans Petter Selasky {
1116b633e08cSHans Petter Selasky mlx5_ib_dbg(dev, "MLX5_LIB_CAP_4K_UAR = %s\n",
1117b633e08cSHans Petter Selasky caps & MLX5_LIB_CAP_4K_UAR ? "y" : "n");
1118b633e08cSHans Petter Selasky }
1119b633e08cSHans Petter Selasky
calc_dynamic_bfregs(int uars_per_sys_page)1120f8f5b459SHans Petter Selasky static u16 calc_dynamic_bfregs(int uars_per_sys_page)
1121f8f5b459SHans Petter Selasky {
1122f8f5b459SHans Petter Selasky /* Large page with non 4k uar support might limit the dynamic size */
1123f8f5b459SHans Petter Selasky if (uars_per_sys_page == 1 && PAGE_SIZE > 4096)
1124f8f5b459SHans Petter Selasky return MLX5_MIN_DYN_BFREGS;
1125f8f5b459SHans Petter Selasky
1126f8f5b459SHans Petter Selasky return MLX5_MAX_DYN_BFREGS;
1127f8f5b459SHans Petter Selasky }
1128f8f5b459SHans Petter Selasky
calc_total_bfregs(struct mlx5_ib_dev * dev,bool lib_uar_4k,struct mlx5_ib_alloc_ucontext_req_v2 * req,struct mlx5_bfreg_info * bfregi)1129f8f5b459SHans Petter Selasky static int calc_total_bfregs(struct mlx5_ib_dev *dev, bool lib_uar_4k,
1130f8f5b459SHans Petter Selasky struct mlx5_ib_alloc_ucontext_req_v2 *req,
1131f8f5b459SHans Petter Selasky struct mlx5_bfreg_info *bfregi)
1132f8f5b459SHans Petter Selasky {
1133f8f5b459SHans Petter Selasky int uars_per_sys_page;
1134f8f5b459SHans Petter Selasky int bfregs_per_sys_page;
1135f8f5b459SHans Petter Selasky int ref_bfregs = req->total_num_bfregs;
1136f8f5b459SHans Petter Selasky
1137f8f5b459SHans Petter Selasky if (req->total_num_bfregs == 0)
1138f8f5b459SHans Petter Selasky return -EINVAL;
1139f8f5b459SHans Petter Selasky
1140f8f5b459SHans Petter Selasky BUILD_BUG_ON(MLX5_MAX_BFREGS % MLX5_NON_FP_BFREGS_IN_PAGE);
1141f8f5b459SHans Petter Selasky BUILD_BUG_ON(MLX5_MAX_BFREGS < MLX5_NON_FP_BFREGS_IN_PAGE);
1142f8f5b459SHans Petter Selasky
1143f8f5b459SHans Petter Selasky if (req->total_num_bfregs > MLX5_MAX_BFREGS)
1144f8f5b459SHans Petter Selasky return -ENOMEM;
1145f8f5b459SHans Petter Selasky
1146f8f5b459SHans Petter Selasky uars_per_sys_page = get_uars_per_sys_page(dev, lib_uar_4k);
1147f8f5b459SHans Petter Selasky bfregs_per_sys_page = uars_per_sys_page * MLX5_NON_FP_BFREGS_PER_UAR;
1148f8f5b459SHans Petter Selasky /* This holds the required static allocation asked by the user */
1149f8f5b459SHans Petter Selasky req->total_num_bfregs = ALIGN(req->total_num_bfregs, bfregs_per_sys_page);
1150f8f5b459SHans Petter Selasky if (req->num_low_latency_bfregs > req->total_num_bfregs - 1)
1151f8f5b459SHans Petter Selasky return -EINVAL;
1152f8f5b459SHans Petter Selasky
1153f8f5b459SHans Petter Selasky bfregi->num_static_sys_pages = req->total_num_bfregs / bfregs_per_sys_page;
1154f8f5b459SHans Petter Selasky bfregi->num_dyn_bfregs = ALIGN(calc_dynamic_bfregs(uars_per_sys_page), bfregs_per_sys_page);
1155f8f5b459SHans Petter Selasky bfregi->total_num_bfregs = req->total_num_bfregs + bfregi->num_dyn_bfregs;
1156f8f5b459SHans Petter Selasky bfregi->num_sys_pages = bfregi->total_num_bfregs / bfregs_per_sys_page;
1157f8f5b459SHans Petter Selasky
1158f8f5b459SHans Petter Selasky mlx5_ib_dbg(dev, "uar_4k: fw support %s, lib support %s, user requested %d bfregs, allocated %d, total bfregs %d, using %d sys pages\n",
1159f8f5b459SHans Petter Selasky MLX5_CAP_GEN(dev->mdev, uar_4k) ? "yes" : "no",
1160f8f5b459SHans Petter Selasky lib_uar_4k ? "yes" : "no", ref_bfregs,
1161f8f5b459SHans Petter Selasky req->total_num_bfregs, bfregi->total_num_bfregs,
1162f8f5b459SHans Petter Selasky bfregi->num_sys_pages);
1163f8f5b459SHans Petter Selasky
1164f8f5b459SHans Petter Selasky return 0;
1165f8f5b459SHans Petter Selasky }
1166f8f5b459SHans Petter Selasky
allocate_uars(struct mlx5_ib_dev * dev,struct mlx5_ib_ucontext * context)1167f8f5b459SHans Petter Selasky static int allocate_uars(struct mlx5_ib_dev *dev, struct mlx5_ib_ucontext *context)
1168f8f5b459SHans Petter Selasky {
1169f8f5b459SHans Petter Selasky struct mlx5_bfreg_info *bfregi;
1170f8f5b459SHans Petter Selasky int err;
1171f8f5b459SHans Petter Selasky int i;
1172f8f5b459SHans Petter Selasky
1173f8f5b459SHans Petter Selasky bfregi = &context->bfregi;
1174f8f5b459SHans Petter Selasky for (i = 0; i < bfregi->num_static_sys_pages; i++) {
1175f8f5b459SHans Petter Selasky err = mlx5_cmd_alloc_uar(dev->mdev, &bfregi->sys_pages[i]);
1176f8f5b459SHans Petter Selasky if (err)
1177f8f5b459SHans Petter Selasky goto error;
1178f8f5b459SHans Petter Selasky
1179f8f5b459SHans Petter Selasky mlx5_ib_dbg(dev, "allocated uar %d\n", bfregi->sys_pages[i]);
1180f8f5b459SHans Petter Selasky }
1181f8f5b459SHans Petter Selasky
1182f8f5b459SHans Petter Selasky for (i = bfregi->num_static_sys_pages; i < bfregi->num_sys_pages; i++)
1183f8f5b459SHans Petter Selasky bfregi->sys_pages[i] = MLX5_IB_INVALID_UAR_INDEX;
1184f8f5b459SHans Petter Selasky
1185f8f5b459SHans Petter Selasky return 0;
1186f8f5b459SHans Petter Selasky
1187f8f5b459SHans Petter Selasky error:
1188f8f5b459SHans Petter Selasky for (--i; i >= 0; i--)
1189f8f5b459SHans Petter Selasky if (mlx5_cmd_free_uar(dev->mdev, bfregi->sys_pages[i]))
1190f8f5b459SHans Petter Selasky mlx5_ib_warn(dev, "failed to free uar %d\n", i);
1191f8f5b459SHans Petter Selasky
1192f8f5b459SHans Petter Selasky return err;
1193f8f5b459SHans Petter Selasky }
1194f8f5b459SHans Petter Selasky
deallocate_uars(struct mlx5_ib_dev * dev,struct mlx5_ib_ucontext * context)1195f8f5b459SHans Petter Selasky static void deallocate_uars(struct mlx5_ib_dev *dev,
1196f8f5b459SHans Petter Selasky struct mlx5_ib_ucontext *context)
1197f8f5b459SHans Petter Selasky {
1198f8f5b459SHans Petter Selasky struct mlx5_bfreg_info *bfregi;
1199f8f5b459SHans Petter Selasky int i;
1200f8f5b459SHans Petter Selasky
1201f8f5b459SHans Petter Selasky bfregi = &context->bfregi;
1202f8f5b459SHans Petter Selasky for (i = 0; i < bfregi->num_sys_pages; i++)
1203f8f5b459SHans Petter Selasky if (i < bfregi->num_static_sys_pages ||
1204f8f5b459SHans Petter Selasky bfregi->sys_pages[i] != MLX5_IB_INVALID_UAR_INDEX)
1205f8f5b459SHans Petter Selasky mlx5_cmd_free_uar(dev->mdev, bfregi->sys_pages[i]);
1206f8f5b459SHans Petter Selasky }
1207f8f5b459SHans Petter Selasky
mlx5_ib_alloc_transport_domain(struct mlx5_ib_dev * dev,u32 * tdn,u16 uid)1208b633e08cSHans Petter Selasky static int mlx5_ib_alloc_transport_domain(struct mlx5_ib_dev *dev, u32 *tdn,
1209b633e08cSHans Petter Selasky u16 uid)
1210b633e08cSHans Petter Selasky {
1211b633e08cSHans Petter Selasky int err;
1212b633e08cSHans Petter Selasky
1213b633e08cSHans Petter Selasky if (!MLX5_CAP_GEN(dev->mdev, log_max_transport_domain))
1214b633e08cSHans Petter Selasky return 0;
1215b633e08cSHans Petter Selasky
1216b633e08cSHans Petter Selasky err = mlx5_alloc_transport_domain(dev->mdev, tdn, uid);
1217b633e08cSHans Petter Selasky if (err)
1218b633e08cSHans Petter Selasky return err;
1219b633e08cSHans Petter Selasky
1220ea00d7e8SHans Petter Selasky if ((MLX5_CAP_GEN(dev->mdev, port_type) != MLX5_CAP_PORT_TYPE_ETH) ||
1221ea00d7e8SHans Petter Selasky (!MLX5_CAP_GEN(dev->mdev, disable_local_lb_uc) &&
1222ea00d7e8SHans Petter Selasky !MLX5_CAP_GEN(dev->mdev, disable_local_lb_mc)))
1223b633e08cSHans Petter Selasky return 0;
1224ea00d7e8SHans Petter Selasky
1225ea00d7e8SHans Petter Selasky mutex_lock(&dev->lb_mutex);
1226ea00d7e8SHans Petter Selasky dev->user_td++;
1227ea00d7e8SHans Petter Selasky
1228ea00d7e8SHans Petter Selasky if (dev->user_td == 2)
1229ea00d7e8SHans Petter Selasky err = mlx5_nic_vport_update_local_lb(dev->mdev, true);
1230ea00d7e8SHans Petter Selasky
1231ea00d7e8SHans Petter Selasky mutex_unlock(&dev->lb_mutex);
1232ea00d7e8SHans Petter Selasky
1233ea00d7e8SHans Petter Selasky if (err != 0)
1234ea00d7e8SHans Petter Selasky mlx5_dealloc_transport_domain(dev->mdev, *tdn, uid);
1235ea00d7e8SHans Petter Selasky return err;
1236b633e08cSHans Petter Selasky }
1237b633e08cSHans Petter Selasky
mlx5_ib_dealloc_transport_domain(struct mlx5_ib_dev * dev,u32 tdn,u16 uid)1238b633e08cSHans Petter Selasky static void mlx5_ib_dealloc_transport_domain(struct mlx5_ib_dev *dev, u32 tdn,
1239b633e08cSHans Petter Selasky u16 uid)
1240b633e08cSHans Petter Selasky {
1241b633e08cSHans Petter Selasky if (!MLX5_CAP_GEN(dev->mdev, log_max_transport_domain))
1242b633e08cSHans Petter Selasky return;
1243b633e08cSHans Petter Selasky
1244b633e08cSHans Petter Selasky mlx5_dealloc_transport_domain(dev->mdev, tdn, uid);
1245ea00d7e8SHans Petter Selasky
1246ea00d7e8SHans Petter Selasky if ((MLX5_CAP_GEN(dev->mdev, port_type) != MLX5_CAP_PORT_TYPE_ETH) ||
1247ea00d7e8SHans Petter Selasky (!MLX5_CAP_GEN(dev->mdev, disable_local_lb_uc) &&
1248ea00d7e8SHans Petter Selasky !MLX5_CAP_GEN(dev->mdev, disable_local_lb_mc)))
1249ea00d7e8SHans Petter Selasky return;
1250ea00d7e8SHans Petter Selasky
1251ea00d7e8SHans Petter Selasky mutex_lock(&dev->lb_mutex);
1252ea00d7e8SHans Petter Selasky dev->user_td--;
1253ea00d7e8SHans Petter Selasky
1254ea00d7e8SHans Petter Selasky if (dev->user_td < 2)
1255ea00d7e8SHans Petter Selasky mlx5_nic_vport_update_local_lb(dev->mdev, false);
1256ea00d7e8SHans Petter Selasky
1257ea00d7e8SHans Petter Selasky mutex_unlock(&dev->lb_mutex);
1258b633e08cSHans Petter Selasky }
1259b633e08cSHans Petter Selasky
mlx5_ib_alloc_ucontext(struct ib_ucontext * uctx,struct ib_udata * udata)1260b633e08cSHans Petter Selasky static int mlx5_ib_alloc_ucontext(struct ib_ucontext *uctx,
126112515907SHans Petter Selasky struct ib_udata *udata)
126212515907SHans Petter Selasky {
1263b633e08cSHans Petter Selasky struct ib_device *ibdev = uctx->device;
126412515907SHans Petter Selasky struct mlx5_ib_dev *dev = to_mdev(ibdev);
12658e6e287fSHans Petter Selasky struct mlx5_ib_alloc_ucontext_req_v2 req = {};
12668e6e287fSHans Petter Selasky struct mlx5_ib_alloc_ucontext_resp resp = {};
1267b633e08cSHans Petter Selasky struct mlx5_ib_ucontext *context = to_mucontext(uctx);
1268f8f5b459SHans Petter Selasky struct mlx5_bfreg_info *bfregi;
126912515907SHans Petter Selasky int ver;
127012515907SHans Petter Selasky int err;
12718e6e287fSHans Petter Selasky size_t min_req_v2 = offsetof(struct mlx5_ib_alloc_ucontext_req_v2,
12728e6e287fSHans Petter Selasky max_cqe_version);
1273f8f5b459SHans Petter Selasky bool lib_uar_4k;
1274f8f5b459SHans Petter Selasky bool lib_uar_dyn;
127512515907SHans Petter Selasky
127612515907SHans Petter Selasky if (!dev->ib_active)
1277b633e08cSHans Petter Selasky return -EAGAIN;
127812515907SHans Petter Selasky
1279b633e08cSHans Petter Selasky if (udata->inlen == sizeof(struct mlx5_ib_alloc_ucontext_req))
128012515907SHans Petter Selasky ver = 0;
1281b633e08cSHans Petter Selasky else if (udata->inlen >= min_req_v2)
128212515907SHans Petter Selasky ver = 2;
12838e6e287fSHans Petter Selasky else
1284b633e08cSHans Petter Selasky return -EINVAL;
128512515907SHans Petter Selasky
1286b633e08cSHans Petter Selasky err = ib_copy_from_udata(&req, udata, min(udata->inlen, sizeof(req)));
12878e6e287fSHans Petter Selasky if (err)
1288b633e08cSHans Petter Selasky return err;
128912515907SHans Petter Selasky
1290b633e08cSHans Petter Selasky if (req.flags & ~MLX5_IB_ALLOC_UCTX_DEVX)
1291b633e08cSHans Petter Selasky return -EOPNOTSUPP;
129212515907SHans Petter Selasky
12938e6e287fSHans Petter Selasky if (req.comp_mask || req.reserved0 || req.reserved1 || req.reserved2)
1294b633e08cSHans Petter Selasky return -EOPNOTSUPP;
12958e6e287fSHans Petter Selasky
1296f8f5b459SHans Petter Selasky req.total_num_bfregs = ALIGN(req.total_num_bfregs,
1297f8f5b459SHans Petter Selasky MLX5_NON_FP_BFREGS_PER_UAR);
1298f8f5b459SHans Petter Selasky if (req.num_low_latency_bfregs > req.total_num_bfregs - 1)
1299b633e08cSHans Petter Selasky return -EINVAL;
130012515907SHans Petter Selasky
130112515907SHans Petter Selasky resp.qp_tab_size = 1 << MLX5_CAP_GEN(dev->mdev, log_max_qp);
130212515907SHans Petter Selasky if (mlx5_core_is_pf(dev->mdev) && MLX5_CAP_GEN(dev->mdev, bf))
130312515907SHans Petter Selasky resp.bf_reg_size = 1 << MLX5_CAP_GEN(dev->mdev, log_bf_reg_size);
13048e6e287fSHans Petter Selasky resp.cache_line_size = cache_line_size();
130512515907SHans Petter Selasky resp.max_sq_desc_sz = MLX5_CAP_GEN(dev->mdev, max_wqe_sz_sq);
130612515907SHans Petter Selasky resp.max_rq_desc_sz = MLX5_CAP_GEN(dev->mdev, max_wqe_sz_rq);
130712515907SHans Petter Selasky resp.max_send_wqebb = 1 << MLX5_CAP_GEN(dev->mdev, log_max_qp_sz);
130812515907SHans Petter Selasky resp.max_recv_wr = 1 << MLX5_CAP_GEN(dev->mdev, log_max_qp_sz);
130912515907SHans Petter Selasky resp.max_srq_recv_wr = 1 << MLX5_CAP_GEN(dev->mdev, log_max_srq_sz);
13108e6e287fSHans Petter Selasky resp.cqe_version = min_t(__u8,
13118e6e287fSHans Petter Selasky (__u8)MLX5_CAP_GEN(dev->mdev, cqe_version),
13128e6e287fSHans Petter Selasky req.max_cqe_version);
1313f8f5b459SHans Petter Selasky resp.log_uar_size = MLX5_CAP_GEN(dev->mdev, uar_4k) ?
1314f8f5b459SHans Petter Selasky MLX5_ADAPTER_PAGE_SHIFT : PAGE_SHIFT;
1315f8f5b459SHans Petter Selasky resp.num_uars_per_page = MLX5_CAP_GEN(dev->mdev, uar_4k) ?
1316f8f5b459SHans Petter Selasky MLX5_CAP_GEN(dev->mdev, num_of_uars_per_page) : 1;
13178e6e287fSHans Petter Selasky resp.response_length = min(offsetof(typeof(resp), response_length) +
13188e6e287fSHans Petter Selasky sizeof(resp.response_length), udata->outlen);
131912515907SHans Petter Selasky
1320f8f5b459SHans Petter Selasky lib_uar_4k = req.lib_caps & MLX5_LIB_CAP_4K_UAR;
1321f8f5b459SHans Petter Selasky lib_uar_dyn = req.lib_caps & MLX5_LIB_CAP_DYN_UAR;
1322f8f5b459SHans Petter Selasky bfregi = &context->bfregi;
1323f8f5b459SHans Petter Selasky
1324f8f5b459SHans Petter Selasky if (lib_uar_dyn) {
1325f8f5b459SHans Petter Selasky bfregi->lib_uar_dyn = lib_uar_dyn;
1326f8f5b459SHans Petter Selasky goto uar_done;
1327f8f5b459SHans Petter Selasky }
1328f8f5b459SHans Petter Selasky
1329f8f5b459SHans Petter Selasky /* updates req->total_num_bfregs */
1330f8f5b459SHans Petter Selasky err = calc_total_bfregs(dev, lib_uar_4k, &req, bfregi);
1331f8f5b459SHans Petter Selasky if (err)
1332f8f5b459SHans Petter Selasky goto out_ctx;
1333f8f5b459SHans Petter Selasky
1334f8f5b459SHans Petter Selasky mutex_init(&bfregi->lock);
1335f8f5b459SHans Petter Selasky bfregi->lib_uar_4k = lib_uar_4k;
1336f8f5b459SHans Petter Selasky bfregi->count = kcalloc(bfregi->total_num_bfregs, sizeof(*bfregi->count),
1337f8f5b459SHans Petter Selasky GFP_KERNEL);
1338f8f5b459SHans Petter Selasky if (!bfregi->count) {
133912515907SHans Petter Selasky err = -ENOMEM;
134012515907SHans Petter Selasky goto out_ctx;
134112515907SHans Petter Selasky }
134212515907SHans Petter Selasky
1343f8f5b459SHans Petter Selasky bfregi->sys_pages = kcalloc(bfregi->num_sys_pages,
1344f8f5b459SHans Petter Selasky sizeof(*bfregi->sys_pages),
134512515907SHans Petter Selasky GFP_KERNEL);
1346f8f5b459SHans Petter Selasky if (!bfregi->sys_pages) {
134712515907SHans Petter Selasky err = -ENOMEM;
13488e6e287fSHans Petter Selasky goto out_count;
13498e6e287fSHans Petter Selasky }
13508e6e287fSHans Petter Selasky
1351f8f5b459SHans Petter Selasky err = allocate_uars(dev, context);
1352f8f5b459SHans Petter Selasky if (err)
1353f8f5b459SHans Petter Selasky goto out_sys_pages;
1354f8f5b459SHans Petter Selasky
1355f8f5b459SHans Petter Selasky uar_done:
1356b633e08cSHans Petter Selasky if (req.flags & MLX5_IB_ALLOC_UCTX_DEVX) {
1357b633e08cSHans Petter Selasky err = mlx5_ib_devx_create(dev, true);
1358b633e08cSHans Petter Selasky if (err < 0)
135912515907SHans Petter Selasky goto out_uars;
1360b633e08cSHans Petter Selasky context->devx_uid = err;
136112515907SHans Petter Selasky }
136212515907SHans Petter Selasky
1363b633e08cSHans Petter Selasky err = mlx5_ib_alloc_transport_domain(dev, &context->tdn,
1364b633e08cSHans Petter Selasky context->devx_uid);
1365b633e08cSHans Petter Selasky if (err)
1366b633e08cSHans Petter Selasky goto out_devx;
1367b633e08cSHans Petter Selasky
136812515907SHans Petter Selasky INIT_LIST_HEAD(&context->db_page_list);
136912515907SHans Petter Selasky mutex_init(&context->db_page_mutex);
137012515907SHans Petter Selasky
1371f8f5b459SHans Petter Selasky resp.tot_bfregs = lib_uar_dyn ? 0 : req.total_num_bfregs;
137212515907SHans Petter Selasky resp.num_ports = MLX5_CAP_GEN(dev->mdev, num_ports);
13738e6e287fSHans Petter Selasky
13748e6e287fSHans Petter Selasky if (field_avail(typeof(resp), cqe_version, udata->outlen))
13758e6e287fSHans Petter Selasky resp.response_length += sizeof(resp.cqe_version);
13768e6e287fSHans Petter Selasky
13778e6e287fSHans Petter Selasky if (field_avail(typeof(resp), cmds_supp_uhw, udata->outlen)) {
13781456d97cSHans Petter Selasky resp.cmds_supp_uhw |= MLX5_USER_CMDS_SUPP_UHW_QUERY_DEVICE |
13791456d97cSHans Petter Selasky MLX5_USER_CMDS_SUPP_UHW_CREATE_AH;
13808e6e287fSHans Petter Selasky resp.response_length += sizeof(resp.cmds_supp_uhw);
13818e6e287fSHans Petter Selasky }
13828e6e287fSHans Petter Selasky
13838e6e287fSHans Petter Selasky /*
13848e6e287fSHans Petter Selasky * We don't want to expose information from the PCI bar that is located
13858e6e287fSHans Petter Selasky * after 4096 bytes, so if the arch only supports larger pages, let's
13868e6e287fSHans Petter Selasky * pretend we don't support reading the HCA's core clock. This is also
13878e6e287fSHans Petter Selasky * forced by mmap function.
13888e6e287fSHans Petter Selasky */
1389f8f5b459SHans Petter Selasky if (offsetofend(typeof(resp), hca_core_clock_offset) <= udata->outlen) {
1390f8f5b459SHans Petter Selasky if (PAGE_SIZE <= 4096) {
13918e6e287fSHans Petter Selasky resp.comp_mask |=
13928e6e287fSHans Petter Selasky MLX5_IB_ALLOC_UCONTEXT_RESP_MASK_CORE_CLOCK_OFFSET;
13938e6e287fSHans Petter Selasky resp.hca_core_clock_offset =
1394f8f5b459SHans Petter Selasky offsetof(struct mlx5_init_seg, internal_timer_h) % PAGE_SIZE;
1395f8f5b459SHans Petter Selasky }
1396f8f5b459SHans Petter Selasky resp.response_length += sizeof(resp.hca_core_clock_offset);
1397f8f5b459SHans Petter Selasky }
1398f8f5b459SHans Petter Selasky
1399f8f5b459SHans Petter Selasky if (offsetofend(typeof(resp), log_uar_size) <= udata->outlen)
1400f8f5b459SHans Petter Selasky resp.response_length += sizeof(resp.log_uar_size);
1401f8f5b459SHans Petter Selasky
1402f8f5b459SHans Petter Selasky if (offsetofend(typeof(resp), num_uars_per_page) <= udata->outlen)
1403f8f5b459SHans Petter Selasky resp.response_length += sizeof(resp.num_uars_per_page);
1404f8f5b459SHans Petter Selasky
1405f8f5b459SHans Petter Selasky if (offsetofend(typeof(resp), num_dyn_bfregs) <= udata->outlen) {
1406f8f5b459SHans Petter Selasky resp.num_dyn_bfregs = bfregi->num_dyn_bfregs;
1407f8f5b459SHans Petter Selasky resp.response_length += sizeof(resp.num_dyn_bfregs);
14088e6e287fSHans Petter Selasky }
14098e6e287fSHans Petter Selasky
14108e6e287fSHans Petter Selasky err = ib_copy_to_udata(udata, &resp, resp.response_length);
141112515907SHans Petter Selasky if (err)
1412b633e08cSHans Petter Selasky goto out_mdev;
141312515907SHans Petter Selasky
1414f8f5b459SHans Petter Selasky bfregi->ver = ver;
1415f8f5b459SHans Petter Selasky bfregi->num_low_latency_bfregs = req.num_low_latency_bfregs;
14168e6e287fSHans Petter Selasky context->cqe_version = resp.cqe_version;
1417b633e08cSHans Petter Selasky context->lib_caps = req.lib_caps;
1418b633e08cSHans Petter Selasky print_lib_caps(dev, context->lib_caps);
141912515907SHans Petter Selasky
1420b633e08cSHans Petter Selasky return 0;
142112515907SHans Petter Selasky
1422b633e08cSHans Petter Selasky out_mdev:
1423b633e08cSHans Petter Selasky mlx5_ib_dealloc_transport_domain(dev, context->tdn, context->devx_uid);
1424b633e08cSHans Petter Selasky out_devx:
1425b633e08cSHans Petter Selasky if (req.flags & MLX5_IB_ALLOC_UCTX_DEVX)
1426b633e08cSHans Petter Selasky mlx5_ib_devx_destroy(dev, context->devx_uid);
14278e6e287fSHans Petter Selasky
142812515907SHans Petter Selasky out_uars:
1429f8f5b459SHans Petter Selasky deallocate_uars(dev, context);
1430f8f5b459SHans Petter Selasky
1431f8f5b459SHans Petter Selasky out_sys_pages:
1432f8f5b459SHans Petter Selasky kfree(bfregi->sys_pages);
1433f8f5b459SHans Petter Selasky
14348e6e287fSHans Petter Selasky out_count:
1435f8f5b459SHans Petter Selasky kfree(bfregi->count);
143612515907SHans Petter Selasky
143712515907SHans Petter Selasky out_ctx:
1438b633e08cSHans Petter Selasky return err;
143912515907SHans Petter Selasky }
144012515907SHans Petter Selasky
mlx5_ib_dealloc_ucontext(struct ib_ucontext * ibcontext)1441b633e08cSHans Petter Selasky static void mlx5_ib_dealloc_ucontext(struct ib_ucontext *ibcontext)
144212515907SHans Petter Selasky {
144312515907SHans Petter Selasky struct mlx5_ib_ucontext *context = to_mucontext(ibcontext);
144412515907SHans Petter Selasky struct mlx5_ib_dev *dev = to_mdev(ibcontext->device);
1445f8f5b459SHans Petter Selasky struct mlx5_bfreg_info *bfregi;
144612515907SHans Petter Selasky
1447f8f5b459SHans Petter Selasky bfregi = &context->bfregi;
1448b633e08cSHans Petter Selasky mlx5_ib_dealloc_transport_domain(dev, context->tdn, context->devx_uid);
1449b633e08cSHans Petter Selasky
1450b633e08cSHans Petter Selasky if (context->devx_uid)
1451b633e08cSHans Petter Selasky mlx5_ib_devx_destroy(dev, context->devx_uid);
145212515907SHans Petter Selasky
1453f8f5b459SHans Petter Selasky deallocate_uars(dev, context);
1454f8f5b459SHans Petter Selasky kfree(bfregi->sys_pages);
1455f8f5b459SHans Petter Selasky kfree(bfregi->count);
145612515907SHans Petter Selasky }
145712515907SHans Petter Selasky
uar_index2pfn(struct mlx5_ib_dev * dev,int uar_idx)1458f8f5b459SHans Petter Selasky static phys_addr_t uar_index2pfn(struct mlx5_ib_dev *dev,
1459f8f5b459SHans Petter Selasky int uar_idx)
146012515907SHans Petter Selasky {
1461f8f5b459SHans Petter Selasky int fw_uars_per_page;
1462f8f5b459SHans Petter Selasky
1463f8f5b459SHans Petter Selasky fw_uars_per_page = MLX5_CAP_GEN(dev->mdev, uar_4k) ? MLX5_UARS_IN_PAGE : 1;
1464f8f5b459SHans Petter Selasky
1465f8f5b459SHans Petter Selasky return (pci_resource_start(dev->mdev->pdev, 0) >> PAGE_SHIFT) + uar_idx / fw_uars_per_page;
146612515907SHans Petter Selasky }
146712515907SHans Petter Selasky
get_command(unsigned long offset)146812515907SHans Petter Selasky static int get_command(unsigned long offset)
146912515907SHans Petter Selasky {
147012515907SHans Petter Selasky return (offset >> MLX5_IB_MMAP_CMD_SHIFT) & MLX5_IB_MMAP_CMD_MASK;
147112515907SHans Petter Selasky }
147212515907SHans Petter Selasky
get_arg(unsigned long offset)147312515907SHans Petter Selasky static int get_arg(unsigned long offset)
147412515907SHans Petter Selasky {
147512515907SHans Petter Selasky return offset & ((1 << MLX5_IB_MMAP_CMD_SHIFT) - 1);
147612515907SHans Petter Selasky }
147712515907SHans Petter Selasky
get_index(unsigned long offset)147812515907SHans Petter Selasky static int get_index(unsigned long offset)
147912515907SHans Petter Selasky {
148012515907SHans Petter Selasky return get_arg(offset);
148112515907SHans Petter Selasky }
148212515907SHans Petter Selasky
1483f8f5b459SHans Petter Selasky /* Index resides in an extra byte to enable larger values than 255 */
get_extended_index(unsigned long offset)1484f8f5b459SHans Petter Selasky static int get_extended_index(unsigned long offset)
1485f8f5b459SHans Petter Selasky {
1486f8f5b459SHans Petter Selasky return get_arg(offset) | ((offset >> 16) & 0xff) << 8;
1487f8f5b459SHans Petter Selasky }
1488f8f5b459SHans Petter Selasky
14898e6e287fSHans Petter Selasky
mlx5_ib_disassociate_ucontext(struct ib_ucontext * ibcontext)1490a3a31fdeSHans Petter Selasky static void mlx5_ib_disassociate_ucontext(struct ib_ucontext *ibcontext)
1491a3a31fdeSHans Petter Selasky {
1492a3a31fdeSHans Petter Selasky }
1493a3a31fdeSHans Petter Selasky
mmap_cmd2str(enum mlx5_ib_mmap_cmd cmd)14948e6e287fSHans Petter Selasky static inline char *mmap_cmd2str(enum mlx5_ib_mmap_cmd cmd)
14958e6e287fSHans Petter Selasky {
14968e6e287fSHans Petter Selasky switch (cmd) {
14978e6e287fSHans Petter Selasky case MLX5_IB_MMAP_WC_PAGE:
14988e6e287fSHans Petter Selasky return "WC";
14998e6e287fSHans Petter Selasky case MLX5_IB_MMAP_REGULAR_PAGE:
15008e6e287fSHans Petter Selasky return "best effort WC";
15018e6e287fSHans Petter Selasky case MLX5_IB_MMAP_NC_PAGE:
15028e6e287fSHans Petter Selasky return "NC";
15038e6e287fSHans Petter Selasky default:
15048e6e287fSHans Petter Selasky return NULL;
15058e6e287fSHans Petter Selasky }
15068e6e287fSHans Petter Selasky }
15078e6e287fSHans Petter Selasky
mlx5_ib_mmap_clock_info_page(struct mlx5_ib_dev * dev,struct vm_area_struct * vma,struct mlx5_ib_ucontext * context)1508b633e08cSHans Petter Selasky static int mlx5_ib_mmap_clock_info_page(struct mlx5_ib_dev *dev,
1509b633e08cSHans Petter Selasky struct vm_area_struct *vma,
1510b633e08cSHans Petter Selasky struct mlx5_ib_ucontext *context)
1511b633e08cSHans Petter Selasky {
1512b633e08cSHans Petter Selasky if ((vma->vm_end - vma->vm_start != PAGE_SIZE) ||
1513b633e08cSHans Petter Selasky !(vma->vm_flags & VM_SHARED))
1514b633e08cSHans Petter Selasky return -EINVAL;
1515b633e08cSHans Petter Selasky
1516b633e08cSHans Petter Selasky if (get_index(vma->vm_pgoff) != MLX5_IB_CLOCK_INFO_V1)
1517b633e08cSHans Petter Selasky return -EOPNOTSUPP;
1518b633e08cSHans Petter Selasky
1519b633e08cSHans Petter Selasky if (vma->vm_flags & (VM_WRITE | VM_EXEC))
1520b633e08cSHans Petter Selasky return -EPERM;
1521b633e08cSHans Petter Selasky
1522b633e08cSHans Petter Selasky return -EOPNOTSUPP;
1523b633e08cSHans Petter Selasky }
1524b633e08cSHans Petter Selasky
mlx5_ib_mmap_free(struct rdma_user_mmap_entry * entry)1525b633e08cSHans Petter Selasky static void mlx5_ib_mmap_free(struct rdma_user_mmap_entry *entry)
1526b633e08cSHans Petter Selasky {
1527b633e08cSHans Petter Selasky struct mlx5_user_mmap_entry *mentry = to_mmmap(entry);
1528b633e08cSHans Petter Selasky struct mlx5_ib_dev *dev = to_mdev(entry->ucontext->device);
1529b633e08cSHans Petter Selasky
1530b633e08cSHans Petter Selasky switch (mentry->mmap_flag) {
1531b633e08cSHans Petter Selasky case MLX5_IB_MMAP_TYPE_UAR_WC:
1532b633e08cSHans Petter Selasky case MLX5_IB_MMAP_TYPE_UAR_NC:
1533b633e08cSHans Petter Selasky mlx5_cmd_free_uar(dev->mdev, mentry->page_idx);
1534b633e08cSHans Petter Selasky kfree(mentry);
1535b633e08cSHans Petter Selasky break;
1536b633e08cSHans Petter Selasky default:
1537b633e08cSHans Petter Selasky WARN_ON(true);
1538b633e08cSHans Petter Selasky }
1539b633e08cSHans Petter Selasky }
1540b633e08cSHans Petter Selasky
uar_mmap(struct mlx5_ib_dev * dev,enum mlx5_ib_mmap_cmd cmd,struct vm_area_struct * vma,struct mlx5_ib_ucontext * context)15418e6e287fSHans Petter Selasky static int uar_mmap(struct mlx5_ib_dev *dev, enum mlx5_ib_mmap_cmd cmd,
15428e6e287fSHans Petter Selasky struct vm_area_struct *vma,
154312515907SHans Petter Selasky struct mlx5_ib_ucontext *context)
154412515907SHans Petter Selasky {
1545f8f5b459SHans Petter Selasky struct mlx5_bfreg_info *bfregi = &context->bfregi;
15468e6e287fSHans Petter Selasky int err;
154712515907SHans Petter Selasky unsigned long idx;
1548f8f5b459SHans Petter Selasky phys_addr_t pfn;
15498e6e287fSHans Petter Selasky pgprot_t prot;
1550f8f5b459SHans Petter Selasky u32 bfreg_dyn_idx = 0;
1551f8f5b459SHans Petter Selasky u32 uar_index;
1552b633e08cSHans Petter Selasky int dyn_uar = (cmd == MLX5_IB_MMAP_ALLOC_WC);
1553f8f5b459SHans Petter Selasky int max_valid_idx = dyn_uar ? bfregi->num_sys_pages :
1554f8f5b459SHans Petter Selasky bfregi->num_static_sys_pages;
1555f8f5b459SHans Petter Selasky
1556f8f5b459SHans Petter Selasky if (bfregi->lib_uar_dyn)
1557f8f5b459SHans Petter Selasky return -EINVAL;
1558f8f5b459SHans Petter Selasky
1559f8f5b459SHans Petter Selasky if (vma->vm_end - vma->vm_start != PAGE_SIZE)
1560f8f5b459SHans Petter Selasky return -EINVAL;
1561f8f5b459SHans Petter Selasky
1562f8f5b459SHans Petter Selasky if (dyn_uar)
1563f8f5b459SHans Petter Selasky idx = get_extended_index(vma->vm_pgoff) + bfregi->num_static_sys_pages;
1564f8f5b459SHans Petter Selasky else
1565f8f5b459SHans Petter Selasky idx = get_index(vma->vm_pgoff);
1566f8f5b459SHans Petter Selasky
1567f8f5b459SHans Petter Selasky if (idx >= max_valid_idx) {
1568f8f5b459SHans Petter Selasky mlx5_ib_warn(dev, "invalid uar index %lu, max=%d\n",
1569f8f5b459SHans Petter Selasky idx, max_valid_idx);
1570f8f5b459SHans Petter Selasky return -EINVAL;
1571f8f5b459SHans Petter Selasky }
157212515907SHans Petter Selasky
15738e6e287fSHans Petter Selasky switch (cmd) {
15748e6e287fSHans Petter Selasky case MLX5_IB_MMAP_WC_PAGE:
1575b633e08cSHans Petter Selasky case MLX5_IB_MMAP_ALLOC_WC:
15768e6e287fSHans Petter Selasky case MLX5_IB_MMAP_REGULAR_PAGE:
15778e6e287fSHans Petter Selasky /* For MLX5_IB_MMAP_REGULAR_PAGE do the best effort to get WC */
15788e6e287fSHans Petter Selasky prot = pgprot_writecombine(vma->vm_page_prot);
15798e6e287fSHans Petter Selasky break;
15808e6e287fSHans Petter Selasky case MLX5_IB_MMAP_NC_PAGE:
15818e6e287fSHans Petter Selasky prot = pgprot_noncached(vma->vm_page_prot);
15828e6e287fSHans Petter Selasky break;
15838e6e287fSHans Petter Selasky default:
158412515907SHans Petter Selasky return -EINVAL;
158512515907SHans Petter Selasky }
158612515907SHans Petter Selasky
1587f8f5b459SHans Petter Selasky if (dyn_uar) {
1588f8f5b459SHans Petter Selasky int uars_per_page;
15898e6e287fSHans Petter Selasky
1590f8f5b459SHans Petter Selasky uars_per_page = get_uars_per_sys_page(dev, bfregi->lib_uar_4k);
1591f8f5b459SHans Petter Selasky bfreg_dyn_idx = idx * (uars_per_page * MLX5_NON_FP_BFREGS_PER_UAR);
1592f8f5b459SHans Petter Selasky if (bfreg_dyn_idx >= bfregi->total_num_bfregs) {
1593f8f5b459SHans Petter Selasky mlx5_ib_warn(dev, "invalid bfreg_dyn_idx %u, max=%u\n",
1594f8f5b459SHans Petter Selasky bfreg_dyn_idx, bfregi->total_num_bfregs);
159512515907SHans Petter Selasky return -EINVAL;
1596f8f5b459SHans Petter Selasky }
159712515907SHans Petter Selasky
1598f8f5b459SHans Petter Selasky mutex_lock(&bfregi->lock);
1599f8f5b459SHans Petter Selasky /* Fail if uar already allocated, first bfreg index of each
1600f8f5b459SHans Petter Selasky * page holds its count.
1601f8f5b459SHans Petter Selasky */
1602f8f5b459SHans Petter Selasky if (bfregi->count[bfreg_dyn_idx]) {
1603f8f5b459SHans Petter Selasky mlx5_ib_warn(dev, "wrong offset, idx %lu is busy, bfregn=%u\n", idx, bfreg_dyn_idx);
1604f8f5b459SHans Petter Selasky mutex_unlock(&bfregi->lock);
1605f8f5b459SHans Petter Selasky return -EINVAL;
1606f8f5b459SHans Petter Selasky }
1607f8f5b459SHans Petter Selasky
1608f8f5b459SHans Petter Selasky bfregi->count[bfreg_dyn_idx]++;
1609f8f5b459SHans Petter Selasky mutex_unlock(&bfregi->lock);
1610f8f5b459SHans Petter Selasky
1611f8f5b459SHans Petter Selasky err = mlx5_cmd_alloc_uar(dev->mdev, &uar_index);
1612f8f5b459SHans Petter Selasky if (err) {
1613f8f5b459SHans Petter Selasky mlx5_ib_warn(dev, "UAR alloc failed\n");
1614f8f5b459SHans Petter Selasky goto free_bfreg;
1615f8f5b459SHans Petter Selasky }
1616f8f5b459SHans Petter Selasky } else {
1617f8f5b459SHans Petter Selasky uar_index = bfregi->sys_pages[idx];
1618f8f5b459SHans Petter Selasky }
1619f8f5b459SHans Petter Selasky
1620f8f5b459SHans Petter Selasky pfn = uar_index2pfn(dev, uar_index);
16218e6e287fSHans Petter Selasky mlx5_ib_dbg(dev, "uar idx 0x%lx, pfn %pa\n", idx, &pfn);
162212515907SHans Petter Selasky
1623b633e08cSHans Petter Selasky err = rdma_user_mmap_io(&context->ibucontext, vma, pfn, PAGE_SIZE,
1624b633e08cSHans Petter Selasky prot, NULL);
16258e6e287fSHans Petter Selasky if (err) {
1626b633e08cSHans Petter Selasky mlx5_ib_err(dev,
1627b633e08cSHans Petter Selasky "rdma_user_mmap_io failed with error=%d, mmap_cmd=%s\n",
1628b633e08cSHans Petter Selasky err, mmap_cmd2str(cmd));
1629f8f5b459SHans Petter Selasky goto err;
163012515907SHans Petter Selasky }
163112515907SHans Petter Selasky
1632f8f5b459SHans Petter Selasky if (dyn_uar)
1633f8f5b459SHans Petter Selasky bfregi->sys_pages[idx] = uar_index;
1634b633e08cSHans Petter Selasky return 0;
1635f8f5b459SHans Petter Selasky
1636f8f5b459SHans Petter Selasky err:
1637f8f5b459SHans Petter Selasky if (!dyn_uar)
1638f8f5b459SHans Petter Selasky return err;
1639f8f5b459SHans Petter Selasky
1640f8f5b459SHans Petter Selasky mlx5_cmd_free_uar(dev->mdev, idx);
1641f8f5b459SHans Petter Selasky
1642f8f5b459SHans Petter Selasky free_bfreg:
1643f8f5b459SHans Petter Selasky mlx5_ib_free_bfreg(dev, bfregi, bfreg_dyn_idx);
1644f8f5b459SHans Petter Selasky
1645f8f5b459SHans Petter Selasky return err;
164612515907SHans Petter Selasky }
164712515907SHans Petter Selasky
mlx5_vma_to_pgoff(struct vm_area_struct * vma)1648b633e08cSHans Petter Selasky static unsigned long mlx5_vma_to_pgoff(struct vm_area_struct *vma)
1649b633e08cSHans Petter Selasky {
1650b633e08cSHans Petter Selasky unsigned long idx;
1651b633e08cSHans Petter Selasky u8 command;
1652b633e08cSHans Petter Selasky
1653b633e08cSHans Petter Selasky command = get_command(vma->vm_pgoff);
1654b633e08cSHans Petter Selasky idx = get_extended_index(vma->vm_pgoff);
1655b633e08cSHans Petter Selasky
1656b633e08cSHans Petter Selasky return (command << 16 | idx);
1657b633e08cSHans Petter Selasky }
1658b633e08cSHans Petter Selasky
mlx5_ib_mmap_offset(struct mlx5_ib_dev * dev,struct vm_area_struct * vma,struct ib_ucontext * ucontext)1659b633e08cSHans Petter Selasky static int mlx5_ib_mmap_offset(struct mlx5_ib_dev *dev,
1660b633e08cSHans Petter Selasky struct vm_area_struct *vma,
1661b633e08cSHans Petter Selasky struct ib_ucontext *ucontext)
1662b633e08cSHans Petter Selasky {
1663b633e08cSHans Petter Selasky struct mlx5_user_mmap_entry *mentry;
1664b633e08cSHans Petter Selasky struct rdma_user_mmap_entry *entry;
1665b633e08cSHans Petter Selasky unsigned long pgoff;
1666b633e08cSHans Petter Selasky pgprot_t prot;
1667b633e08cSHans Petter Selasky phys_addr_t pfn;
1668b633e08cSHans Petter Selasky int ret;
1669b633e08cSHans Petter Selasky
1670b633e08cSHans Petter Selasky pgoff = mlx5_vma_to_pgoff(vma);
1671b633e08cSHans Petter Selasky entry = rdma_user_mmap_entry_get_pgoff(ucontext, pgoff);
1672b633e08cSHans Petter Selasky if (!entry)
1673b633e08cSHans Petter Selasky return -EINVAL;
1674b633e08cSHans Petter Selasky
1675b633e08cSHans Petter Selasky mentry = to_mmmap(entry);
1676b633e08cSHans Petter Selasky pfn = (mentry->address >> PAGE_SHIFT);
1677b633e08cSHans Petter Selasky if (mentry->mmap_flag == MLX5_IB_MMAP_TYPE_VAR ||
1678b633e08cSHans Petter Selasky mentry->mmap_flag == MLX5_IB_MMAP_TYPE_UAR_NC)
1679b633e08cSHans Petter Selasky prot = pgprot_noncached(vma->vm_page_prot);
1680b633e08cSHans Petter Selasky else
1681b633e08cSHans Petter Selasky prot = pgprot_writecombine(vma->vm_page_prot);
1682b633e08cSHans Petter Selasky ret = rdma_user_mmap_io(ucontext, vma, pfn,
1683b633e08cSHans Petter Selasky entry->npages * PAGE_SIZE,
1684b633e08cSHans Petter Selasky prot,
1685b633e08cSHans Petter Selasky entry);
1686b633e08cSHans Petter Selasky rdma_user_mmap_entry_put(&mentry->rdma_entry);
1687b633e08cSHans Petter Selasky return ret;
1688b633e08cSHans Petter Selasky }
1689b633e08cSHans Petter Selasky
mlx5_ib_mmap(struct ib_ucontext * ibcontext,struct vm_area_struct * vma)169012515907SHans Petter Selasky static int mlx5_ib_mmap(struct ib_ucontext *ibcontext, struct vm_area_struct *vma)
169112515907SHans Petter Selasky {
169212515907SHans Petter Selasky struct mlx5_ib_ucontext *context = to_mucontext(ibcontext);
169312515907SHans Petter Selasky struct mlx5_ib_dev *dev = to_mdev(ibcontext->device);
169412515907SHans Petter Selasky unsigned long command;
16958e6e287fSHans Petter Selasky phys_addr_t pfn;
169612515907SHans Petter Selasky
169712515907SHans Petter Selasky command = get_command(vma->vm_pgoff);
169812515907SHans Petter Selasky switch (command) {
169912515907SHans Petter Selasky case MLX5_IB_MMAP_WC_PAGE:
1700b633e08cSHans Petter Selasky case MLX5_IB_MMAP_ALLOC_WC:
1701b633e08cSHans Petter Selasky if (!dev->wc_support)
1702b633e08cSHans Petter Selasky return -EPERM;
1703b633e08cSHans Petter Selasky /* FALLTHROUGH */
170412515907SHans Petter Selasky case MLX5_IB_MMAP_NC_PAGE:
17058e6e287fSHans Petter Selasky case MLX5_IB_MMAP_REGULAR_PAGE:
17068e6e287fSHans Petter Selasky return uar_mmap(dev, command, vma, context);
17078e6e287fSHans Petter Selasky
17088e6e287fSHans Petter Selasky case MLX5_IB_MMAP_GET_CONTIGUOUS_PAGES:
17098e6e287fSHans Petter Selasky return -ENOSYS;
17108e6e287fSHans Petter Selasky
17118e6e287fSHans Petter Selasky case MLX5_IB_MMAP_CORE_CLOCK:
17128e6e287fSHans Petter Selasky if (vma->vm_end - vma->vm_start != PAGE_SIZE)
17138e6e287fSHans Petter Selasky return -EINVAL;
17148e6e287fSHans Petter Selasky
17158e6e287fSHans Petter Selasky if (vma->vm_flags & VM_WRITE)
17168e6e287fSHans Petter Selasky return -EPERM;
17178e6e287fSHans Petter Selasky
17188e6e287fSHans Petter Selasky /* Don't expose to user-space information it shouldn't have */
17198e6e287fSHans Petter Selasky if (PAGE_SIZE > 4096)
17208e6e287fSHans Petter Selasky return -EOPNOTSUPP;
17218e6e287fSHans Petter Selasky
17228e6e287fSHans Petter Selasky pfn = (dev->mdev->iseg_base +
17238e6e287fSHans Petter Selasky offsetof(struct mlx5_init_seg, internal_timer_h)) >>
17248e6e287fSHans Petter Selasky PAGE_SHIFT;
1725b633e08cSHans Petter Selasky return rdma_user_mmap_io(&context->ibucontext, vma, pfn,
1726b633e08cSHans Petter Selasky PAGE_SIZE,
1727b633e08cSHans Petter Selasky pgprot_noncached(vma->vm_page_prot),
1728b633e08cSHans Petter Selasky NULL);
1729b633e08cSHans Petter Selasky case MLX5_IB_MMAP_CLOCK_INFO:
1730b633e08cSHans Petter Selasky return mlx5_ib_mmap_clock_info_page(dev, vma, context);
173112515907SHans Petter Selasky
173212515907SHans Petter Selasky default:
1733b633e08cSHans Petter Selasky return mlx5_ib_mmap_offset(dev, vma, ibcontext);
173412515907SHans Petter Selasky }
173512515907SHans Petter Selasky
173612515907SHans Petter Selasky return 0;
173712515907SHans Petter Selasky }
173812515907SHans Petter Selasky
mlx5_ib_alloc_pd(struct ib_pd * ibpd,struct ib_udata * udata)1739b633e08cSHans Petter Selasky static int mlx5_ib_alloc_pd(struct ib_pd *ibpd, struct ib_udata *udata)
174012515907SHans Petter Selasky {
1741b633e08cSHans Petter Selasky struct mlx5_ib_pd *pd = to_mpd(ibpd);
1742b633e08cSHans Petter Selasky struct ib_device *ibdev = ibpd->device;
174312515907SHans Petter Selasky struct mlx5_ib_alloc_pd_resp resp;
174412515907SHans Petter Selasky int err;
1745b633e08cSHans Petter Selasky struct mlx5_ib_ucontext *context = rdma_udata_to_drv_context(
1746b633e08cSHans Petter Selasky udata, struct mlx5_ib_ucontext, ibucontext);
1747b633e08cSHans Petter Selasky u16 uid = context ? context->devx_uid : 0;
174812515907SHans Petter Selasky
1749b633e08cSHans Petter Selasky err = mlx5_core_alloc_pd(to_mdev(ibdev)->mdev, &pd->pdn, uid);
1750b633e08cSHans Petter Selasky if (err)
1751b633e08cSHans Petter Selasky return (err);
175212515907SHans Petter Selasky
1753b633e08cSHans Petter Selasky pd->uid = uid;
1754b633e08cSHans Petter Selasky if (udata) {
175512515907SHans Petter Selasky resp.pdn = pd->pdn;
175612515907SHans Petter Selasky if (ib_copy_to_udata(udata, &resp, sizeof(resp))) {
1757b633e08cSHans Petter Selasky mlx5_core_dealloc_pd(to_mdev(ibdev)->mdev, pd->pdn, uid);
1758b633e08cSHans Petter Selasky return -EFAULT;
175912515907SHans Petter Selasky }
176012515907SHans Petter Selasky }
176112515907SHans Petter Selasky
1762b633e08cSHans Petter Selasky return 0;
176312515907SHans Petter Selasky }
176412515907SHans Petter Selasky
mlx5_ib_dealloc_pd(struct ib_pd * pd,struct ib_udata * udata)1765b633e08cSHans Petter Selasky static void mlx5_ib_dealloc_pd(struct ib_pd *pd, struct ib_udata *udata)
176612515907SHans Petter Selasky {
176712515907SHans Petter Selasky struct mlx5_ib_dev *mdev = to_mdev(pd->device);
176812515907SHans Petter Selasky struct mlx5_ib_pd *mpd = to_mpd(pd);
176912515907SHans Petter Selasky
1770b633e08cSHans Petter Selasky mlx5_core_dealloc_pd(mdev->mdev, mpd->pdn, mpd->uid);
177112515907SHans Petter Selasky }
177212515907SHans Petter Selasky
17738e6e287fSHans Petter Selasky enum {
17748e6e287fSHans Petter Selasky MATCH_CRITERIA_ENABLE_OUTER_BIT,
17758e6e287fSHans Petter Selasky MATCH_CRITERIA_ENABLE_MISC_BIT,
17768e6e287fSHans Petter Selasky MATCH_CRITERIA_ENABLE_INNER_BIT
17778e6e287fSHans Petter Selasky };
17788e6e287fSHans Petter Selasky
17798e6e287fSHans Petter Selasky #define HEADER_IS_ZERO(match_criteria, headers) \
17808e6e287fSHans Petter Selasky !(memchr_inv(MLX5_ADDR_OF(fte_match_param, match_criteria, headers), \
17818e6e287fSHans Petter Selasky 0, MLX5_FLD_SZ_BYTES(fte_match_param, headers))) \
17828e6e287fSHans Petter Selasky
get_match_criteria_enable(u32 * match_criteria)17838e6e287fSHans Petter Selasky static u8 get_match_criteria_enable(u32 *match_criteria)
17848e6e287fSHans Petter Selasky {
17858e6e287fSHans Petter Selasky u8 match_criteria_enable;
17868e6e287fSHans Petter Selasky
17878e6e287fSHans Petter Selasky match_criteria_enable =
17888e6e287fSHans Petter Selasky (!HEADER_IS_ZERO(match_criteria, outer_headers)) <<
17898e6e287fSHans Petter Selasky MATCH_CRITERIA_ENABLE_OUTER_BIT;
17908e6e287fSHans Petter Selasky match_criteria_enable |=
17918e6e287fSHans Petter Selasky (!HEADER_IS_ZERO(match_criteria, misc_parameters)) <<
17928e6e287fSHans Petter Selasky MATCH_CRITERIA_ENABLE_MISC_BIT;
17938e6e287fSHans Petter Selasky match_criteria_enable |=
17948e6e287fSHans Petter Selasky (!HEADER_IS_ZERO(match_criteria, inner_headers)) <<
17958e6e287fSHans Petter Selasky MATCH_CRITERIA_ENABLE_INNER_BIT;
17968e6e287fSHans Petter Selasky
17978e6e287fSHans Petter Selasky return match_criteria_enable;
17988e6e287fSHans Petter Selasky }
17998e6e287fSHans Petter Selasky
set_proto(void * outer_c,void * outer_v,u8 mask,u8 val)18008e6e287fSHans Petter Selasky static void set_proto(void *outer_c, void *outer_v, u8 mask, u8 val)
18018e6e287fSHans Petter Selasky {
18028e6e287fSHans Petter Selasky MLX5_SET(fte_match_set_lyr_2_4, outer_c, ip_protocol, mask);
18038e6e287fSHans Petter Selasky MLX5_SET(fte_match_set_lyr_2_4, outer_v, ip_protocol, val);
18048e6e287fSHans Petter Selasky }
18058e6e287fSHans Petter Selasky
set_tos(void * outer_c,void * outer_v,u8 mask,u8 val)18068e6e287fSHans Petter Selasky static void set_tos(void *outer_c, void *outer_v, u8 mask, u8 val)
18078e6e287fSHans Petter Selasky {
18088e6e287fSHans Petter Selasky MLX5_SET(fte_match_set_lyr_2_4, outer_c, ip_ecn, mask);
18098e6e287fSHans Petter Selasky MLX5_SET(fte_match_set_lyr_2_4, outer_v, ip_ecn, val);
18108e6e287fSHans Petter Selasky MLX5_SET(fte_match_set_lyr_2_4, outer_c, ip_dscp, mask >> 2);
18118e6e287fSHans Petter Selasky MLX5_SET(fte_match_set_lyr_2_4, outer_v, ip_dscp, val >> 2);
18128e6e287fSHans Petter Selasky }
18138e6e287fSHans Petter Selasky
18148e6e287fSHans Petter Selasky #define LAST_ETH_FIELD vlan_tag
18158e6e287fSHans Petter Selasky #define LAST_IB_FIELD sl
18168e6e287fSHans Petter Selasky #define LAST_IPV4_FIELD tos
18178e6e287fSHans Petter Selasky #define LAST_IPV6_FIELD traffic_class
18188e6e287fSHans Petter Selasky #define LAST_TCP_UDP_FIELD src_port
18198e6e287fSHans Petter Selasky
18208e6e287fSHans Petter Selasky /* Field is the last supported field */
18218e6e287fSHans Petter Selasky #define FIELDS_NOT_SUPPORTED(filter, field)\
18228e6e287fSHans Petter Selasky memchr_inv((void *)&filter.field +\
18238e6e287fSHans Petter Selasky sizeof(filter.field), 0,\
18248e6e287fSHans Petter Selasky sizeof(filter) -\
18258e6e287fSHans Petter Selasky offsetof(typeof(filter), field) -\
18268e6e287fSHans Petter Selasky sizeof(filter.field))
18278e6e287fSHans Petter Selasky
parse_flow_attr(u32 * match_c,u32 * match_v,const union ib_flow_spec * ib_spec)18288e6e287fSHans Petter Selasky static int parse_flow_attr(u32 *match_c, u32 *match_v,
18298e6e287fSHans Petter Selasky const union ib_flow_spec *ib_spec)
18308e6e287fSHans Petter Selasky {
18318e6e287fSHans Petter Selasky void *outer_headers_c = MLX5_ADDR_OF(fte_match_param, match_c,
18328e6e287fSHans Petter Selasky outer_headers);
18338e6e287fSHans Petter Selasky void *outer_headers_v = MLX5_ADDR_OF(fte_match_param, match_v,
18348e6e287fSHans Petter Selasky outer_headers);
18358e6e287fSHans Petter Selasky void *misc_params_c = MLX5_ADDR_OF(fte_match_param, match_c,
18368e6e287fSHans Petter Selasky misc_parameters);
18378e6e287fSHans Petter Selasky void *misc_params_v = MLX5_ADDR_OF(fte_match_param, match_v,
18388e6e287fSHans Petter Selasky misc_parameters);
18398e6e287fSHans Petter Selasky
18408e6e287fSHans Petter Selasky switch (ib_spec->type) {
18418e6e287fSHans Petter Selasky case IB_FLOW_SPEC_ETH:
18428e6e287fSHans Petter Selasky if (FIELDS_NOT_SUPPORTED(ib_spec->eth.mask, LAST_ETH_FIELD))
18438e6e287fSHans Petter Selasky return -ENOTSUPP;
18448e6e287fSHans Petter Selasky
18458e6e287fSHans Petter Selasky ether_addr_copy(MLX5_ADDR_OF(fte_match_set_lyr_2_4, outer_headers_c,
18468e6e287fSHans Petter Selasky dmac_47_16),
18478e6e287fSHans Petter Selasky ib_spec->eth.mask.dst_mac);
18488e6e287fSHans Petter Selasky ether_addr_copy(MLX5_ADDR_OF(fte_match_set_lyr_2_4, outer_headers_v,
18498e6e287fSHans Petter Selasky dmac_47_16),
18508e6e287fSHans Petter Selasky ib_spec->eth.val.dst_mac);
18518e6e287fSHans Petter Selasky
18528e6e287fSHans Petter Selasky ether_addr_copy(MLX5_ADDR_OF(fte_match_set_lyr_2_4, outer_headers_c,
18538e6e287fSHans Petter Selasky smac_47_16),
18548e6e287fSHans Petter Selasky ib_spec->eth.mask.src_mac);
18558e6e287fSHans Petter Selasky ether_addr_copy(MLX5_ADDR_OF(fte_match_set_lyr_2_4, outer_headers_v,
18568e6e287fSHans Petter Selasky smac_47_16),
18578e6e287fSHans Petter Selasky ib_spec->eth.val.src_mac);
18588e6e287fSHans Petter Selasky
18598e6e287fSHans Petter Selasky if (ib_spec->eth.mask.vlan_tag) {
18608e6e287fSHans Petter Selasky MLX5_SET(fte_match_set_lyr_2_4, outer_headers_c,
18618e6e287fSHans Petter Selasky cvlan_tag, 1);
18628e6e287fSHans Petter Selasky MLX5_SET(fte_match_set_lyr_2_4, outer_headers_v,
18638e6e287fSHans Petter Selasky cvlan_tag, 1);
18648e6e287fSHans Petter Selasky
18658e6e287fSHans Petter Selasky MLX5_SET(fte_match_set_lyr_2_4, outer_headers_c,
18668e6e287fSHans Petter Selasky first_vid, ntohs(ib_spec->eth.mask.vlan_tag));
18678e6e287fSHans Petter Selasky MLX5_SET(fte_match_set_lyr_2_4, outer_headers_v,
18688e6e287fSHans Petter Selasky first_vid, ntohs(ib_spec->eth.val.vlan_tag));
18698e6e287fSHans Petter Selasky
18708e6e287fSHans Petter Selasky MLX5_SET(fte_match_set_lyr_2_4, outer_headers_c,
18718e6e287fSHans Petter Selasky first_cfi,
18728e6e287fSHans Petter Selasky ntohs(ib_spec->eth.mask.vlan_tag) >> 12);
18738e6e287fSHans Petter Selasky MLX5_SET(fte_match_set_lyr_2_4, outer_headers_v,
18748e6e287fSHans Petter Selasky first_cfi,
18758e6e287fSHans Petter Selasky ntohs(ib_spec->eth.val.vlan_tag) >> 12);
18768e6e287fSHans Petter Selasky
18778e6e287fSHans Petter Selasky MLX5_SET(fte_match_set_lyr_2_4, outer_headers_c,
18788e6e287fSHans Petter Selasky first_prio,
18798e6e287fSHans Petter Selasky ntohs(ib_spec->eth.mask.vlan_tag) >> 13);
18808e6e287fSHans Petter Selasky MLX5_SET(fte_match_set_lyr_2_4, outer_headers_v,
18818e6e287fSHans Petter Selasky first_prio,
18828e6e287fSHans Petter Selasky ntohs(ib_spec->eth.val.vlan_tag) >> 13);
18838e6e287fSHans Petter Selasky }
18848e6e287fSHans Petter Selasky MLX5_SET(fte_match_set_lyr_2_4, outer_headers_c,
18858e6e287fSHans Petter Selasky ethertype, ntohs(ib_spec->eth.mask.ether_type));
18868e6e287fSHans Petter Selasky MLX5_SET(fte_match_set_lyr_2_4, outer_headers_v,
18878e6e287fSHans Petter Selasky ethertype, ntohs(ib_spec->eth.val.ether_type));
18888e6e287fSHans Petter Selasky break;
18898e6e287fSHans Petter Selasky case IB_FLOW_SPEC_IPV4:
18908e6e287fSHans Petter Selasky if (FIELDS_NOT_SUPPORTED(ib_spec->ipv4.mask, LAST_IPV4_FIELD))
18918e6e287fSHans Petter Selasky return -ENOTSUPP;
18928e6e287fSHans Petter Selasky
18938e6e287fSHans Petter Selasky MLX5_SET(fte_match_set_lyr_2_4, outer_headers_c,
18948e6e287fSHans Petter Selasky ethertype, 0xffff);
18958e6e287fSHans Petter Selasky MLX5_SET(fte_match_set_lyr_2_4, outer_headers_v,
18968e6e287fSHans Petter Selasky ethertype, ETH_P_IP);
18978e6e287fSHans Petter Selasky
18988e6e287fSHans Petter Selasky memcpy(MLX5_ADDR_OF(fte_match_set_lyr_2_4, outer_headers_c,
18998e6e287fSHans Petter Selasky src_ipv4_src_ipv6.ipv4_layout.ipv4),
19008e6e287fSHans Petter Selasky &ib_spec->ipv4.mask.src_ip,
19018e6e287fSHans Petter Selasky sizeof(ib_spec->ipv4.mask.src_ip));
19028e6e287fSHans Petter Selasky memcpy(MLX5_ADDR_OF(fte_match_set_lyr_2_4, outer_headers_v,
19038e6e287fSHans Petter Selasky src_ipv4_src_ipv6.ipv4_layout.ipv4),
19048e6e287fSHans Petter Selasky &ib_spec->ipv4.val.src_ip,
19058e6e287fSHans Petter Selasky sizeof(ib_spec->ipv4.val.src_ip));
19068e6e287fSHans Petter Selasky memcpy(MLX5_ADDR_OF(fte_match_set_lyr_2_4, outer_headers_c,
19078e6e287fSHans Petter Selasky dst_ipv4_dst_ipv6.ipv4_layout.ipv4),
19088e6e287fSHans Petter Selasky &ib_spec->ipv4.mask.dst_ip,
19098e6e287fSHans Petter Selasky sizeof(ib_spec->ipv4.mask.dst_ip));
19108e6e287fSHans Petter Selasky memcpy(MLX5_ADDR_OF(fte_match_set_lyr_2_4, outer_headers_v,
19118e6e287fSHans Petter Selasky dst_ipv4_dst_ipv6.ipv4_layout.ipv4),
19128e6e287fSHans Petter Selasky &ib_spec->ipv4.val.dst_ip,
19138e6e287fSHans Petter Selasky sizeof(ib_spec->ipv4.val.dst_ip));
19148e6e287fSHans Petter Selasky
19158e6e287fSHans Petter Selasky set_tos(outer_headers_c, outer_headers_v,
19168e6e287fSHans Petter Selasky ib_spec->ipv4.mask.tos, ib_spec->ipv4.val.tos);
19178e6e287fSHans Petter Selasky
19188e6e287fSHans Petter Selasky set_proto(outer_headers_c, outer_headers_v,
19198e6e287fSHans Petter Selasky ib_spec->ipv4.mask.proto, ib_spec->ipv4.val.proto);
19208e6e287fSHans Petter Selasky break;
19218e6e287fSHans Petter Selasky case IB_FLOW_SPEC_IPV6:
19228e6e287fSHans Petter Selasky if (FIELDS_NOT_SUPPORTED(ib_spec->ipv6.mask, LAST_IPV6_FIELD))
19238e6e287fSHans Petter Selasky return -ENOTSUPP;
19248e6e287fSHans Petter Selasky
19258e6e287fSHans Petter Selasky MLX5_SET(fte_match_set_lyr_2_4, outer_headers_c,
19268e6e287fSHans Petter Selasky ethertype, 0xffff);
19278e6e287fSHans Petter Selasky MLX5_SET(fte_match_set_lyr_2_4, outer_headers_v,
19283b173281SKa Ho Ng ethertype, ETH_P_IPV6);
19298e6e287fSHans Petter Selasky
19308e6e287fSHans Petter Selasky memcpy(MLX5_ADDR_OF(fte_match_set_lyr_2_4, outer_headers_c,
19318e6e287fSHans Petter Selasky src_ipv4_src_ipv6.ipv6_layout.ipv6),
19328e6e287fSHans Petter Selasky &ib_spec->ipv6.mask.src_ip,
19338e6e287fSHans Petter Selasky sizeof(ib_spec->ipv6.mask.src_ip));
19348e6e287fSHans Petter Selasky memcpy(MLX5_ADDR_OF(fte_match_set_lyr_2_4, outer_headers_v,
19358e6e287fSHans Petter Selasky src_ipv4_src_ipv6.ipv6_layout.ipv6),
19368e6e287fSHans Petter Selasky &ib_spec->ipv6.val.src_ip,
19378e6e287fSHans Petter Selasky sizeof(ib_spec->ipv6.val.src_ip));
19388e6e287fSHans Petter Selasky memcpy(MLX5_ADDR_OF(fte_match_set_lyr_2_4, outer_headers_c,
19398e6e287fSHans Petter Selasky dst_ipv4_dst_ipv6.ipv6_layout.ipv6),
19408e6e287fSHans Petter Selasky &ib_spec->ipv6.mask.dst_ip,
19418e6e287fSHans Petter Selasky sizeof(ib_spec->ipv6.mask.dst_ip));
19428e6e287fSHans Petter Selasky memcpy(MLX5_ADDR_OF(fte_match_set_lyr_2_4, outer_headers_v,
19438e6e287fSHans Petter Selasky dst_ipv4_dst_ipv6.ipv6_layout.ipv6),
19448e6e287fSHans Petter Selasky &ib_spec->ipv6.val.dst_ip,
19458e6e287fSHans Petter Selasky sizeof(ib_spec->ipv6.val.dst_ip));
19468e6e287fSHans Petter Selasky
19478e6e287fSHans Petter Selasky set_tos(outer_headers_c, outer_headers_v,
19488e6e287fSHans Petter Selasky ib_spec->ipv6.mask.traffic_class,
19498e6e287fSHans Petter Selasky ib_spec->ipv6.val.traffic_class);
19508e6e287fSHans Petter Selasky
19518e6e287fSHans Petter Selasky set_proto(outer_headers_c, outer_headers_v,
19528e6e287fSHans Petter Selasky ib_spec->ipv6.mask.next_hdr,
19538e6e287fSHans Petter Selasky ib_spec->ipv6.val.next_hdr);
19548e6e287fSHans Petter Selasky
19558e6e287fSHans Petter Selasky MLX5_SET(fte_match_set_misc, misc_params_c,
19568e6e287fSHans Petter Selasky outer_ipv6_flow_label,
19578e6e287fSHans Petter Selasky ntohl(ib_spec->ipv6.mask.flow_label));
19588e6e287fSHans Petter Selasky MLX5_SET(fte_match_set_misc, misc_params_v,
19598e6e287fSHans Petter Selasky outer_ipv6_flow_label,
19608e6e287fSHans Petter Selasky ntohl(ib_spec->ipv6.val.flow_label));
19618e6e287fSHans Petter Selasky break;
19628e6e287fSHans Petter Selasky case IB_FLOW_SPEC_TCP:
19638e6e287fSHans Petter Selasky if (FIELDS_NOT_SUPPORTED(ib_spec->tcp_udp.mask,
19648e6e287fSHans Petter Selasky LAST_TCP_UDP_FIELD))
19658e6e287fSHans Petter Selasky return -ENOTSUPP;
19668e6e287fSHans Petter Selasky
19678e6e287fSHans Petter Selasky MLX5_SET(fte_match_set_lyr_2_4, outer_headers_c, ip_protocol,
19688e6e287fSHans Petter Selasky 0xff);
19698e6e287fSHans Petter Selasky MLX5_SET(fte_match_set_lyr_2_4, outer_headers_v, ip_protocol,
19708e6e287fSHans Petter Selasky IPPROTO_TCP);
19718e6e287fSHans Petter Selasky
19728e6e287fSHans Petter Selasky MLX5_SET(fte_match_set_lyr_2_4, outer_headers_c, tcp_sport,
19738e6e287fSHans Petter Selasky ntohs(ib_spec->tcp_udp.mask.src_port));
19748e6e287fSHans Petter Selasky MLX5_SET(fte_match_set_lyr_2_4, outer_headers_v, tcp_sport,
19758e6e287fSHans Petter Selasky ntohs(ib_spec->tcp_udp.val.src_port));
19768e6e287fSHans Petter Selasky
19778e6e287fSHans Petter Selasky MLX5_SET(fte_match_set_lyr_2_4, outer_headers_c, tcp_dport,
19788e6e287fSHans Petter Selasky ntohs(ib_spec->tcp_udp.mask.dst_port));
19798e6e287fSHans Petter Selasky MLX5_SET(fte_match_set_lyr_2_4, outer_headers_v, tcp_dport,
19808e6e287fSHans Petter Selasky ntohs(ib_spec->tcp_udp.val.dst_port));
19818e6e287fSHans Petter Selasky break;
19828e6e287fSHans Petter Selasky case IB_FLOW_SPEC_UDP:
19838e6e287fSHans Petter Selasky if (FIELDS_NOT_SUPPORTED(ib_spec->tcp_udp.mask,
19848e6e287fSHans Petter Selasky LAST_TCP_UDP_FIELD))
19858e6e287fSHans Petter Selasky return -ENOTSUPP;
19868e6e287fSHans Petter Selasky
19878e6e287fSHans Petter Selasky MLX5_SET(fte_match_set_lyr_2_4, outer_headers_c, ip_protocol,
19888e6e287fSHans Petter Selasky 0xff);
19898e6e287fSHans Petter Selasky MLX5_SET(fte_match_set_lyr_2_4, outer_headers_v, ip_protocol,
19908e6e287fSHans Petter Selasky IPPROTO_UDP);
19918e6e287fSHans Petter Selasky
19928e6e287fSHans Petter Selasky MLX5_SET(fte_match_set_lyr_2_4, outer_headers_c, udp_sport,
19938e6e287fSHans Petter Selasky ntohs(ib_spec->tcp_udp.mask.src_port));
19948e6e287fSHans Petter Selasky MLX5_SET(fte_match_set_lyr_2_4, outer_headers_v, udp_sport,
19958e6e287fSHans Petter Selasky ntohs(ib_spec->tcp_udp.val.src_port));
19968e6e287fSHans Petter Selasky
19978e6e287fSHans Petter Selasky MLX5_SET(fte_match_set_lyr_2_4, outer_headers_c, udp_dport,
19988e6e287fSHans Petter Selasky ntohs(ib_spec->tcp_udp.mask.dst_port));
19998e6e287fSHans Petter Selasky MLX5_SET(fte_match_set_lyr_2_4, outer_headers_v, udp_dport,
20008e6e287fSHans Petter Selasky ntohs(ib_spec->tcp_udp.val.dst_port));
20018e6e287fSHans Petter Selasky break;
20028e6e287fSHans Petter Selasky default:
20038e6e287fSHans Petter Selasky return -EINVAL;
20048e6e287fSHans Petter Selasky }
20058e6e287fSHans Petter Selasky
20068e6e287fSHans Petter Selasky return 0;
20078e6e287fSHans Petter Selasky }
20088e6e287fSHans Petter Selasky
20098e6e287fSHans Petter Selasky /* If a flow could catch both multicast and unicast packets,
20108e6e287fSHans Petter Selasky * it won't fall into the multicast flow steering table and this rule
20118e6e287fSHans Petter Selasky * could steal other multicast packets.
20128e6e287fSHans Petter Selasky */
flow_is_multicast_only(struct ib_flow_attr * ib_attr)20138e6e287fSHans Petter Selasky static bool flow_is_multicast_only(struct ib_flow_attr *ib_attr)
20148e6e287fSHans Petter Selasky {
20158e6e287fSHans Petter Selasky struct ib_flow_spec_eth *eth_spec;
20168e6e287fSHans Petter Selasky
20178e6e287fSHans Petter Selasky if (ib_attr->type != IB_FLOW_ATTR_NORMAL ||
20188e6e287fSHans Petter Selasky ib_attr->size < sizeof(struct ib_flow_attr) +
20198e6e287fSHans Petter Selasky sizeof(struct ib_flow_spec_eth) ||
20208e6e287fSHans Petter Selasky ib_attr->num_of_specs < 1)
20218e6e287fSHans Petter Selasky return false;
20228e6e287fSHans Petter Selasky
20238e6e287fSHans Petter Selasky eth_spec = (struct ib_flow_spec_eth *)(ib_attr + 1);
20248e6e287fSHans Petter Selasky if (eth_spec->type != IB_FLOW_SPEC_ETH ||
20258e6e287fSHans Petter Selasky eth_spec->size != sizeof(*eth_spec))
20268e6e287fSHans Petter Selasky return false;
20278e6e287fSHans Petter Selasky
20288e6e287fSHans Petter Selasky return is_multicast_ether_addr(eth_spec->mask.dst_mac) &&
20298e6e287fSHans Petter Selasky is_multicast_ether_addr(eth_spec->val.dst_mac);
20308e6e287fSHans Petter Selasky }
20318e6e287fSHans Petter Selasky
is_valid_attr(const struct ib_flow_attr * flow_attr)20328e6e287fSHans Petter Selasky static bool is_valid_attr(const struct ib_flow_attr *flow_attr)
20338e6e287fSHans Petter Selasky {
20348e6e287fSHans Petter Selasky union ib_flow_spec *ib_spec = (union ib_flow_spec *)(flow_attr + 1);
20358e6e287fSHans Petter Selasky bool has_ipv4_spec = false;
20368e6e287fSHans Petter Selasky bool eth_type_ipv4 = true;
20378e6e287fSHans Petter Selasky unsigned int spec_index;
20388e6e287fSHans Petter Selasky
20398e6e287fSHans Petter Selasky /* Validate that ethertype is correct */
20408e6e287fSHans Petter Selasky for (spec_index = 0; spec_index < flow_attr->num_of_specs; spec_index++) {
20418e6e287fSHans Petter Selasky if (ib_spec->type == IB_FLOW_SPEC_ETH &&
20428e6e287fSHans Petter Selasky ib_spec->eth.mask.ether_type) {
20438e6e287fSHans Petter Selasky if (!((ib_spec->eth.mask.ether_type == htons(0xffff)) &&
20448e6e287fSHans Petter Selasky ib_spec->eth.val.ether_type == htons(ETH_P_IP)))
20458e6e287fSHans Petter Selasky eth_type_ipv4 = false;
20468e6e287fSHans Petter Selasky } else if (ib_spec->type == IB_FLOW_SPEC_IPV4) {
20478e6e287fSHans Petter Selasky has_ipv4_spec = true;
20488e6e287fSHans Petter Selasky }
20498e6e287fSHans Petter Selasky ib_spec = (void *)ib_spec + ib_spec->size;
20508e6e287fSHans Petter Selasky }
20518e6e287fSHans Petter Selasky return !has_ipv4_spec || eth_type_ipv4;
20528e6e287fSHans Petter Selasky }
20538e6e287fSHans Petter Selasky
put_flow_table(struct mlx5_ib_dev * dev,struct mlx5_ib_flow_prio * prio,bool ft_added)20548e6e287fSHans Petter Selasky static void put_flow_table(struct mlx5_ib_dev *dev,
20558e6e287fSHans Petter Selasky struct mlx5_ib_flow_prio *prio, bool ft_added)
20568e6e287fSHans Petter Selasky {
20578e6e287fSHans Petter Selasky prio->refcount -= !!ft_added;
20588e6e287fSHans Petter Selasky if (!prio->refcount) {
20598e6e287fSHans Petter Selasky mlx5_destroy_flow_table(prio->flow_table);
20608e6e287fSHans Petter Selasky prio->flow_table = NULL;
20618e6e287fSHans Petter Selasky }
20628e6e287fSHans Petter Selasky }
20638e6e287fSHans Petter Selasky
mlx5_ib_destroy_flow(struct ib_flow * flow_id)20648e6e287fSHans Petter Selasky static int mlx5_ib_destroy_flow(struct ib_flow *flow_id)
20658e6e287fSHans Petter Selasky {
20668e6e287fSHans Petter Selasky struct mlx5_ib_dev *dev = to_mdev(flow_id->qp->device);
20678e6e287fSHans Petter Selasky struct mlx5_ib_flow_handler *handler = container_of(flow_id,
20688e6e287fSHans Petter Selasky struct mlx5_ib_flow_handler,
20698e6e287fSHans Petter Selasky ibflow);
20708e6e287fSHans Petter Selasky struct mlx5_ib_flow_handler *iter, *tmp;
20718e6e287fSHans Petter Selasky
20728e6e287fSHans Petter Selasky mutex_lock(&dev->flow_db.lock);
20738e6e287fSHans Petter Selasky
20748e6e287fSHans Petter Selasky list_for_each_entry_safe(iter, tmp, &handler->list, list) {
2075b0b87d91SHans Petter Selasky mlx5_del_flow_rule(&iter->rule);
20768e6e287fSHans Petter Selasky put_flow_table(dev, iter->prio, true);
20778e6e287fSHans Petter Selasky list_del(&iter->list);
20788e6e287fSHans Petter Selasky kfree(iter);
20798e6e287fSHans Petter Selasky }
20808e6e287fSHans Petter Selasky
2081b0b87d91SHans Petter Selasky mlx5_del_flow_rule(&handler->rule);
20828e6e287fSHans Petter Selasky put_flow_table(dev, handler->prio, true);
20838e6e287fSHans Petter Selasky mutex_unlock(&dev->flow_db.lock);
20848e6e287fSHans Petter Selasky
20858e6e287fSHans Petter Selasky kfree(handler);
20868e6e287fSHans Petter Selasky
20878e6e287fSHans Petter Selasky return 0;
20888e6e287fSHans Petter Selasky }
20898e6e287fSHans Petter Selasky
ib_prio_to_core_prio(unsigned int priority,bool dont_trap)20908e6e287fSHans Petter Selasky static int ib_prio_to_core_prio(unsigned int priority, bool dont_trap)
20918e6e287fSHans Petter Selasky {
20928e6e287fSHans Petter Selasky priority *= 2;
20938e6e287fSHans Petter Selasky if (!dont_trap)
20948e6e287fSHans Petter Selasky priority++;
20958e6e287fSHans Petter Selasky return priority;
20968e6e287fSHans Petter Selasky }
20978e6e287fSHans Petter Selasky
20988e6e287fSHans Petter Selasky enum flow_table_type {
20998e6e287fSHans Petter Selasky MLX5_IB_FT_RX,
21008e6e287fSHans Petter Selasky MLX5_IB_FT_TX
21018e6e287fSHans Petter Selasky };
21028e6e287fSHans Petter Selasky
21038e6e287fSHans Petter Selasky #define MLX5_FS_MAX_TYPES 10
21048e6e287fSHans Petter Selasky #define MLX5_FS_MAX_ENTRIES 32000UL
get_flow_table(struct mlx5_ib_dev * dev,struct ib_flow_attr * flow_attr,enum flow_table_type ft_type)21058e6e287fSHans Petter Selasky static struct mlx5_ib_flow_prio *get_flow_table(struct mlx5_ib_dev *dev,
21068e6e287fSHans Petter Selasky struct ib_flow_attr *flow_attr,
21078e6e287fSHans Petter Selasky enum flow_table_type ft_type)
21088e6e287fSHans Petter Selasky {
21098e6e287fSHans Petter Selasky bool dont_trap = flow_attr->flags & IB_FLOW_ATTR_FLAGS_DONT_TRAP;
21108e6e287fSHans Petter Selasky struct mlx5_flow_namespace *ns = NULL;
21118e6e287fSHans Petter Selasky struct mlx5_ib_flow_prio *prio;
21128e6e287fSHans Petter Selasky struct mlx5_flow_table *ft;
21138e6e287fSHans Petter Selasky int num_entries;
21148e6e287fSHans Petter Selasky int num_groups;
21158e6e287fSHans Petter Selasky int priority;
21168e6e287fSHans Petter Selasky int err = 0;
21178e6e287fSHans Petter Selasky
21188e6e287fSHans Petter Selasky if (flow_attr->type == IB_FLOW_ATTR_NORMAL) {
21198e6e287fSHans Petter Selasky if (flow_is_multicast_only(flow_attr) &&
21208e6e287fSHans Petter Selasky !dont_trap)
21218e6e287fSHans Petter Selasky priority = MLX5_IB_FLOW_MCAST_PRIO;
21228e6e287fSHans Petter Selasky else
21238e6e287fSHans Petter Selasky priority = ib_prio_to_core_prio(flow_attr->priority,
21248e6e287fSHans Petter Selasky dont_trap);
21258e6e287fSHans Petter Selasky ns = mlx5_get_flow_namespace(dev->mdev,
21268e6e287fSHans Petter Selasky MLX5_FLOW_NAMESPACE_BYPASS);
21278e6e287fSHans Petter Selasky num_entries = MLX5_FS_MAX_ENTRIES;
21288e6e287fSHans Petter Selasky num_groups = MLX5_FS_MAX_TYPES;
21298e6e287fSHans Petter Selasky prio = &dev->flow_db.prios[priority];
21308e6e287fSHans Petter Selasky } else if (flow_attr->type == IB_FLOW_ATTR_ALL_DEFAULT ||
21318e6e287fSHans Petter Selasky flow_attr->type == IB_FLOW_ATTR_MC_DEFAULT) {
21328e6e287fSHans Petter Selasky ns = mlx5_get_flow_namespace(dev->mdev,
21338e6e287fSHans Petter Selasky MLX5_FLOW_NAMESPACE_LEFTOVERS);
21348e6e287fSHans Petter Selasky build_leftovers_ft_param("bypass", &priority,
21358e6e287fSHans Petter Selasky &num_entries,
21368e6e287fSHans Petter Selasky &num_groups);
21378e6e287fSHans Petter Selasky prio = &dev->flow_db.prios[MLX5_IB_FLOW_LEFTOVERS_PRIO];
21388e6e287fSHans Petter Selasky } else if (flow_attr->type == IB_FLOW_ATTR_SNIFFER) {
21398e6e287fSHans Petter Selasky if (!MLX5_CAP_FLOWTABLE(dev->mdev,
21408e6e287fSHans Petter Selasky allow_sniffer_and_nic_rx_shared_tir))
21418e6e287fSHans Petter Selasky return ERR_PTR(-ENOTSUPP);
21428e6e287fSHans Petter Selasky
21438e6e287fSHans Petter Selasky ns = mlx5_get_flow_namespace(dev->mdev, ft_type == MLX5_IB_FT_RX ?
21448e6e287fSHans Petter Selasky MLX5_FLOW_NAMESPACE_SNIFFER_RX :
21458e6e287fSHans Petter Selasky MLX5_FLOW_NAMESPACE_SNIFFER_TX);
21468e6e287fSHans Petter Selasky
21478e6e287fSHans Petter Selasky prio = &dev->flow_db.sniffer[ft_type];
21488e6e287fSHans Petter Selasky priority = 0;
21498e6e287fSHans Petter Selasky num_entries = 1;
21508e6e287fSHans Petter Selasky num_groups = 1;
21518e6e287fSHans Petter Selasky }
21528e6e287fSHans Petter Selasky
21538e6e287fSHans Petter Selasky if (!ns)
21548e6e287fSHans Petter Selasky return ERR_PTR(-ENOTSUPP);
21558e6e287fSHans Petter Selasky
21568e6e287fSHans Petter Selasky ft = prio->flow_table;
21578e6e287fSHans Petter Selasky if (!ft) {
21588e6e287fSHans Petter Selasky ft = mlx5_create_auto_grouped_flow_table(ns, priority, "bypass",
21598e6e287fSHans Petter Selasky num_entries,
2160b94ef2a3SKonstantin Belousov num_groups,
2161b94ef2a3SKonstantin Belousov 0);
21628e6e287fSHans Petter Selasky
21638e6e287fSHans Petter Selasky if (!IS_ERR(ft)) {
21648e6e287fSHans Petter Selasky prio->refcount = 0;
21658e6e287fSHans Petter Selasky prio->flow_table = ft;
21668e6e287fSHans Petter Selasky } else {
21678e6e287fSHans Petter Selasky err = PTR_ERR(ft);
21688e6e287fSHans Petter Selasky }
21698e6e287fSHans Petter Selasky }
21708e6e287fSHans Petter Selasky
21718e6e287fSHans Petter Selasky return err ? ERR_PTR(err) : prio;
21728e6e287fSHans Petter Selasky }
21738e6e287fSHans Petter Selasky
create_flow_rule(struct mlx5_ib_dev * dev,struct mlx5_ib_flow_prio * ft_prio,const struct ib_flow_attr * flow_attr,struct mlx5_flow_destination * dst)21748e6e287fSHans Petter Selasky static struct mlx5_ib_flow_handler *create_flow_rule(struct mlx5_ib_dev *dev,
21758e6e287fSHans Petter Selasky struct mlx5_ib_flow_prio *ft_prio,
21768e6e287fSHans Petter Selasky const struct ib_flow_attr *flow_attr,
21778e6e287fSHans Petter Selasky struct mlx5_flow_destination *dst)
21788e6e287fSHans Petter Selasky {
21798e6e287fSHans Petter Selasky struct mlx5_flow_table *ft = ft_prio->flow_table;
21808e6e287fSHans Petter Selasky struct mlx5_ib_flow_handler *handler;
21818e6e287fSHans Petter Selasky struct mlx5_flow_spec *spec;
21828e6e287fSHans Petter Selasky const void *ib_flow = (const void *)flow_attr + sizeof(*flow_attr);
21838e6e287fSHans Petter Selasky unsigned int spec_index;
2184cb054a49SMark Bloch struct mlx5_flow_act flow_act = {
2185cb054a49SMark Bloch .actions = MLX5_FLOW_ACT_ACTIONS_FLOW_TAG,
2186cb054a49SMark Bloch .flow_tag = MLX5_FS_DEFAULT_FLOW_TAG,
2187cb054a49SMark Bloch };
21888e6e287fSHans Petter Selasky u32 action;
21898e6e287fSHans Petter Selasky int err = 0;
21908e6e287fSHans Petter Selasky
21918e6e287fSHans Petter Selasky if (!is_valid_attr(flow_attr))
21928e6e287fSHans Petter Selasky return ERR_PTR(-EINVAL);
21938e6e287fSHans Petter Selasky
21948e6e287fSHans Petter Selasky spec = mlx5_vzalloc(sizeof(*spec));
21958e6e287fSHans Petter Selasky handler = kzalloc(sizeof(*handler), GFP_KERNEL);
21968e6e287fSHans Petter Selasky if (!handler || !spec) {
21978e6e287fSHans Petter Selasky err = -ENOMEM;
21988e6e287fSHans Petter Selasky goto free;
21998e6e287fSHans Petter Selasky }
22008e6e287fSHans Petter Selasky
22018e6e287fSHans Petter Selasky INIT_LIST_HEAD(&handler->list);
22028e6e287fSHans Petter Selasky
22038e6e287fSHans Petter Selasky for (spec_index = 0; spec_index < flow_attr->num_of_specs; spec_index++) {
22048e6e287fSHans Petter Selasky err = parse_flow_attr(spec->match_criteria,
22058e6e287fSHans Petter Selasky spec->match_value, ib_flow);
22068e6e287fSHans Petter Selasky if (err < 0)
22078e6e287fSHans Petter Selasky goto free;
22088e6e287fSHans Petter Selasky
22098e6e287fSHans Petter Selasky ib_flow += ((union ib_flow_spec *)ib_flow)->size;
22108e6e287fSHans Petter Selasky }
22118e6e287fSHans Petter Selasky
22128e6e287fSHans Petter Selasky spec->match_criteria_enable = get_match_criteria_enable(spec->match_criteria);
221376ed99edSMark Bloch action = dst ? MLX5_FLOW_RULE_FWD_ACTION_DEST : 0;
22148e6e287fSHans Petter Selasky handler->rule = mlx5_add_flow_rule(ft, spec->match_criteria_enable,
22158e6e287fSHans Petter Selasky spec->match_criteria,
22168e6e287fSHans Petter Selasky spec->match_value,
22178e6e287fSHans Petter Selasky action,
2218cb054a49SMark Bloch &flow_act,
22198e6e287fSHans Petter Selasky dst);
22208e6e287fSHans Petter Selasky
22218e6e287fSHans Petter Selasky if (IS_ERR(handler->rule)) {
22228e6e287fSHans Petter Selasky err = PTR_ERR(handler->rule);
22238e6e287fSHans Petter Selasky goto free;
22248e6e287fSHans Petter Selasky }
22258e6e287fSHans Petter Selasky
22268e6e287fSHans Petter Selasky ft_prio->refcount++;
22278e6e287fSHans Petter Selasky handler->prio = ft_prio;
22288e6e287fSHans Petter Selasky
22298e6e287fSHans Petter Selasky ft_prio->flow_table = ft;
22308e6e287fSHans Petter Selasky free:
22318e6e287fSHans Petter Selasky if (err)
22328e6e287fSHans Petter Selasky kfree(handler);
22338e6e287fSHans Petter Selasky kvfree(spec);
22348e6e287fSHans Petter Selasky return err ? ERR_PTR(err) : handler;
22358e6e287fSHans Petter Selasky }
22368e6e287fSHans Petter Selasky
create_dont_trap_rule(struct mlx5_ib_dev * dev,struct mlx5_ib_flow_prio * ft_prio,struct ib_flow_attr * flow_attr,struct mlx5_flow_destination * dst)22378e6e287fSHans Petter Selasky static struct mlx5_ib_flow_handler *create_dont_trap_rule(struct mlx5_ib_dev *dev,
22388e6e287fSHans Petter Selasky struct mlx5_ib_flow_prio *ft_prio,
22398e6e287fSHans Petter Selasky struct ib_flow_attr *flow_attr,
22408e6e287fSHans Petter Selasky struct mlx5_flow_destination *dst)
22418e6e287fSHans Petter Selasky {
22428e6e287fSHans Petter Selasky struct mlx5_ib_flow_handler *handler_dst = NULL;
22438e6e287fSHans Petter Selasky struct mlx5_ib_flow_handler *handler = NULL;
22448e6e287fSHans Petter Selasky
22458e6e287fSHans Petter Selasky handler = create_flow_rule(dev, ft_prio, flow_attr, NULL);
22468e6e287fSHans Petter Selasky if (!IS_ERR(handler)) {
22478e6e287fSHans Petter Selasky handler_dst = create_flow_rule(dev, ft_prio,
22488e6e287fSHans Petter Selasky flow_attr, dst);
22498e6e287fSHans Petter Selasky if (IS_ERR(handler_dst)) {
2250b0b87d91SHans Petter Selasky mlx5_del_flow_rule(&handler->rule);
22518e6e287fSHans Petter Selasky ft_prio->refcount--;
22528e6e287fSHans Petter Selasky kfree(handler);
22538e6e287fSHans Petter Selasky handler = handler_dst;
22548e6e287fSHans Petter Selasky } else {
22558e6e287fSHans Petter Selasky list_add(&handler_dst->list, &handler->list);
22568e6e287fSHans Petter Selasky }
22578e6e287fSHans Petter Selasky }
22588e6e287fSHans Petter Selasky
22598e6e287fSHans Petter Selasky return handler;
22608e6e287fSHans Petter Selasky }
22618e6e287fSHans Petter Selasky enum {
22628e6e287fSHans Petter Selasky LEFTOVERS_MC,
22638e6e287fSHans Petter Selasky LEFTOVERS_UC,
22648e6e287fSHans Petter Selasky };
22658e6e287fSHans Petter Selasky
create_leftovers_rule(struct mlx5_ib_dev * dev,struct mlx5_ib_flow_prio * ft_prio,struct ib_flow_attr * flow_attr,struct mlx5_flow_destination * dst)22668e6e287fSHans Petter Selasky static struct mlx5_ib_flow_handler *create_leftovers_rule(struct mlx5_ib_dev *dev,
22678e6e287fSHans Petter Selasky struct mlx5_ib_flow_prio *ft_prio,
22688e6e287fSHans Petter Selasky struct ib_flow_attr *flow_attr,
22698e6e287fSHans Petter Selasky struct mlx5_flow_destination *dst)
22708e6e287fSHans Petter Selasky {
22718e6e287fSHans Petter Selasky struct mlx5_ib_flow_handler *handler_ucast = NULL;
22728e6e287fSHans Petter Selasky struct mlx5_ib_flow_handler *handler = NULL;
22738e6e287fSHans Petter Selasky
22748e6e287fSHans Petter Selasky static struct {
22758e6e287fSHans Petter Selasky struct ib_flow_attr flow_attr;
22768e6e287fSHans Petter Selasky struct ib_flow_spec_eth eth_flow;
22778e6e287fSHans Petter Selasky } leftovers_specs[] = {
22788e6e287fSHans Petter Selasky [LEFTOVERS_MC] = {
22798e6e287fSHans Petter Selasky .flow_attr = {
22808e6e287fSHans Petter Selasky .num_of_specs = 1,
22818e6e287fSHans Petter Selasky .size = sizeof(leftovers_specs[0])
22828e6e287fSHans Petter Selasky },
22838e6e287fSHans Petter Selasky .eth_flow = {
22848e6e287fSHans Petter Selasky .type = IB_FLOW_SPEC_ETH,
22858e6e287fSHans Petter Selasky .size = sizeof(struct ib_flow_spec_eth),
22868e6e287fSHans Petter Selasky .mask = {.dst_mac = {0x1} },
22878e6e287fSHans Petter Selasky .val = {.dst_mac = {0x1} }
22888e6e287fSHans Petter Selasky }
22898e6e287fSHans Petter Selasky },
22908e6e287fSHans Petter Selasky [LEFTOVERS_UC] = {
22918e6e287fSHans Petter Selasky .flow_attr = {
22928e6e287fSHans Petter Selasky .num_of_specs = 1,
22938e6e287fSHans Petter Selasky .size = sizeof(leftovers_specs[0])
22948e6e287fSHans Petter Selasky },
22958e6e287fSHans Petter Selasky .eth_flow = {
22968e6e287fSHans Petter Selasky .type = IB_FLOW_SPEC_ETH,
22978e6e287fSHans Petter Selasky .size = sizeof(struct ib_flow_spec_eth),
22988e6e287fSHans Petter Selasky .mask = {.dst_mac = {0x1} },
22998e6e287fSHans Petter Selasky .val = {.dst_mac = {} }
23008e6e287fSHans Petter Selasky }
23018e6e287fSHans Petter Selasky }
23028e6e287fSHans Petter Selasky };
23038e6e287fSHans Petter Selasky
23048e6e287fSHans Petter Selasky handler = create_flow_rule(dev, ft_prio,
23058e6e287fSHans Petter Selasky &leftovers_specs[LEFTOVERS_MC].flow_attr,
23068e6e287fSHans Petter Selasky dst);
23078e6e287fSHans Petter Selasky if (!IS_ERR(handler) &&
23088e6e287fSHans Petter Selasky flow_attr->type == IB_FLOW_ATTR_ALL_DEFAULT) {
23098e6e287fSHans Petter Selasky handler_ucast = create_flow_rule(dev, ft_prio,
23108e6e287fSHans Petter Selasky &leftovers_specs[LEFTOVERS_UC].flow_attr,
23118e6e287fSHans Petter Selasky dst);
23128e6e287fSHans Petter Selasky if (IS_ERR(handler_ucast)) {
2313b0b87d91SHans Petter Selasky mlx5_del_flow_rule(&handler->rule);
23148e6e287fSHans Petter Selasky ft_prio->refcount--;
23158e6e287fSHans Petter Selasky kfree(handler);
23168e6e287fSHans Petter Selasky handler = handler_ucast;
23178e6e287fSHans Petter Selasky } else {
23188e6e287fSHans Petter Selasky list_add(&handler_ucast->list, &handler->list);
23198e6e287fSHans Petter Selasky }
23208e6e287fSHans Petter Selasky }
23218e6e287fSHans Petter Selasky
23228e6e287fSHans Petter Selasky return handler;
23238e6e287fSHans Petter Selasky }
23248e6e287fSHans Petter Selasky
create_sniffer_rule(struct mlx5_ib_dev * dev,struct mlx5_ib_flow_prio * ft_rx,struct mlx5_ib_flow_prio * ft_tx,struct mlx5_flow_destination * dst)23258e6e287fSHans Petter Selasky static struct mlx5_ib_flow_handler *create_sniffer_rule(struct mlx5_ib_dev *dev,
23268e6e287fSHans Petter Selasky struct mlx5_ib_flow_prio *ft_rx,
23278e6e287fSHans Petter Selasky struct mlx5_ib_flow_prio *ft_tx,
23288e6e287fSHans Petter Selasky struct mlx5_flow_destination *dst)
23298e6e287fSHans Petter Selasky {
23308e6e287fSHans Petter Selasky struct mlx5_ib_flow_handler *handler_rx;
23318e6e287fSHans Petter Selasky struct mlx5_ib_flow_handler *handler_tx;
23328e6e287fSHans Petter Selasky int err;
23338e6e287fSHans Petter Selasky static const struct ib_flow_attr flow_attr = {
23348e6e287fSHans Petter Selasky .num_of_specs = 0,
2335a30f7170SHans Petter Selasky .type = IB_FLOW_ATTR_SNIFFER,
23368e6e287fSHans Petter Selasky .size = sizeof(flow_attr)
23378e6e287fSHans Petter Selasky };
23388e6e287fSHans Petter Selasky
23398e6e287fSHans Petter Selasky handler_rx = create_flow_rule(dev, ft_rx, &flow_attr, dst);
23408e6e287fSHans Petter Selasky if (IS_ERR(handler_rx)) {
23418e6e287fSHans Petter Selasky err = PTR_ERR(handler_rx);
23428e6e287fSHans Petter Selasky goto err;
23438e6e287fSHans Petter Selasky }
23448e6e287fSHans Petter Selasky
23458e6e287fSHans Petter Selasky handler_tx = create_flow_rule(dev, ft_tx, &flow_attr, dst);
23468e6e287fSHans Petter Selasky if (IS_ERR(handler_tx)) {
23478e6e287fSHans Petter Selasky err = PTR_ERR(handler_tx);
23488e6e287fSHans Petter Selasky goto err_tx;
23498e6e287fSHans Petter Selasky }
23508e6e287fSHans Petter Selasky
23518e6e287fSHans Petter Selasky list_add(&handler_tx->list, &handler_rx->list);
23528e6e287fSHans Petter Selasky
23538e6e287fSHans Petter Selasky return handler_rx;
23548e6e287fSHans Petter Selasky
23558e6e287fSHans Petter Selasky err_tx:
2356b0b87d91SHans Petter Selasky mlx5_del_flow_rule(&handler_rx->rule);
23578e6e287fSHans Petter Selasky ft_rx->refcount--;
23588e6e287fSHans Petter Selasky kfree(handler_rx);
23598e6e287fSHans Petter Selasky err:
23608e6e287fSHans Petter Selasky return ERR_PTR(err);
23618e6e287fSHans Petter Selasky }
23628e6e287fSHans Petter Selasky
mlx5_ib_create_flow(struct ib_qp * qp,struct ib_flow_attr * flow_attr,int domain,struct ib_udata * udata)23638e6e287fSHans Petter Selasky static struct ib_flow *mlx5_ib_create_flow(struct ib_qp *qp,
23648e6e287fSHans Petter Selasky struct ib_flow_attr *flow_attr,
2365b633e08cSHans Petter Selasky int domain,
2366b633e08cSHans Petter Selasky struct ib_udata *udata)
23678e6e287fSHans Petter Selasky {
23688e6e287fSHans Petter Selasky struct mlx5_ib_dev *dev = to_mdev(qp->device);
23698e6e287fSHans Petter Selasky struct mlx5_ib_qp *mqp = to_mqp(qp);
23708e6e287fSHans Petter Selasky struct mlx5_ib_flow_handler *handler = NULL;
23718e6e287fSHans Petter Selasky struct mlx5_flow_destination *dst = NULL;
23728e6e287fSHans Petter Selasky struct mlx5_ib_flow_prio *ft_prio_tx = NULL;
23738e6e287fSHans Petter Selasky struct mlx5_ib_flow_prio *ft_prio;
2374a30f7170SHans Petter Selasky struct mlx5_ib_create_flow *ucmd = NULL, ucmd_hdr;
2375a30f7170SHans Petter Selasky size_t min_ucmd_sz, required_ucmd_sz;
23768e6e287fSHans Petter Selasky int err;
23778e6e287fSHans Petter Selasky
2378a30f7170SHans Petter Selasky if (udata && udata->inlen) {
2379a30f7170SHans Petter Selasky min_ucmd_sz = offsetofend(struct mlx5_ib_create_flow, reserved);
2380a30f7170SHans Petter Selasky if (udata->inlen < min_ucmd_sz)
2381a30f7170SHans Petter Selasky return ERR_PTR(-EOPNOTSUPP);
23828e6e287fSHans Petter Selasky
2383a30f7170SHans Petter Selasky err = ib_copy_from_udata(&ucmd_hdr, udata, min_ucmd_sz);
2384a30f7170SHans Petter Selasky if (err)
2385a30f7170SHans Petter Selasky return ERR_PTR(err);
2386a30f7170SHans Petter Selasky
2387a30f7170SHans Petter Selasky /* currently supports only one counters data */
2388a30f7170SHans Petter Selasky if (ucmd_hdr.ncounters_data > 1)
23898e6e287fSHans Petter Selasky return ERR_PTR(-EINVAL);
23908e6e287fSHans Petter Selasky
2391a30f7170SHans Petter Selasky required_ucmd_sz = min_ucmd_sz +
2392a30f7170SHans Petter Selasky sizeof(struct mlx5_ib_flow_counters_data) *
2393a30f7170SHans Petter Selasky ucmd_hdr.ncounters_data;
2394a30f7170SHans Petter Selasky if (udata->inlen > required_ucmd_sz &&
2395a30f7170SHans Petter Selasky !ib_is_udata_cleared(udata, required_ucmd_sz,
2396a30f7170SHans Petter Selasky udata->inlen - required_ucmd_sz))
2397a30f7170SHans Petter Selasky return ERR_PTR(-EOPNOTSUPP);
2398a30f7170SHans Petter Selasky
2399a30f7170SHans Petter Selasky ucmd = kzalloc(required_ucmd_sz, GFP_KERNEL);
2400a30f7170SHans Petter Selasky if (!ucmd)
24018e6e287fSHans Petter Selasky return ERR_PTR(-ENOMEM);
24028e6e287fSHans Petter Selasky
2403a30f7170SHans Petter Selasky err = ib_copy_from_udata(ucmd, udata, required_ucmd_sz);
2404a30f7170SHans Petter Selasky if (err)
2405a30f7170SHans Petter Selasky goto free_ucmd;
2406a30f7170SHans Petter Selasky }
2407a30f7170SHans Petter Selasky
2408a30f7170SHans Petter Selasky if (flow_attr->priority > MLX5_IB_FLOW_LAST_PRIO) {
2409a30f7170SHans Petter Selasky err = -ENOMEM;
2410a30f7170SHans Petter Selasky goto free_ucmd;
2411a30f7170SHans Petter Selasky }
2412a30f7170SHans Petter Selasky
2413a30f7170SHans Petter Selasky if (flow_attr->flags & ~IB_FLOW_ATTR_FLAGS_DONT_TRAP) {
2414a30f7170SHans Petter Selasky err = -EINVAL;
2415a30f7170SHans Petter Selasky goto free_ucmd;
2416a30f7170SHans Petter Selasky }
2417a30f7170SHans Petter Selasky
2418a30f7170SHans Petter Selasky dst = kzalloc(sizeof(*dst), GFP_KERNEL);
2419a30f7170SHans Petter Selasky if (!dst) {
2420a30f7170SHans Petter Selasky err = -ENOMEM;
2421a30f7170SHans Petter Selasky goto free_ucmd;
2422a30f7170SHans Petter Selasky }
2423a30f7170SHans Petter Selasky
24248e6e287fSHans Petter Selasky mutex_lock(&dev->flow_db.lock);
24258e6e287fSHans Petter Selasky
24268e6e287fSHans Petter Selasky ft_prio = get_flow_table(dev, flow_attr, MLX5_IB_FT_RX);
24278e6e287fSHans Petter Selasky if (IS_ERR(ft_prio)) {
24288e6e287fSHans Petter Selasky err = PTR_ERR(ft_prio);
24298e6e287fSHans Petter Selasky goto unlock;
24308e6e287fSHans Petter Selasky }
24318e6e287fSHans Petter Selasky if (flow_attr->type == IB_FLOW_ATTR_SNIFFER) {
24328e6e287fSHans Petter Selasky ft_prio_tx = get_flow_table(dev, flow_attr, MLX5_IB_FT_TX);
24338e6e287fSHans Petter Selasky if (IS_ERR(ft_prio_tx)) {
24348e6e287fSHans Petter Selasky err = PTR_ERR(ft_prio_tx);
24358e6e287fSHans Petter Selasky ft_prio_tx = NULL;
24368e6e287fSHans Petter Selasky goto destroy_ft;
24378e6e287fSHans Petter Selasky }
24388e6e287fSHans Petter Selasky }
24398e6e287fSHans Petter Selasky
24408e6e287fSHans Petter Selasky dst->type = MLX5_FLOW_DESTINATION_TYPE_TIR;
24418e6e287fSHans Petter Selasky if (mqp->flags & MLX5_IB_QP_RSS)
24428e6e287fSHans Petter Selasky dst->tir_num = mqp->rss_qp.tirn;
24438e6e287fSHans Petter Selasky else
24448e6e287fSHans Petter Selasky dst->tir_num = mqp->raw_packet_qp.rq.tirn;
24458e6e287fSHans Petter Selasky
2446a30f7170SHans Petter Selasky switch (flow_attr->type) {
2447a30f7170SHans Petter Selasky case IB_FLOW_ATTR_NORMAL:
2448a30f7170SHans Petter Selasky if (mqp->flags & IB_QP_CREATE_SOURCE_QPN) {
2449a30f7170SHans Petter Selasky err = -EOPNOTSUPP;
2450a30f7170SHans Petter Selasky goto destroy_ft;
24518e6e287fSHans Petter Selasky }
2452a30f7170SHans Petter Selasky if (flow_attr->flags & IB_FLOW_ATTR_FLAGS_DONT_TRAP) {
2453a30f7170SHans Petter Selasky handler = create_dont_trap_rule(dev, ft_prio, flow_attr, dst);
24548e6e287fSHans Petter Selasky } else {
2455a30f7170SHans Petter Selasky handler = create_flow_rule(dev, ft_prio, flow_attr, dst);
2456a30f7170SHans Petter Selasky }
2457a30f7170SHans Petter Selasky break;
2458a30f7170SHans Petter Selasky case IB_FLOW_ATTR_ALL_DEFAULT:
2459a30f7170SHans Petter Selasky case IB_FLOW_ATTR_MC_DEFAULT:
2460a30f7170SHans Petter Selasky handler = create_leftovers_rule(dev, ft_prio, flow_attr, dst);
2461a30f7170SHans Petter Selasky break;
2462a30f7170SHans Petter Selasky case IB_FLOW_ATTR_SNIFFER:
2463a30f7170SHans Petter Selasky handler = create_sniffer_rule(dev, ft_prio, ft_prio_tx, dst);
2464a30f7170SHans Petter Selasky break;
2465a30f7170SHans Petter Selasky default:
24668e6e287fSHans Petter Selasky err = -EINVAL;
24678e6e287fSHans Petter Selasky goto destroy_ft;
24688e6e287fSHans Petter Selasky }
24698e6e287fSHans Petter Selasky
24708e6e287fSHans Petter Selasky if (IS_ERR(handler)) {
24718e6e287fSHans Petter Selasky err = PTR_ERR(handler);
24728e6e287fSHans Petter Selasky handler = NULL;
24738e6e287fSHans Petter Selasky goto destroy_ft;
24748e6e287fSHans Petter Selasky }
24758e6e287fSHans Petter Selasky
24768e6e287fSHans Petter Selasky mutex_unlock(&dev->flow_db.lock);
24778e6e287fSHans Petter Selasky kfree(dst);
2478a30f7170SHans Petter Selasky kfree(ucmd);
24798e6e287fSHans Petter Selasky
24808e6e287fSHans Petter Selasky return &handler->ibflow;
24818e6e287fSHans Petter Selasky
24828e6e287fSHans Petter Selasky destroy_ft:
24838e6e287fSHans Petter Selasky put_flow_table(dev, ft_prio, false);
24848e6e287fSHans Petter Selasky if (ft_prio_tx)
24858e6e287fSHans Petter Selasky put_flow_table(dev, ft_prio_tx, false);
24868e6e287fSHans Petter Selasky unlock:
24878e6e287fSHans Petter Selasky mutex_unlock(&dev->flow_db.lock);
24888e6e287fSHans Petter Selasky kfree(dst);
2489a30f7170SHans Petter Selasky free_ucmd:
2490a30f7170SHans Petter Selasky kfree(ucmd);
24918e6e287fSHans Petter Selasky return ERR_PTR(err);
24928e6e287fSHans Petter Selasky }
24938e6e287fSHans Petter Selasky
mlx5_ib_mcg_attach(struct ib_qp * ibqp,union ib_gid * gid,u16 lid)249412515907SHans Petter Selasky static int mlx5_ib_mcg_attach(struct ib_qp *ibqp, union ib_gid *gid, u16 lid)
249512515907SHans Petter Selasky {
249612515907SHans Petter Selasky struct mlx5_ib_dev *dev = to_mdev(ibqp->device);
249712515907SHans Petter Selasky int err;
249812515907SHans Petter Selasky
249912515907SHans Petter Selasky err = mlx5_core_attach_mcg(dev->mdev, gid, ibqp->qp_num);
250012515907SHans Petter Selasky if (err)
250112515907SHans Petter Selasky mlx5_ib_warn(dev, "failed attaching QPN 0x%x, MGID %pI6\n",
250212515907SHans Petter Selasky ibqp->qp_num, gid->raw);
250312515907SHans Petter Selasky
250412515907SHans Petter Selasky return err;
250512515907SHans Petter Selasky }
250612515907SHans Petter Selasky
mlx5_ib_mcg_detach(struct ib_qp * ibqp,union ib_gid * gid,u16 lid)250712515907SHans Petter Selasky static int mlx5_ib_mcg_detach(struct ib_qp *ibqp, union ib_gid *gid, u16 lid)
250812515907SHans Petter Selasky {
250912515907SHans Petter Selasky struct mlx5_ib_dev *dev = to_mdev(ibqp->device);
251012515907SHans Petter Selasky int err;
251112515907SHans Petter Selasky
251212515907SHans Petter Selasky err = mlx5_core_detach_mcg(dev->mdev, gid, ibqp->qp_num);
251312515907SHans Petter Selasky if (err)
251412515907SHans Petter Selasky mlx5_ib_warn(dev, "failed detaching QPN 0x%x, MGID %pI6\n",
251512515907SHans Petter Selasky ibqp->qp_num, gid->raw);
251612515907SHans Petter Selasky
251712515907SHans Petter Selasky return err;
251812515907SHans Petter Selasky }
251912515907SHans Petter Selasky
init_node_data(struct mlx5_ib_dev * dev)252012515907SHans Petter Selasky static int init_node_data(struct mlx5_ib_dev *dev)
252112515907SHans Petter Selasky {
252212515907SHans Petter Selasky int err;
252312515907SHans Petter Selasky
252412515907SHans Petter Selasky err = mlx5_query_node_desc(dev, dev->ib_dev.node_desc);
252512515907SHans Petter Selasky if (err)
252612515907SHans Petter Selasky return err;
252712515907SHans Petter Selasky
252812515907SHans Petter Selasky return mlx5_query_node_guid(dev, &dev->ib_dev.node_guid);
252912515907SHans Petter Selasky }
253012515907SHans Petter Selasky
show_fw_pages(struct device * device,struct device_attribute * attr,char * buf)253112515907SHans Petter Selasky static ssize_t show_fw_pages(struct device *device, struct device_attribute *attr,
253212515907SHans Petter Selasky char *buf)
253312515907SHans Petter Selasky {
253412515907SHans Petter Selasky struct mlx5_ib_dev *dev =
253512515907SHans Petter Selasky container_of(device, struct mlx5_ib_dev, ib_dev.dev);
253612515907SHans Petter Selasky
253712515907SHans Petter Selasky return sprintf(buf, "%lld\n", (long long)dev->mdev->priv.fw_pages);
253812515907SHans Petter Selasky }
253912515907SHans Petter Selasky
show_reg_pages(struct device * device,struct device_attribute * attr,char * buf)254012515907SHans Petter Selasky static ssize_t show_reg_pages(struct device *device,
254112515907SHans Petter Selasky struct device_attribute *attr, char *buf)
254212515907SHans Petter Selasky {
254312515907SHans Petter Selasky struct mlx5_ib_dev *dev =
254412515907SHans Petter Selasky container_of(device, struct mlx5_ib_dev, ib_dev.dev);
254512515907SHans Petter Selasky
254612515907SHans Petter Selasky return sprintf(buf, "%d\n", atomic_read(&dev->mdev->priv.reg_pages));
254712515907SHans Petter Selasky }
254812515907SHans Petter Selasky
show_hca(struct device * device,struct device_attribute * attr,char * buf)254912515907SHans Petter Selasky static ssize_t show_hca(struct device *device, struct device_attribute *attr,
255012515907SHans Petter Selasky char *buf)
255112515907SHans Petter Selasky {
255212515907SHans Petter Selasky struct mlx5_ib_dev *dev =
255312515907SHans Petter Selasky container_of(device, struct mlx5_ib_dev, ib_dev.dev);
255412515907SHans Petter Selasky return sprintf(buf, "MT%d\n", dev->mdev->pdev->device);
255512515907SHans Petter Selasky }
255612515907SHans Petter Selasky
show_rev(struct device * device,struct device_attribute * attr,char * buf)255712515907SHans Petter Selasky static ssize_t show_rev(struct device *device, struct device_attribute *attr,
255812515907SHans Petter Selasky char *buf)
255912515907SHans Petter Selasky {
256012515907SHans Petter Selasky struct mlx5_ib_dev *dev =
256112515907SHans Petter Selasky container_of(device, struct mlx5_ib_dev, ib_dev.dev);
25628e6e287fSHans Petter Selasky return sprintf(buf, "%x\n", dev->mdev->pdev->revision);
256312515907SHans Petter Selasky }
256412515907SHans Petter Selasky
show_board(struct device * device,struct device_attribute * attr,char * buf)256512515907SHans Petter Selasky static ssize_t show_board(struct device *device, struct device_attribute *attr,
256612515907SHans Petter Selasky char *buf)
256712515907SHans Petter Selasky {
256812515907SHans Petter Selasky struct mlx5_ib_dev *dev =
256912515907SHans Petter Selasky container_of(device, struct mlx5_ib_dev, ib_dev.dev);
257012515907SHans Petter Selasky return sprintf(buf, "%.*s\n", MLX5_BOARD_ID_LEN,
257112515907SHans Petter Selasky dev->mdev->board_id);
257212515907SHans Petter Selasky }
257312515907SHans Petter Selasky
257412515907SHans Petter Selasky static DEVICE_ATTR(hw_rev, S_IRUGO, show_rev, NULL);
257512515907SHans Petter Selasky static DEVICE_ATTR(hca_type, S_IRUGO, show_hca, NULL);
257612515907SHans Petter Selasky static DEVICE_ATTR(board_id, S_IRUGO, show_board, NULL);
257712515907SHans Petter Selasky static DEVICE_ATTR(fw_pages, S_IRUGO, show_fw_pages, NULL);
257812515907SHans Petter Selasky static DEVICE_ATTR(reg_pages, S_IRUGO, show_reg_pages, NULL);
257912515907SHans Petter Selasky
258012515907SHans Petter Selasky static struct device_attribute *mlx5_class_attributes[] = {
258112515907SHans Petter Selasky &dev_attr_hw_rev,
258212515907SHans Petter Selasky &dev_attr_hca_type,
258312515907SHans Petter Selasky &dev_attr_board_id,
258412515907SHans Petter Selasky &dev_attr_fw_pages,
258512515907SHans Petter Selasky &dev_attr_reg_pages,
258612515907SHans Petter Selasky };
258712515907SHans Petter Selasky
pkey_change_handler(struct work_struct * work)25888e6e287fSHans Petter Selasky static void pkey_change_handler(struct work_struct *work)
25898e6e287fSHans Petter Selasky {
25908e6e287fSHans Petter Selasky struct mlx5_ib_port_resources *ports =
25918e6e287fSHans Petter Selasky container_of(work, struct mlx5_ib_port_resources,
25928e6e287fSHans Petter Selasky pkey_change_work);
25938e6e287fSHans Petter Selasky
25948e6e287fSHans Petter Selasky mutex_lock(&ports->devr->mutex);
25958e6e287fSHans Petter Selasky mlx5_ib_gsi_pkey_change(ports->gsi);
25968e6e287fSHans Petter Selasky mutex_unlock(&ports->devr->mutex);
25978e6e287fSHans Petter Selasky }
25988e6e287fSHans Petter Selasky
mlx5_ib_handle_internal_error(struct mlx5_ib_dev * ibdev)259912515907SHans Petter Selasky static void mlx5_ib_handle_internal_error(struct mlx5_ib_dev *ibdev)
260012515907SHans Petter Selasky {
260112515907SHans Petter Selasky struct mlx5_ib_qp *mqp;
260212515907SHans Petter Selasky struct mlx5_ib_cq *send_mcq, *recv_mcq;
260312515907SHans Petter Selasky struct mlx5_core_cq *mcq;
260412515907SHans Petter Selasky struct list_head cq_armed_list;
260512515907SHans Petter Selasky unsigned long flags_qp;
260612515907SHans Petter Selasky unsigned long flags_cq;
260712515907SHans Petter Selasky unsigned long flags;
260812515907SHans Petter Selasky
260912515907SHans Petter Selasky INIT_LIST_HEAD(&cq_armed_list);
261012515907SHans Petter Selasky
261112515907SHans Petter Selasky /* Go over qp list reside on that ibdev, sync with create/destroy qp.*/
261212515907SHans Petter Selasky spin_lock_irqsave(&ibdev->reset_flow_resource_lock, flags);
261312515907SHans Petter Selasky list_for_each_entry(mqp, &ibdev->qp_list, qps_list) {
261412515907SHans Petter Selasky spin_lock_irqsave(&mqp->sq.lock, flags_qp);
261512515907SHans Petter Selasky if (mqp->sq.tail != mqp->sq.head) {
261612515907SHans Petter Selasky send_mcq = to_mcq(mqp->ibqp.send_cq);
261712515907SHans Petter Selasky spin_lock_irqsave(&send_mcq->lock, flags_cq);
261812515907SHans Petter Selasky if (send_mcq->mcq.comp &&
261912515907SHans Petter Selasky mqp->ibqp.send_cq->comp_handler) {
262012515907SHans Petter Selasky if (!send_mcq->mcq.reset_notify_added) {
262112515907SHans Petter Selasky send_mcq->mcq.reset_notify_added = 1;
262212515907SHans Petter Selasky list_add_tail(&send_mcq->mcq.reset_notify,
262312515907SHans Petter Selasky &cq_armed_list);
262412515907SHans Petter Selasky }
262512515907SHans Petter Selasky }
262612515907SHans Petter Selasky spin_unlock_irqrestore(&send_mcq->lock, flags_cq);
262712515907SHans Petter Selasky }
262812515907SHans Petter Selasky spin_unlock_irqrestore(&mqp->sq.lock, flags_qp);
262912515907SHans Petter Selasky spin_lock_irqsave(&mqp->rq.lock, flags_qp);
263012515907SHans Petter Selasky /* no handling is needed for SRQ */
263112515907SHans Petter Selasky if (!mqp->ibqp.srq) {
263212515907SHans Petter Selasky if (mqp->rq.tail != mqp->rq.head) {
263312515907SHans Petter Selasky recv_mcq = to_mcq(mqp->ibqp.recv_cq);
263412515907SHans Petter Selasky spin_lock_irqsave(&recv_mcq->lock, flags_cq);
263512515907SHans Petter Selasky if (recv_mcq->mcq.comp &&
263612515907SHans Petter Selasky mqp->ibqp.recv_cq->comp_handler) {
263712515907SHans Petter Selasky if (!recv_mcq->mcq.reset_notify_added) {
263812515907SHans Petter Selasky recv_mcq->mcq.reset_notify_added = 1;
263912515907SHans Petter Selasky list_add_tail(&recv_mcq->mcq.reset_notify,
264012515907SHans Petter Selasky &cq_armed_list);
264112515907SHans Petter Selasky }
264212515907SHans Petter Selasky }
264312515907SHans Petter Selasky spin_unlock_irqrestore(&recv_mcq->lock,
264412515907SHans Petter Selasky flags_cq);
264512515907SHans Petter Selasky }
264612515907SHans Petter Selasky }
264712515907SHans Petter Selasky spin_unlock_irqrestore(&mqp->rq.lock, flags_qp);
264812515907SHans Petter Selasky }
264912515907SHans Petter Selasky /*At that point all inflight post send were put to be executed as of we
265012515907SHans Petter Selasky * lock/unlock above locks Now need to arm all involved CQs.
265112515907SHans Petter Selasky */
265212515907SHans Petter Selasky list_for_each_entry(mcq, &cq_armed_list, reset_notify) {
2653f34f0a65SHans Petter Selasky mcq->comp(mcq, NULL);
265412515907SHans Petter Selasky }
265512515907SHans Petter Selasky spin_unlock_irqrestore(&ibdev->reset_flow_resource_lock, flags);
265612515907SHans Petter Selasky }
265712515907SHans Petter Selasky
mlx5_ib_event(struct mlx5_core_dev * dev,void * context,enum mlx5_dev_event event,unsigned long param)265812515907SHans Petter Selasky static void mlx5_ib_event(struct mlx5_core_dev *dev, void *context,
265912515907SHans Petter Selasky enum mlx5_dev_event event, unsigned long param)
266012515907SHans Petter Selasky {
266112515907SHans Petter Selasky struct mlx5_ib_dev *ibdev = (struct mlx5_ib_dev *)context;
266212515907SHans Petter Selasky struct ib_event ibev;
26638e6e287fSHans Petter Selasky bool fatal = false;
2664843a89d3SSlava Shwartsman u8 port = (u8)param;
266512515907SHans Petter Selasky
266612515907SHans Petter Selasky switch (event) {
266712515907SHans Petter Selasky case MLX5_DEV_EVENT_SYS_ERROR:
266812515907SHans Petter Selasky ibev.event = IB_EVENT_DEVICE_FATAL;
266912515907SHans Petter Selasky mlx5_ib_handle_internal_error(ibdev);
26708e6e287fSHans Petter Selasky fatal = true;
267112515907SHans Petter Selasky break;
267212515907SHans Petter Selasky
267312515907SHans Petter Selasky case MLX5_DEV_EVENT_PORT_UP:
267412515907SHans Petter Selasky case MLX5_DEV_EVENT_PORT_DOWN:
267512515907SHans Petter Selasky case MLX5_DEV_EVENT_PORT_INITIALIZED:
26768e6e287fSHans Petter Selasky /* In RoCE, port up/down events are handled in
26778e6e287fSHans Petter Selasky * mlx5_netdev_event().
26788e6e287fSHans Petter Selasky */
26798e6e287fSHans Petter Selasky if (mlx5_ib_port_link_layer(&ibdev->ib_dev, port) ==
26808e6e287fSHans Petter Selasky IB_LINK_LAYER_ETHERNET)
26818e6e287fSHans Petter Selasky return;
26828e6e287fSHans Petter Selasky
26838e6e287fSHans Petter Selasky ibev.event = (event == MLX5_DEV_EVENT_PORT_UP) ?
26848e6e287fSHans Petter Selasky IB_EVENT_PORT_ACTIVE : IB_EVENT_PORT_ERR;
268512515907SHans Petter Selasky break;
268612515907SHans Petter Selasky
268712515907SHans Petter Selasky case MLX5_DEV_EVENT_LID_CHANGE:
268812515907SHans Petter Selasky ibev.event = IB_EVENT_LID_CHANGE;
268912515907SHans Petter Selasky break;
269012515907SHans Petter Selasky
269112515907SHans Petter Selasky case MLX5_DEV_EVENT_PKEY_CHANGE:
269212515907SHans Petter Selasky ibev.event = IB_EVENT_PKEY_CHANGE;
26938e6e287fSHans Petter Selasky
26948e6e287fSHans Petter Selasky schedule_work(&ibdev->devr.ports[port - 1].pkey_change_work);
269512515907SHans Petter Selasky break;
269612515907SHans Petter Selasky
269712515907SHans Petter Selasky case MLX5_DEV_EVENT_GUID_CHANGE:
269812515907SHans Petter Selasky ibev.event = IB_EVENT_GID_CHANGE;
269912515907SHans Petter Selasky break;
270012515907SHans Petter Selasky
270112515907SHans Petter Selasky case MLX5_DEV_EVENT_CLIENT_REREG:
270212515907SHans Petter Selasky ibev.event = IB_EVENT_CLIENT_REREGISTER;
270312515907SHans Petter Selasky break;
270412515907SHans Petter Selasky
270512515907SHans Petter Selasky default:
270662bfa774SHans Petter Selasky /* unsupported event */
270762bfa774SHans Petter Selasky return;
270812515907SHans Petter Selasky }
270912515907SHans Petter Selasky
271012515907SHans Petter Selasky ibev.device = &ibdev->ib_dev;
271112515907SHans Petter Selasky ibev.element.port_num = port;
271212515907SHans Petter Selasky
2713843a89d3SSlava Shwartsman if (!rdma_is_port_valid(&ibdev->ib_dev, port)) {
271462bfa774SHans Petter Selasky mlx5_ib_warn(ibdev, "warning: event(%d) on port %d\n", event, port);
271512515907SHans Petter Selasky return;
271612515907SHans Petter Selasky }
271712515907SHans Petter Selasky
271812515907SHans Petter Selasky if (ibdev->ib_active)
271912515907SHans Petter Selasky ib_dispatch_event(&ibev);
27208e6e287fSHans Petter Selasky
27218e6e287fSHans Petter Selasky if (fatal)
27228e6e287fSHans Petter Selasky ibdev->ib_active = false;
272312515907SHans Petter Selasky }
272412515907SHans Petter Selasky
get_ext_port_caps(struct mlx5_ib_dev * dev)272512515907SHans Petter Selasky static void get_ext_port_caps(struct mlx5_ib_dev *dev)
272612515907SHans Petter Selasky {
272712515907SHans Petter Selasky int port;
272812515907SHans Petter Selasky
272912515907SHans Petter Selasky for (port = 1; port <= MLX5_CAP_GEN(dev->mdev, num_ports); port++)
273012515907SHans Petter Selasky mlx5_query_ext_port_caps(dev, port);
273112515907SHans Petter Selasky }
273212515907SHans Petter Selasky
get_port_caps(struct mlx5_ib_dev * dev)273312515907SHans Petter Selasky static int get_port_caps(struct mlx5_ib_dev *dev)
273412515907SHans Petter Selasky {
273512515907SHans Petter Selasky struct ib_device_attr *dprops = NULL;
273612515907SHans Petter Selasky struct ib_port_attr *pprops = NULL;
273712515907SHans Petter Selasky int err = -ENOMEM;
273812515907SHans Petter Selasky int port;
27398e6e287fSHans Petter Selasky struct ib_udata uhw = {.inlen = 0, .outlen = 0};
274012515907SHans Petter Selasky
274112515907SHans Petter Selasky pprops = kmalloc(sizeof(*pprops), GFP_KERNEL);
274212515907SHans Petter Selasky if (!pprops)
274312515907SHans Petter Selasky goto out;
274412515907SHans Petter Selasky
274512515907SHans Petter Selasky dprops = kmalloc(sizeof(*dprops), GFP_KERNEL);
274612515907SHans Petter Selasky if (!dprops)
274712515907SHans Petter Selasky goto out;
274812515907SHans Petter Selasky
27498e6e287fSHans Petter Selasky err = mlx5_ib_query_device(&dev->ib_dev, dprops, &uhw);
275012515907SHans Petter Selasky if (err) {
275112515907SHans Petter Selasky mlx5_ib_warn(dev, "query_device failed %d\n", err);
275212515907SHans Petter Selasky goto out;
275312515907SHans Petter Selasky }
275412515907SHans Petter Selasky
275512515907SHans Petter Selasky for (port = 1; port <= MLX5_CAP_GEN(dev->mdev, num_ports); port++) {
275612515907SHans Petter Selasky err = mlx5_ib_query_port(&dev->ib_dev, port, pprops);
275712515907SHans Petter Selasky if (err) {
275812515907SHans Petter Selasky mlx5_ib_warn(dev, "query_port %d failed %d\n",
275912515907SHans Petter Selasky port, err);
276012515907SHans Petter Selasky break;
276112515907SHans Petter Selasky }
27628e6e287fSHans Petter Selasky dev->mdev->port_caps[port - 1].pkey_table_len =
27638e6e287fSHans Petter Selasky dprops->max_pkeys;
27648e6e287fSHans Petter Selasky dev->mdev->port_caps[port - 1].gid_table_len =
27658e6e287fSHans Petter Selasky pprops->gid_tbl_len;
276612515907SHans Petter Selasky mlx5_ib_dbg(dev, "pkey_table_len %d, gid_table_len %d\n",
276712515907SHans Petter Selasky dprops->max_pkeys, pprops->gid_tbl_len);
276812515907SHans Petter Selasky }
276912515907SHans Petter Selasky
277012515907SHans Petter Selasky out:
277112515907SHans Petter Selasky kfree(pprops);
277212515907SHans Petter Selasky kfree(dprops);
277312515907SHans Petter Selasky
277412515907SHans Petter Selasky return err;
277512515907SHans Petter Selasky }
277612515907SHans Petter Selasky
destroy_umrc_res(struct mlx5_ib_dev * dev)277712515907SHans Petter Selasky static void destroy_umrc_res(struct mlx5_ib_dev *dev)
277812515907SHans Petter Selasky {
277912515907SHans Petter Selasky int err;
278012515907SHans Petter Selasky
278112515907SHans Petter Selasky err = mlx5_mr_cache_cleanup(dev);
278212515907SHans Petter Selasky if (err)
278312515907SHans Petter Selasky mlx5_ib_warn(dev, "mr cache cleanup failed\n");
278412515907SHans Petter Selasky
2785b633e08cSHans Petter Selasky if (dev->umrc.qp)
2786b633e08cSHans Petter Selasky mlx5_ib_destroy_qp(dev->umrc.qp, NULL);
2787b633e08cSHans Petter Selasky if (dev->umrc.cq)
27888e6e287fSHans Petter Selasky ib_free_cq(dev->umrc.cq);
2789b633e08cSHans Petter Selasky if (dev->umrc.pd)
279012515907SHans Petter Selasky ib_dealloc_pd(dev->umrc.pd);
279112515907SHans Petter Selasky }
279212515907SHans Petter Selasky
279312515907SHans Petter Selasky enum {
279412515907SHans Petter Selasky MAX_UMR_WR = 128,
279512515907SHans Petter Selasky };
279612515907SHans Petter Selasky
create_umr_res(struct mlx5_ib_dev * dev)279712515907SHans Petter Selasky static int create_umr_res(struct mlx5_ib_dev *dev)
279812515907SHans Petter Selasky {
27998e6e287fSHans Petter Selasky struct ib_qp_init_attr *init_attr = NULL;
28008e6e287fSHans Petter Selasky struct ib_qp_attr *attr = NULL;
280112515907SHans Petter Selasky struct ib_pd *pd;
28028e6e287fSHans Petter Selasky struct ib_cq *cq;
28038e6e287fSHans Petter Selasky struct ib_qp *qp;
280412515907SHans Petter Selasky int ret;
280512515907SHans Petter Selasky
28068e6e287fSHans Petter Selasky attr = kzalloc(sizeof(*attr), GFP_KERNEL);
28078e6e287fSHans Petter Selasky init_attr = kzalloc(sizeof(*init_attr), GFP_KERNEL);
28088e6e287fSHans Petter Selasky if (!attr || !init_attr) {
28098e6e287fSHans Petter Selasky ret = -ENOMEM;
28108e6e287fSHans Petter Selasky goto error_0;
28118e6e287fSHans Petter Selasky }
28128e6e287fSHans Petter Selasky
28138e6e287fSHans Petter Selasky pd = ib_alloc_pd(&dev->ib_dev, 0);
281412515907SHans Petter Selasky if (IS_ERR(pd)) {
281512515907SHans Petter Selasky mlx5_ib_dbg(dev, "Couldn't create PD for sync UMR QP\n");
281612515907SHans Petter Selasky ret = PTR_ERR(pd);
281712515907SHans Petter Selasky goto error_0;
281812515907SHans Petter Selasky }
281912515907SHans Petter Selasky
28208e6e287fSHans Petter Selasky cq = ib_alloc_cq(&dev->ib_dev, NULL, 128, 0, IB_POLL_SOFTIRQ);
28218e6e287fSHans Petter Selasky if (IS_ERR(cq)) {
28228e6e287fSHans Petter Selasky mlx5_ib_dbg(dev, "Couldn't create CQ for sync UMR QP\n");
28238e6e287fSHans Petter Selasky ret = PTR_ERR(cq);
28248e6e287fSHans Petter Selasky goto error_2;
282512515907SHans Petter Selasky }
282612515907SHans Petter Selasky
28278e6e287fSHans Petter Selasky init_attr->send_cq = cq;
28288e6e287fSHans Petter Selasky init_attr->recv_cq = cq;
28298e6e287fSHans Petter Selasky init_attr->sq_sig_type = IB_SIGNAL_ALL_WR;
28308e6e287fSHans Petter Selasky init_attr->cap.max_send_wr = MAX_UMR_WR;
28318e6e287fSHans Petter Selasky init_attr->cap.max_send_sge = 1;
28328e6e287fSHans Petter Selasky init_attr->qp_type = MLX5_IB_QPT_REG_UMR;
28338e6e287fSHans Petter Selasky init_attr->port_num = 1;
28348e6e287fSHans Petter Selasky qp = mlx5_ib_create_qp(pd, init_attr, NULL);
28358e6e287fSHans Petter Selasky if (IS_ERR(qp)) {
28368e6e287fSHans Petter Selasky mlx5_ib_dbg(dev, "Couldn't create sync UMR QP\n");
28378e6e287fSHans Petter Selasky ret = PTR_ERR(qp);
28388e6e287fSHans Petter Selasky goto error_3;
28398e6e287fSHans Petter Selasky }
28408e6e287fSHans Petter Selasky qp->device = &dev->ib_dev;
28418e6e287fSHans Petter Selasky qp->real_qp = qp;
28428e6e287fSHans Petter Selasky qp->uobject = NULL;
28438e6e287fSHans Petter Selasky qp->qp_type = MLX5_IB_QPT_REG_UMR;
28448e6e287fSHans Petter Selasky
28458e6e287fSHans Petter Selasky attr->qp_state = IB_QPS_INIT;
28468e6e287fSHans Petter Selasky attr->port_num = 1;
28478e6e287fSHans Petter Selasky ret = mlx5_ib_modify_qp(qp, attr, IB_QP_STATE | IB_QP_PKEY_INDEX |
28488e6e287fSHans Petter Selasky IB_QP_PORT, NULL);
28498e6e287fSHans Petter Selasky if (ret) {
28508e6e287fSHans Petter Selasky mlx5_ib_dbg(dev, "Couldn't modify UMR QP\n");
28518e6e287fSHans Petter Selasky goto error_4;
28528e6e287fSHans Petter Selasky }
28538e6e287fSHans Petter Selasky
28548e6e287fSHans Petter Selasky memset(attr, 0, sizeof(*attr));
28558e6e287fSHans Petter Selasky attr->qp_state = IB_QPS_RTR;
28568e6e287fSHans Petter Selasky attr->path_mtu = IB_MTU_256;
28578e6e287fSHans Petter Selasky
28588e6e287fSHans Petter Selasky ret = mlx5_ib_modify_qp(qp, attr, IB_QP_STATE, NULL);
28598e6e287fSHans Petter Selasky if (ret) {
28608e6e287fSHans Petter Selasky mlx5_ib_dbg(dev, "Couldn't modify umr QP to rtr\n");
28618e6e287fSHans Petter Selasky goto error_4;
28628e6e287fSHans Petter Selasky }
28638e6e287fSHans Petter Selasky
28648e6e287fSHans Petter Selasky memset(attr, 0, sizeof(*attr));
28658e6e287fSHans Petter Selasky attr->qp_state = IB_QPS_RTS;
28668e6e287fSHans Petter Selasky ret = mlx5_ib_modify_qp(qp, attr, IB_QP_STATE, NULL);
28678e6e287fSHans Petter Selasky if (ret) {
28688e6e287fSHans Petter Selasky mlx5_ib_dbg(dev, "Couldn't modify umr QP to rts\n");
28698e6e287fSHans Petter Selasky goto error_4;
28708e6e287fSHans Petter Selasky }
28718e6e287fSHans Petter Selasky
28728e6e287fSHans Petter Selasky dev->umrc.qp = qp;
28738e6e287fSHans Petter Selasky dev->umrc.cq = cq;
287412515907SHans Petter Selasky dev->umrc.pd = pd;
287512515907SHans Petter Selasky
28768e6e287fSHans Petter Selasky sema_init(&dev->umrc.sem, MAX_UMR_WR);
287712515907SHans Petter Selasky ret = mlx5_mr_cache_init(dev);
287812515907SHans Petter Selasky if (ret) {
287912515907SHans Petter Selasky mlx5_ib_warn(dev, "mr cache init failed %d\n", ret);
288012515907SHans Petter Selasky goto error_4;
288112515907SHans Petter Selasky }
288212515907SHans Petter Selasky
28838e6e287fSHans Petter Selasky kfree(attr);
28848e6e287fSHans Petter Selasky kfree(init_attr);
28858e6e287fSHans Petter Selasky
288612515907SHans Petter Selasky return 0;
288712515907SHans Petter Selasky
288812515907SHans Petter Selasky error_4:
2889b633e08cSHans Petter Selasky mlx5_ib_destroy_qp(qp, NULL);
2890b633e08cSHans Petter Selasky dev->umrc.qp = NULL;
28918e6e287fSHans Petter Selasky
28928e6e287fSHans Petter Selasky error_3:
28938e6e287fSHans Petter Selasky ib_free_cq(cq);
2894b633e08cSHans Petter Selasky dev->umrc.cq = NULL;
28958e6e287fSHans Petter Selasky
28968e6e287fSHans Petter Selasky error_2:
289712515907SHans Petter Selasky ib_dealloc_pd(pd);
2898b633e08cSHans Petter Selasky dev->umrc.pd = NULL;
28998e6e287fSHans Petter Selasky
290012515907SHans Petter Selasky error_0:
29018e6e287fSHans Petter Selasky kfree(attr);
29028e6e287fSHans Petter Selasky kfree(init_attr);
290312515907SHans Petter Selasky return ret;
290412515907SHans Petter Selasky }
290512515907SHans Petter Selasky
create_dev_resources(struct mlx5_ib_resources * devr)290612515907SHans Petter Selasky static int create_dev_resources(struct mlx5_ib_resources *devr)
290712515907SHans Petter Selasky {
290812515907SHans Petter Selasky struct ib_srq_init_attr attr;
290912515907SHans Petter Selasky struct mlx5_ib_dev *dev;
2910b633e08cSHans Petter Selasky struct ib_device *ibdev;
291112515907SHans Petter Selasky struct ib_cq_init_attr cq_attr = {.cqe = 1};
29128e6e287fSHans Petter Selasky int port;
29138e6e287fSHans Petter Selasky int ret = 0;
291412515907SHans Petter Selasky
291512515907SHans Petter Selasky dev = container_of(devr, struct mlx5_ib_dev, devr);
2916b633e08cSHans Petter Selasky ibdev = &dev->ib_dev;
291712515907SHans Petter Selasky
29188e6e287fSHans Petter Selasky mutex_init(&devr->mutex);
29198e6e287fSHans Petter Selasky
2920b633e08cSHans Petter Selasky devr->p0 = rdma_zalloc_drv_obj(ibdev, ib_pd);
2921b633e08cSHans Petter Selasky if (!devr->p0)
2922b633e08cSHans Petter Selasky return -ENOMEM;
2923b633e08cSHans Petter Selasky
2924b633e08cSHans Petter Selasky devr->p0->device = ibdev;
292512515907SHans Petter Selasky devr->p0->uobject = NULL;
292612515907SHans Petter Selasky atomic_set(&devr->p0->usecnt, 0);
292712515907SHans Petter Selasky
2928b633e08cSHans Petter Selasky ret = mlx5_ib_alloc_pd(devr->p0, NULL);
2929b633e08cSHans Petter Selasky if (ret)
2930b633e08cSHans Petter Selasky goto error0;
2931b633e08cSHans Petter Selasky
2932b633e08cSHans Petter Selasky devr->c0 = rdma_zalloc_drv_obj(ibdev, ib_cq);
2933b633e08cSHans Petter Selasky if (!devr->c0) {
2934b633e08cSHans Petter Selasky ret = -ENOMEM;
293512515907SHans Petter Selasky goto error1;
293612515907SHans Petter Selasky }
2937b633e08cSHans Petter Selasky
293812515907SHans Petter Selasky devr->c0->device = &dev->ib_dev;
293912515907SHans Petter Selasky atomic_set(&devr->c0->usecnt, 0);
294012515907SHans Petter Selasky
2941b633e08cSHans Petter Selasky ret = mlx5_ib_create_cq(devr->c0, &cq_attr, NULL);
2942b633e08cSHans Petter Selasky if (ret)
2943b633e08cSHans Petter Selasky goto err_create_cq;
2944b633e08cSHans Petter Selasky
2945b633e08cSHans Petter Selasky devr->x0 = mlx5_ib_alloc_xrcd(&dev->ib_dev, NULL);
294612515907SHans Petter Selasky if (IS_ERR(devr->x0)) {
294712515907SHans Petter Selasky ret = PTR_ERR(devr->x0);
294812515907SHans Petter Selasky goto error2;
294912515907SHans Petter Selasky }
295012515907SHans Petter Selasky devr->x0->device = &dev->ib_dev;
295112515907SHans Petter Selasky devr->x0->inode = NULL;
295212515907SHans Petter Selasky atomic_set(&devr->x0->usecnt, 0);
295312515907SHans Petter Selasky mutex_init(&devr->x0->tgt_qp_mutex);
295412515907SHans Petter Selasky INIT_LIST_HEAD(&devr->x0->tgt_qp_list);
295512515907SHans Petter Selasky
2956b633e08cSHans Petter Selasky devr->x1 = mlx5_ib_alloc_xrcd(&dev->ib_dev, NULL);
295712515907SHans Petter Selasky if (IS_ERR(devr->x1)) {
295812515907SHans Petter Selasky ret = PTR_ERR(devr->x1);
295912515907SHans Petter Selasky goto error3;
296012515907SHans Petter Selasky }
296112515907SHans Petter Selasky devr->x1->device = &dev->ib_dev;
296212515907SHans Petter Selasky devr->x1->inode = NULL;
296312515907SHans Petter Selasky atomic_set(&devr->x1->usecnt, 0);
296412515907SHans Petter Selasky mutex_init(&devr->x1->tgt_qp_mutex);
296512515907SHans Petter Selasky INIT_LIST_HEAD(&devr->x1->tgt_qp_list);
296612515907SHans Petter Selasky
296712515907SHans Petter Selasky memset(&attr, 0, sizeof(attr));
296812515907SHans Petter Selasky attr.attr.max_sge = 1;
296912515907SHans Petter Selasky attr.attr.max_wr = 1;
297012515907SHans Petter Selasky attr.srq_type = IB_SRQT_XRC;
2971b633e08cSHans Petter Selasky attr.ext.cq = devr->c0;
297212515907SHans Petter Selasky attr.ext.xrc.xrcd = devr->x0;
297312515907SHans Petter Selasky
2974b633e08cSHans Petter Selasky devr->s0 = rdma_zalloc_drv_obj(ibdev, ib_srq);
2975b633e08cSHans Petter Selasky if (!devr->s0) {
2976b633e08cSHans Petter Selasky ret = -ENOMEM;
297712515907SHans Petter Selasky goto error4;
297812515907SHans Petter Selasky }
2979b633e08cSHans Petter Selasky
298012515907SHans Petter Selasky devr->s0->device = &dev->ib_dev;
298112515907SHans Petter Selasky devr->s0->pd = devr->p0;
298212515907SHans Petter Selasky devr->s0->srq_type = IB_SRQT_XRC;
298312515907SHans Petter Selasky devr->s0->ext.xrc.xrcd = devr->x0;
2984b633e08cSHans Petter Selasky devr->s0->ext.cq = devr->c0;
2985b633e08cSHans Petter Selasky ret = mlx5_ib_create_srq(devr->s0, &attr, NULL);
2986b633e08cSHans Petter Selasky if (ret)
2987b633e08cSHans Petter Selasky goto err_create;
2988b633e08cSHans Petter Selasky
298912515907SHans Petter Selasky atomic_inc(&devr->s0->ext.xrc.xrcd->usecnt);
2990b633e08cSHans Petter Selasky atomic_inc(&devr->s0->ext.cq->usecnt);
299112515907SHans Petter Selasky atomic_inc(&devr->p0->usecnt);
299212515907SHans Petter Selasky atomic_set(&devr->s0->usecnt, 0);
299312515907SHans Petter Selasky
299412515907SHans Petter Selasky memset(&attr, 0, sizeof(attr));
299512515907SHans Petter Selasky attr.attr.max_sge = 1;
299612515907SHans Petter Selasky attr.attr.max_wr = 1;
299712515907SHans Petter Selasky attr.srq_type = IB_SRQT_BASIC;
2998b633e08cSHans Petter Selasky devr->s1 = rdma_zalloc_drv_obj(ibdev, ib_srq);
2999b633e08cSHans Petter Selasky if (!devr->s1) {
3000b633e08cSHans Petter Selasky ret = -ENOMEM;
300112515907SHans Petter Selasky goto error5;
300212515907SHans Petter Selasky }
3003b633e08cSHans Petter Selasky
300412515907SHans Petter Selasky devr->s1->device = &dev->ib_dev;
300512515907SHans Petter Selasky devr->s1->pd = devr->p0;
300612515907SHans Petter Selasky devr->s1->srq_type = IB_SRQT_BASIC;
3007b633e08cSHans Petter Selasky devr->s1->ext.cq = devr->c0;
3008b633e08cSHans Petter Selasky
3009b633e08cSHans Petter Selasky ret = mlx5_ib_create_srq(devr->s1, &attr, NULL);
3010b633e08cSHans Petter Selasky if (ret)
3011b633e08cSHans Petter Selasky goto error6;
3012b633e08cSHans Petter Selasky
301312515907SHans Petter Selasky atomic_inc(&devr->p0->usecnt);
3014b633e08cSHans Petter Selasky atomic_set(&devr->s1->usecnt, 0);
30158e6e287fSHans Petter Selasky
30168e6e287fSHans Petter Selasky for (port = 0; port < ARRAY_SIZE(devr->ports); ++port) {
30178e6e287fSHans Petter Selasky INIT_WORK(&devr->ports[port].pkey_change_work,
30188e6e287fSHans Petter Selasky pkey_change_handler);
30198e6e287fSHans Petter Selasky devr->ports[port].devr = devr;
30208e6e287fSHans Petter Selasky }
302112515907SHans Petter Selasky
302212515907SHans Petter Selasky return 0;
302312515907SHans Petter Selasky
3024b633e08cSHans Petter Selasky error6:
3025b633e08cSHans Petter Selasky kfree(devr->s1);
302612515907SHans Petter Selasky error5:
3027b633e08cSHans Petter Selasky mlx5_ib_destroy_srq(devr->s0, NULL);
3028b633e08cSHans Petter Selasky err_create:
3029b633e08cSHans Petter Selasky kfree(devr->s0);
303012515907SHans Petter Selasky error4:
3031b633e08cSHans Petter Selasky mlx5_ib_dealloc_xrcd(devr->x1, NULL);
303212515907SHans Petter Selasky error3:
3033b633e08cSHans Petter Selasky mlx5_ib_dealloc_xrcd(devr->x0, NULL);
303412515907SHans Petter Selasky error2:
3035b633e08cSHans Petter Selasky mlx5_ib_destroy_cq(devr->c0, NULL);
3036b633e08cSHans Petter Selasky err_create_cq:
3037b633e08cSHans Petter Selasky kfree(devr->c0);
303812515907SHans Petter Selasky error1:
3039b633e08cSHans Petter Selasky mlx5_ib_dealloc_pd(devr->p0, NULL);
304012515907SHans Petter Selasky error0:
3041b633e08cSHans Petter Selasky kfree(devr->p0);
304212515907SHans Petter Selasky return ret;
304312515907SHans Petter Selasky }
304412515907SHans Petter Selasky
destroy_dev_resources(struct mlx5_ib_resources * devr)304512515907SHans Petter Selasky static void destroy_dev_resources(struct mlx5_ib_resources *devr)
304612515907SHans Petter Selasky {
30478e6e287fSHans Petter Selasky int port;
30488e6e287fSHans Petter Selasky
3049b633e08cSHans Petter Selasky mlx5_ib_destroy_srq(devr->s1, NULL);
3050b633e08cSHans Petter Selasky kfree(devr->s1);
3051b633e08cSHans Petter Selasky mlx5_ib_destroy_srq(devr->s0, NULL);
3052b633e08cSHans Petter Selasky kfree(devr->s0);
3053b633e08cSHans Petter Selasky mlx5_ib_dealloc_xrcd(devr->x0, NULL);
3054b633e08cSHans Petter Selasky mlx5_ib_dealloc_xrcd(devr->x1, NULL);
3055b633e08cSHans Petter Selasky mlx5_ib_destroy_cq(devr->c0, NULL);
3056b633e08cSHans Petter Selasky kfree(devr->c0);
3057b633e08cSHans Petter Selasky mlx5_ib_dealloc_pd(devr->p0, NULL);
3058b633e08cSHans Petter Selasky kfree(devr->p0);
30598e6e287fSHans Petter Selasky
30608e6e287fSHans Petter Selasky /* Make sure no change P_Key work items are still executing */
3061b633e08cSHans Petter Selasky for (port = 0; port < ARRAY_SIZE(devr->ports); ++port)
30628e6e287fSHans Petter Selasky cancel_work_sync(&devr->ports[port].pkey_change_work);
306312515907SHans Petter Selasky }
306412515907SHans Petter Selasky
get_core_cap_flags(struct ib_device * ibdev)3065d05554bbSHans Petter Selasky static u32 get_core_cap_flags(struct ib_device *ibdev)
3066d05554bbSHans Petter Selasky {
3067d05554bbSHans Petter Selasky struct mlx5_ib_dev *dev = to_mdev(ibdev);
3068d05554bbSHans Petter Selasky enum rdma_link_layer ll = mlx5_ib_port_link_layer(ibdev, 1);
3069d05554bbSHans Petter Selasky u8 l3_type_cap = MLX5_CAP_ROCE(dev->mdev, l3_type);
3070d05554bbSHans Petter Selasky u8 roce_version_cap = MLX5_CAP_ROCE(dev->mdev, roce_version);
3071d05554bbSHans Petter Selasky u32 ret = 0;
3072d05554bbSHans Petter Selasky
3073d05554bbSHans Petter Selasky if (ll == IB_LINK_LAYER_INFINIBAND)
3074d05554bbSHans Petter Selasky return RDMA_CORE_PORT_IBA_IB;
3075d05554bbSHans Petter Selasky
3076d05554bbSHans Petter Selasky if (!(l3_type_cap & MLX5_ROCE_L3_TYPE_IPV4_CAP))
30778e6e287fSHans Petter Selasky return 0;
3078d05554bbSHans Petter Selasky
3079d05554bbSHans Petter Selasky if (!(l3_type_cap & MLX5_ROCE_L3_TYPE_IPV6_CAP))
30808e6e287fSHans Petter Selasky return 0;
3081d05554bbSHans Petter Selasky
3082d05554bbSHans Petter Selasky if (roce_version_cap & MLX5_ROCE_VERSION_1_CAP)
3083d05554bbSHans Petter Selasky ret |= RDMA_CORE_PORT_IBA_ROCE;
3084d05554bbSHans Petter Selasky
3085d05554bbSHans Petter Selasky if (roce_version_cap & MLX5_ROCE_VERSION_2_CAP)
3086d05554bbSHans Petter Selasky ret |= RDMA_CORE_PORT_IBA_ROCE_UDP_ENCAP;
3087d05554bbSHans Petter Selasky
3088d05554bbSHans Petter Selasky return ret;
3089d05554bbSHans Petter Selasky }
3090d05554bbSHans Petter Selasky
mlx5_port_immutable(struct ib_device * ibdev,u8 port_num,struct ib_port_immutable * immutable)3091d05554bbSHans Petter Selasky static int mlx5_port_immutable(struct ib_device *ibdev, u8 port_num,
3092d05554bbSHans Petter Selasky struct ib_port_immutable *immutable)
3093d05554bbSHans Petter Selasky {
3094d05554bbSHans Petter Selasky struct ib_port_attr attr;
3095626cb01dSHans Petter Selasky struct mlx5_ib_dev *dev = to_mdev(ibdev);
3096626cb01dSHans Petter Selasky enum rdma_link_layer ll = mlx5_ib_port_link_layer(ibdev, port_num);
3097d05554bbSHans Petter Selasky int err;
3098d05554bbSHans Petter Selasky
30998e6e287fSHans Petter Selasky err = mlx5_ib_query_port(ibdev, port_num, &attr);
3100d05554bbSHans Petter Selasky if (err)
3101d05554bbSHans Petter Selasky return err;
3102d05554bbSHans Petter Selasky
3103d05554bbSHans Petter Selasky immutable->pkey_tbl_len = attr.pkey_tbl_len;
3104d05554bbSHans Petter Selasky immutable->gid_tbl_len = attr.gid_tbl_len;
3105d05554bbSHans Petter Selasky immutable->core_cap_flags = get_core_cap_flags(ibdev);
3106626cb01dSHans Petter Selasky if ((ll == IB_LINK_LAYER_INFINIBAND) || MLX5_CAP_GEN(dev->mdev, roce))
3107d05554bbSHans Petter Selasky immutable->max_mad_size = IB_MGMT_MAD_SIZE;
3108d05554bbSHans Petter Selasky
3109d05554bbSHans Petter Selasky return 0;
3110d05554bbSHans Petter Selasky }
3111d05554bbSHans Petter Selasky
get_dev_fw_str(struct ib_device * ibdev,char * str,size_t str_len)31128e6e287fSHans Petter Selasky static void get_dev_fw_str(struct ib_device *ibdev, char *str,
31138e6e287fSHans Petter Selasky size_t str_len)
311412515907SHans Petter Selasky {
31158e6e287fSHans Petter Selasky struct mlx5_ib_dev *dev =
31168e6e287fSHans Petter Selasky container_of(ibdev, struct mlx5_ib_dev, ib_dev);
31178e6e287fSHans Petter Selasky snprintf(str, str_len, "%d.%d.%04d", fw_rev_maj(dev->mdev),
31188e6e287fSHans Petter Selasky fw_rev_min(dev->mdev), fw_rev_sub(dev->mdev));
31198e6e287fSHans Petter Selasky }
31208e6e287fSHans Petter Selasky
mlx5_roce_lag_init(struct mlx5_ib_dev * dev)31218e6e287fSHans Petter Selasky static int mlx5_roce_lag_init(struct mlx5_ib_dev *dev)
31228e6e287fSHans Petter Selasky {
31238e6e287fSHans Petter Selasky return 0;
31248e6e287fSHans Petter Selasky }
31258e6e287fSHans Petter Selasky
mlx5_roce_lag_cleanup(struct mlx5_ib_dev * dev)31268e6e287fSHans Petter Selasky static void mlx5_roce_lag_cleanup(struct mlx5_ib_dev *dev)
31278e6e287fSHans Petter Selasky {
31288e6e287fSHans Petter Selasky }
31298e6e287fSHans Petter Selasky
mlx5_remove_roce_notifier(struct mlx5_ib_dev * dev)31308e6e287fSHans Petter Selasky static void mlx5_remove_roce_notifier(struct mlx5_ib_dev *dev)
31318e6e287fSHans Petter Selasky {
31328e6e287fSHans Petter Selasky if (dev->roce.nb.notifier_call) {
31338e6e287fSHans Petter Selasky unregister_netdevice_notifier(&dev->roce.nb);
31348e6e287fSHans Petter Selasky dev->roce.nb.notifier_call = NULL;
31358e6e287fSHans Petter Selasky }
31368e6e287fSHans Petter Selasky }
31378e6e287fSHans Petter Selasky
31385dc00f00SJustin Hibbits static int
mlx5_enable_roce_if_cb(if_t ifp,void * arg)31395dc00f00SJustin Hibbits mlx5_enable_roce_if_cb(if_t ifp, void *arg)
31405dc00f00SJustin Hibbits {
31415dc00f00SJustin Hibbits struct mlx5_ib_dev *dev = arg;
31425dc00f00SJustin Hibbits
31435dc00f00SJustin Hibbits /* check if network interface belongs to mlx5en */
31445dc00f00SJustin Hibbits if (!mlx5_netdev_match(ifp, dev->mdev, "mce"))
31455dc00f00SJustin Hibbits return (0);
31465dc00f00SJustin Hibbits
31475dc00f00SJustin Hibbits write_lock(&dev->roce.netdev_lock);
31485dc00f00SJustin Hibbits dev->roce.netdev = ifp;
31495dc00f00SJustin Hibbits write_unlock(&dev->roce.netdev_lock);
31505dc00f00SJustin Hibbits
31515dc00f00SJustin Hibbits return (0);
31525dc00f00SJustin Hibbits }
31535dc00f00SJustin Hibbits
mlx5_enable_roce(struct mlx5_ib_dev * dev)31548e6e287fSHans Petter Selasky static int mlx5_enable_roce(struct mlx5_ib_dev *dev)
31558e6e287fSHans Petter Selasky {
315670510800SJustin Hibbits struct epoch_tracker et;
31578e6e287fSHans Petter Selasky VNET_ITERATOR_DECL(vnet_iter);
315812515907SHans Petter Selasky int err;
315912515907SHans Petter Selasky
31608e6e287fSHans Petter Selasky /* Check if mlx5en net device already exists */
31618e6e287fSHans Petter Selasky VNET_LIST_RLOCK();
316270510800SJustin Hibbits NET_EPOCH_ENTER(et);
31638e6e287fSHans Petter Selasky VNET_FOREACH(vnet_iter) {
31648e6e287fSHans Petter Selasky CURVNET_SET_QUIET(vnet_iter);
31655dc00f00SJustin Hibbits if_foreach(mlx5_enable_roce_if_cb, dev);
31668e6e287fSHans Petter Selasky CURVNET_RESTORE();
316712515907SHans Petter Selasky }
316870510800SJustin Hibbits NET_EPOCH_EXIT(et);
31698e6e287fSHans Petter Selasky VNET_LIST_RUNLOCK();
317012515907SHans Petter Selasky
31718e6e287fSHans Petter Selasky dev->roce.nb.notifier_call = mlx5_netdev_event;
31728e6e287fSHans Petter Selasky err = register_netdevice_notifier(&dev->roce.nb);
317312515907SHans Petter Selasky if (err) {
31748e6e287fSHans Petter Selasky dev->roce.nb.notifier_call = NULL;
31758e6e287fSHans Petter Selasky return err;
317612515907SHans Petter Selasky }
317712515907SHans Petter Selasky
3178626cb01dSHans Petter Selasky if (MLX5_CAP_GEN(dev->mdev, roce)) {
31798e6e287fSHans Petter Selasky err = mlx5_nic_vport_enable_roce(dev->mdev);
31808e6e287fSHans Petter Selasky if (err)
31818e6e287fSHans Petter Selasky goto err_unregister_netdevice_notifier;
3182626cb01dSHans Petter Selasky }
318312515907SHans Petter Selasky
31848e6e287fSHans Petter Selasky err = mlx5_roce_lag_init(dev);
31858e6e287fSHans Petter Selasky if (err)
31868e6e287fSHans Petter Selasky goto err_disable_roce;
318712515907SHans Petter Selasky
318812515907SHans Petter Selasky return 0;
318912515907SHans Petter Selasky
31908e6e287fSHans Petter Selasky err_disable_roce:
3191626cb01dSHans Petter Selasky if (MLX5_CAP_GEN(dev->mdev, roce))
31928e6e287fSHans Petter Selasky mlx5_nic_vport_disable_roce(dev->mdev);
319312515907SHans Petter Selasky
31948e6e287fSHans Petter Selasky err_unregister_netdevice_notifier:
31958e6e287fSHans Petter Selasky mlx5_remove_roce_notifier(dev);
31968e6e287fSHans Petter Selasky return err;
319712515907SHans Petter Selasky }
319812515907SHans Petter Selasky
mlx5_disable_roce(struct mlx5_ib_dev * dev)31998e6e287fSHans Petter Selasky static void mlx5_disable_roce(struct mlx5_ib_dev *dev)
320012515907SHans Petter Selasky {
32018e6e287fSHans Petter Selasky mlx5_roce_lag_cleanup(dev);
3202626cb01dSHans Petter Selasky if (MLX5_CAP_GEN(dev->mdev, roce))
32038e6e287fSHans Petter Selasky mlx5_nic_vport_disable_roce(dev->mdev);
320412515907SHans Petter Selasky }
320512515907SHans Petter Selasky
mlx5_ib_dealloc_q_port_counter(struct mlx5_ib_dev * dev,u8 port_num)320612515907SHans Petter Selasky static void mlx5_ib_dealloc_q_port_counter(struct mlx5_ib_dev *dev, u8 port_num)
320712515907SHans Petter Selasky {
320812515907SHans Petter Selasky mlx5_vport_dealloc_q_counter(dev->mdev,
320912515907SHans Petter Selasky MLX5_INTERFACE_PROTOCOL_IB,
321012515907SHans Petter Selasky dev->port[port_num].q_cnt_id);
321112515907SHans Petter Selasky dev->port[port_num].q_cnt_id = 0;
321212515907SHans Petter Selasky }
321312515907SHans Petter Selasky
mlx5_ib_dealloc_q_counters(struct mlx5_ib_dev * dev)321412515907SHans Petter Selasky static void mlx5_ib_dealloc_q_counters(struct mlx5_ib_dev *dev)
321512515907SHans Petter Selasky {
321612515907SHans Petter Selasky unsigned int i;
321712515907SHans Petter Selasky
321812515907SHans Petter Selasky for (i = 0; i < dev->num_ports; i++)
321912515907SHans Petter Selasky mlx5_ib_dealloc_q_port_counter(dev, i);
322012515907SHans Petter Selasky }
322112515907SHans Petter Selasky
mlx5_ib_alloc_q_counters(struct mlx5_ib_dev * dev)322212515907SHans Petter Selasky static int mlx5_ib_alloc_q_counters(struct mlx5_ib_dev *dev)
322312515907SHans Petter Selasky {
322412515907SHans Petter Selasky int i;
322512515907SHans Petter Selasky int ret;
322612515907SHans Petter Selasky
322712515907SHans Petter Selasky for (i = 0; i < dev->num_ports; i++) {
322812515907SHans Petter Selasky ret = mlx5_vport_alloc_q_counter(dev->mdev,
322912515907SHans Petter Selasky MLX5_INTERFACE_PROTOCOL_IB,
323012515907SHans Petter Selasky &dev->port[i].q_cnt_id);
323112515907SHans Petter Selasky if (ret) {
323212515907SHans Petter Selasky mlx5_ib_warn(dev,
32338e6e287fSHans Petter Selasky "couldn't allocate queue counter for port %d, err %d\n",
32348e6e287fSHans Petter Selasky i + 1, ret);
323512515907SHans Petter Selasky goto dealloc_counters;
323612515907SHans Petter Selasky }
323712515907SHans Petter Selasky }
323812515907SHans Petter Selasky
323912515907SHans Petter Selasky return 0;
324012515907SHans Petter Selasky
324112515907SHans Petter Selasky dealloc_counters:
324212515907SHans Petter Selasky while (--i >= 0)
324312515907SHans Petter Selasky mlx5_ib_dealloc_q_port_counter(dev, i);
324412515907SHans Petter Selasky
324512515907SHans Petter Selasky return ret;
324612515907SHans Petter Selasky }
324712515907SHans Petter Selasky
32488e6e287fSHans Petter Selasky static const char * const names[] = {
32498e6e287fSHans Petter Selasky "rx_write_requests",
32508e6e287fSHans Petter Selasky "rx_read_requests",
32518e6e287fSHans Petter Selasky "rx_atomic_requests",
32528e6e287fSHans Petter Selasky "out_of_buffer",
32538e6e287fSHans Petter Selasky "out_of_sequence",
32548e6e287fSHans Petter Selasky "duplicate_request",
32558e6e287fSHans Petter Selasky "rnr_nak_retry_err",
32568e6e287fSHans Petter Selasky "packet_seq_err",
32578e6e287fSHans Petter Selasky "implied_nak_seq_err",
32588e6e287fSHans Petter Selasky "local_ack_timeout_err",
325912515907SHans Petter Selasky };
326012515907SHans Petter Selasky
32618e6e287fSHans Petter Selasky static const size_t stats_offsets[] = {
32628e6e287fSHans Petter Selasky MLX5_BYTE_OFF(query_q_counter_out, rx_write_requests),
32638e6e287fSHans Petter Selasky MLX5_BYTE_OFF(query_q_counter_out, rx_read_requests),
32648e6e287fSHans Petter Selasky MLX5_BYTE_OFF(query_q_counter_out, rx_atomic_requests),
32658e6e287fSHans Petter Selasky MLX5_BYTE_OFF(query_q_counter_out, out_of_buffer),
32668e6e287fSHans Petter Selasky MLX5_BYTE_OFF(query_q_counter_out, out_of_sequence),
32678e6e287fSHans Petter Selasky MLX5_BYTE_OFF(query_q_counter_out, duplicate_request),
32688e6e287fSHans Petter Selasky MLX5_BYTE_OFF(query_q_counter_out, rnr_nak_retry_err),
32698e6e287fSHans Petter Selasky MLX5_BYTE_OFF(query_q_counter_out, packet_seq_err),
32708e6e287fSHans Petter Selasky MLX5_BYTE_OFF(query_q_counter_out, implied_nak_seq_err),
32718e6e287fSHans Petter Selasky MLX5_BYTE_OFF(query_q_counter_out, local_ack_timeout_err),
327212515907SHans Petter Selasky };
327312515907SHans Petter Selasky
mlx5_ib_alloc_hw_stats(struct ib_device * ibdev,u8 port_num)32748e6e287fSHans Petter Selasky static struct rdma_hw_stats *mlx5_ib_alloc_hw_stats(struct ib_device *ibdev,
32758e6e287fSHans Petter Selasky u8 port_num)
327612515907SHans Petter Selasky {
32778e6e287fSHans Petter Selasky BUILD_BUG_ON(ARRAY_SIZE(names) != ARRAY_SIZE(stats_offsets));
327812515907SHans Petter Selasky
32798e6e287fSHans Petter Selasky /* We support only per port stats */
32808e6e287fSHans Petter Selasky if (port_num == 0)
32818e6e287fSHans Petter Selasky return NULL;
328212515907SHans Petter Selasky
32838e6e287fSHans Petter Selasky return rdma_alloc_hw_stats_struct(names, ARRAY_SIZE(names),
32848e6e287fSHans Petter Selasky RDMA_HW_STATS_DEFAULT_LIFESPAN);
328512515907SHans Petter Selasky }
328612515907SHans Petter Selasky
mlx5_ib_get_hw_stats(struct ib_device * ibdev,struct rdma_hw_stats * stats,u8 port,int index)32878e6e287fSHans Petter Selasky static int mlx5_ib_get_hw_stats(struct ib_device *ibdev,
32888e6e287fSHans Petter Selasky struct rdma_hw_stats *stats,
32898e6e287fSHans Petter Selasky u8 port, int index)
329012515907SHans Petter Selasky {
32918e6e287fSHans Petter Selasky struct mlx5_ib_dev *dev = to_mdev(ibdev);
329212515907SHans Petter Selasky int outlen = MLX5_ST_SZ_BYTES(query_q_counter_out);
329312515907SHans Petter Selasky void *out;
32948e6e287fSHans Petter Selasky __be32 val;
329512515907SHans Petter Selasky int ret;
32968e6e287fSHans Petter Selasky int i;
32978e6e287fSHans Petter Selasky
32988e6e287fSHans Petter Selasky if (!port || !stats)
32998e6e287fSHans Petter Selasky return -ENOSYS;
330012515907SHans Petter Selasky
330112515907SHans Petter Selasky out = mlx5_vzalloc(outlen);
330212515907SHans Petter Selasky if (!out)
330312515907SHans Petter Selasky return -ENOMEM;
330412515907SHans Petter Selasky
33058e6e287fSHans Petter Selasky ret = mlx5_vport_query_q_counter(dev->mdev,
33068e6e287fSHans Petter Selasky dev->port[port - 1].q_cnt_id, 0,
330712515907SHans Petter Selasky out, outlen);
330812515907SHans Petter Selasky if (ret)
330912515907SHans Petter Selasky goto free;
331012515907SHans Petter Selasky
33118e6e287fSHans Petter Selasky for (i = 0; i < ARRAY_SIZE(names); i++) {
33128e6e287fSHans Petter Selasky val = *(__be32 *)(out + stats_offsets[i]);
33138e6e287fSHans Petter Selasky stats->value[i] = (u64)be32_to_cpu(val);
33148e6e287fSHans Petter Selasky }
331512515907SHans Petter Selasky free:
33168e6e287fSHans Petter Selasky kvfree(out);
33178e6e287fSHans Petter Selasky return ARRAY_SIZE(names);
331812515907SHans Petter Selasky }
331912515907SHans Petter Selasky
mlx5_ib_stage_bfreg_init(struct mlx5_ib_dev * dev)3320e47a6525SHans Petter Selasky static int mlx5_ib_stage_bfreg_init(struct mlx5_ib_dev *dev)
3321e47a6525SHans Petter Selasky {
3322e47a6525SHans Petter Selasky int err;
3323e47a6525SHans Petter Selasky
3324e47a6525SHans Petter Selasky err = mlx5_alloc_bfreg(dev->mdev, &dev->bfreg, false, false);
3325e47a6525SHans Petter Selasky if (err)
3326e47a6525SHans Petter Selasky return err;
3327e47a6525SHans Petter Selasky
3328e47a6525SHans Petter Selasky err = mlx5_alloc_bfreg(dev->mdev, &dev->fp_bfreg, false, true);
3329e47a6525SHans Petter Selasky if (err) {
3330e47a6525SHans Petter Selasky mlx5_free_bfreg(dev->mdev, &dev->bfreg);
3331e47a6525SHans Petter Selasky return err;
3332e47a6525SHans Petter Selasky }
3333e47a6525SHans Petter Selasky
3334e47a6525SHans Petter Selasky err = mlx5_alloc_bfreg(dev->mdev, &dev->wc_bfreg, true, false);
3335e47a6525SHans Petter Selasky if (err) {
3336e47a6525SHans Petter Selasky mlx5_free_bfreg(dev->mdev, &dev->fp_bfreg);
3337e47a6525SHans Petter Selasky mlx5_free_bfreg(dev->mdev, &dev->bfreg);
3338e47a6525SHans Petter Selasky }
3339e47a6525SHans Petter Selasky
3340e47a6525SHans Petter Selasky return err;
3341e47a6525SHans Petter Selasky }
3342e47a6525SHans Petter Selasky
mlx5_ib_stage_bfreg_cleanup(struct mlx5_ib_dev * dev)3343e47a6525SHans Petter Selasky static void mlx5_ib_stage_bfreg_cleanup(struct mlx5_ib_dev *dev)
3344e47a6525SHans Petter Selasky {
3345e47a6525SHans Petter Selasky mlx5_free_bfreg(dev->mdev, &dev->wc_bfreg);
3346e47a6525SHans Petter Selasky mlx5_free_bfreg(dev->mdev, &dev->fp_bfreg);
3347e47a6525SHans Petter Selasky mlx5_free_bfreg(dev->mdev, &dev->bfreg);
3348e47a6525SHans Petter Selasky }
3349e47a6525SHans Petter Selasky
mlx5_ib_add(struct mlx5_core_dev * mdev)335012515907SHans Petter Selasky static void *mlx5_ib_add(struct mlx5_core_dev *mdev)
335112515907SHans Petter Selasky {
335212515907SHans Petter Selasky struct mlx5_ib_dev *dev;
33538e6e287fSHans Petter Selasky enum rdma_link_layer ll;
33548e6e287fSHans Petter Selasky int port_type_cap;
335512515907SHans Petter Selasky int err;
335612515907SHans Petter Selasky int i;
335712515907SHans Petter Selasky
33588e6e287fSHans Petter Selasky port_type_cap = MLX5_CAP_GEN(mdev, port_type);
33598e6e287fSHans Petter Selasky ll = mlx5_port_type_cap_to_rdma_ll(port_type_cap);
33608e6e287fSHans Petter Selasky
336112515907SHans Petter Selasky dev = (struct mlx5_ib_dev *)ib_alloc_device(sizeof(*dev));
336212515907SHans Petter Selasky if (!dev)
336312515907SHans Petter Selasky return NULL;
336412515907SHans Petter Selasky
336512515907SHans Petter Selasky dev->mdev = mdev;
336612515907SHans Petter Selasky
336712515907SHans Petter Selasky dev->port = kcalloc(MLX5_CAP_GEN(mdev, num_ports), sizeof(*dev->port),
336812515907SHans Petter Selasky GFP_KERNEL);
336912515907SHans Petter Selasky if (!dev->port)
337012515907SHans Petter Selasky goto err_dealloc;
337112515907SHans Petter Selasky
33728e6e287fSHans Petter Selasky rwlock_init(&dev->roce.netdev_lock);
337312515907SHans Petter Selasky err = get_port_caps(dev);
337412515907SHans Petter Selasky if (err)
337512515907SHans Petter Selasky goto err_free_port;
337612515907SHans Petter Selasky
337712515907SHans Petter Selasky if (mlx5_use_mad_ifc(dev))
337812515907SHans Petter Selasky get_ext_port_caps(dev);
337912515907SHans Petter Selasky
338012515907SHans Petter Selasky MLX5_INIT_DOORBELL_LOCK(&dev->uar_lock);
338112515907SHans Petter Selasky
3382ea00d7e8SHans Petter Selasky mutex_init(&dev->lb_mutex);
3383ea00d7e8SHans Petter Selasky
3384b633e08cSHans Petter Selasky INIT_IB_DEVICE_OPS(&dev->ib_dev.ops, mlx5, MLX5);
338514a1b9bdSHans Petter Selasky snprintf(dev->ib_dev.name, IB_DEVICE_NAME_MAX, "mlx5_%d", device_get_unit(mdev->pdev->dev.bsddev));
338612515907SHans Petter Selasky dev->ib_dev.owner = THIS_MODULE;
338712515907SHans Petter Selasky dev->ib_dev.node_type = RDMA_NODE_IB_CA;
33888e6e287fSHans Petter Selasky dev->ib_dev.local_dma_lkey = 0 /* not supported for now */;
338912515907SHans Petter Selasky dev->num_ports = MLX5_CAP_GEN(mdev, num_ports);
339012515907SHans Petter Selasky dev->ib_dev.phys_port_cnt = dev->num_ports;
339112515907SHans Petter Selasky dev->ib_dev.num_comp_vectors =
339212515907SHans Petter Selasky dev->mdev->priv.eq_table.num_comp_vectors;
339312515907SHans Petter Selasky dev->ib_dev.dma_device = &mdev->pdev->dev;
339412515907SHans Petter Selasky
339512515907SHans Petter Selasky dev->ib_dev.uverbs_abi_ver = MLX5_IB_UVERBS_ABI_VERSION;
339612515907SHans Petter Selasky dev->ib_dev.uverbs_cmd_mask =
339712515907SHans Petter Selasky (1ull << IB_USER_VERBS_CMD_GET_CONTEXT) |
339812515907SHans Petter Selasky (1ull << IB_USER_VERBS_CMD_QUERY_DEVICE) |
339912515907SHans Petter Selasky (1ull << IB_USER_VERBS_CMD_QUERY_PORT) |
340012515907SHans Petter Selasky (1ull << IB_USER_VERBS_CMD_ALLOC_PD) |
340112515907SHans Petter Selasky (1ull << IB_USER_VERBS_CMD_DEALLOC_PD) |
34021456d97cSHans Petter Selasky (1ull << IB_USER_VERBS_CMD_CREATE_AH) |
34031456d97cSHans Petter Selasky (1ull << IB_USER_VERBS_CMD_DESTROY_AH) |
340412515907SHans Petter Selasky (1ull << IB_USER_VERBS_CMD_REG_MR) |
34058e6e287fSHans Petter Selasky (1ull << IB_USER_VERBS_CMD_REREG_MR) |
340612515907SHans Petter Selasky (1ull << IB_USER_VERBS_CMD_DEREG_MR) |
340712515907SHans Petter Selasky (1ull << IB_USER_VERBS_CMD_CREATE_COMP_CHANNEL) |
340812515907SHans Petter Selasky (1ull << IB_USER_VERBS_CMD_CREATE_CQ) |
340912515907SHans Petter Selasky (1ull << IB_USER_VERBS_CMD_RESIZE_CQ) |
341012515907SHans Petter Selasky (1ull << IB_USER_VERBS_CMD_DESTROY_CQ) |
341112515907SHans Petter Selasky (1ull << IB_USER_VERBS_CMD_CREATE_QP) |
341212515907SHans Petter Selasky (1ull << IB_USER_VERBS_CMD_MODIFY_QP) |
341312515907SHans Petter Selasky (1ull << IB_USER_VERBS_CMD_QUERY_QP) |
341412515907SHans Petter Selasky (1ull << IB_USER_VERBS_CMD_DESTROY_QP) |
341512515907SHans Petter Selasky (1ull << IB_USER_VERBS_CMD_ATTACH_MCAST) |
341612515907SHans Petter Selasky (1ull << IB_USER_VERBS_CMD_DETACH_MCAST) |
341712515907SHans Petter Selasky (1ull << IB_USER_VERBS_CMD_CREATE_SRQ) |
341812515907SHans Petter Selasky (1ull << IB_USER_VERBS_CMD_MODIFY_SRQ) |
341912515907SHans Petter Selasky (1ull << IB_USER_VERBS_CMD_QUERY_SRQ) |
342012515907SHans Petter Selasky (1ull << IB_USER_VERBS_CMD_DESTROY_SRQ) |
342112515907SHans Petter Selasky (1ull << IB_USER_VERBS_CMD_CREATE_XSRQ) |
342212515907SHans Petter Selasky (1ull << IB_USER_VERBS_CMD_OPEN_QP);
34238e6e287fSHans Petter Selasky dev->ib_dev.uverbs_ex_cmd_mask =
34248e6e287fSHans Petter Selasky (1ull << IB_USER_VERBS_EX_CMD_QUERY_DEVICE) |
34258e6e287fSHans Petter Selasky (1ull << IB_USER_VERBS_EX_CMD_CREATE_CQ) |
34268e6e287fSHans Petter Selasky (1ull << IB_USER_VERBS_EX_CMD_CREATE_QP);
342712515907SHans Petter Selasky
342812515907SHans Petter Selasky dev->ib_dev.query_device = mlx5_ib_query_device;
342912515907SHans Petter Selasky dev->ib_dev.query_port = mlx5_ib_query_port;
343012515907SHans Petter Selasky dev->ib_dev.get_link_layer = mlx5_ib_port_link_layer;
34318e6e287fSHans Petter Selasky if (ll == IB_LINK_LAYER_ETHERNET)
343212515907SHans Petter Selasky dev->ib_dev.get_netdev = mlx5_ib_get_netdev;
343312515907SHans Petter Selasky dev->ib_dev.query_gid = mlx5_ib_query_gid;
34348e6e287fSHans Petter Selasky dev->ib_dev.add_gid = mlx5_ib_add_gid;
34358e6e287fSHans Petter Selasky dev->ib_dev.del_gid = mlx5_ib_del_gid;
343612515907SHans Petter Selasky dev->ib_dev.query_pkey = mlx5_ib_query_pkey;
343712515907SHans Petter Selasky dev->ib_dev.modify_device = mlx5_ib_modify_device;
343812515907SHans Petter Selasky dev->ib_dev.modify_port = mlx5_ib_modify_port;
343912515907SHans Petter Selasky dev->ib_dev.alloc_ucontext = mlx5_ib_alloc_ucontext;
344012515907SHans Petter Selasky dev->ib_dev.dealloc_ucontext = mlx5_ib_dealloc_ucontext;
344112515907SHans Petter Selasky dev->ib_dev.mmap = mlx5_ib_mmap;
3442b633e08cSHans Petter Selasky dev->ib_dev.mmap_free = mlx5_ib_mmap_free;
344312515907SHans Petter Selasky dev->ib_dev.alloc_pd = mlx5_ib_alloc_pd;
344412515907SHans Petter Selasky dev->ib_dev.dealloc_pd = mlx5_ib_dealloc_pd;
344512515907SHans Petter Selasky dev->ib_dev.create_ah = mlx5_ib_create_ah;
344612515907SHans Petter Selasky dev->ib_dev.query_ah = mlx5_ib_query_ah;
344712515907SHans Petter Selasky dev->ib_dev.destroy_ah = mlx5_ib_destroy_ah;
344812515907SHans Petter Selasky dev->ib_dev.create_srq = mlx5_ib_create_srq;
344912515907SHans Petter Selasky dev->ib_dev.modify_srq = mlx5_ib_modify_srq;
345012515907SHans Petter Selasky dev->ib_dev.query_srq = mlx5_ib_query_srq;
345112515907SHans Petter Selasky dev->ib_dev.destroy_srq = mlx5_ib_destroy_srq;
345212515907SHans Petter Selasky dev->ib_dev.post_srq_recv = mlx5_ib_post_srq_recv;
345312515907SHans Petter Selasky dev->ib_dev.create_qp = mlx5_ib_create_qp;
345412515907SHans Petter Selasky dev->ib_dev.modify_qp = mlx5_ib_modify_qp;
345512515907SHans Petter Selasky dev->ib_dev.query_qp = mlx5_ib_query_qp;
345612515907SHans Petter Selasky dev->ib_dev.destroy_qp = mlx5_ib_destroy_qp;
345712515907SHans Petter Selasky dev->ib_dev.post_send = mlx5_ib_post_send;
345812515907SHans Petter Selasky dev->ib_dev.post_recv = mlx5_ib_post_recv;
345912515907SHans Petter Selasky dev->ib_dev.create_cq = mlx5_ib_create_cq;
346012515907SHans Petter Selasky dev->ib_dev.modify_cq = mlx5_ib_modify_cq;
346112515907SHans Petter Selasky dev->ib_dev.resize_cq = mlx5_ib_resize_cq;
346212515907SHans Petter Selasky dev->ib_dev.destroy_cq = mlx5_ib_destroy_cq;
346312515907SHans Petter Selasky dev->ib_dev.poll_cq = mlx5_ib_poll_cq;
346412515907SHans Petter Selasky dev->ib_dev.req_notify_cq = mlx5_ib_arm_cq;
346512515907SHans Petter Selasky dev->ib_dev.get_dma_mr = mlx5_ib_get_dma_mr;
346612515907SHans Petter Selasky dev->ib_dev.reg_user_mr = mlx5_ib_reg_user_mr;
34678e6e287fSHans Petter Selasky dev->ib_dev.rereg_user_mr = mlx5_ib_rereg_user_mr;
346812515907SHans Petter Selasky dev->ib_dev.dereg_mr = mlx5_ib_dereg_mr;
346912515907SHans Petter Selasky dev->ib_dev.attach_mcast = mlx5_ib_mcg_attach;
347012515907SHans Petter Selasky dev->ib_dev.detach_mcast = mlx5_ib_mcg_detach;
347112515907SHans Petter Selasky dev->ib_dev.process_mad = mlx5_ib_process_mad;
34728e6e287fSHans Petter Selasky dev->ib_dev.alloc_mr = mlx5_ib_alloc_mr;
34738e6e287fSHans Petter Selasky dev->ib_dev.map_mr_sg = mlx5_ib_map_mr_sg;
34748e6e287fSHans Petter Selasky dev->ib_dev.check_mr_status = mlx5_ib_check_mr_status;
3475d05554bbSHans Petter Selasky dev->ib_dev.get_port_immutable = mlx5_port_immutable;
34768e6e287fSHans Petter Selasky dev->ib_dev.get_dev_fw_str = get_dev_fw_str;
34778e6e287fSHans Petter Selasky if (mlx5_core_is_pf(mdev)) {
34788e6e287fSHans Petter Selasky dev->ib_dev.get_vf_config = mlx5_ib_get_vf_config;
34798e6e287fSHans Petter Selasky dev->ib_dev.set_vf_link_state = mlx5_ib_set_vf_link_state;
34808e6e287fSHans Petter Selasky dev->ib_dev.get_vf_stats = mlx5_ib_get_vf_stats;
34818e6e287fSHans Petter Selasky dev->ib_dev.set_vf_guid = mlx5_ib_set_vf_guid;
34828e6e287fSHans Petter Selasky }
34838e6e287fSHans Petter Selasky
3484a3a31fdeSHans Petter Selasky dev->ib_dev.disassociate_ucontext = mlx5_ib_disassociate_ucontext;
3485a3a31fdeSHans Petter Selasky
34868e6e287fSHans Petter Selasky mlx5_ib_internal_fill_odp_caps(dev);
34878e6e287fSHans Petter Selasky
34888e6e287fSHans Petter Selasky if (MLX5_CAP_GEN(mdev, imaicl)) {
34898e6e287fSHans Petter Selasky dev->ib_dev.alloc_mw = mlx5_ib_alloc_mw;
34908e6e287fSHans Petter Selasky dev->ib_dev.dealloc_mw = mlx5_ib_dealloc_mw;
34918e6e287fSHans Petter Selasky dev->ib_dev.uverbs_cmd_mask |=
34928e6e287fSHans Petter Selasky (1ull << IB_USER_VERBS_CMD_ALLOC_MW) |
34938e6e287fSHans Petter Selasky (1ull << IB_USER_VERBS_CMD_DEALLOC_MW);
34948e6e287fSHans Petter Selasky }
34958e6e287fSHans Petter Selasky
34968e6e287fSHans Petter Selasky if (MLX5_CAP_GEN(dev->mdev, out_of_seq_cnt) &&
34978e6e287fSHans Petter Selasky MLX5_CAP_GEN(dev->mdev, retransmission_q_counters)) {
34988e6e287fSHans Petter Selasky dev->ib_dev.get_hw_stats = mlx5_ib_get_hw_stats;
34998e6e287fSHans Petter Selasky dev->ib_dev.alloc_hw_stats = mlx5_ib_alloc_hw_stats;
35008e6e287fSHans Petter Selasky }
350112515907SHans Petter Selasky
350212515907SHans Petter Selasky if (MLX5_CAP_GEN(mdev, xrc)) {
350312515907SHans Petter Selasky dev->ib_dev.alloc_xrcd = mlx5_ib_alloc_xrcd;
350412515907SHans Petter Selasky dev->ib_dev.dealloc_xrcd = mlx5_ib_dealloc_xrcd;
350512515907SHans Petter Selasky dev->ib_dev.uverbs_cmd_mask |=
350612515907SHans Petter Selasky (1ull << IB_USER_VERBS_CMD_OPEN_XRCD) |
350712515907SHans Petter Selasky (1ull << IB_USER_VERBS_CMD_CLOSE_XRCD);
350812515907SHans Petter Selasky }
350912515907SHans Petter Selasky
35108e6e287fSHans Petter Selasky if (mlx5_ib_port_link_layer(&dev->ib_dev, 1) ==
35118e6e287fSHans Petter Selasky IB_LINK_LAYER_ETHERNET) {
35128e6e287fSHans Petter Selasky dev->ib_dev.create_flow = mlx5_ib_create_flow;
35138e6e287fSHans Petter Selasky dev->ib_dev.destroy_flow = mlx5_ib_destroy_flow;
35148e6e287fSHans Petter Selasky dev->ib_dev.create_wq = mlx5_ib_create_wq;
35158e6e287fSHans Petter Selasky dev->ib_dev.modify_wq = mlx5_ib_modify_wq;
35168e6e287fSHans Petter Selasky dev->ib_dev.destroy_wq = mlx5_ib_destroy_wq;
35178e6e287fSHans Petter Selasky dev->ib_dev.create_rwq_ind_table = mlx5_ib_create_rwq_ind_table;
35188e6e287fSHans Petter Selasky dev->ib_dev.destroy_rwq_ind_table = mlx5_ib_destroy_rwq_ind_table;
35198e6e287fSHans Petter Selasky dev->ib_dev.uverbs_ex_cmd_mask |=
35208e6e287fSHans Petter Selasky (1ull << IB_USER_VERBS_EX_CMD_CREATE_FLOW) |
35218e6e287fSHans Petter Selasky (1ull << IB_USER_VERBS_EX_CMD_DESTROY_FLOW) |
35228e6e287fSHans Petter Selasky (1ull << IB_USER_VERBS_EX_CMD_CREATE_WQ) |
35238e6e287fSHans Petter Selasky (1ull << IB_USER_VERBS_EX_CMD_MODIFY_WQ) |
35248e6e287fSHans Petter Selasky (1ull << IB_USER_VERBS_EX_CMD_DESTROY_WQ) |
35258e6e287fSHans Petter Selasky (1ull << IB_USER_VERBS_EX_CMD_CREATE_RWQ_IND_TBL) |
35268e6e287fSHans Petter Selasky (1ull << IB_USER_VERBS_EX_CMD_DESTROY_RWQ_IND_TBL);
35278e6e287fSHans Petter Selasky }
352812515907SHans Petter Selasky err = init_node_data(dev);
352912515907SHans Petter Selasky if (err)
35308e6e287fSHans Petter Selasky goto err_free_port;
353112515907SHans Petter Selasky
35328e6e287fSHans Petter Selasky mutex_init(&dev->flow_db.lock);
353312515907SHans Petter Selasky mutex_init(&dev->cap_mask_mutex);
353412515907SHans Petter Selasky INIT_LIST_HEAD(&dev->qp_list);
353512515907SHans Petter Selasky spin_lock_init(&dev->reset_flow_resource_lock);
353612515907SHans Petter Selasky
35378e6e287fSHans Petter Selasky if (ll == IB_LINK_LAYER_ETHERNET) {
35388e6e287fSHans Petter Selasky err = mlx5_enable_roce(dev);
35398e6e287fSHans Petter Selasky if (err)
35408e6e287fSHans Petter Selasky goto err_free_port;
35418e6e287fSHans Petter Selasky }
35428e6e287fSHans Petter Selasky
354312515907SHans Petter Selasky err = create_dev_resources(&dev->devr);
354412515907SHans Petter Selasky if (err)
354512515907SHans Petter Selasky goto err_disable_roce;
354612515907SHans Petter Selasky
35478e6e287fSHans Petter Selasky err = mlx5_ib_odp_init_one(dev);
35488e6e287fSHans Petter Selasky if (err)
35498e6e287fSHans Petter Selasky goto err_rsrc;
355012515907SHans Petter Selasky
355112515907SHans Petter Selasky err = mlx5_ib_alloc_q_counters(dev);
355212515907SHans Petter Selasky if (err)
355312515907SHans Petter Selasky goto err_odp;
355412515907SHans Petter Selasky
3555e47a6525SHans Petter Selasky err = mlx5_ib_stage_bfreg_init(dev);
355612515907SHans Petter Selasky if (err)
355712515907SHans Petter Selasky goto err_q_cnt;
355812515907SHans Petter Selasky
3559e47a6525SHans Petter Selasky err = ib_register_device(&dev->ib_dev, NULL);
3560e47a6525SHans Petter Selasky if (err)
3561e47a6525SHans Petter Selasky goto err_bfreg;
3562e47a6525SHans Petter Selasky
356312515907SHans Petter Selasky err = create_umr_res(dev);
356412515907SHans Petter Selasky if (err)
356512515907SHans Petter Selasky goto err_dev;
356612515907SHans Petter Selasky
356712515907SHans Petter Selasky for (i = 0; i < ARRAY_SIZE(mlx5_class_attributes); i++) {
356812515907SHans Petter Selasky err = device_create_file(&dev->ib_dev.dev,
356912515907SHans Petter Selasky mlx5_class_attributes[i]);
357012515907SHans Petter Selasky if (err)
35718e6e287fSHans Petter Selasky goto err_umrc;
357212515907SHans Petter Selasky }
357312515907SHans Petter Selasky
3574118063fbSHans Petter Selasky err = mlx5_ib_init_congestion(dev);
3575118063fbSHans Petter Selasky if (err)
3576118063fbSHans Petter Selasky goto err_umrc;
3577118063fbSHans Petter Selasky
357812515907SHans Petter Selasky dev->ib_active = true;
357912515907SHans Petter Selasky
358012515907SHans Petter Selasky return dev;
358112515907SHans Petter Selasky
35828e6e287fSHans Petter Selasky err_umrc:
358312515907SHans Petter Selasky destroy_umrc_res(dev);
358412515907SHans Petter Selasky
358512515907SHans Petter Selasky err_dev:
358612515907SHans Petter Selasky ib_unregister_device(&dev->ib_dev);
358712515907SHans Petter Selasky
3588e47a6525SHans Petter Selasky err_bfreg:
3589e47a6525SHans Petter Selasky mlx5_ib_stage_bfreg_cleanup(dev);
3590e47a6525SHans Petter Selasky
359112515907SHans Petter Selasky err_q_cnt:
359212515907SHans Petter Selasky mlx5_ib_dealloc_q_counters(dev);
359312515907SHans Petter Selasky
359412515907SHans Petter Selasky err_odp:
35958e6e287fSHans Petter Selasky mlx5_ib_odp_remove_one(dev);
35968e6e287fSHans Petter Selasky
35978e6e287fSHans Petter Selasky err_rsrc:
359812515907SHans Petter Selasky destroy_dev_resources(&dev->devr);
359912515907SHans Petter Selasky
360012515907SHans Petter Selasky err_disable_roce:
36018e6e287fSHans Petter Selasky if (ll == IB_LINK_LAYER_ETHERNET) {
36028e6e287fSHans Petter Selasky mlx5_disable_roce(dev);
36038e6e287fSHans Petter Selasky mlx5_remove_roce_notifier(dev);
36048e6e287fSHans Petter Selasky }
36058e6e287fSHans Petter Selasky
360612515907SHans Petter Selasky err_free_port:
360712515907SHans Petter Selasky kfree(dev->port);
360812515907SHans Petter Selasky
360912515907SHans Petter Selasky err_dealloc:
361012515907SHans Petter Selasky ib_dealloc_device((struct ib_device *)dev);
361112515907SHans Petter Selasky
361212515907SHans Petter Selasky return NULL;
361312515907SHans Petter Selasky }
361412515907SHans Petter Selasky
mlx5_ib_remove(struct mlx5_core_dev * mdev,void * context)361512515907SHans Petter Selasky static void mlx5_ib_remove(struct mlx5_core_dev *mdev, void *context)
361612515907SHans Petter Selasky {
361712515907SHans Petter Selasky struct mlx5_ib_dev *dev = context;
36188e6e287fSHans Petter Selasky enum rdma_link_layer ll = mlx5_ib_port_link_layer(&dev->ib_dev, 1);
361912515907SHans Petter Selasky
3620118063fbSHans Petter Selasky mlx5_ib_cleanup_congestion(dev);
36218e6e287fSHans Petter Selasky mlx5_remove_roce_notifier(dev);
362212515907SHans Petter Selasky ib_unregister_device(&dev->ib_dev);
3623e47a6525SHans Petter Selasky mlx5_ib_stage_bfreg_cleanup(dev);
36248e6e287fSHans Petter Selasky mlx5_ib_dealloc_q_counters(dev);
362512515907SHans Petter Selasky destroy_umrc_res(dev);
36268e6e287fSHans Petter Selasky mlx5_ib_odp_remove_one(dev);
362712515907SHans Petter Selasky destroy_dev_resources(&dev->devr);
36288e6e287fSHans Petter Selasky if (ll == IB_LINK_LAYER_ETHERNET)
36298e6e287fSHans Petter Selasky mlx5_disable_roce(dev);
363012515907SHans Petter Selasky kfree(dev->port);
363112515907SHans Petter Selasky ib_dealloc_device(&dev->ib_dev);
363212515907SHans Petter Selasky }
363312515907SHans Petter Selasky
363412515907SHans Petter Selasky static struct mlx5_interface mlx5_ib_interface = {
363512515907SHans Petter Selasky .add = mlx5_ib_add,
363612515907SHans Petter Selasky .remove = mlx5_ib_remove,
363712515907SHans Petter Selasky .event = mlx5_ib_event,
363812515907SHans Petter Selasky .protocol = MLX5_INTERFACE_PROTOCOL_IB,
363912515907SHans Petter Selasky };
364012515907SHans Petter Selasky
mlx5_ib_init(void)364112515907SHans Petter Selasky static int __init mlx5_ib_init(void)
364212515907SHans Petter Selasky {
364312515907SHans Petter Selasky int err;
364412515907SHans Petter Selasky
36458e6e287fSHans Petter Selasky err = mlx5_ib_odp_init();
36468e6e287fSHans Petter Selasky if (err)
36478e6e287fSHans Petter Selasky return err;
364812515907SHans Petter Selasky
364912515907SHans Petter Selasky err = mlx5_register_interface(&mlx5_ib_interface);
365012515907SHans Petter Selasky if (err)
365112515907SHans Petter Selasky goto clean_odp;
365212515907SHans Petter Selasky
365312515907SHans Petter Selasky return err;
365412515907SHans Petter Selasky
365512515907SHans Petter Selasky clean_odp:
36568e6e287fSHans Petter Selasky mlx5_ib_odp_cleanup();
365712515907SHans Petter Selasky return err;
365812515907SHans Petter Selasky }
365912515907SHans Petter Selasky
mlx5_ib_cleanup(void)366012515907SHans Petter Selasky static void __exit mlx5_ib_cleanup(void)
366112515907SHans Petter Selasky {
366212515907SHans Petter Selasky mlx5_unregister_interface(&mlx5_ib_interface);
36638e6e287fSHans Petter Selasky mlx5_ib_odp_cleanup();
366412515907SHans Petter Selasky }
366512515907SHans Petter Selasky
36661866c98eSHans Petter Selasky module_init_order(mlx5_ib_init, SI_ORDER_SEVENTH);
36671866c98eSHans Petter Selasky module_exit_order(mlx5_ib_cleanup, SI_ORDER_SEVENTH);
3668