197549c34SHans Petter Selasky /* 297549c34SHans Petter Selasky * Copyright (c) 2006, 2007 Cisco Systems, Inc. All rights reserved. 397549c34SHans Petter Selasky * Copyright (c) 2005, 2014 Mellanox Technologies. All rights reserved. 497549c34SHans Petter Selasky * 597549c34SHans Petter Selasky * This software is available to you under a choice of one of two 697549c34SHans Petter Selasky * licenses. You may choose to be licensed under the terms of the GNU 797549c34SHans Petter Selasky * General Public License (GPL) Version 2, available from the file 897549c34SHans Petter Selasky * COPYING in the main directory of this source tree, or the 997549c34SHans Petter Selasky * OpenIB.org BSD license below: 1097549c34SHans Petter Selasky * 1197549c34SHans Petter Selasky * Redistribution and use in source and binary forms, with or 1297549c34SHans Petter Selasky * without modification, are permitted provided that the following 1397549c34SHans Petter Selasky * conditions are met: 1497549c34SHans Petter Selasky * 1597549c34SHans Petter Selasky * - Redistributions of source code must retain the above 1697549c34SHans Petter Selasky * copyright notice, this list of conditions and the following 1797549c34SHans Petter Selasky * disclaimer. 1897549c34SHans Petter Selasky * 1997549c34SHans Petter Selasky * - Redistributions in binary form must reproduce the above 2097549c34SHans Petter Selasky * copyright notice, this list of conditions and the following 2197549c34SHans Petter Selasky * disclaimer in the documentation and/or other materials 2297549c34SHans Petter Selasky * provided with the distribution. 2397549c34SHans Petter Selasky * 2497549c34SHans Petter Selasky * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, 2597549c34SHans Petter Selasky * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF 2697549c34SHans Petter Selasky * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND 2797549c34SHans Petter Selasky * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS 2897549c34SHans Petter Selasky * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN 2997549c34SHans Petter Selasky * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN 3097549c34SHans Petter Selasky * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 3197549c34SHans Petter Selasky * SOFTWARE. 3297549c34SHans Petter Selasky */ 3397549c34SHans Petter Selasky 3497549c34SHans Petter Selasky #include <linux/errno.h> 3597549c34SHans Petter Selasky #include <linux/module.h> 3697549c34SHans Petter Selasky #include <linux/io-mapping.h> 3797549c34SHans Petter Selasky 3897549c34SHans Petter Selasky #include <linux/page.h> 3997549c34SHans Petter Selasky 4097549c34SHans Petter Selasky #include "mlx4.h" 4197549c34SHans Petter Selasky #include "icm.h" 4297549c34SHans Petter Selasky 4397549c34SHans Petter Selasky enum { 4497549c34SHans Petter Selasky MLX4_NUM_RESERVED_UARS = 8 4597549c34SHans Petter Selasky }; 4697549c34SHans Petter Selasky 4797549c34SHans Petter Selasky int mlx4_pd_alloc(struct mlx4_dev *dev, u32 *pdn) 4897549c34SHans Petter Selasky { 4997549c34SHans Petter Selasky struct mlx4_priv *priv = mlx4_priv(dev); 5097549c34SHans Petter Selasky 5197549c34SHans Petter Selasky *pdn = mlx4_bitmap_alloc(&priv->pd_bitmap); 5297549c34SHans Petter Selasky if (*pdn == -1) 5397549c34SHans Petter Selasky return -ENOMEM; 5497549c34SHans Petter Selasky 5597549c34SHans Petter Selasky return 0; 5697549c34SHans Petter Selasky } 5797549c34SHans Petter Selasky EXPORT_SYMBOL_GPL(mlx4_pd_alloc); 5897549c34SHans Petter Selasky 5997549c34SHans Petter Selasky void mlx4_pd_free(struct mlx4_dev *dev, u32 pdn) 6097549c34SHans Petter Selasky { 6197549c34SHans Petter Selasky mlx4_bitmap_free(&mlx4_priv(dev)->pd_bitmap, pdn, MLX4_USE_RR); 6297549c34SHans Petter Selasky } 6397549c34SHans Petter Selasky EXPORT_SYMBOL_GPL(mlx4_pd_free); 6497549c34SHans Petter Selasky 6597549c34SHans Petter Selasky int __mlx4_xrcd_alloc(struct mlx4_dev *dev, u32 *xrcdn) 6697549c34SHans Petter Selasky { 6797549c34SHans Petter Selasky struct mlx4_priv *priv = mlx4_priv(dev); 6897549c34SHans Petter Selasky 6997549c34SHans Petter Selasky *xrcdn = mlx4_bitmap_alloc(&priv->xrcd_bitmap); 7097549c34SHans Petter Selasky if (*xrcdn == -1) 7197549c34SHans Petter Selasky return -ENOMEM; 7297549c34SHans Petter Selasky 7397549c34SHans Petter Selasky return 0; 7497549c34SHans Petter Selasky } 7597549c34SHans Petter Selasky 7697549c34SHans Petter Selasky int mlx4_xrcd_alloc(struct mlx4_dev *dev, u32 *xrcdn) 7797549c34SHans Petter Selasky { 7897549c34SHans Petter Selasky u64 out_param; 7997549c34SHans Petter Selasky int err; 8097549c34SHans Petter Selasky 8197549c34SHans Petter Selasky if (mlx4_is_mfunc(dev)) { 8297549c34SHans Petter Selasky err = mlx4_cmd_imm(dev, 0, &out_param, 8397549c34SHans Petter Selasky RES_XRCD, RES_OP_RESERVE, 8497549c34SHans Petter Selasky MLX4_CMD_ALLOC_RES, 8597549c34SHans Petter Selasky MLX4_CMD_TIME_CLASS_A, MLX4_CMD_WRAPPED); 8697549c34SHans Petter Selasky if (err) 8797549c34SHans Petter Selasky return err; 8897549c34SHans Petter Selasky 8997549c34SHans Petter Selasky *xrcdn = get_param_l(&out_param); 9097549c34SHans Petter Selasky return 0; 9197549c34SHans Petter Selasky } 9297549c34SHans Petter Selasky return __mlx4_xrcd_alloc(dev, xrcdn); 9397549c34SHans Petter Selasky } 9497549c34SHans Petter Selasky EXPORT_SYMBOL_GPL(mlx4_xrcd_alloc); 9597549c34SHans Petter Selasky 9697549c34SHans Petter Selasky void __mlx4_xrcd_free(struct mlx4_dev *dev, u32 xrcdn) 9797549c34SHans Petter Selasky { 9897549c34SHans Petter Selasky mlx4_bitmap_free(&mlx4_priv(dev)->xrcd_bitmap, xrcdn, MLX4_USE_RR); 9997549c34SHans Petter Selasky } 10097549c34SHans Petter Selasky 10197549c34SHans Petter Selasky void mlx4_xrcd_free(struct mlx4_dev *dev, u32 xrcdn) 10297549c34SHans Petter Selasky { 10397549c34SHans Petter Selasky u64 in_param = 0; 10497549c34SHans Petter Selasky int err; 10597549c34SHans Petter Selasky 10697549c34SHans Petter Selasky if (mlx4_is_mfunc(dev)) { 10797549c34SHans Petter Selasky set_param_l(&in_param, xrcdn); 10897549c34SHans Petter Selasky err = mlx4_cmd(dev, in_param, RES_XRCD, 10997549c34SHans Petter Selasky RES_OP_RESERVE, MLX4_CMD_FREE_RES, 11097549c34SHans Petter Selasky MLX4_CMD_TIME_CLASS_A, MLX4_CMD_WRAPPED); 11197549c34SHans Petter Selasky if (err) 11297549c34SHans Petter Selasky mlx4_warn(dev, "Failed to release xrcdn %d\n", xrcdn); 11397549c34SHans Petter Selasky } else 11497549c34SHans Petter Selasky __mlx4_xrcd_free(dev, xrcdn); 11597549c34SHans Petter Selasky } 11697549c34SHans Petter Selasky EXPORT_SYMBOL_GPL(mlx4_xrcd_free); 11797549c34SHans Petter Selasky 11897549c34SHans Petter Selasky int mlx4_init_pd_table(struct mlx4_dev *dev) 11997549c34SHans Petter Selasky { 12097549c34SHans Petter Selasky struct mlx4_priv *priv = mlx4_priv(dev); 12197549c34SHans Petter Selasky 12297549c34SHans Petter Selasky return mlx4_bitmap_init(&priv->pd_bitmap, dev->caps.num_pds, 12397549c34SHans Petter Selasky (1 << NOT_MASKED_PD_BITS) - 1, 12497549c34SHans Petter Selasky dev->caps.reserved_pds, 0); 12597549c34SHans Petter Selasky } 12697549c34SHans Petter Selasky 12797549c34SHans Petter Selasky void mlx4_cleanup_pd_table(struct mlx4_dev *dev) 12897549c34SHans Petter Selasky { 12997549c34SHans Petter Selasky mlx4_bitmap_cleanup(&mlx4_priv(dev)->pd_bitmap); 13097549c34SHans Petter Selasky } 13197549c34SHans Petter Selasky 13297549c34SHans Petter Selasky int mlx4_init_xrcd_table(struct mlx4_dev *dev) 13397549c34SHans Petter Selasky { 13497549c34SHans Petter Selasky struct mlx4_priv *priv = mlx4_priv(dev); 13597549c34SHans Petter Selasky 13697549c34SHans Petter Selasky return mlx4_bitmap_init(&priv->xrcd_bitmap, (1 << 16), 13797549c34SHans Petter Selasky (1 << 16) - 1, dev->caps.reserved_xrcds + 1, 0); 13897549c34SHans Petter Selasky } 13997549c34SHans Petter Selasky 14097549c34SHans Petter Selasky void mlx4_cleanup_xrcd_table(struct mlx4_dev *dev) 14197549c34SHans Petter Selasky { 14297549c34SHans Petter Selasky mlx4_bitmap_cleanup(&mlx4_priv(dev)->xrcd_bitmap); 14397549c34SHans Petter Selasky } 14497549c34SHans Petter Selasky 14597549c34SHans Petter Selasky int mlx4_uar_alloc(struct mlx4_dev *dev, struct mlx4_uar *uar) 14697549c34SHans Petter Selasky { 14797549c34SHans Petter Selasky int offset; 14897549c34SHans Petter Selasky 14997549c34SHans Petter Selasky uar->index = mlx4_bitmap_alloc(&mlx4_priv(dev)->uar_table.bitmap); 15097549c34SHans Petter Selasky if (uar->index == -1) 15197549c34SHans Petter Selasky return -ENOMEM; 15297549c34SHans Petter Selasky 15397549c34SHans Petter Selasky if (mlx4_is_slave(dev)) 154c3191c2eSHans Petter Selasky offset = uar->index % ((int)pci_resource_len(dev->persist->pdev, 155c3191c2eSHans Petter Selasky 2) / 15697549c34SHans Petter Selasky dev->caps.uar_page_size); 15797549c34SHans Petter Selasky else 15897549c34SHans Petter Selasky offset = uar->index; 159c3191c2eSHans Petter Selasky uar->pfn = (pci_resource_start(dev->persist->pdev, 2) >> PAGE_SHIFT) 160c3191c2eSHans Petter Selasky + offset; 16197549c34SHans Petter Selasky uar->map = NULL; 16297549c34SHans Petter Selasky return 0; 16397549c34SHans Petter Selasky } 16497549c34SHans Petter Selasky EXPORT_SYMBOL_GPL(mlx4_uar_alloc); 16597549c34SHans Petter Selasky 16697549c34SHans Petter Selasky void mlx4_uar_free(struct mlx4_dev *dev, struct mlx4_uar *uar) 16797549c34SHans Petter Selasky { 16897549c34SHans Petter Selasky mlx4_bitmap_free(&mlx4_priv(dev)->uar_table.bitmap, uar->index, MLX4_USE_RR); 16997549c34SHans Petter Selasky } 17097549c34SHans Petter Selasky EXPORT_SYMBOL_GPL(mlx4_uar_free); 17197549c34SHans Petter Selasky 17297549c34SHans Petter Selasky int mlx4_bf_alloc(struct mlx4_dev *dev, struct mlx4_bf *bf, int node) 17397549c34SHans Petter Selasky { 17497549c34SHans Petter Selasky struct mlx4_priv *priv = mlx4_priv(dev); 17597549c34SHans Petter Selasky struct mlx4_uar *uar; 17697549c34SHans Petter Selasky int err = 0; 17797549c34SHans Petter Selasky int idx; 17897549c34SHans Petter Selasky 17997549c34SHans Petter Selasky if (!priv->bf_mapping) 18097549c34SHans Petter Selasky return -ENOMEM; 18197549c34SHans Petter Selasky 18297549c34SHans Petter Selasky mutex_lock(&priv->bf_mutex); 18397549c34SHans Petter Selasky if (!list_empty(&priv->bf_list)) 18497549c34SHans Petter Selasky uar = list_entry(priv->bf_list.next, struct mlx4_uar, bf_list); 18597549c34SHans Petter Selasky else { 18697549c34SHans Petter Selasky if (mlx4_bitmap_avail(&priv->uar_table.bitmap) < MLX4_NUM_RESERVED_UARS) { 18797549c34SHans Petter Selasky err = -ENOMEM; 18897549c34SHans Petter Selasky goto out; 18997549c34SHans Petter Selasky } 190c3191c2eSHans Petter Selasky uar = kmalloc_node(sizeof(*uar), GFP_KERNEL, node); 19197549c34SHans Petter Selasky if (!uar) { 192c3191c2eSHans Petter Selasky uar = kmalloc(sizeof(*uar), GFP_KERNEL); 19397549c34SHans Petter Selasky if (!uar) { 19497549c34SHans Petter Selasky err = -ENOMEM; 19597549c34SHans Petter Selasky goto out; 19697549c34SHans Petter Selasky } 19797549c34SHans Petter Selasky } 19897549c34SHans Petter Selasky err = mlx4_uar_alloc(dev, uar); 19997549c34SHans Petter Selasky if (err) 20097549c34SHans Petter Selasky goto free_kmalloc; 20197549c34SHans Petter Selasky 20297549c34SHans Petter Selasky uar->map = ioremap(uar->pfn << PAGE_SHIFT, PAGE_SIZE); 20397549c34SHans Petter Selasky if (!uar->map) { 20497549c34SHans Petter Selasky err = -ENOMEM; 20597549c34SHans Petter Selasky goto free_uar; 20697549c34SHans Petter Selasky } 20797549c34SHans Petter Selasky 208c3191c2eSHans Petter Selasky uar->bf_map = io_mapping_map_wc(priv->bf_mapping, 209c3191c2eSHans Petter Selasky uar->index << PAGE_SHIFT, 210c3191c2eSHans Petter Selasky PAGE_SIZE); 21197549c34SHans Petter Selasky if (!uar->bf_map) { 21297549c34SHans Petter Selasky err = -ENOMEM; 21397549c34SHans Petter Selasky goto unamp_uar; 21497549c34SHans Petter Selasky } 21597549c34SHans Petter Selasky uar->free_bf_bmap = 0; 21697549c34SHans Petter Selasky list_add(&uar->bf_list, &priv->bf_list); 21797549c34SHans Petter Selasky } 21897549c34SHans Petter Selasky 21997549c34SHans Petter Selasky idx = ffz(uar->free_bf_bmap); 22097549c34SHans Petter Selasky uar->free_bf_bmap |= 1 << idx; 22197549c34SHans Petter Selasky bf->uar = uar; 22297549c34SHans Petter Selasky bf->offset = 0; 22397549c34SHans Petter Selasky bf->buf_size = dev->caps.bf_reg_size / 2; 22497549c34SHans Petter Selasky bf->reg = uar->bf_map + idx * dev->caps.bf_reg_size; 22597549c34SHans Petter Selasky if (uar->free_bf_bmap == (1 << dev->caps.bf_regs_per_page) - 1) 22697549c34SHans Petter Selasky list_del_init(&uar->bf_list); 22797549c34SHans Petter Selasky 22897549c34SHans Petter Selasky goto out; 22997549c34SHans Petter Selasky 23097549c34SHans Petter Selasky unamp_uar: 23197549c34SHans Petter Selasky bf->uar = NULL; 23297549c34SHans Petter Selasky iounmap(uar->map); 23397549c34SHans Petter Selasky 23497549c34SHans Petter Selasky free_uar: 23597549c34SHans Petter Selasky mlx4_uar_free(dev, uar); 23697549c34SHans Petter Selasky 23797549c34SHans Petter Selasky free_kmalloc: 23897549c34SHans Petter Selasky kfree(uar); 23997549c34SHans Petter Selasky 24097549c34SHans Petter Selasky out: 24197549c34SHans Petter Selasky mutex_unlock(&priv->bf_mutex); 24297549c34SHans Petter Selasky return err; 24397549c34SHans Petter Selasky } 24497549c34SHans Petter Selasky EXPORT_SYMBOL_GPL(mlx4_bf_alloc); 24597549c34SHans Petter Selasky 24697549c34SHans Petter Selasky void mlx4_bf_free(struct mlx4_dev *dev, struct mlx4_bf *bf) 24797549c34SHans Petter Selasky { 24897549c34SHans Petter Selasky struct mlx4_priv *priv = mlx4_priv(dev); 24997549c34SHans Petter Selasky int idx; 25097549c34SHans Petter Selasky 25197549c34SHans Petter Selasky if (!bf->uar || !bf->uar->bf_map) 25297549c34SHans Petter Selasky return; 25397549c34SHans Petter Selasky 25497549c34SHans Petter Selasky mutex_lock(&priv->bf_mutex); 25597549c34SHans Petter Selasky idx = (bf->reg - bf->uar->bf_map) / dev->caps.bf_reg_size; 25697549c34SHans Petter Selasky bf->uar->free_bf_bmap &= ~(1 << idx); 25797549c34SHans Petter Selasky if (!bf->uar->free_bf_bmap) { 25897549c34SHans Petter Selasky if (!list_empty(&bf->uar->bf_list)) 25997549c34SHans Petter Selasky list_del(&bf->uar->bf_list); 26097549c34SHans Petter Selasky 26197549c34SHans Petter Selasky io_mapping_unmap(bf->uar->bf_map); 26297549c34SHans Petter Selasky iounmap(bf->uar->map); 26397549c34SHans Petter Selasky mlx4_uar_free(dev, bf->uar); 26497549c34SHans Petter Selasky kfree(bf->uar); 26597549c34SHans Petter Selasky } else if (list_empty(&bf->uar->bf_list)) 26697549c34SHans Petter Selasky list_add(&bf->uar->bf_list, &priv->bf_list); 26797549c34SHans Petter Selasky 26897549c34SHans Petter Selasky mutex_unlock(&priv->bf_mutex); 26997549c34SHans Petter Selasky } 27097549c34SHans Petter Selasky EXPORT_SYMBOL_GPL(mlx4_bf_free); 27197549c34SHans Petter Selasky 27297549c34SHans Petter Selasky int mlx4_init_uar_table(struct mlx4_dev *dev) 27397549c34SHans Petter Selasky { 274c3191c2eSHans Petter Selasky int num_reserved_uar = mlx4_get_num_reserved_uar(dev); 275c3191c2eSHans Petter Selasky 276c3191c2eSHans Petter Selasky mlx4_dbg(dev, "uar_page_shift = %d", dev->uar_page_shift); 277c3191c2eSHans Petter Selasky mlx4_dbg(dev, "Effective reserved_uars=%d", dev->caps.reserved_uars); 278c3191c2eSHans Petter Selasky 279c3191c2eSHans Petter Selasky if (dev->caps.num_uars <= num_reserved_uar) { 280c3191c2eSHans Petter Selasky mlx4_err(dev, "Only %d UAR pages (need more than %d)\n", 281c3191c2eSHans Petter Selasky dev->caps.num_uars, num_reserved_uar); 28297549c34SHans Petter Selasky mlx4_err(dev, "Increase firmware log2_uar_bar_megabytes?\n"); 28397549c34SHans Petter Selasky return -ENODEV; 28497549c34SHans Petter Selasky } 28597549c34SHans Petter Selasky 28697549c34SHans Petter Selasky return mlx4_bitmap_init(&mlx4_priv(dev)->uar_table.bitmap, 28797549c34SHans Petter Selasky dev->caps.num_uars, dev->caps.num_uars - 1, 28897549c34SHans Petter Selasky dev->caps.reserved_uars, 0); 28997549c34SHans Petter Selasky } 29097549c34SHans Petter Selasky 29197549c34SHans Petter Selasky void mlx4_cleanup_uar_table(struct mlx4_dev *dev) 29297549c34SHans Petter Selasky { 29397549c34SHans Petter Selasky mlx4_bitmap_cleanup(&mlx4_priv(dev)->uar_table.bitmap); 29497549c34SHans Petter Selasky } 295