1cdcd52d4SBartosz Sobczak /*- 2cdcd52d4SBartosz Sobczak * SPDX-License-Identifier: GPL-2.0 or Linux-OpenIB 3cdcd52d4SBartosz Sobczak * 4cdcd52d4SBartosz Sobczak * Copyright (c) 2015 - 2021 Intel Corporation 5cdcd52d4SBartosz Sobczak * 6cdcd52d4SBartosz Sobczak * This software is available to you under a choice of one of two 7cdcd52d4SBartosz Sobczak * licenses. You may choose to be licensed under the terms of the GNU 8cdcd52d4SBartosz Sobczak * General Public License (GPL) Version 2, available from the file 9cdcd52d4SBartosz Sobczak * COPYING in the main directory of this source tree, or the 10cdcd52d4SBartosz Sobczak * OpenFabrics.org BSD license below: 11cdcd52d4SBartosz Sobczak * 12cdcd52d4SBartosz Sobczak * Redistribution and use in source and binary forms, with or 13cdcd52d4SBartosz Sobczak * without modification, are permitted provided that the following 14cdcd52d4SBartosz Sobczak * conditions are met: 15cdcd52d4SBartosz Sobczak * 16cdcd52d4SBartosz Sobczak * - Redistributions of source code must retain the above 17cdcd52d4SBartosz Sobczak * copyright notice, this list of conditions and the following 18cdcd52d4SBartosz Sobczak * disclaimer. 19cdcd52d4SBartosz Sobczak * 20cdcd52d4SBartosz Sobczak * - Redistributions in binary form must reproduce the above 21cdcd52d4SBartosz Sobczak * copyright notice, this list of conditions and the following 22cdcd52d4SBartosz Sobczak * disclaimer in the documentation and/or other materials 23cdcd52d4SBartosz Sobczak * provided with the distribution. 24cdcd52d4SBartosz Sobczak * 25cdcd52d4SBartosz Sobczak * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, 26cdcd52d4SBartosz Sobczak * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF 27cdcd52d4SBartosz Sobczak * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND 28cdcd52d4SBartosz Sobczak * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS 29cdcd52d4SBartosz Sobczak * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN 30cdcd52d4SBartosz Sobczak * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN 31cdcd52d4SBartosz Sobczak * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 32cdcd52d4SBartosz Sobczak * SOFTWARE. 33cdcd52d4SBartosz Sobczak */ 34cdcd52d4SBartosz Sobczak /*$FreeBSD$*/ 35cdcd52d4SBartosz Sobczak 36cdcd52d4SBartosz Sobczak #include "osdep.h" 37cdcd52d4SBartosz Sobczak #include "irdma_hmc.h" 38cdcd52d4SBartosz Sobczak #include "irdma_defs.h" 39cdcd52d4SBartosz Sobczak #include "irdma_type.h" 40cdcd52d4SBartosz Sobczak #include "irdma_protos.h" 41cdcd52d4SBartosz Sobczak #include "irdma_pble.h" 42cdcd52d4SBartosz Sobczak 43cdcd52d4SBartosz Sobczak static int add_pble_prm(struct irdma_hmc_pble_rsrc *pble_rsrc); 44cdcd52d4SBartosz Sobczak 45cdcd52d4SBartosz Sobczak /** 46cdcd52d4SBartosz Sobczak * irdma_destroy_pble_prm - destroy prm during module unload 47cdcd52d4SBartosz Sobczak * @pble_rsrc: pble resources 48cdcd52d4SBartosz Sobczak */ 49cdcd52d4SBartosz Sobczak void 50cdcd52d4SBartosz Sobczak irdma_destroy_pble_prm(struct irdma_hmc_pble_rsrc *pble_rsrc) 51cdcd52d4SBartosz Sobczak { 52cdcd52d4SBartosz Sobczak struct irdma_chunk *chunk; 53cdcd52d4SBartosz Sobczak struct irdma_pble_prm *pinfo = &pble_rsrc->pinfo; 54cdcd52d4SBartosz Sobczak 55cdcd52d4SBartosz Sobczak while (!list_empty(&pinfo->clist)) { 56cdcd52d4SBartosz Sobczak chunk = (struct irdma_chunk *)(&pinfo->clist)->next; 57cdcd52d4SBartosz Sobczak list_del(&chunk->list); 58cdcd52d4SBartosz Sobczak if (chunk->type == PBLE_SD_PAGED) 59cdcd52d4SBartosz Sobczak irdma_pble_free_paged_mem(chunk); 60cdcd52d4SBartosz Sobczak if (chunk->bitmapbuf) 61cdcd52d4SBartosz Sobczak irdma_prm_rem_bitmapmem(pble_rsrc->dev->hw, chunk); 62cdcd52d4SBartosz Sobczak kfree(chunk->chunkmem.va); 63cdcd52d4SBartosz Sobczak } 64cdcd52d4SBartosz Sobczak spin_lock_destroy(&pinfo->prm_lock); 65cdcd52d4SBartosz Sobczak mutex_destroy(&pble_rsrc->pble_mutex_lock); 66cdcd52d4SBartosz Sobczak } 67cdcd52d4SBartosz Sobczak 68cdcd52d4SBartosz Sobczak /** 69cdcd52d4SBartosz Sobczak * irdma_hmc_init_pble - Initialize pble resources during module load 70cdcd52d4SBartosz Sobczak * @dev: irdma_sc_dev struct 71cdcd52d4SBartosz Sobczak * @pble_rsrc: pble resources 72cdcd52d4SBartosz Sobczak */ 73cdcd52d4SBartosz Sobczak int 74cdcd52d4SBartosz Sobczak irdma_hmc_init_pble(struct irdma_sc_dev *dev, 75cdcd52d4SBartosz Sobczak struct irdma_hmc_pble_rsrc *pble_rsrc) 76cdcd52d4SBartosz Sobczak { 77cdcd52d4SBartosz Sobczak struct irdma_hmc_info *hmc_info; 78cdcd52d4SBartosz Sobczak u32 fpm_idx = 0; 79cdcd52d4SBartosz Sobczak int status = 0; 80cdcd52d4SBartosz Sobczak 81cdcd52d4SBartosz Sobczak hmc_info = dev->hmc_info; 82cdcd52d4SBartosz Sobczak pble_rsrc->dev = dev; 83cdcd52d4SBartosz Sobczak pble_rsrc->fpm_base_addr = hmc_info->hmc_obj[IRDMA_HMC_IW_PBLE].base; 84cdcd52d4SBartosz Sobczak /* Start pble' on 4k boundary */ 85cdcd52d4SBartosz Sobczak if (pble_rsrc->fpm_base_addr & 0xfff) 86cdcd52d4SBartosz Sobczak fpm_idx = (4096 - (pble_rsrc->fpm_base_addr & 0xfff)) >> 3; 87cdcd52d4SBartosz Sobczak pble_rsrc->unallocated_pble = 88cdcd52d4SBartosz Sobczak hmc_info->hmc_obj[IRDMA_HMC_IW_PBLE].cnt - fpm_idx; 89cdcd52d4SBartosz Sobczak pble_rsrc->next_fpm_addr = pble_rsrc->fpm_base_addr + (fpm_idx << 3); 90cdcd52d4SBartosz Sobczak pble_rsrc->pinfo.pble_shift = PBLE_SHIFT; 91cdcd52d4SBartosz Sobczak 92cdcd52d4SBartosz Sobczak mutex_init(&pble_rsrc->pble_mutex_lock); 93cdcd52d4SBartosz Sobczak 94cdcd52d4SBartosz Sobczak spin_lock_init(&pble_rsrc->pinfo.prm_lock); 95cdcd52d4SBartosz Sobczak INIT_LIST_HEAD(&pble_rsrc->pinfo.clist); 96cdcd52d4SBartosz Sobczak if (add_pble_prm(pble_rsrc)) { 97cdcd52d4SBartosz Sobczak irdma_destroy_pble_prm(pble_rsrc); 98cdcd52d4SBartosz Sobczak status = -ENOMEM; 99cdcd52d4SBartosz Sobczak } 100cdcd52d4SBartosz Sobczak 101cdcd52d4SBartosz Sobczak return status; 102cdcd52d4SBartosz Sobczak } 103cdcd52d4SBartosz Sobczak 104cdcd52d4SBartosz Sobczak /** 105cdcd52d4SBartosz Sobczak * get_sd_pd_idx - Returns sd index, pd index and rel_pd_idx from fpm address 106cdcd52d4SBartosz Sobczak * @pble_rsrc: structure containing fpm address 107cdcd52d4SBartosz Sobczak * @idx: where to return indexes 108cdcd52d4SBartosz Sobczak */ 109cdcd52d4SBartosz Sobczak static void 110cdcd52d4SBartosz Sobczak get_sd_pd_idx(struct irdma_hmc_pble_rsrc *pble_rsrc, 111cdcd52d4SBartosz Sobczak struct sd_pd_idx *idx) 112cdcd52d4SBartosz Sobczak { 113cdcd52d4SBartosz Sobczak idx->sd_idx = (u32)pble_rsrc->next_fpm_addr / IRDMA_HMC_DIRECT_BP_SIZE; 114cdcd52d4SBartosz Sobczak idx->pd_idx = (u32)(pble_rsrc->next_fpm_addr / IRDMA_HMC_PAGED_BP_SIZE); 115cdcd52d4SBartosz Sobczak idx->rel_pd_idx = (idx->pd_idx % IRDMA_HMC_PD_CNT_IN_SD); 116cdcd52d4SBartosz Sobczak } 117cdcd52d4SBartosz Sobczak 118cdcd52d4SBartosz Sobczak /** 119cdcd52d4SBartosz Sobczak * add_sd_direct - add sd direct for pble 120cdcd52d4SBartosz Sobczak * @pble_rsrc: pble resource ptr 121cdcd52d4SBartosz Sobczak * @info: page info for sd 122cdcd52d4SBartosz Sobczak */ 123cdcd52d4SBartosz Sobczak static int 124cdcd52d4SBartosz Sobczak add_sd_direct(struct irdma_hmc_pble_rsrc *pble_rsrc, 125cdcd52d4SBartosz Sobczak struct irdma_add_page_info *info) 126cdcd52d4SBartosz Sobczak { 127cdcd52d4SBartosz Sobczak struct irdma_sc_dev *dev = pble_rsrc->dev; 128cdcd52d4SBartosz Sobczak int ret_code = 0; 129cdcd52d4SBartosz Sobczak struct sd_pd_idx *idx = &info->idx; 130cdcd52d4SBartosz Sobczak struct irdma_chunk *chunk = info->chunk; 131cdcd52d4SBartosz Sobczak struct irdma_hmc_info *hmc_info = info->hmc_info; 132cdcd52d4SBartosz Sobczak struct irdma_hmc_sd_entry *sd_entry = info->sd_entry; 133cdcd52d4SBartosz Sobczak u32 offset = 0; 134cdcd52d4SBartosz Sobczak 135cdcd52d4SBartosz Sobczak if (!sd_entry->valid) { 136cdcd52d4SBartosz Sobczak ret_code = irdma_add_sd_table_entry(dev->hw, hmc_info, 137cdcd52d4SBartosz Sobczak info->idx.sd_idx, 138cdcd52d4SBartosz Sobczak IRDMA_SD_TYPE_DIRECT, 139cdcd52d4SBartosz Sobczak IRDMA_HMC_DIRECT_BP_SIZE); 140cdcd52d4SBartosz Sobczak if (ret_code) 141cdcd52d4SBartosz Sobczak return ret_code; 142cdcd52d4SBartosz Sobczak 143cdcd52d4SBartosz Sobczak chunk->type = PBLE_SD_CONTIGOUS; 144cdcd52d4SBartosz Sobczak } 145cdcd52d4SBartosz Sobczak 146cdcd52d4SBartosz Sobczak offset = idx->rel_pd_idx << HMC_PAGED_BP_SHIFT; 147cdcd52d4SBartosz Sobczak chunk->size = info->pages << HMC_PAGED_BP_SHIFT; 148cdcd52d4SBartosz Sobczak chunk->vaddr = (u8 *)sd_entry->u.bp.addr.va + offset; 149cdcd52d4SBartosz Sobczak chunk->fpm_addr = pble_rsrc->next_fpm_addr; 150cdcd52d4SBartosz Sobczak irdma_debug(dev, IRDMA_DEBUG_PBLE, 151cdcd52d4SBartosz Sobczak "chunk_size[%ld] = 0x%lx vaddr=0x%p fpm_addr = %lx\n", 152cdcd52d4SBartosz Sobczak chunk->size, chunk->size, chunk->vaddr, chunk->fpm_addr); 153cdcd52d4SBartosz Sobczak 154cdcd52d4SBartosz Sobczak return 0; 155cdcd52d4SBartosz Sobczak } 156cdcd52d4SBartosz Sobczak 157cdcd52d4SBartosz Sobczak /** 158cdcd52d4SBartosz Sobczak * fpm_to_idx - given fpm address, get pble index 159cdcd52d4SBartosz Sobczak * @pble_rsrc: pble resource management 160cdcd52d4SBartosz Sobczak * @addr: fpm address for index 161cdcd52d4SBartosz Sobczak */ 162cdcd52d4SBartosz Sobczak static u32 fpm_to_idx(struct irdma_hmc_pble_rsrc *pble_rsrc, u64 addr){ 163cdcd52d4SBartosz Sobczak u64 idx; 164cdcd52d4SBartosz Sobczak 165cdcd52d4SBartosz Sobczak idx = (addr - (pble_rsrc->fpm_base_addr)) >> 3; 166cdcd52d4SBartosz Sobczak 167cdcd52d4SBartosz Sobczak return (u32)idx; 168cdcd52d4SBartosz Sobczak } 169cdcd52d4SBartosz Sobczak 170cdcd52d4SBartosz Sobczak /** 171cdcd52d4SBartosz Sobczak * add_bp_pages - add backing pages for sd 172cdcd52d4SBartosz Sobczak * @pble_rsrc: pble resource management 173cdcd52d4SBartosz Sobczak * @info: page info for sd 174cdcd52d4SBartosz Sobczak */ 175cdcd52d4SBartosz Sobczak static int 176cdcd52d4SBartosz Sobczak add_bp_pages(struct irdma_hmc_pble_rsrc *pble_rsrc, 177cdcd52d4SBartosz Sobczak struct irdma_add_page_info *info) 178cdcd52d4SBartosz Sobczak { 179cdcd52d4SBartosz Sobczak struct irdma_sc_dev *dev = pble_rsrc->dev; 180cdcd52d4SBartosz Sobczak u8 *addr; 181cdcd52d4SBartosz Sobczak struct irdma_dma_mem mem; 182cdcd52d4SBartosz Sobczak struct irdma_hmc_pd_entry *pd_entry; 183cdcd52d4SBartosz Sobczak struct irdma_hmc_sd_entry *sd_entry = info->sd_entry; 184cdcd52d4SBartosz Sobczak struct irdma_hmc_info *hmc_info = info->hmc_info; 185cdcd52d4SBartosz Sobczak struct irdma_chunk *chunk = info->chunk; 186cdcd52d4SBartosz Sobczak int status = 0; 187cdcd52d4SBartosz Sobczak u32 rel_pd_idx = info->idx.rel_pd_idx; 188cdcd52d4SBartosz Sobczak u32 pd_idx = info->idx.pd_idx; 189cdcd52d4SBartosz Sobczak u32 i; 190cdcd52d4SBartosz Sobczak 191cdcd52d4SBartosz Sobczak if (irdma_pble_get_paged_mem(chunk, info->pages)) 192cdcd52d4SBartosz Sobczak return -ENOMEM; 193cdcd52d4SBartosz Sobczak 194cdcd52d4SBartosz Sobczak status = irdma_add_sd_table_entry(dev->hw, hmc_info, info->idx.sd_idx, 195cdcd52d4SBartosz Sobczak IRDMA_SD_TYPE_PAGED, 196cdcd52d4SBartosz Sobczak IRDMA_HMC_DIRECT_BP_SIZE); 197cdcd52d4SBartosz Sobczak if (status) 198cdcd52d4SBartosz Sobczak goto error; 199cdcd52d4SBartosz Sobczak 200cdcd52d4SBartosz Sobczak addr = chunk->vaddr; 201cdcd52d4SBartosz Sobczak for (i = 0; i < info->pages; i++) { 202cdcd52d4SBartosz Sobczak mem.pa = (u64)chunk->dmainfo.dmaaddrs[i]; 203cdcd52d4SBartosz Sobczak mem.size = 4096; 204cdcd52d4SBartosz Sobczak mem.va = addr; 205cdcd52d4SBartosz Sobczak pd_entry = &sd_entry->u.pd_table.pd_entry[rel_pd_idx++]; 206cdcd52d4SBartosz Sobczak if (!pd_entry->valid) { 207cdcd52d4SBartosz Sobczak status = irdma_add_pd_table_entry(dev, hmc_info, 208cdcd52d4SBartosz Sobczak pd_idx++, &mem); 209cdcd52d4SBartosz Sobczak if (status) 210cdcd52d4SBartosz Sobczak goto error; 211cdcd52d4SBartosz Sobczak 212cdcd52d4SBartosz Sobczak addr += 4096; 213cdcd52d4SBartosz Sobczak } 214cdcd52d4SBartosz Sobczak } 215cdcd52d4SBartosz Sobczak 216cdcd52d4SBartosz Sobczak chunk->fpm_addr = pble_rsrc->next_fpm_addr; 217cdcd52d4SBartosz Sobczak return 0; 218cdcd52d4SBartosz Sobczak 219cdcd52d4SBartosz Sobczak error: 220cdcd52d4SBartosz Sobczak irdma_pble_free_paged_mem(chunk); 221cdcd52d4SBartosz Sobczak 222cdcd52d4SBartosz Sobczak return status; 223cdcd52d4SBartosz Sobczak } 224cdcd52d4SBartosz Sobczak 225cdcd52d4SBartosz Sobczak /** 226cdcd52d4SBartosz Sobczak * irdma_get_type - add a sd entry type for sd 227cdcd52d4SBartosz Sobczak * @dev: irdma_sc_dev struct 228cdcd52d4SBartosz Sobczak * @idx: index of sd 229cdcd52d4SBartosz Sobczak * @pages: pages in the sd 230cdcd52d4SBartosz Sobczak */ 231cdcd52d4SBartosz Sobczak static enum irdma_sd_entry_type 232cdcd52d4SBartosz Sobczak irdma_get_type(struct irdma_sc_dev *dev, 233cdcd52d4SBartosz Sobczak struct sd_pd_idx *idx, u32 pages) 234cdcd52d4SBartosz Sobczak { 235cdcd52d4SBartosz Sobczak enum irdma_sd_entry_type sd_entry_type; 236cdcd52d4SBartosz Sobczak 237cdcd52d4SBartosz Sobczak sd_entry_type = !idx->rel_pd_idx && pages == IRDMA_HMC_PD_CNT_IN_SD ? 238cdcd52d4SBartosz Sobczak IRDMA_SD_TYPE_DIRECT : IRDMA_SD_TYPE_PAGED; 239cdcd52d4SBartosz Sobczak return sd_entry_type; 240cdcd52d4SBartosz Sobczak } 241cdcd52d4SBartosz Sobczak 242cdcd52d4SBartosz Sobczak /** 243cdcd52d4SBartosz Sobczak * add_pble_prm - add a sd entry for pble resoure 244cdcd52d4SBartosz Sobczak * @pble_rsrc: pble resource management 245cdcd52d4SBartosz Sobczak */ 246cdcd52d4SBartosz Sobczak static int 247cdcd52d4SBartosz Sobczak add_pble_prm(struct irdma_hmc_pble_rsrc *pble_rsrc) 248cdcd52d4SBartosz Sobczak { 249cdcd52d4SBartosz Sobczak struct irdma_sc_dev *dev = pble_rsrc->dev; 250cdcd52d4SBartosz Sobczak struct irdma_hmc_sd_entry *sd_entry; 251cdcd52d4SBartosz Sobczak struct irdma_hmc_info *hmc_info; 252cdcd52d4SBartosz Sobczak struct irdma_chunk *chunk; 253cdcd52d4SBartosz Sobczak struct irdma_add_page_info info; 254cdcd52d4SBartosz Sobczak struct sd_pd_idx *idx = &info.idx; 255cdcd52d4SBartosz Sobczak int ret_code = 0; 256cdcd52d4SBartosz Sobczak enum irdma_sd_entry_type sd_entry_type; 257cdcd52d4SBartosz Sobczak u64 sd_reg_val = 0; 258cdcd52d4SBartosz Sobczak struct irdma_virt_mem chunkmem; 259cdcd52d4SBartosz Sobczak u32 pages; 260cdcd52d4SBartosz Sobczak 261cdcd52d4SBartosz Sobczak if (pble_rsrc->unallocated_pble < PBLE_PER_PAGE) 262cdcd52d4SBartosz Sobczak return -ENOMEM; 263cdcd52d4SBartosz Sobczak 264cdcd52d4SBartosz Sobczak if (pble_rsrc->next_fpm_addr & 0xfff) 265cdcd52d4SBartosz Sobczak return -EINVAL; 266cdcd52d4SBartosz Sobczak 267cdcd52d4SBartosz Sobczak chunkmem.size = sizeof(*chunk); 268777e472cSBartosz Sobczak chunkmem.va = kzalloc(chunkmem.size, GFP_KERNEL); 269cdcd52d4SBartosz Sobczak if (!chunkmem.va) 270cdcd52d4SBartosz Sobczak return -ENOMEM; 271cdcd52d4SBartosz Sobczak 272cdcd52d4SBartosz Sobczak chunk = chunkmem.va; 273cdcd52d4SBartosz Sobczak chunk->chunkmem = chunkmem; 274cdcd52d4SBartosz Sobczak hmc_info = dev->hmc_info; 275cdcd52d4SBartosz Sobczak chunk->dev = dev; 276cdcd52d4SBartosz Sobczak chunk->fpm_addr = pble_rsrc->next_fpm_addr; 277cdcd52d4SBartosz Sobczak get_sd_pd_idx(pble_rsrc, idx); 278cdcd52d4SBartosz Sobczak sd_entry = &hmc_info->sd_table.sd_entry[idx->sd_idx]; 279cdcd52d4SBartosz Sobczak pages = (idx->rel_pd_idx) ? (IRDMA_HMC_PD_CNT_IN_SD - idx->rel_pd_idx) : 280cdcd52d4SBartosz Sobczak IRDMA_HMC_PD_CNT_IN_SD; 281cdcd52d4SBartosz Sobczak pages = min(pages, pble_rsrc->unallocated_pble >> PBLE_512_SHIFT); 282cdcd52d4SBartosz Sobczak info.chunk = chunk; 283cdcd52d4SBartosz Sobczak info.hmc_info = hmc_info; 284cdcd52d4SBartosz Sobczak info.pages = pages; 285cdcd52d4SBartosz Sobczak info.sd_entry = sd_entry; 286cdcd52d4SBartosz Sobczak if (!sd_entry->valid) 287cdcd52d4SBartosz Sobczak sd_entry_type = irdma_get_type(dev, idx, pages); 288cdcd52d4SBartosz Sobczak else 289cdcd52d4SBartosz Sobczak sd_entry_type = sd_entry->entry_type; 290cdcd52d4SBartosz Sobczak 291cdcd52d4SBartosz Sobczak irdma_debug(dev, IRDMA_DEBUG_PBLE, 292cdcd52d4SBartosz Sobczak "pages = %d, unallocated_pble[%d] current_fpm_addr = %lx\n", 293cdcd52d4SBartosz Sobczak pages, pble_rsrc->unallocated_pble, pble_rsrc->next_fpm_addr); 294cdcd52d4SBartosz Sobczak irdma_debug(dev, IRDMA_DEBUG_PBLE, "sd_entry_type = %d\n", 295cdcd52d4SBartosz Sobczak sd_entry_type); 296cdcd52d4SBartosz Sobczak if (sd_entry_type == IRDMA_SD_TYPE_DIRECT) 297cdcd52d4SBartosz Sobczak ret_code = add_sd_direct(pble_rsrc, &info); 298cdcd52d4SBartosz Sobczak 299cdcd52d4SBartosz Sobczak if (ret_code) 300cdcd52d4SBartosz Sobczak sd_entry_type = IRDMA_SD_TYPE_PAGED; 301cdcd52d4SBartosz Sobczak else 302cdcd52d4SBartosz Sobczak pble_rsrc->stats_direct_sds++; 303cdcd52d4SBartosz Sobczak 304cdcd52d4SBartosz Sobczak if (sd_entry_type == IRDMA_SD_TYPE_PAGED) { 305cdcd52d4SBartosz Sobczak ret_code = add_bp_pages(pble_rsrc, &info); 306cdcd52d4SBartosz Sobczak if (ret_code) 307cdcd52d4SBartosz Sobczak goto error; 308cdcd52d4SBartosz Sobczak else 309cdcd52d4SBartosz Sobczak pble_rsrc->stats_paged_sds++; 310cdcd52d4SBartosz Sobczak } 311cdcd52d4SBartosz Sobczak 312cdcd52d4SBartosz Sobczak ret_code = irdma_prm_add_pble_mem(&pble_rsrc->pinfo, chunk); 313cdcd52d4SBartosz Sobczak if (ret_code) 314cdcd52d4SBartosz Sobczak goto error; 315cdcd52d4SBartosz Sobczak 316cdcd52d4SBartosz Sobczak pble_rsrc->next_fpm_addr += chunk->size; 317cdcd52d4SBartosz Sobczak irdma_debug(dev, IRDMA_DEBUG_PBLE, 318cdcd52d4SBartosz Sobczak "next_fpm_addr = %lx chunk_size[%lu] = 0x%lx\n", 319cdcd52d4SBartosz Sobczak pble_rsrc->next_fpm_addr, chunk->size, chunk->size); 320cdcd52d4SBartosz Sobczak pble_rsrc->unallocated_pble -= (u32)(chunk->size >> 3); 321cdcd52d4SBartosz Sobczak sd_reg_val = (sd_entry_type == IRDMA_SD_TYPE_PAGED) ? 322cdcd52d4SBartosz Sobczak sd_entry->u.pd_table.pd_page_addr.pa : 323cdcd52d4SBartosz Sobczak sd_entry->u.bp.addr.pa; 324cdcd52d4SBartosz Sobczak if (!sd_entry->valid) { 325cdcd52d4SBartosz Sobczak ret_code = irdma_hmc_sd_one(dev, hmc_info->hmc_fn_id, sd_reg_val, 326cdcd52d4SBartosz Sobczak idx->sd_idx, sd_entry->entry_type, true); 327cdcd52d4SBartosz Sobczak if (ret_code) 328cdcd52d4SBartosz Sobczak goto error; 329cdcd52d4SBartosz Sobczak } 330cdcd52d4SBartosz Sobczak 331cdcd52d4SBartosz Sobczak sd_entry->valid = true; 332cdcd52d4SBartosz Sobczak list_add(&chunk->list, &pble_rsrc->pinfo.clist); 333cdcd52d4SBartosz Sobczak return 0; 334cdcd52d4SBartosz Sobczak 335cdcd52d4SBartosz Sobczak error: 336cdcd52d4SBartosz Sobczak if (chunk->bitmapbuf) 337cdcd52d4SBartosz Sobczak irdma_prm_rem_bitmapmem(pble_rsrc->dev->hw, chunk); 338cdcd52d4SBartosz Sobczak kfree(chunk->chunkmem.va); 339cdcd52d4SBartosz Sobczak 340cdcd52d4SBartosz Sobczak return ret_code; 341cdcd52d4SBartosz Sobczak } 342cdcd52d4SBartosz Sobczak 343cdcd52d4SBartosz Sobczak /** 344cdcd52d4SBartosz Sobczak * free_lvl2 - fee level 2 pble 345cdcd52d4SBartosz Sobczak * @pble_rsrc: pble resource management 346cdcd52d4SBartosz Sobczak * @palloc: level 2 pble allocation 347cdcd52d4SBartosz Sobczak */ 348cdcd52d4SBartosz Sobczak static void 349cdcd52d4SBartosz Sobczak free_lvl2(struct irdma_hmc_pble_rsrc *pble_rsrc, 350cdcd52d4SBartosz Sobczak struct irdma_pble_alloc *palloc) 351cdcd52d4SBartosz Sobczak { 352cdcd52d4SBartosz Sobczak u32 i; 353cdcd52d4SBartosz Sobczak struct irdma_pble_level2 *lvl2 = &palloc->level2; 354cdcd52d4SBartosz Sobczak struct irdma_pble_info *root = &lvl2->root; 355cdcd52d4SBartosz Sobczak struct irdma_pble_info *leaf = lvl2->leaf; 356cdcd52d4SBartosz Sobczak 357cdcd52d4SBartosz Sobczak for (i = 0; i < lvl2->leaf_cnt; i++, leaf++) { 358cdcd52d4SBartosz Sobczak if (leaf->addr) 359cdcd52d4SBartosz Sobczak irdma_prm_return_pbles(&pble_rsrc->pinfo, 360cdcd52d4SBartosz Sobczak &leaf->chunkinfo); 361cdcd52d4SBartosz Sobczak else 362cdcd52d4SBartosz Sobczak break; 363cdcd52d4SBartosz Sobczak } 364cdcd52d4SBartosz Sobczak 365cdcd52d4SBartosz Sobczak if (root->addr) 366cdcd52d4SBartosz Sobczak irdma_prm_return_pbles(&pble_rsrc->pinfo, &root->chunkinfo); 367cdcd52d4SBartosz Sobczak 368cdcd52d4SBartosz Sobczak kfree(lvl2->leafmem.va); 369cdcd52d4SBartosz Sobczak lvl2->leaf = NULL; 370cdcd52d4SBartosz Sobczak } 371cdcd52d4SBartosz Sobczak 372cdcd52d4SBartosz Sobczak /** 373cdcd52d4SBartosz Sobczak * get_lvl2_pble - get level 2 pble resource 374cdcd52d4SBartosz Sobczak * @pble_rsrc: pble resource management 375cdcd52d4SBartosz Sobczak * @palloc: level 2 pble allocation 376cdcd52d4SBartosz Sobczak */ 377cdcd52d4SBartosz Sobczak static int 378cdcd52d4SBartosz Sobczak get_lvl2_pble(struct irdma_hmc_pble_rsrc *pble_rsrc, 379cdcd52d4SBartosz Sobczak struct irdma_pble_alloc *palloc) 380cdcd52d4SBartosz Sobczak { 381cdcd52d4SBartosz Sobczak u32 lf4k, lflast, total, i; 382cdcd52d4SBartosz Sobczak u32 pblcnt = PBLE_PER_PAGE; 383cdcd52d4SBartosz Sobczak u64 *addr; 384cdcd52d4SBartosz Sobczak struct irdma_pble_level2 *lvl2 = &palloc->level2; 385cdcd52d4SBartosz Sobczak struct irdma_pble_info *root = &lvl2->root; 386cdcd52d4SBartosz Sobczak struct irdma_pble_info *leaf; 387cdcd52d4SBartosz Sobczak int ret_code; 388cdcd52d4SBartosz Sobczak u64 fpm_addr; 389cdcd52d4SBartosz Sobczak 390cdcd52d4SBartosz Sobczak /* number of full 512 (4K) leafs) */ 391cdcd52d4SBartosz Sobczak lf4k = palloc->total_cnt >> 9; 392cdcd52d4SBartosz Sobczak lflast = palloc->total_cnt % PBLE_PER_PAGE; 393cdcd52d4SBartosz Sobczak total = (lflast == 0) ? lf4k : lf4k + 1; 394cdcd52d4SBartosz Sobczak lvl2->leaf_cnt = total; 395cdcd52d4SBartosz Sobczak 396cdcd52d4SBartosz Sobczak lvl2->leafmem.size = (sizeof(*leaf) * total); 397777e472cSBartosz Sobczak lvl2->leafmem.va = kzalloc(lvl2->leafmem.size, GFP_KERNEL); 398cdcd52d4SBartosz Sobczak if (!lvl2->leafmem.va) 399cdcd52d4SBartosz Sobczak return -ENOMEM; 400cdcd52d4SBartosz Sobczak 401cdcd52d4SBartosz Sobczak lvl2->leaf = lvl2->leafmem.va; 402cdcd52d4SBartosz Sobczak leaf = lvl2->leaf; 403cdcd52d4SBartosz Sobczak ret_code = irdma_prm_get_pbles(&pble_rsrc->pinfo, &root->chunkinfo, 404cdcd52d4SBartosz Sobczak total << 3, &root->addr, &fpm_addr); 405cdcd52d4SBartosz Sobczak if (ret_code) { 406cdcd52d4SBartosz Sobczak kfree(lvl2->leafmem.va); 407cdcd52d4SBartosz Sobczak lvl2->leaf = NULL; 408cdcd52d4SBartosz Sobczak return -ENOMEM; 409cdcd52d4SBartosz Sobczak } 410cdcd52d4SBartosz Sobczak 411cdcd52d4SBartosz Sobczak root->idx = fpm_to_idx(pble_rsrc, fpm_addr); 412cdcd52d4SBartosz Sobczak root->cnt = total; 413cdcd52d4SBartosz Sobczak addr = root->addr; 414cdcd52d4SBartosz Sobczak for (i = 0; i < total; i++, leaf++) { 415cdcd52d4SBartosz Sobczak pblcnt = (lflast && ((i + 1) == total)) ? 416cdcd52d4SBartosz Sobczak lflast : PBLE_PER_PAGE; 417cdcd52d4SBartosz Sobczak ret_code = irdma_prm_get_pbles(&pble_rsrc->pinfo, 418cdcd52d4SBartosz Sobczak &leaf->chunkinfo, pblcnt << 3, 419cdcd52d4SBartosz Sobczak &leaf->addr, &fpm_addr); 420cdcd52d4SBartosz Sobczak if (ret_code) 421cdcd52d4SBartosz Sobczak goto error; 422cdcd52d4SBartosz Sobczak 423cdcd52d4SBartosz Sobczak leaf->idx = fpm_to_idx(pble_rsrc, fpm_addr); 424cdcd52d4SBartosz Sobczak 425cdcd52d4SBartosz Sobczak leaf->cnt = pblcnt; 426cdcd52d4SBartosz Sobczak *addr = (u64)leaf->idx; 427cdcd52d4SBartosz Sobczak addr++; 428cdcd52d4SBartosz Sobczak } 429cdcd52d4SBartosz Sobczak 430cdcd52d4SBartosz Sobczak palloc->level = PBLE_LEVEL_2; 431cdcd52d4SBartosz Sobczak pble_rsrc->stats_lvl2++; 432cdcd52d4SBartosz Sobczak return 0; 433cdcd52d4SBartosz Sobczak 434cdcd52d4SBartosz Sobczak error: 435cdcd52d4SBartosz Sobczak free_lvl2(pble_rsrc, palloc); 436cdcd52d4SBartosz Sobczak 437cdcd52d4SBartosz Sobczak return -ENOMEM; 438cdcd52d4SBartosz Sobczak } 439cdcd52d4SBartosz Sobczak 440cdcd52d4SBartosz Sobczak /** 441cdcd52d4SBartosz Sobczak * get_lvl1_pble - get level 1 pble resource 442cdcd52d4SBartosz Sobczak * @pble_rsrc: pble resource management 443cdcd52d4SBartosz Sobczak * @palloc: level 1 pble allocation 444cdcd52d4SBartosz Sobczak */ 445cdcd52d4SBartosz Sobczak static int 446cdcd52d4SBartosz Sobczak get_lvl1_pble(struct irdma_hmc_pble_rsrc *pble_rsrc, 447cdcd52d4SBartosz Sobczak struct irdma_pble_alloc *palloc) 448cdcd52d4SBartosz Sobczak { 449cdcd52d4SBartosz Sobczak int ret_code; 450cdcd52d4SBartosz Sobczak u64 fpm_addr; 451cdcd52d4SBartosz Sobczak struct irdma_pble_info *lvl1 = &palloc->level1; 452cdcd52d4SBartosz Sobczak 453cdcd52d4SBartosz Sobczak ret_code = irdma_prm_get_pbles(&pble_rsrc->pinfo, &lvl1->chunkinfo, 454cdcd52d4SBartosz Sobczak palloc->total_cnt << 3, &lvl1->addr, 455cdcd52d4SBartosz Sobczak &fpm_addr); 456cdcd52d4SBartosz Sobczak if (ret_code) 457cdcd52d4SBartosz Sobczak return -ENOMEM; 458cdcd52d4SBartosz Sobczak 459cdcd52d4SBartosz Sobczak palloc->level = PBLE_LEVEL_1; 460cdcd52d4SBartosz Sobczak lvl1->idx = fpm_to_idx(pble_rsrc, fpm_addr); 461cdcd52d4SBartosz Sobczak lvl1->cnt = palloc->total_cnt; 462cdcd52d4SBartosz Sobczak pble_rsrc->stats_lvl1++; 463cdcd52d4SBartosz Sobczak 464cdcd52d4SBartosz Sobczak return 0; 465cdcd52d4SBartosz Sobczak } 466cdcd52d4SBartosz Sobczak 467cdcd52d4SBartosz Sobczak /** 468cdcd52d4SBartosz Sobczak * get_lvl1_lvl2_pble - calls get_lvl1 and get_lvl2 pble routine 469cdcd52d4SBartosz Sobczak * @pble_rsrc: pble resources 470cdcd52d4SBartosz Sobczak * @palloc: contains all inforamtion regarding pble (idx + pble addr) 471cdcd52d4SBartosz Sobczak * @level1_only: flag for a level 1 PBLE 472cdcd52d4SBartosz Sobczak */ 473cdcd52d4SBartosz Sobczak static int 474cdcd52d4SBartosz Sobczak get_lvl1_lvl2_pble(struct irdma_hmc_pble_rsrc *pble_rsrc, 475cdcd52d4SBartosz Sobczak struct irdma_pble_alloc *palloc, bool level1_only) 476cdcd52d4SBartosz Sobczak { 477cdcd52d4SBartosz Sobczak int status = 0; 478cdcd52d4SBartosz Sobczak 479cdcd52d4SBartosz Sobczak status = get_lvl1_pble(pble_rsrc, palloc); 480cdcd52d4SBartosz Sobczak if (!status || level1_only || palloc->total_cnt <= PBLE_PER_PAGE) 481cdcd52d4SBartosz Sobczak return status; 482cdcd52d4SBartosz Sobczak 483cdcd52d4SBartosz Sobczak status = get_lvl2_pble(pble_rsrc, palloc); 484cdcd52d4SBartosz Sobczak 485cdcd52d4SBartosz Sobczak return status; 486cdcd52d4SBartosz Sobczak } 487cdcd52d4SBartosz Sobczak 488cdcd52d4SBartosz Sobczak /** 489cdcd52d4SBartosz Sobczak * irdma_get_pble - allocate pbles from the prm 490cdcd52d4SBartosz Sobczak * @pble_rsrc: pble resources 491cdcd52d4SBartosz Sobczak * @palloc: contains all inforamtion regarding pble (idx + pble addr) 492cdcd52d4SBartosz Sobczak * @pble_cnt: #of pbles requested 493cdcd52d4SBartosz Sobczak * @level1_only: true if only pble level 1 to acquire 494cdcd52d4SBartosz Sobczak */ 495cdcd52d4SBartosz Sobczak int 496cdcd52d4SBartosz Sobczak irdma_get_pble(struct irdma_hmc_pble_rsrc *pble_rsrc, 497cdcd52d4SBartosz Sobczak struct irdma_pble_alloc *palloc, u32 pble_cnt, 498cdcd52d4SBartosz Sobczak bool level1_only) 499cdcd52d4SBartosz Sobczak { 500cdcd52d4SBartosz Sobczak int status = 0; 501cdcd52d4SBartosz Sobczak int max_sds = 0; 502cdcd52d4SBartosz Sobczak int i; 503cdcd52d4SBartosz Sobczak 504cdcd52d4SBartosz Sobczak palloc->total_cnt = pble_cnt; 505cdcd52d4SBartosz Sobczak palloc->level = PBLE_LEVEL_0; 506cdcd52d4SBartosz Sobczak 507cdcd52d4SBartosz Sobczak mutex_lock(&pble_rsrc->pble_mutex_lock); 508cdcd52d4SBartosz Sobczak 509cdcd52d4SBartosz Sobczak /* 510cdcd52d4SBartosz Sobczak * check first to see if we can get pble's without acquiring additional sd's 511cdcd52d4SBartosz Sobczak */ 512cdcd52d4SBartosz Sobczak status = get_lvl1_lvl2_pble(pble_rsrc, palloc, level1_only); 513cdcd52d4SBartosz Sobczak if (!status) 514cdcd52d4SBartosz Sobczak goto exit; 515cdcd52d4SBartosz Sobczak 516cdcd52d4SBartosz Sobczak max_sds = (palloc->total_cnt >> 18) + 1; 517cdcd52d4SBartosz Sobczak for (i = 0; i < max_sds; i++) { 518cdcd52d4SBartosz Sobczak status = add_pble_prm(pble_rsrc); 519cdcd52d4SBartosz Sobczak if (status) 520cdcd52d4SBartosz Sobczak break; 521cdcd52d4SBartosz Sobczak 522cdcd52d4SBartosz Sobczak status = get_lvl1_lvl2_pble(pble_rsrc, palloc, level1_only); 523cdcd52d4SBartosz Sobczak /* if level1_only, only go through it once */ 524cdcd52d4SBartosz Sobczak if (!status || level1_only) 525cdcd52d4SBartosz Sobczak break; 526cdcd52d4SBartosz Sobczak } 527cdcd52d4SBartosz Sobczak 528cdcd52d4SBartosz Sobczak exit: 529cdcd52d4SBartosz Sobczak if (!status) { 530cdcd52d4SBartosz Sobczak pble_rsrc->allocdpbles += pble_cnt; 531cdcd52d4SBartosz Sobczak pble_rsrc->stats_alloc_ok++; 532cdcd52d4SBartosz Sobczak } else { 533cdcd52d4SBartosz Sobczak pble_rsrc->stats_alloc_fail++; 534cdcd52d4SBartosz Sobczak } 535cdcd52d4SBartosz Sobczak mutex_unlock(&pble_rsrc->pble_mutex_lock); 536cdcd52d4SBartosz Sobczak 537cdcd52d4SBartosz Sobczak return status; 538cdcd52d4SBartosz Sobczak } 539cdcd52d4SBartosz Sobczak 540cdcd52d4SBartosz Sobczak /** 541cdcd52d4SBartosz Sobczak * irdma_free_pble - put pbles back into prm 542cdcd52d4SBartosz Sobczak * @pble_rsrc: pble resources 543cdcd52d4SBartosz Sobczak * @palloc: contains all information regarding pble resource being freed 544cdcd52d4SBartosz Sobczak */ 545cdcd52d4SBartosz Sobczak void 546cdcd52d4SBartosz Sobczak irdma_free_pble(struct irdma_hmc_pble_rsrc *pble_rsrc, 547cdcd52d4SBartosz Sobczak struct irdma_pble_alloc *palloc) 548cdcd52d4SBartosz Sobczak { 549cdcd52d4SBartosz Sobczak pble_rsrc->freedpbles += palloc->total_cnt; 550cdcd52d4SBartosz Sobczak 551cdcd52d4SBartosz Sobczak if (palloc->level == PBLE_LEVEL_2) 552cdcd52d4SBartosz Sobczak free_lvl2(pble_rsrc, palloc); 553cdcd52d4SBartosz Sobczak else 554cdcd52d4SBartosz Sobczak irdma_prm_return_pbles(&pble_rsrc->pinfo, 555cdcd52d4SBartosz Sobczak &palloc->level1.chunkinfo); 556cdcd52d4SBartosz Sobczak pble_rsrc->stats_alloc_freed++; 557cdcd52d4SBartosz Sobczak } 558