822 	mr->ibmr.lkey = mr->mmkey.key;  in mlx5_ib_get_dma_mr()
823 	mr->ibmr.rkey = mr->mmkey.key;  in mlx5_ib_get_dma_mr()
826 	return &mr->ibmr;  in mlx5_ib_get_dma_mr()
914 	mr->ibmr.lkey = mr->mmkey.key;  in set_mr_fields()
915 	mr->ibmr.rkey = mr->mmkey.key;  in set_mr_fields()
916 	mr->ibmr.length = length;  in set_mr_fields()
917 	mr->ibmr.device = &dev->ib_dev;  in set_mr_fields()
973 	mr->ibmr.pd = pd;  in alloc_cacheable_mr()
1086 	wr->pd = mr->ibmr.pd;  in mlx5_ib_create_xlt_wr()
1289 	mr->ibmr.pd = pd;  in reg_create()
1390 	return &mr->ibmr;  in mlx5_ib_get_dm_mr()
1485 			mlx5_ib_dereg_mr(&mr->ibmr, NULL);  in create_real_mr()
1489 	return &mr->ibmr;  in create_real_mr()
1516 		return &mr->ibmr;  in create_user_odp_mr()
1542 	return &mr->ibmr;  in create_user_odp_mr()
1545 	mlx5_ib_dereg_mr(&mr->ibmr, NULL);  in create_user_odp_mr()
1639 	return &mr->ibmr;  in mlx5_ib_reg_user_mr_dmabuf()
1642 	mlx5_ib_dereg_mr(&mr->ibmr, NULL);  in mlx5_ib_reg_user_mr_dmabuf()
1691 	struct mlx5_ib_dev *dev = to_mdev(mr->ibmr.device);  in umr_rereg_pd_access()
1718 	struct mlx5_ib_dev *dev = to_mdev(mr->ibmr.device);  in can_use_umr_rereg_pas()
1738 	struct mlx5_ib_dev *dev = to_mdev(mr->ibmr.device);  in umr_rereg_pas()
1753 		mr->ibmr.pd = pd;  in umr_rereg_pas()
1762 	mr->ibmr.length = new_umem->length;  in umr_rereg_pas()
1915 		struct ib_device *device = mr->ibmr.device;  in mlx5_free_priv_descs()
1926 int mlx5_ib_dereg_mr(struct ib_mr *ibmr, struct ib_udata *udata)  in mlx5_ib_dereg_mr()  argument
1928 	struct mlx5_ib_mr *mr = to_mmr(ibmr);  in mlx5_ib_dereg_mr()
1929 	struct mlx5_ib_dev *dev = to_mdev(ibmr->device);  in mlx5_ib_dereg_mr()
1942 	if (ibmr->type == IB_MR_TYPE_INTEGRITY) {  in mlx5_ib_dereg_mr()
1943 		xa_cmpxchg(&dev->sig_mrs, mlx5_base_mkey(mr->mmkey.key), ibmr,  in mlx5_ib_dereg_mr()
1947 			rc = mlx5_ib_dereg_mr(&mr->mtt_mr->ibmr, NULL);  in mlx5_ib_dereg_mr()
1953 			rc = mlx5_ib_dereg_mr(&mr->klm_mr->ibmr, NULL);  in mlx5_ib_dereg_mr()
1980 		rc = destroy_mkey(to_mdev(mr->ibmr.device), mr);  in mlx5_ib_dereg_mr()
2044 	mr->ibmr.lkey = mr->mmkey.key;  in _mlx5_alloc_mkey_descs()
2045 	mr->ibmr.rkey = mr->mmkey.key;  in _mlx5_alloc_mkey_descs()
2069 	mr->ibmr.pd = pd;  in mlx5_ib_alloc_pi_mr()
2070 	mr->ibmr.device = pd->device;  in mlx5_ib_alloc_pi_mr()
2173 	mlx5_ib_dereg_mr(&mr->mtt_mr->ibmr, NULL);  in mlx5_alloc_integrity_descs()
2176 	mlx5_ib_dereg_mr(&mr->klm_mr->ibmr, NULL);  in mlx5_alloc_integrity_descs()
2212 	mr->ibmr.device = pd->device;  in __mlx5_ib_alloc_mr()
2236 	return &mr->ibmr;  in __mlx5_ib_alloc_mr()
2352 int mlx5_ib_check_mr_status(struct ib_mr *ibmr, u32 check_mask,  in mlx5_ib_check_mr_status()  argument
2355 	struct mlx5_ib_mr *mmr = to_mmr(ibmr);  in mlx5_ib_check_mr_status()
2376 		if (ibmr->lkey == mmr->sig->err_item.key)  in mlx5_ib_check_mr_status()
2394 mlx5_ib_map_pa_mr_sg_pi(struct ib_mr *ibmr, struct scatterlist *data_sg,  in mlx5_ib_map_pa_mr_sg_pi()  argument
2399 	struct mlx5_ib_mr *mr = to_mmr(ibmr);  in mlx5_ib_map_pa_mr_sg_pi()
2421 		ibmr->length = mr->data_length + mr->meta_length;  in mlx5_ib_map_pa_mr_sg_pi()
2439 	u32 lkey = mr->ibmr.pd->local_dma_lkey;  in mlx5_ib_sg_to_klms()
2442 	mr->ibmr.iova = sg_dma_address(sg) + sg_offset;  in mlx5_ib_sg_to_klms()
2443 	mr->ibmr.length = 0;  in mlx5_ib_sg_to_klms()
2451 		mr->ibmr.length += sg_dma_len(sg) - sg_offset;  in mlx5_ib_sg_to_klms()
2460 	mr->data_length = mr->ibmr.length;  in mlx5_ib_sg_to_klms()
2473 			mr->ibmr.length += sg_dma_len(sg) - sg_offset;  in mlx5_ib_sg_to_klms()
2481 		mr->meta_length = mr->ibmr.length - mr->data_length;  in mlx5_ib_sg_to_klms()
2487 static int mlx5_set_page(struct ib_mr *ibmr, u64 addr)  in mlx5_set_page()  argument
2489 	struct mlx5_ib_mr *mr = to_mmr(ibmr);  in mlx5_set_page()
2501 static int mlx5_set_page_pi(struct ib_mr *ibmr, u64 addr)  in mlx5_set_page_pi()  argument
2503 	struct mlx5_ib_mr *mr = to_mmr(ibmr);  in mlx5_set_page_pi()
2517 mlx5_ib_map_mtt_mr_sg_pi(struct ib_mr *ibmr, struct scatterlist *data_sg,  in mlx5_ib_map_mtt_mr_sg_pi()  argument
2522 	struct mlx5_ib_mr *mr = to_mmr(ibmr);  in mlx5_ib_map_mtt_mr_sg_pi()
2530 	ib_dma_sync_single_for_cpu(ibmr->device, pi_mr->desc_map,  in mlx5_ib_map_mtt_mr_sg_pi()
2534 	pi_mr->ibmr.page_size = ibmr->page_size;  in mlx5_ib_map_mtt_mr_sg_pi()
2535 	n = ib_sg_to_pages(&pi_mr->ibmr, data_sg, data_sg_nents, data_sg_offset,  in mlx5_ib_map_mtt_mr_sg_pi()
2540 	pi_mr->data_iova = pi_mr->ibmr.iova;  in mlx5_ib_map_mtt_mr_sg_pi()
2541 	pi_mr->data_length = pi_mr->ibmr.length;  in mlx5_ib_map_mtt_mr_sg_pi()
2542 	pi_mr->ibmr.length = pi_mr->data_length;  in mlx5_ib_map_mtt_mr_sg_pi()
2543 	ibmr->length = pi_mr->data_length;  in mlx5_ib_map_mtt_mr_sg_pi()
2546 		u64 page_mask = ~((u64)ibmr->page_size - 1);  in mlx5_ib_map_mtt_mr_sg_pi()
2549 		n += ib_sg_to_pages(&pi_mr->ibmr, meta_sg, meta_sg_nents,  in mlx5_ib_map_mtt_mr_sg_pi()
2552 		pi_mr->meta_length = pi_mr->ibmr.length;  in mlx5_ib_map_mtt_mr_sg_pi()
2560 				 pi_mr->ndescs * ibmr->page_size +  in mlx5_ib_map_mtt_mr_sg_pi()
2561 				 (pi_mr->ibmr.iova & ~page_mask);  in mlx5_ib_map_mtt_mr_sg_pi()
2569 		pi_mr->ibmr.length = pi_mr->pi_iova + pi_mr->meta_length - iova;  in mlx5_ib_map_mtt_mr_sg_pi()
2570 		pi_mr->ibmr.iova = iova;  in mlx5_ib_map_mtt_mr_sg_pi()
2571 		ibmr->length += pi_mr->meta_length;  in mlx5_ib_map_mtt_mr_sg_pi()
2574 	ib_dma_sync_single_for_device(ibmr->device, pi_mr->desc_map,  in mlx5_ib_map_mtt_mr_sg_pi()
2582 mlx5_ib_map_klm_mr_sg_pi(struct ib_mr *ibmr, struct scatterlist *data_sg,  in mlx5_ib_map_klm_mr_sg_pi()  argument
2587 	struct mlx5_ib_mr *mr = to_mmr(ibmr);  in mlx5_ib_map_klm_mr_sg_pi()
2595 	ib_dma_sync_single_for_cpu(ibmr->device, pi_mr->desc_map,  in mlx5_ib_map_klm_mr_sg_pi()
2602 	ib_dma_sync_single_for_device(ibmr->device, pi_mr->desc_map,  in mlx5_ib_map_klm_mr_sg_pi()
2608 	pi_mr->ibmr.iova = 0;  in mlx5_ib_map_klm_mr_sg_pi()
2610 	ibmr->length = pi_mr->ibmr.length;  in mlx5_ib_map_klm_mr_sg_pi()
2615 int mlx5_ib_map_mr_sg_pi(struct ib_mr *ibmr, struct scatterlist *data_sg,  in mlx5_ib_map_mr_sg_pi()  argument
2620 	struct mlx5_ib_mr *mr = to_mmr(ibmr);  in mlx5_ib_map_mr_sg_pi()
2624 	WARN_ON(ibmr->type != IB_MR_TYPE_INTEGRITY);  in mlx5_ib_map_mr_sg_pi()
2637 	n = mlx5_ib_map_pa_mr_sg_pi(ibmr, data_sg, data_sg_nents,  in mlx5_ib_map_mr_sg_pi()
2651 	n = mlx5_ib_map_mtt_mr_sg_pi(ibmr, data_sg, data_sg_nents,  in mlx5_ib_map_mr_sg_pi()
2658 	n = mlx5_ib_map_klm_mr_sg_pi(ibmr, data_sg, data_sg_nents,  in mlx5_ib_map_mr_sg_pi()
2666 	ibmr->iova = 0;  in mlx5_ib_map_mr_sg_pi()
2669 		ibmr->sig_attrs->meta_length = pi_mr->meta_length;  in mlx5_ib_map_mr_sg_pi()
2671 		ibmr->sig_attrs->meta_length = mr->meta_length;  in mlx5_ib_map_mr_sg_pi()
2676 int mlx5_ib_map_mr_sg(struct ib_mr *ibmr, struct scatterlist *sg, int sg_nents,  in mlx5_ib_map_mr_sg()  argument
2679 	struct mlx5_ib_mr *mr = to_mmr(ibmr);  in mlx5_ib_map_mr_sg()
2684 	ib_dma_sync_single_for_cpu(ibmr->device, mr->desc_map,  in mlx5_ib_map_mr_sg()
2692 		n = ib_sg_to_pages(ibmr, sg, sg_nents, sg_offset,  in mlx5_ib_map_mr_sg()
2695 	ib_dma_sync_single_for_device(ibmr->device, mr->desc_map,  in mlx5_ib_map_mr_sg()