198 		     const struct nvkm_vmm_desc *desc, u32 ptei, u32 ptes)  in nvkm_vmm_unref_sptes()  argument
209 	for (lpti = ptei >> sptb; ptes; spti = 0, lpti++) {  in nvkm_vmm_unref_sptes()
210 		const u32 pten = min(sptn - spti, ptes);  in nvkm_vmm_unref_sptes()
212 		ptes -= pten;  in nvkm_vmm_unref_sptes()
222 			for (ptes = 1, ptei++; ptei < lpti; ptes++, ptei++) {  in nvkm_vmm_unref_sptes()
236 		for (ptes = 1, ptei++; ptei < lpti; ptes++, ptei++) {  in nvkm_vmm_unref_sptes()
243 			TRA(it, "LPTE %05x: U -> S %d PTEs", pteb, ptes);  in nvkm_vmm_unref_sptes()
244 			pair->func->sparse(vmm, pgt->pt[0], pteb, ptes);  in nvkm_vmm_unref_sptes()
251 			TRA(it, "LPTE %05x: U -> I %d PTEs", pteb, ptes);  in nvkm_vmm_unref_sptes()
252 			pair->func->invalid(vmm, pgt->pt[0], pteb, ptes);  in nvkm_vmm_unref_sptes()
258 nvkm_vmm_unref_ptes(struct nvkm_vmm_iter *it, bool pfn, u32 ptei, u32 ptes)  in nvkm_vmm_unref_ptes()  argument
267 		dma = desc->func->pfn_clear(it->vmm, pgt->pt[type], ptei, ptes);  in nvkm_vmm_unref_ptes()
272 			desc->func->pfn_unmap(it->vmm, pgt->pt[type], ptei, ptes);  in nvkm_vmm_unref_ptes()
277 	pgt->refs[type] -= ptes;  in nvkm_vmm_unref_ptes()
281 		nvkm_vmm_unref_sptes(it, pgt, desc, ptei, ptes);  in nvkm_vmm_unref_ptes()
297 		   const struct nvkm_vmm_desc *desc, u32 ptei, u32 ptes)  in nvkm_vmm_ref_sptes()  argument
308 	for (lpti = ptei >> sptb; ptes; spti = 0, lpti++) {  in nvkm_vmm_ref_sptes()
309 		const u32 pten = min(sptn - spti, ptes);  in nvkm_vmm_ref_sptes()
311 		ptes -= pten;  in nvkm_vmm_ref_sptes()
321 			for (ptes = 1, ptei++; ptei < lpti; ptes++, ptei++) {  in nvkm_vmm_ref_sptes()
335 		for (ptes = 1, ptei++; ptei < lpti; ptes++, ptei++) {  in nvkm_vmm_ref_sptes()
343 			const u32 sptc = ptes * sptn;  in nvkm_vmm_ref_sptes()
350 			TRA(it, "LPTE %05x: S -> U %d PTEs", pteb, ptes);  in nvkm_vmm_ref_sptes()
351 			pair->func->unmap(vmm, pgt->pt[0], pteb, ptes);  in nvkm_vmm_ref_sptes()
357 			TRA(it, "LPTE %05x: I -> U %d PTEs", pteb, ptes);  in nvkm_vmm_ref_sptes()
358 			pair->func->unmap(vmm, pgt->pt[0], pteb, ptes);  in nvkm_vmm_ref_sptes()
364 nvkm_vmm_ref_ptes(struct nvkm_vmm_iter *it, bool pfn, u32 ptei, u32 ptes)  in nvkm_vmm_ref_ptes()  argument
371 	pgt->refs[type] += ptes;  in nvkm_vmm_ref_ptes()
375 		nvkm_vmm_ref_sptes(it, pgt, desc, ptei, ptes);  in nvkm_vmm_ref_ptes()
382 		     struct nvkm_vmm_pt *pgt, u32 ptei, u32 ptes)  in nvkm_vmm_sparse_ptes()  argument
385 		while (ptes--)  in nvkm_vmm_sparse_ptes()
389 		memset(&pgt->pte[ptei], NVKM_VMM_PTE_SPARSE, ptes);  in nvkm_vmm_sparse_ptes()
394 nvkm_vmm_sparse_unref_ptes(struct nvkm_vmm_iter *it, bool pfn, u32 ptei, u32 ptes)  in nvkm_vmm_sparse_unref_ptes()  argument
398 		memset(&pt->pde[ptei], 0x00, sizeof(pt->pde[0]) * ptes);  in nvkm_vmm_sparse_unref_ptes()
401 		memset(&pt->pte[ptei], 0x00, sizeof(pt->pte[0]) * ptes);  in nvkm_vmm_sparse_unref_ptes()
402 	return nvkm_vmm_unref_ptes(it, pfn, ptei, ptes);  in nvkm_vmm_sparse_unref_ptes()
406 nvkm_vmm_sparse_ref_ptes(struct nvkm_vmm_iter *it, bool pfn, u32 ptei, u32 ptes)  in nvkm_vmm_sparse_ref_ptes()  argument
408 	nvkm_vmm_sparse_ptes(it->desc, it->pt[0], ptei, ptes);  in nvkm_vmm_sparse_ref_ptes()
409 	return nvkm_vmm_ref_ptes(it, pfn, ptei, ptes);  in nvkm_vmm_sparse_ref_ptes()
423 	u32 pteb, ptei, ptes;  in nvkm_vmm_ref_hwpt()  local
449 			for (ptes = 1, ptei++; ptei < pten; ptes++, ptei++) {  in nvkm_vmm_ref_hwpt()
457 					desc->func->sparse(vmm, pt, pteb, ptes);  in nvkm_vmm_ref_hwpt()
459 					desc->func->invalid(vmm, pt, pteb, ptes);  in nvkm_vmm_ref_hwpt()
460 				memset(&pgt->pte[pteb], 0x00, ptes);  in nvkm_vmm_ref_hwpt()
462 				desc->func->unmap(vmm, pt, pteb, ptes);  in nvkm_vmm_ref_hwpt()
463 				while (ptes--)  in nvkm_vmm_ref_hwpt()
536 		const u32 ptes = min_t(u64, it.cnt, pten - ptei);  in nvkm_vmm_iter()  local
563 		if (!REF_PTES || REF_PTES(&it, pfn, ptei, ptes)) {  in nvkm_vmm_iter()
567 					MAP_PTES(vmm, pt, ptei, ptes, map);  in nvkm_vmm_iter()
569 					CLR_PTES(vmm, pt, ptei, ptes);  in nvkm_vmm_iter()
575 		it.pte[it.lvl] += ptes;  in nvkm_vmm_iter()
576 		it.cnt -= ptes;  in nvkm_vmm_iter()
1915 nvkm_vmm_boot_ptes(struct nvkm_vmm_iter *it, bool pfn, u32 ptei, u32 ptes)  in nvkm_vmm_boot_ptes()  argument