64     ibmf_qp_handle_t ibmf_qp_handle, ibmf_wqe_mgt_t *wqe_mgt,
77 	ibmf_wqe_mgt_t		*wqe_mgt;  in ibmf_send_wqe_cache_constructor()  local
102 	wqe_mgt = cip->ci_wqe_mgt_list;  in ibmf_send_wqe_cache_constructor()
105 	while (wqe_mgt != NULL) {  in ibmf_send_wqe_cache_constructor()
106 		mutex_enter(&wqe_mgt->wqes_mutex);  in ibmf_send_wqe_cache_constructor()
107 		if ((send_wqe->send_sg_mem >= wqe_mgt->wqes_ib_mem) &&  in ibmf_send_wqe_cache_constructor()
108 		    (send_wqe->send_sg_mem < (wqe_mgt->wqes_ib_mem +  in ibmf_send_wqe_cache_constructor()
109 		    wqe_mgt->wqes_kmem_sz))) {  in ibmf_send_wqe_cache_constructor()
110 			mutex_exit(&wqe_mgt->wqes_mutex);  in ibmf_send_wqe_cache_constructor()
113 		mutex_exit(&wqe_mgt->wqes_mutex);  in ibmf_send_wqe_cache_constructor()
114 		wqe_mgt = wqe_mgt->wqe_mgt_next;  in ibmf_send_wqe_cache_constructor()
117 	if (wqe_mgt == NULL) {  in ibmf_send_wqe_cache_constructor()
131 	mutex_enter(&wqe_mgt->wqes_mutex);  in ibmf_send_wqe_cache_constructor()
133 	send_wqe->send_mem = (caddr_t)((uintptr_t)wqe_mgt->wqes_kmem +  in ibmf_send_wqe_cache_constructor()
134 	    (uintptr_t)(send_wqe->send_sg_mem - wqe_mgt->wqes_ib_mem));  in ibmf_send_wqe_cache_constructor()
136 	send_wqe->send_sg_lkey = wqe_mgt->wqes_ib_lkey;  in ibmf_send_wqe_cache_constructor()
137 	send_wqe->send_mem_hdl = wqe_mgt->wqes_ib_mem_hdl;  in ibmf_send_wqe_cache_constructor()
141 	mutex_exit(&wqe_mgt->wqes_mutex);  in ibmf_send_wqe_cache_constructor()
189 	ibmf_wqe_mgt_t		*wqe_mgt;  in ibmf_recv_wqe_cache_constructor()  local
214 	wqe_mgt = cip->ci_wqe_mgt_list;  in ibmf_recv_wqe_cache_constructor()
217 	while (wqe_mgt != NULL) {  in ibmf_recv_wqe_cache_constructor()
218 		mutex_enter(&wqe_mgt->wqes_mutex);  in ibmf_recv_wqe_cache_constructor()
219 		if ((recv_wqe->recv_sg_mem >= wqe_mgt->wqes_ib_mem) &&  in ibmf_recv_wqe_cache_constructor()
220 		    (recv_wqe->recv_sg_mem < (wqe_mgt->wqes_ib_mem +  in ibmf_recv_wqe_cache_constructor()
221 		    wqe_mgt->wqes_kmem_sz))) {  in ibmf_recv_wqe_cache_constructor()
222 			mutex_exit(&wqe_mgt->wqes_mutex);  in ibmf_recv_wqe_cache_constructor()
225 		mutex_exit(&wqe_mgt->wqes_mutex);  in ibmf_recv_wqe_cache_constructor()
226 		wqe_mgt = wqe_mgt->wqe_mgt_next;  in ibmf_recv_wqe_cache_constructor()
229 	if (wqe_mgt == NULL) {  in ibmf_recv_wqe_cache_constructor()
243 	mutex_enter(&wqe_mgt->wqes_mutex);  in ibmf_recv_wqe_cache_constructor()
245 	recv_wqe->recv_mem = (caddr_t)((uintptr_t)wqe_mgt->wqes_kmem +  in ibmf_recv_wqe_cache_constructor()
246 	    (uintptr_t)(recv_wqe->recv_sg_mem - wqe_mgt->wqes_ib_mem));  in ibmf_recv_wqe_cache_constructor()
248 	recv_wqe->recv_sg_lkey = wqe_mgt->wqes_ib_lkey;  in ibmf_recv_wqe_cache_constructor()
249 	recv_wqe->recv_mem_hdl = wqe_mgt->wqes_ib_mem_hdl;  in ibmf_recv_wqe_cache_constructor()
254 	mutex_exit(&wqe_mgt->wqes_mutex);  in ibmf_recv_wqe_cache_constructor()
302 	ibmf_wqe_mgt_t	*wqe_mgt;  in ibmf_altqp_send_wqe_cache_constructor()  local
328 	wqe_mgt = qp_ctx->isq_wqe_mgt_list;  in ibmf_altqp_send_wqe_cache_constructor()
331 	while (wqe_mgt != NULL) {  in ibmf_altqp_send_wqe_cache_constructor()
332 		mutex_enter(&wqe_mgt->wqes_mutex);  in ibmf_altqp_send_wqe_cache_constructor()
333 		if ((send_wqe->send_sg_mem >= wqe_mgt->wqes_ib_mem) &&  in ibmf_altqp_send_wqe_cache_constructor()
334 		    (send_wqe->send_sg_mem < (wqe_mgt->wqes_ib_mem +  in ibmf_altqp_send_wqe_cache_constructor()
335 		    wqe_mgt->wqes_kmem_sz))) {  in ibmf_altqp_send_wqe_cache_constructor()
336 			mutex_exit(&wqe_mgt->wqes_mutex);  in ibmf_altqp_send_wqe_cache_constructor()
339 		mutex_exit(&wqe_mgt->wqes_mutex);  in ibmf_altqp_send_wqe_cache_constructor()
340 		wqe_mgt = wqe_mgt->wqe_mgt_next;  in ibmf_altqp_send_wqe_cache_constructor()
343 	if (wqe_mgt == NULL) {  in ibmf_altqp_send_wqe_cache_constructor()
358 	mutex_enter(&wqe_mgt->wqes_mutex);  in ibmf_altqp_send_wqe_cache_constructor()
360 	send_wqe->send_mem = (caddr_t)((uintptr_t)wqe_mgt->wqes_kmem +  in ibmf_altqp_send_wqe_cache_constructor()
361 	    (uintptr_t)(send_wqe->send_sg_mem - wqe_mgt->wqes_ib_mem));  in ibmf_altqp_send_wqe_cache_constructor()
363 	send_wqe->send_sg_lkey = wqe_mgt->wqes_ib_lkey;  in ibmf_altqp_send_wqe_cache_constructor()
364 	send_wqe->send_mem_hdl = wqe_mgt->wqes_ib_mem_hdl;  in ibmf_altqp_send_wqe_cache_constructor()
367 	mutex_exit(&wqe_mgt->wqes_mutex);  in ibmf_altqp_send_wqe_cache_constructor()
415 	ibmf_wqe_mgt_t	*wqe_mgt;  in ibmf_altqp_recv_wqe_cache_constructor()  local
441 	wqe_mgt = qp_ctx->isq_wqe_mgt_list;  in ibmf_altqp_recv_wqe_cache_constructor()
444 	while (wqe_mgt != NULL) {  in ibmf_altqp_recv_wqe_cache_constructor()
445 		mutex_enter(&wqe_mgt->wqes_mutex);  in ibmf_altqp_recv_wqe_cache_constructor()
446 		if ((recv_wqe->recv_sg_mem >= wqe_mgt->wqes_ib_mem) &&  in ibmf_altqp_recv_wqe_cache_constructor()
447 		    (recv_wqe->recv_sg_mem < (wqe_mgt->wqes_ib_mem +  in ibmf_altqp_recv_wqe_cache_constructor()
448 		    wqe_mgt->wqes_kmem_sz))) {  in ibmf_altqp_recv_wqe_cache_constructor()
449 			mutex_exit(&wqe_mgt->wqes_mutex);  in ibmf_altqp_recv_wqe_cache_constructor()
452 		mutex_exit(&wqe_mgt->wqes_mutex);  in ibmf_altqp_recv_wqe_cache_constructor()
453 		wqe_mgt = wqe_mgt->wqe_mgt_next;  in ibmf_altqp_recv_wqe_cache_constructor()
456 	if (wqe_mgt == NULL) {  in ibmf_altqp_recv_wqe_cache_constructor()
470 	mutex_enter(&wqe_mgt->wqes_mutex);  in ibmf_altqp_recv_wqe_cache_constructor()
472 	recv_wqe->recv_mem = (caddr_t)((uintptr_t)wqe_mgt->wqes_kmem +  in ibmf_altqp_recv_wqe_cache_constructor()
473 	    (uintptr_t)(recv_wqe->recv_sg_mem - wqe_mgt->wqes_ib_mem));  in ibmf_altqp_recv_wqe_cache_constructor()
475 	recv_wqe->recv_sg_lkey = wqe_mgt->wqes_ib_lkey;  in ibmf_altqp_recv_wqe_cache_constructor()
476 	recv_wqe->recv_mem_hdl = wqe_mgt->wqes_ib_mem_hdl;  in ibmf_altqp_recv_wqe_cache_constructor()
479 	mutex_exit(&wqe_mgt->wqes_mutex);  in ibmf_altqp_recv_wqe_cache_constructor()
616 	ibmf_wqe_mgt_t	*wqe_mgt;  in ibmf_i_fini_wqes()  local
627 	wqe_mgt = cip->ci_wqe_mgt_list;  in ibmf_i_fini_wqes()
628 	while (wqe_mgt != NULL) {  in ibmf_i_fini_wqes()
630 		cip->ci_wqe_mgt_list = wqe_mgt->wqe_mgt_next;  in ibmf_i_fini_wqes()
633 		mutex_enter(&wqe_mgt->wqes_mutex);  in ibmf_i_fini_wqes()
634 		wqe_ib_mem_hdl = wqe_mgt->wqes_ib_mem_hdl;  in ibmf_i_fini_wqes()
635 		wqe_kmem = wqe_mgt->wqes_kmem;  in ibmf_i_fini_wqes()
636 		wqe_kmem_sz = wqe_mgt->wqes_kmem_sz;  in ibmf_i_fini_wqes()
637 		mutex_exit(&wqe_mgt->wqes_mutex);  in ibmf_i_fini_wqes()
646 		mutex_destroy(&wqe_mgt->wqes_mutex);  in ibmf_i_fini_wqes()
649 		kmem_free(wqe_mgt, sizeof (ibmf_wqe_mgt_t));  in ibmf_i_fini_wqes()
652 		wqe_mgt = cip->ci_wqe_mgt_list;  in ibmf_i_fini_wqes()
781 	ibmf_wqe_mgt_t	*wqe_mgt;  in ibmf_i_fini_altqp_wqes()  local
791 	wqe_mgt = qp_ctx->isq_wqe_mgt_list;  in ibmf_i_fini_altqp_wqes()
792 	while (wqe_mgt != NULL) {  in ibmf_i_fini_altqp_wqes()
794 		qp_ctx->isq_wqe_mgt_list = wqe_mgt->wqe_mgt_next;  in ibmf_i_fini_altqp_wqes()
797 		mutex_enter(&wqe_mgt->wqes_mutex);  in ibmf_i_fini_altqp_wqes()
798 		wqe_ib_mem_hdl = wqe_mgt->wqes_ib_mem_hdl;  in ibmf_i_fini_altqp_wqes()
799 		wqe_kmem = wqe_mgt->wqes_kmem;  in ibmf_i_fini_altqp_wqes()
800 		wqe_kmem_sz = wqe_mgt->wqes_kmem_sz;  in ibmf_i_fini_altqp_wqes()
801 		mutex_exit(&wqe_mgt->wqes_mutex);  in ibmf_i_fini_altqp_wqes()
812 		mutex_destroy(&wqe_mgt->wqes_mutex);  in ibmf_i_fini_altqp_wqes()
815 		kmem_free(wqe_mgt, sizeof (ibmf_wqe_mgt_t));  in ibmf_i_fini_altqp_wqes()
818 		wqe_mgt = qp_ctx->isq_wqe_mgt_list;  in ibmf_i_fini_altqp_wqes()
1073 	ibmf_wqe_mgt_t		*wqe_mgt;  in ibmf_i_extend_wqe_cache()  local
1084 	wqe_mgt = kmem_zalloc(sizeof (ibmf_wqe_mgt_t),  in ibmf_i_extend_wqe_cache()
1086 	if (wqe_mgt == NULL) {  in ibmf_i_extend_wqe_cache()
1096 	mutex_init(&wqe_mgt->wqes_mutex, NULL, MUTEX_DRIVER, NULL);  in ibmf_i_extend_wqe_cache()
1097 	_NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(*wqe_mgt))  in ibmf_i_extend_wqe_cache()
1100 	if (ibmf_i_extend_wqe_mem(cip, ibmf_qp_handle, wqe_mgt,  in ibmf_i_extend_wqe_cache()
1102 		mutex_destroy(&wqe_mgt->wqes_mutex);  in ibmf_i_extend_wqe_cache()
1103 		kmem_free(wqe_mgt, sizeof (ibmf_wqe_mgt_t));  in ibmf_i_extend_wqe_cache()
1127     ibmf_wqe_mgt_t *wqe_mgt, boolean_t block)  in ibmf_i_extend_wqe_mem()  argument
1137 	_NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(*wqe_mgt))  in ibmf_i_extend_wqe_mem()
1144 	    tnf_opaque, wqe_mgt, wqe_mgt, tnf_uint, block, block);  in ibmf_i_extend_wqe_mem()
1150 	wqe_mgt->wqes_kmem_sz = cip->ci_nports * 2 *  in ibmf_i_extend_wqe_mem()
1153 	wqe_mgt->wqes_kmem = kmem_zalloc(wqe_mgt->wqes_kmem_sz,  in ibmf_i_extend_wqe_mem()
1155 	if (wqe_mgt->wqes_kmem == NULL) {  in ibmf_i_extend_wqe_mem()
1166 	mem_attr.mr_vaddr = (ib_vaddr_t)(uintptr_t)wqe_mgt->wqes_kmem;  in ibmf_i_extend_wqe_mem()
1167 	mem_attr.mr_len = wqe_mgt->wqes_kmem_sz;  in ibmf_i_extend_wqe_mem()
1176 		kmem_free(wqe_mgt->wqes_kmem, wqe_mgt->wqes_kmem_sz);  in ibmf_i_extend_wqe_mem()
1188 	wqe_mgt->wqes_ib_mem = mem_desc.md_vaddr;  in ibmf_i_extend_wqe_mem()
1189 	wqe_mgt->wqes_ib_lkey = mem_desc.md_lkey;  in ibmf_i_extend_wqe_mem()
1190 	wqe_mgt->wqes_ib_mem_hdl = mem_hdl;  in ibmf_i_extend_wqe_mem()
1201 	if (vmem_add(wqe_vmem_arena, (void *)(uintptr_t)wqe_mgt->wqes_ib_mem,  in ibmf_i_extend_wqe_mem()
1202 	    wqe_mgt->wqes_kmem_sz,  in ibmf_i_extend_wqe_mem()
1205 		    wqe_mgt->wqes_ib_mem_hdl);  in ibmf_i_extend_wqe_mem()
1206 		kmem_free(wqe_mgt->wqes_kmem, wqe_mgt->wqes_kmem_sz);  in ibmf_i_extend_wqe_mem()
1225 		pwqe_mgt->wqe_mgt_next = wqe_mgt;  in ibmf_i_extend_wqe_mem()
1235 		pwqe_mgt->wqe_mgt_next = wqe_mgt;  in ibmf_i_extend_wqe_mem()