520 static inline void init_inc_context(struct inc_context *ic)  in init_inc_context()  argument
522 	ic->bitmap_block = NULL;  in init_inc_context()
523 	ic->bitmap = NULL;  in init_inc_context()
524 	ic->overflow_leaf = NULL;  in init_inc_context()
527 static inline void exit_inc_context(struct ll_disk *ll, struct inc_context *ic)  in exit_inc_context()  argument
529 	if (ic->bitmap_block)  in exit_inc_context()
530 		dm_tm_unlock(ll->tm, ic->bitmap_block);  in exit_inc_context()
531 	if (ic->overflow_leaf)  in exit_inc_context()
532 		dm_tm_unlock(ll->tm, ic->overflow_leaf);  in exit_inc_context()
535 static inline void reset_inc_context(struct ll_disk *ll, struct inc_context *ic)  in reset_inc_context()  argument
537 	exit_inc_context(ll, ic);  in reset_inc_context()
538 	init_inc_context(ic);  in reset_inc_context()
551 static int __sm_ll_inc_overflow(struct ll_disk *ll, dm_block_t b, struct inc_context *ic)  in __sm_ll_inc_overflow()  argument
563 	reset_inc_context(ll, ic);  in __sm_ll_inc_overflow()
566 				     b, &index, &ll->ref_count_root, &ic->overflow_leaf);  in __sm_ll_inc_overflow()
570 	n = dm_block_data(ic->overflow_leaf);  in __sm_ll_inc_overflow()
584 static int sm_ll_inc_overflow(struct ll_disk *ll, dm_block_t b, struct inc_context *ic)  in sm_ll_inc_overflow()  argument
594 	if (ic->overflow_leaf) {  in sm_ll_inc_overflow()
595 		n = dm_block_data(ic->overflow_leaf);  in sm_ll_inc_overflow()
606 	return __sm_ll_inc_overflow(ll, b, ic);  in sm_ll_inc_overflow()
609 static inline int shadow_bitmap(struct ll_disk *ll, struct inc_context *ic)  in shadow_bitmap()  argument
613 	r = dm_tm_shadow_block(ll->tm, le64_to_cpu(ic->ie_disk.blocknr),  in shadow_bitmap()
614 			       &dm_sm_bitmap_validator, &ic->bitmap_block, &inc);  in shadow_bitmap()
619 	ic->ie_disk.blocknr = cpu_to_le64(dm_block_location(ic->bitmap_block));  in shadow_bitmap()
620 	ic->bitmap = dm_bitmap_data(ic->bitmap_block);  in shadow_bitmap()
629 static inline int ensure_bitmap(struct ll_disk *ll, struct inc_context *ic)  in ensure_bitmap()  argument
631 	if (!ic->bitmap_block) {  in ensure_bitmap()
632 		int r = dm_bm_write_lock(dm_tm_get_bm(ll->tm), le64_to_cpu(ic->ie_disk.blocknr),  in ensure_bitmap()
633 					 &dm_sm_bitmap_validator, &ic->bitmap_block);  in ensure_bitmap()
638 		ic->bitmap = dm_bitmap_data(ic->bitmap_block);  in ensure_bitmap()
650 				   struct inc_context *ic)  in sm_ll_inc_bitmap()  argument
662 		r = ensure_bitmap(ll, ic);  in sm_ll_inc_bitmap()
666 		old = sm_lookup_bitmap(ic->bitmap, bit);  in sm_ll_inc_bitmap()
670 			sm_set_bitmap(ic->bitmap, bit, 1);  in sm_ll_inc_bitmap()
673 			le32_add_cpu(&ic->ie_disk.nr_free, -1);  in sm_ll_inc_bitmap()
674 			if (le32_to_cpu(ic->ie_disk.none_free_before) == bit)  in sm_ll_inc_bitmap()
675 				ic->ie_disk.none_free_before = cpu_to_le32(bit + 1);  in sm_ll_inc_bitmap()
680 			sm_set_bitmap(ic->bitmap, bit, 2);  in sm_ll_inc_bitmap()
685 			sm_set_bitmap(ic->bitmap, bit, 3);  in sm_ll_inc_bitmap()
686 			reset_inc_context(ll, ic);  in sm_ll_inc_bitmap()
702 			r = sm_ll_inc_overflow(ll, b, ic);  in sm_ll_inc_bitmap()
720 	struct inc_context ic;  in __sm_ll_inc()  local
724 	init_inc_context(&ic);  in __sm_ll_inc()
727 	r = ll->load_ie(ll, index, &ic.ie_disk);  in __sm_ll_inc()
731 	r = shadow_bitmap(ll, &ic);  in __sm_ll_inc()
736 	r = sm_ll_inc_bitmap(ll, b, bit, bit_end, nr_allocations, new_b, &ic);  in __sm_ll_inc()
738 	exit_inc_context(ll, &ic);  in __sm_ll_inc()
743 	return ll->save_ie(ll, index, &ic.ie_disk);  in __sm_ll_inc()
763 				struct inc_context *ic)  in __sm_ll_del_overflow()  argument
765 	reset_inc_context(ll, ic);  in __sm_ll_del_overflow()
771 				struct inc_context *ic, uint32_t *old_rc)  in __sm_ll_dec_overflow()  argument
779 	reset_inc_context(ll, ic);  in __sm_ll_dec_overflow()
781 				     b, &index, &ll->ref_count_root, &ic->overflow_leaf);  in __sm_ll_dec_overflow()
785 	n = dm_block_data(ic->overflow_leaf);  in __sm_ll_dec_overflow()
797 		return __sm_ll_del_overflow(ll, b, ic);  in __sm_ll_dec_overflow()
805 			      struct inc_context *ic, uint32_t *old_rc)  in sm_ll_dec_overflow()  argument
810 	if (ic->overflow_leaf) {  in sm_ll_dec_overflow()
816 		n = dm_block_data(ic->overflow_leaf);  in sm_ll_dec_overflow()
828 				return __sm_ll_del_overflow(ll, b, ic);  in sm_ll_dec_overflow()
834 	return __sm_ll_dec_overflow(ll, b, ic, old_rc);  in sm_ll_dec_overflow()
842 				   struct inc_context *ic,  in sm_ll_dec_bitmap()  argument
854 		r = ensure_bitmap(ll, ic);  in sm_ll_dec_bitmap()
858 		old = sm_lookup_bitmap(ic->bitmap, bit);  in sm_ll_dec_bitmap()
866 			sm_set_bitmap(ic->bitmap, bit, 0);  in sm_ll_dec_bitmap()
869 			le32_add_cpu(&ic->ie_disk.nr_free, 1);  in sm_ll_dec_bitmap()
870 			ic->ie_disk.none_free_before =  in sm_ll_dec_bitmap()
871 				cpu_to_le32(min(le32_to_cpu(ic->ie_disk.none_free_before), bit));  in sm_ll_dec_bitmap()
876 			sm_set_bitmap(ic->bitmap, bit, 1);  in sm_ll_dec_bitmap()
880 			r = sm_ll_dec_overflow(ll, b, ic, &old);  in sm_ll_dec_bitmap()
885 				r = ensure_bitmap(ll, ic);  in sm_ll_dec_bitmap()
889 				sm_set_bitmap(ic->bitmap, bit, 2);  in sm_ll_dec_bitmap()
904 	struct inc_context ic;  in __sm_ll_dec()  local
907 	init_inc_context(&ic);  in __sm_ll_dec()
910 	r = ll->load_ie(ll, index, &ic.ie_disk);  in __sm_ll_dec()
914 	r = shadow_bitmap(ll, &ic);  in __sm_ll_dec()
919 	r = sm_ll_dec_bitmap(ll, b, bit, bit_end, &ic, nr_allocations, new_b);  in __sm_ll_dec()
920 	exit_inc_context(ll, &ic);  in __sm_ll_dec()
925 	return ll->save_ie(ll, index, &ic.ie_disk);  in __sm_ll_dec()