93 	struct chash_iter iter = CHASH_ITER_INIT(table, 0);  in __chash_table_dump_stats()  local
99 		if (chash_iter_is_valid(iter))  in __chash_table_dump_stats()
101 		else if (chash_iter_is_empty(iter))  in __chash_table_dump_stats()
105 		CHASH_ITER_INC(iter);  in __chash_table_dump_stats()
106 	} while (iter.slot);  in __chash_table_dump_stats()
169 	struct chash_iter iter = CHASH_ITER_INIT(table, 0);  in chash_table_dump()  local
172 		if ((iter.slot & 3) == 0)  in chash_table_dump()
173 			pr_debug("%04x: ", iter.slot);  in chash_table_dump()
175 		if (chash_iter_is_valid(iter))  in chash_table_dump()
176 			pr_debug("[%016llx] ", chash_iter_key(iter));  in chash_table_dump()
177 		else if (chash_iter_is_empty(iter))  in chash_table_dump()
182 		if ((iter.slot & 3) == 3)  in chash_table_dump()
185 		CHASH_ITER_INC(iter);  in chash_table_dump()
186 	} while (iter.slot);  in chash_table_dump()
188 	if ((iter.slot & 3) != 0)  in chash_table_dump()
195 	struct chash_iter iter = CHASH_ITER_INIT(table, 0);  in chash_table_check()  local
199 		if (!chash_iter_is_valid(iter)) {  in chash_table_check()
200 			CHASH_ITER_INC(iter);  in chash_table_check()
204 		hash = chash_iter_hash(iter);  in chash_table_check()
206 		while (cur.slot != iter.slot) {  in chash_table_check()
209 				       iter.slot, hash, cur.slot);  in chash_table_check()
216 		CHASH_ITER_INC(iter);  in chash_table_check()
217 	} while (iter.slot);  in chash_table_check()
273 static int chash_table_find(struct chash_iter *iter, u64 key,  in chash_table_find()  argument
279 	u32 hash = iter->slot;  in chash_table_find()
280 	struct chash_iter first_redundant = CHASH_ITER_INIT(iter->table, -1);  in chash_table_find()
283 	while (!chash_iter_is_valid(*iter) || chash_iter_key(*iter) != key) {  in chash_table_find()
284 		if (chash_iter_is_empty(*iter)) {  in chash_table_find()
292 			while (first_redundant.slot != iter->slot) {  in chash_table_find()
298 			chash_table_check(iter->table);  in chash_table_find()
301 		} else if (!chash_iter_is_valid(*iter)) {  in chash_table_find()
307 				first_avail = iter->slot;  in chash_table_find()
312 				CHASH_ITER_SET(first_redundant, iter->slot);  in chash_table_find()
319 			u32 cur_hash = chash_iter_hash(*iter);  in chash_table_find()
321 			if (!CHASH_IN_RANGE(iter->table, cur_hash,  in chash_table_find()
323 					    iter->slot)) {  in chash_table_find()
329 				chash_iter_relocate(first_redundant, *iter);  in chash_table_find()
333 			} else if (cur_hash != iter->slot) {  in chash_table_find()
340 					CHASH_ITER_INIT(iter->table, cur_hash);  in chash_table_find()
342 				while (new_iter.slot != iter->slot &&  in chash_table_find()
346 				if (new_iter.slot != iter->slot)  in chash_table_find()
347 					chash_iter_relocate(new_iter, *iter);  in chash_table_find()
351 		CHASH_ITER_INC(*iter);  in chash_table_find()
352 		if (iter->slot == hash) {  in chash_table_find()
353 			iter->slot = -1;  in chash_table_find()
359 	iter->table->hits++;  in chash_table_find()
360 	iter->table->hits_steps += CHASH_SUB(iter->table, iter->slot, hash) + 1;  in chash_table_find()
365 		chash_iter_relocate(first_redundant, *iter);  in chash_table_find()
366 		iter->slot = first_redundant.slot;  in chash_table_find()
367 		iter->mask = first_redundant.mask;  in chash_table_find()
371 	iter->table->hits_time_ns += local_clock() - ts1;  in chash_table_find()
378 	iter->table->miss++;  in chash_table_find()
379 	iter->table->miss_steps += (iter->slot < 0) ?  in chash_table_find()
380 		(1 << iter->table->bits) :  in chash_table_find()
381 		CHASH_SUB(iter->table, iter->slot, hash) + 1;  in chash_table_find()
385 		CHASH_ITER_SET(*iter, first_avail);  in chash_table_find()
388 	iter->table->miss_time_ns += local_clock() - ts1;  in chash_table_find()
399 	struct chash_iter iter = CHASH_ITER_INIT(table, hash);  in __chash_table_copy_in()  local
400 	int r = chash_table_find(&iter, key, false);  in __chash_table_copy_in()
405 			memcpy(chash_iter_value(iter), value,  in __chash_table_copy_in()
411 	if (iter.slot < 0) {  in __chash_table_copy_in()
416 	chash_iter_set_valid(iter);  in __chash_table_copy_in()
419 		table->keys32[iter.slot] = key;  in __chash_table_copy_in()
421 		table->keys64[iter.slot] = key;  in __chash_table_copy_in()
423 		memcpy(chash_iter_value(iter), value, table->value_size);  in __chash_table_copy_in()
434 	struct chash_iter iter = CHASH_ITER_INIT(table, hash);  in __chash_table_copy_out()  local
435 	int r = chash_table_find(&iter, key, remove);  in __chash_table_copy_out()
441 		memcpy(value, chash_iter_value(iter), table->value_size);  in __chash_table_copy_out()
444 		chash_iter_set_invalid(iter);  in __chash_table_copy_out()
446 	return iter.slot;  in __chash_table_copy_out()