1e7c033c3SPaolo Bonzini /* 2e7c033c3SPaolo Bonzini * Hierarchical Bitmap Data Type 3e7c033c3SPaolo Bonzini * 4e7c033c3SPaolo Bonzini * Copyright Red Hat, Inc., 2012 5e7c033c3SPaolo Bonzini * 6e7c033c3SPaolo Bonzini * Author: Paolo Bonzini <pbonzini@redhat.com> 7e7c033c3SPaolo Bonzini * 8e7c033c3SPaolo Bonzini * This work is licensed under the terms of the GNU GPL, version 2 or 9e7c033c3SPaolo Bonzini * later. See the COPYING file in the top-level directory. 10e7c033c3SPaolo Bonzini */ 11e7c033c3SPaolo Bonzini 12e7c033c3SPaolo Bonzini #include "qemu/osdep.h" 13e7c033c3SPaolo Bonzini #include "qemu/hbitmap.h" 14e7c033c3SPaolo Bonzini #include "qemu/host-utils.h" 15e7c033c3SPaolo Bonzini #include "trace.h" 16a3b52535SVladimir Sementsov-Ogievskiy #include "crypto/hash.h" 17e7c033c3SPaolo Bonzini 18e7c033c3SPaolo Bonzini /* HBitmaps provides an array of bits. The bits are stored as usual in an 19e7c033c3SPaolo Bonzini * array of unsigned longs, but HBitmap is also optimized to provide fast 20e7c033c3SPaolo Bonzini * iteration over set bits; going from one bit to the next is O(logB n) 21e7c033c3SPaolo Bonzini * worst case, with B = sizeof(long) * CHAR_BIT: the result is low enough 22e7c033c3SPaolo Bonzini * that the number of levels is in fact fixed. 23e7c033c3SPaolo Bonzini * 24e7c033c3SPaolo Bonzini * In order to do this, it stacks multiple bitmaps with progressively coarser 25e7c033c3SPaolo Bonzini * granularity; in all levels except the last, bit N is set iff the N-th 26e7c033c3SPaolo Bonzini * unsigned long is nonzero in the immediately next level. When iteration 27e7c033c3SPaolo Bonzini * completes on the last level it can examine the 2nd-last level to quickly 28e7c033c3SPaolo Bonzini * skip entire words, and even do so recursively to skip blocks of 64 words or 29e7c033c3SPaolo Bonzini * powers thereof (32 on 32-bit machines). 30e7c033c3SPaolo Bonzini * 31e7c033c3SPaolo Bonzini * Given an index in the bitmap, it can be split in group of bits like 32e7c033c3SPaolo Bonzini * this (for the 64-bit case): 33e7c033c3SPaolo Bonzini * 34e7c033c3SPaolo Bonzini * bits 0-57 => word in the last bitmap | bits 58-63 => bit in the word 35e7c033c3SPaolo Bonzini * bits 0-51 => word in the 2nd-last bitmap | bits 52-57 => bit in the word 36e7c033c3SPaolo Bonzini * bits 0-45 => word in the 3rd-last bitmap | bits 46-51 => bit in the word 37e7c033c3SPaolo Bonzini * 38e7c033c3SPaolo Bonzini * So it is easy to move up simply by shifting the index right by 39e7c033c3SPaolo Bonzini * log2(BITS_PER_LONG) bits. To move down, you shift the index left 40e7c033c3SPaolo Bonzini * similarly, and add the word index within the group. Iteration uses 41e7c033c3SPaolo Bonzini * ffs (find first set bit) to find the next word to examine; this 42e7c033c3SPaolo Bonzini * operation can be done in constant time in most current architectures. 43e7c033c3SPaolo Bonzini * 44e7c033c3SPaolo Bonzini * Setting or clearing a range of m bits on all levels, the work to perform 45e7c033c3SPaolo Bonzini * is O(m + m/W + m/W^2 + ...), which is O(m) like on a regular bitmap. 46e7c033c3SPaolo Bonzini * 47e7c033c3SPaolo Bonzini * When iterating on a bitmap, each bit (on any level) is only visited 48e7c033c3SPaolo Bonzini * once. Hence, The total cost of visiting a bitmap with m bits in it is 49e7c033c3SPaolo Bonzini * the number of bits that are set in all bitmaps. Unless the bitmap is 50e7c033c3SPaolo Bonzini * extremely sparse, this is also O(m + m/W + m/W^2 + ...), so the amortized 51e7c033c3SPaolo Bonzini * cost of advancing from one bit to the next is usually constant (worst case 52e7c033c3SPaolo Bonzini * O(logB n) as in the non-amortized complexity). 53e7c033c3SPaolo Bonzini */ 54e7c033c3SPaolo Bonzini 55e7c033c3SPaolo Bonzini struct HBitmap { 56e7c033c3SPaolo Bonzini /* Number of total bits in the bottom level. */ 57e7c033c3SPaolo Bonzini uint64_t size; 58e7c033c3SPaolo Bonzini 59e7c033c3SPaolo Bonzini /* Number of set bits in the bottom level. */ 60e7c033c3SPaolo Bonzini uint64_t count; 61e7c033c3SPaolo Bonzini 62e7c033c3SPaolo Bonzini /* A scaling factor. Given a granularity of G, each bit in the bitmap will 63e7c033c3SPaolo Bonzini * will actually represent a group of 2^G elements. Each operation on a 64e7c033c3SPaolo Bonzini * range of bits first rounds the bits to determine which group they land 65e7c033c3SPaolo Bonzini * in, and then affect the entire page; iteration will only visit the first 66e7c033c3SPaolo Bonzini * bit of each group. Here is an example of operations in a size-16, 67e7c033c3SPaolo Bonzini * granularity-1 HBitmap: 68e7c033c3SPaolo Bonzini * 69e7c033c3SPaolo Bonzini * initial state 00000000 70e7c033c3SPaolo Bonzini * set(start=0, count=9) 11111000 (iter: 0, 2, 4, 6, 8) 71e7c033c3SPaolo Bonzini * reset(start=1, count=3) 00111000 (iter: 4, 6, 8) 72e7c033c3SPaolo Bonzini * set(start=9, count=2) 00111100 (iter: 4, 6, 8, 10) 73e7c033c3SPaolo Bonzini * reset(start=5, count=5) 00000000 74e7c033c3SPaolo Bonzini * 75e7c033c3SPaolo Bonzini * From an implementation point of view, when setting or resetting bits, 76e7c033c3SPaolo Bonzini * the bitmap will scale bit numbers right by this amount of bits. When 77e7c033c3SPaolo Bonzini * iterating, the bitmap will scale bit numbers left by this amount of 78e7c033c3SPaolo Bonzini * bits. 79e7c033c3SPaolo Bonzini */ 80e7c033c3SPaolo Bonzini int granularity; 81e7c033c3SPaolo Bonzini 8207ac4cdbSFam Zheng /* A meta dirty bitmap to track the dirtiness of bits in this HBitmap. */ 8307ac4cdbSFam Zheng HBitmap *meta; 8407ac4cdbSFam Zheng 85e7c033c3SPaolo Bonzini /* A number of progressively less coarse bitmaps (i.e. level 0 is the 86e7c033c3SPaolo Bonzini * coarsest). Each bit in level N represents a word in level N+1 that 87e7c033c3SPaolo Bonzini * has a set bit, except the last level where each bit represents the 88e7c033c3SPaolo Bonzini * actual bitmap. 89e7c033c3SPaolo Bonzini * 90e7c033c3SPaolo Bonzini * Note that all bitmaps have the same number of levels. Even a 1-bit 91e7c033c3SPaolo Bonzini * bitmap will still allocate HBITMAP_LEVELS arrays. 92e7c033c3SPaolo Bonzini */ 93e7c033c3SPaolo Bonzini unsigned long *levels[HBITMAP_LEVELS]; 948515efbeSJohn Snow 958515efbeSJohn Snow /* The length of each levels[] array. */ 968515efbeSJohn Snow uint64_t sizes[HBITMAP_LEVELS]; 97e7c033c3SPaolo Bonzini }; 98e7c033c3SPaolo Bonzini 99e7c033c3SPaolo Bonzini /* Advance hbi to the next nonzero word and return it. hbi->pos 100e7c033c3SPaolo Bonzini * is updated. Returns zero if we reach the end of the bitmap. 101e7c033c3SPaolo Bonzini */ 102e7c033c3SPaolo Bonzini unsigned long hbitmap_iter_skip_words(HBitmapIter *hbi) 103e7c033c3SPaolo Bonzini { 104e7c033c3SPaolo Bonzini size_t pos = hbi->pos; 105e7c033c3SPaolo Bonzini const HBitmap *hb = hbi->hb; 106e7c033c3SPaolo Bonzini unsigned i = HBITMAP_LEVELS - 1; 107e7c033c3SPaolo Bonzini 108e7c033c3SPaolo Bonzini unsigned long cur; 109e7c033c3SPaolo Bonzini do { 110f63ea4e9SVladimir Sementsov-Ogievskiy i--; 111e7c033c3SPaolo Bonzini pos >>= BITS_PER_LEVEL; 112f63ea4e9SVladimir Sementsov-Ogievskiy cur = hbi->cur[i] & hb->levels[i][pos]; 113e7c033c3SPaolo Bonzini } while (cur == 0); 114e7c033c3SPaolo Bonzini 115e7c033c3SPaolo Bonzini /* Check for end of iteration. We always use fewer than BITS_PER_LONG 116e7c033c3SPaolo Bonzini * bits in the level 0 bitmap; thus we can repurpose the most significant 117e7c033c3SPaolo Bonzini * bit as a sentinel. The sentinel is set in hbitmap_alloc and ensures 118e7c033c3SPaolo Bonzini * that the above loop ends even without an explicit check on i. 119e7c033c3SPaolo Bonzini */ 120e7c033c3SPaolo Bonzini 121e7c033c3SPaolo Bonzini if (i == 0 && cur == (1UL << (BITS_PER_LONG - 1))) { 122e7c033c3SPaolo Bonzini return 0; 123e7c033c3SPaolo Bonzini } 124e7c033c3SPaolo Bonzini for (; i < HBITMAP_LEVELS - 1; i++) { 125e7c033c3SPaolo Bonzini /* Shift back pos to the left, matching the right shifts above. 126e7c033c3SPaolo Bonzini * The index of this word's least significant set bit provides 127e7c033c3SPaolo Bonzini * the low-order bits. 128e7c033c3SPaolo Bonzini */ 12918331e7cSRichard Henderson assert(cur); 13018331e7cSRichard Henderson pos = (pos << BITS_PER_LEVEL) + ctzl(cur); 131e7c033c3SPaolo Bonzini hbi->cur[i] = cur & (cur - 1); 132e7c033c3SPaolo Bonzini 133e7c033c3SPaolo Bonzini /* Set up next level for iteration. */ 134e7c033c3SPaolo Bonzini cur = hb->levels[i + 1][pos]; 135e7c033c3SPaolo Bonzini } 136e7c033c3SPaolo Bonzini 137e7c033c3SPaolo Bonzini hbi->pos = pos; 138e7c033c3SPaolo Bonzini trace_hbitmap_iter_skip_words(hbi->hb, hbi, pos, cur); 139e7c033c3SPaolo Bonzini 140e7c033c3SPaolo Bonzini assert(cur); 141e7c033c3SPaolo Bonzini return cur; 142e7c033c3SPaolo Bonzini } 143e7c033c3SPaolo Bonzini 144f63ea4e9SVladimir Sementsov-Ogievskiy int64_t hbitmap_iter_next(HBitmapIter *hbi) 145f63ea4e9SVladimir Sementsov-Ogievskiy { 146f63ea4e9SVladimir Sementsov-Ogievskiy unsigned long cur = hbi->cur[HBITMAP_LEVELS - 1] & 147f63ea4e9SVladimir Sementsov-Ogievskiy hbi->hb->levels[HBITMAP_LEVELS - 1][hbi->pos]; 148f63ea4e9SVladimir Sementsov-Ogievskiy int64_t item; 149f63ea4e9SVladimir Sementsov-Ogievskiy 150f63ea4e9SVladimir Sementsov-Ogievskiy if (cur == 0) { 151f63ea4e9SVladimir Sementsov-Ogievskiy cur = hbitmap_iter_skip_words(hbi); 152f63ea4e9SVladimir Sementsov-Ogievskiy if (cur == 0) { 153f63ea4e9SVladimir Sementsov-Ogievskiy return -1; 154f63ea4e9SVladimir Sementsov-Ogievskiy } 155f63ea4e9SVladimir Sementsov-Ogievskiy } 156f63ea4e9SVladimir Sementsov-Ogievskiy 157f63ea4e9SVladimir Sementsov-Ogievskiy /* The next call will resume work from the next bit. */ 158f63ea4e9SVladimir Sementsov-Ogievskiy hbi->cur[HBITMAP_LEVELS - 1] = cur & (cur - 1); 159f63ea4e9SVladimir Sementsov-Ogievskiy item = ((uint64_t)hbi->pos << BITS_PER_LEVEL) + ctzl(cur); 160f63ea4e9SVladimir Sementsov-Ogievskiy 161f63ea4e9SVladimir Sementsov-Ogievskiy return item << hbi->granularity; 162f63ea4e9SVladimir Sementsov-Ogievskiy } 163f63ea4e9SVladimir Sementsov-Ogievskiy 164e7c033c3SPaolo Bonzini void hbitmap_iter_init(HBitmapIter *hbi, const HBitmap *hb, uint64_t first) 165e7c033c3SPaolo Bonzini { 166e7c033c3SPaolo Bonzini unsigned i, bit; 167e7c033c3SPaolo Bonzini uint64_t pos; 168e7c033c3SPaolo Bonzini 169e7c033c3SPaolo Bonzini hbi->hb = hb; 170e7c033c3SPaolo Bonzini pos = first >> hb->granularity; 1711b095244SPaolo Bonzini assert(pos < hb->size); 172e7c033c3SPaolo Bonzini hbi->pos = pos >> BITS_PER_LEVEL; 173e7c033c3SPaolo Bonzini hbi->granularity = hb->granularity; 174e7c033c3SPaolo Bonzini 175e7c033c3SPaolo Bonzini for (i = HBITMAP_LEVELS; i-- > 0; ) { 176e7c033c3SPaolo Bonzini bit = pos & (BITS_PER_LONG - 1); 177e7c033c3SPaolo Bonzini pos >>= BITS_PER_LEVEL; 178e7c033c3SPaolo Bonzini 179e7c033c3SPaolo Bonzini /* Drop bits representing items before first. */ 180e7c033c3SPaolo Bonzini hbi->cur[i] = hb->levels[i][pos] & ~((1UL << bit) - 1); 181e7c033c3SPaolo Bonzini 182e7c033c3SPaolo Bonzini /* We have already added level i+1, so the lowest set bit has 183e7c033c3SPaolo Bonzini * been processed. Clear it. 184e7c033c3SPaolo Bonzini */ 185e7c033c3SPaolo Bonzini if (i != HBITMAP_LEVELS - 1) { 186e7c033c3SPaolo Bonzini hbi->cur[i] &= ~(1UL << bit); 187e7c033c3SPaolo Bonzini } 188e7c033c3SPaolo Bonzini } 189e7c033c3SPaolo Bonzini } 190e7c033c3SPaolo Bonzini 19156207df5SVladimir Sementsov-Ogievskiy int64_t hbitmap_next_zero(const HBitmap *hb, uint64_t start) 19256207df5SVladimir Sementsov-Ogievskiy { 19356207df5SVladimir Sementsov-Ogievskiy size_t pos = (start >> hb->granularity) >> BITS_PER_LEVEL; 19456207df5SVladimir Sementsov-Ogievskiy unsigned long *last_lev = hb->levels[HBITMAP_LEVELS - 1]; 19556207df5SVladimir Sementsov-Ogievskiy uint64_t sz = hb->sizes[HBITMAP_LEVELS - 1]; 19656207df5SVladimir Sementsov-Ogievskiy unsigned long cur = last_lev[pos]; 19756207df5SVladimir Sementsov-Ogievskiy unsigned start_bit_offset = 19856207df5SVladimir Sementsov-Ogievskiy (start >> hb->granularity) & (BITS_PER_LONG - 1); 19956207df5SVladimir Sementsov-Ogievskiy int64_t res; 20056207df5SVladimir Sementsov-Ogievskiy 20156207df5SVladimir Sementsov-Ogievskiy cur |= (1UL << start_bit_offset) - 1; 20256207df5SVladimir Sementsov-Ogievskiy assert((start >> hb->granularity) < hb->size); 20356207df5SVladimir Sementsov-Ogievskiy 20456207df5SVladimir Sementsov-Ogievskiy if (cur == (unsigned long)-1) { 20556207df5SVladimir Sementsov-Ogievskiy do { 20656207df5SVladimir Sementsov-Ogievskiy pos++; 20756207df5SVladimir Sementsov-Ogievskiy } while (pos < sz && last_lev[pos] == (unsigned long)-1); 20856207df5SVladimir Sementsov-Ogievskiy 20956207df5SVladimir Sementsov-Ogievskiy if (pos >= sz) { 21056207df5SVladimir Sementsov-Ogievskiy return -1; 21156207df5SVladimir Sementsov-Ogievskiy } 21256207df5SVladimir Sementsov-Ogievskiy 21356207df5SVladimir Sementsov-Ogievskiy cur = last_lev[pos]; 21456207df5SVladimir Sementsov-Ogievskiy } 21556207df5SVladimir Sementsov-Ogievskiy 21656207df5SVladimir Sementsov-Ogievskiy res = (pos << BITS_PER_LEVEL) + ctol(cur); 21756207df5SVladimir Sementsov-Ogievskiy if (res >= hb->size) { 21856207df5SVladimir Sementsov-Ogievskiy return -1; 21956207df5SVladimir Sementsov-Ogievskiy } 22056207df5SVladimir Sementsov-Ogievskiy 22156207df5SVladimir Sementsov-Ogievskiy res = res << hb->granularity; 22256207df5SVladimir Sementsov-Ogievskiy if (res < start) { 22356207df5SVladimir Sementsov-Ogievskiy assert(((start - res) >> hb->granularity) == 0); 22456207df5SVladimir Sementsov-Ogievskiy return start; 22556207df5SVladimir Sementsov-Ogievskiy } 22656207df5SVladimir Sementsov-Ogievskiy 22756207df5SVladimir Sementsov-Ogievskiy return res; 22856207df5SVladimir Sementsov-Ogievskiy } 22956207df5SVladimir Sementsov-Ogievskiy 230e7c033c3SPaolo Bonzini bool hbitmap_empty(const HBitmap *hb) 231e7c033c3SPaolo Bonzini { 232e7c033c3SPaolo Bonzini return hb->count == 0; 233e7c033c3SPaolo Bonzini } 234e7c033c3SPaolo Bonzini 235e7c033c3SPaolo Bonzini int hbitmap_granularity(const HBitmap *hb) 236e7c033c3SPaolo Bonzini { 237e7c033c3SPaolo Bonzini return hb->granularity; 238e7c033c3SPaolo Bonzini } 239e7c033c3SPaolo Bonzini 240e7c033c3SPaolo Bonzini uint64_t hbitmap_count(const HBitmap *hb) 241e7c033c3SPaolo Bonzini { 242e7c033c3SPaolo Bonzini return hb->count << hb->granularity; 243e7c033c3SPaolo Bonzini } 244e7c033c3SPaolo Bonzini 245e7c033c3SPaolo Bonzini /* Count the number of set bits between start and end, not accounting for 246e7c033c3SPaolo Bonzini * the granularity. Also an example of how to use hbitmap_iter_next_word. 247e7c033c3SPaolo Bonzini */ 248e7c033c3SPaolo Bonzini static uint64_t hb_count_between(HBitmap *hb, uint64_t start, uint64_t last) 249e7c033c3SPaolo Bonzini { 250e7c033c3SPaolo Bonzini HBitmapIter hbi; 251e7c033c3SPaolo Bonzini uint64_t count = 0; 252e7c033c3SPaolo Bonzini uint64_t end = last + 1; 253e7c033c3SPaolo Bonzini unsigned long cur; 254e7c033c3SPaolo Bonzini size_t pos; 255e7c033c3SPaolo Bonzini 256e7c033c3SPaolo Bonzini hbitmap_iter_init(&hbi, hb, start << hb->granularity); 257e7c033c3SPaolo Bonzini for (;;) { 258e7c033c3SPaolo Bonzini pos = hbitmap_iter_next_word(&hbi, &cur); 259e7c033c3SPaolo Bonzini if (pos >= (end >> BITS_PER_LEVEL)) { 260e7c033c3SPaolo Bonzini break; 261e7c033c3SPaolo Bonzini } 262591b320aSPeter Maydell count += ctpopl(cur); 263e7c033c3SPaolo Bonzini } 264e7c033c3SPaolo Bonzini 265e7c033c3SPaolo Bonzini if (pos == (end >> BITS_PER_LEVEL)) { 266e7c033c3SPaolo Bonzini /* Drop bits representing the END-th and subsequent items. */ 267e7c033c3SPaolo Bonzini int bit = end & (BITS_PER_LONG - 1); 268e7c033c3SPaolo Bonzini cur &= (1UL << bit) - 1; 269591b320aSPeter Maydell count += ctpopl(cur); 270e7c033c3SPaolo Bonzini } 271e7c033c3SPaolo Bonzini 272e7c033c3SPaolo Bonzini return count; 273e7c033c3SPaolo Bonzini } 274e7c033c3SPaolo Bonzini 275e7c033c3SPaolo Bonzini /* Setting starts at the last layer and propagates up if an element 27607ac4cdbSFam Zheng * changes. 277e7c033c3SPaolo Bonzini */ 278e7c033c3SPaolo Bonzini static inline bool hb_set_elem(unsigned long *elem, uint64_t start, uint64_t last) 279e7c033c3SPaolo Bonzini { 280e7c033c3SPaolo Bonzini unsigned long mask; 28107ac4cdbSFam Zheng unsigned long old; 282e7c033c3SPaolo Bonzini 283e7c033c3SPaolo Bonzini assert((last >> BITS_PER_LEVEL) == (start >> BITS_PER_LEVEL)); 284e7c033c3SPaolo Bonzini assert(start <= last); 285e7c033c3SPaolo Bonzini 286e7c033c3SPaolo Bonzini mask = 2UL << (last & (BITS_PER_LONG - 1)); 287e7c033c3SPaolo Bonzini mask -= 1UL << (start & (BITS_PER_LONG - 1)); 28807ac4cdbSFam Zheng old = *elem; 289e7c033c3SPaolo Bonzini *elem |= mask; 29007ac4cdbSFam Zheng return old != *elem; 291e7c033c3SPaolo Bonzini } 292e7c033c3SPaolo Bonzini 29307ac4cdbSFam Zheng /* The recursive workhorse (the depth is limited to HBITMAP_LEVELS)... 29407ac4cdbSFam Zheng * Returns true if at least one bit is changed. */ 29507ac4cdbSFam Zheng static bool hb_set_between(HBitmap *hb, int level, uint64_t start, 29607ac4cdbSFam Zheng uint64_t last) 297e7c033c3SPaolo Bonzini { 298e7c033c3SPaolo Bonzini size_t pos = start >> BITS_PER_LEVEL; 299e7c033c3SPaolo Bonzini size_t lastpos = last >> BITS_PER_LEVEL; 300e7c033c3SPaolo Bonzini bool changed = false; 301e7c033c3SPaolo Bonzini size_t i; 302e7c033c3SPaolo Bonzini 303e7c033c3SPaolo Bonzini i = pos; 304e7c033c3SPaolo Bonzini if (i < lastpos) { 305e7c033c3SPaolo Bonzini uint64_t next = (start | (BITS_PER_LONG - 1)) + 1; 306e7c033c3SPaolo Bonzini changed |= hb_set_elem(&hb->levels[level][i], start, next - 1); 307e7c033c3SPaolo Bonzini for (;;) { 308e7c033c3SPaolo Bonzini start = next; 309e7c033c3SPaolo Bonzini next += BITS_PER_LONG; 310e7c033c3SPaolo Bonzini if (++i == lastpos) { 311e7c033c3SPaolo Bonzini break; 312e7c033c3SPaolo Bonzini } 313e7c033c3SPaolo Bonzini changed |= (hb->levels[level][i] == 0); 314e7c033c3SPaolo Bonzini hb->levels[level][i] = ~0UL; 315e7c033c3SPaolo Bonzini } 316e7c033c3SPaolo Bonzini } 317e7c033c3SPaolo Bonzini changed |= hb_set_elem(&hb->levels[level][i], start, last); 318e7c033c3SPaolo Bonzini 319e7c033c3SPaolo Bonzini /* If there was any change in this layer, we may have to update 320e7c033c3SPaolo Bonzini * the one above. 321e7c033c3SPaolo Bonzini */ 322e7c033c3SPaolo Bonzini if (level > 0 && changed) { 323e7c033c3SPaolo Bonzini hb_set_between(hb, level - 1, pos, lastpos); 324e7c033c3SPaolo Bonzini } 32507ac4cdbSFam Zheng return changed; 326e7c033c3SPaolo Bonzini } 327e7c033c3SPaolo Bonzini 328e7c033c3SPaolo Bonzini void hbitmap_set(HBitmap *hb, uint64_t start, uint64_t count) 329e7c033c3SPaolo Bonzini { 330e7c033c3SPaolo Bonzini /* Compute range in the last layer. */ 33107ac4cdbSFam Zheng uint64_t first, n; 332e7c033c3SPaolo Bonzini uint64_t last = start + count - 1; 333e7c033c3SPaolo Bonzini 334e7c033c3SPaolo Bonzini trace_hbitmap_set(hb, start, count, 335e7c033c3SPaolo Bonzini start >> hb->granularity, last >> hb->granularity); 336e7c033c3SPaolo Bonzini 33707ac4cdbSFam Zheng first = start >> hb->granularity; 338e7c033c3SPaolo Bonzini last >>= hb->granularity; 3390e321191SVladimir Sementsov-Ogievskiy assert(last < hb->size); 34007ac4cdbSFam Zheng n = last - first + 1; 341e7c033c3SPaolo Bonzini 34207ac4cdbSFam Zheng hb->count += n - hb_count_between(hb, first, last); 34307ac4cdbSFam Zheng if (hb_set_between(hb, HBITMAP_LEVELS - 1, first, last) && 34407ac4cdbSFam Zheng hb->meta) { 34507ac4cdbSFam Zheng hbitmap_set(hb->meta, start, count); 34607ac4cdbSFam Zheng } 347e7c033c3SPaolo Bonzini } 348e7c033c3SPaolo Bonzini 349e7c033c3SPaolo Bonzini /* Resetting works the other way round: propagate up if the new 350e7c033c3SPaolo Bonzini * value is zero. 351e7c033c3SPaolo Bonzini */ 352e7c033c3SPaolo Bonzini static inline bool hb_reset_elem(unsigned long *elem, uint64_t start, uint64_t last) 353e7c033c3SPaolo Bonzini { 354e7c033c3SPaolo Bonzini unsigned long mask; 355e7c033c3SPaolo Bonzini bool blanked; 356e7c033c3SPaolo Bonzini 357e7c033c3SPaolo Bonzini assert((last >> BITS_PER_LEVEL) == (start >> BITS_PER_LEVEL)); 358e7c033c3SPaolo Bonzini assert(start <= last); 359e7c033c3SPaolo Bonzini 360e7c033c3SPaolo Bonzini mask = 2UL << (last & (BITS_PER_LONG - 1)); 361e7c033c3SPaolo Bonzini mask -= 1UL << (start & (BITS_PER_LONG - 1)); 362e7c033c3SPaolo Bonzini blanked = *elem != 0 && ((*elem & ~mask) == 0); 363e7c033c3SPaolo Bonzini *elem &= ~mask; 364e7c033c3SPaolo Bonzini return blanked; 365e7c033c3SPaolo Bonzini } 366e7c033c3SPaolo Bonzini 36707ac4cdbSFam Zheng /* The recursive workhorse (the depth is limited to HBITMAP_LEVELS)... 36807ac4cdbSFam Zheng * Returns true if at least one bit is changed. */ 36907ac4cdbSFam Zheng static bool hb_reset_between(HBitmap *hb, int level, uint64_t start, 37007ac4cdbSFam Zheng uint64_t last) 371e7c033c3SPaolo Bonzini { 372e7c033c3SPaolo Bonzini size_t pos = start >> BITS_PER_LEVEL; 373e7c033c3SPaolo Bonzini size_t lastpos = last >> BITS_PER_LEVEL; 374e7c033c3SPaolo Bonzini bool changed = false; 375e7c033c3SPaolo Bonzini size_t i; 376e7c033c3SPaolo Bonzini 377e7c033c3SPaolo Bonzini i = pos; 378e7c033c3SPaolo Bonzini if (i < lastpos) { 379e7c033c3SPaolo Bonzini uint64_t next = (start | (BITS_PER_LONG - 1)) + 1; 380e7c033c3SPaolo Bonzini 381e7c033c3SPaolo Bonzini /* Here we need a more complex test than when setting bits. Even if 382e7c033c3SPaolo Bonzini * something was changed, we must not blank bits in the upper level 383e7c033c3SPaolo Bonzini * unless the lower-level word became entirely zero. So, remove pos 384e7c033c3SPaolo Bonzini * from the upper-level range if bits remain set. 385e7c033c3SPaolo Bonzini */ 386e7c033c3SPaolo Bonzini if (hb_reset_elem(&hb->levels[level][i], start, next - 1)) { 387e7c033c3SPaolo Bonzini changed = true; 388e7c033c3SPaolo Bonzini } else { 389e7c033c3SPaolo Bonzini pos++; 390e7c033c3SPaolo Bonzini } 391e7c033c3SPaolo Bonzini 392e7c033c3SPaolo Bonzini for (;;) { 393e7c033c3SPaolo Bonzini start = next; 394e7c033c3SPaolo Bonzini next += BITS_PER_LONG; 395e7c033c3SPaolo Bonzini if (++i == lastpos) { 396e7c033c3SPaolo Bonzini break; 397e7c033c3SPaolo Bonzini } 398e7c033c3SPaolo Bonzini changed |= (hb->levels[level][i] != 0); 399e7c033c3SPaolo Bonzini hb->levels[level][i] = 0UL; 400e7c033c3SPaolo Bonzini } 401e7c033c3SPaolo Bonzini } 402e7c033c3SPaolo Bonzini 403e7c033c3SPaolo Bonzini /* Same as above, this time for lastpos. */ 404e7c033c3SPaolo Bonzini if (hb_reset_elem(&hb->levels[level][i], start, last)) { 405e7c033c3SPaolo Bonzini changed = true; 406e7c033c3SPaolo Bonzini } else { 407e7c033c3SPaolo Bonzini lastpos--; 408e7c033c3SPaolo Bonzini } 409e7c033c3SPaolo Bonzini 410e7c033c3SPaolo Bonzini if (level > 0 && changed) { 411e7c033c3SPaolo Bonzini hb_reset_between(hb, level - 1, pos, lastpos); 412e7c033c3SPaolo Bonzini } 41307ac4cdbSFam Zheng 41407ac4cdbSFam Zheng return changed; 41507ac4cdbSFam Zheng 416e7c033c3SPaolo Bonzini } 417e7c033c3SPaolo Bonzini 418e7c033c3SPaolo Bonzini void hbitmap_reset(HBitmap *hb, uint64_t start, uint64_t count) 419e7c033c3SPaolo Bonzini { 420e7c033c3SPaolo Bonzini /* Compute range in the last layer. */ 42107ac4cdbSFam Zheng uint64_t first; 422e7c033c3SPaolo Bonzini uint64_t last = start + count - 1; 423e7c033c3SPaolo Bonzini 424e7c033c3SPaolo Bonzini trace_hbitmap_reset(hb, start, count, 425e7c033c3SPaolo Bonzini start >> hb->granularity, last >> hb->granularity); 426e7c033c3SPaolo Bonzini 42707ac4cdbSFam Zheng first = start >> hb->granularity; 428e7c033c3SPaolo Bonzini last >>= hb->granularity; 4290e321191SVladimir Sementsov-Ogievskiy assert(last < hb->size); 430e7c033c3SPaolo Bonzini 43107ac4cdbSFam Zheng hb->count -= hb_count_between(hb, first, last); 43207ac4cdbSFam Zheng if (hb_reset_between(hb, HBITMAP_LEVELS - 1, first, last) && 43307ac4cdbSFam Zheng hb->meta) { 43407ac4cdbSFam Zheng hbitmap_set(hb->meta, start, count); 43507ac4cdbSFam Zheng } 436e7c033c3SPaolo Bonzini } 437e7c033c3SPaolo Bonzini 438c6a8c328SWen Congyang void hbitmap_reset_all(HBitmap *hb) 439c6a8c328SWen Congyang { 440c6a8c328SWen Congyang unsigned int i; 441c6a8c328SWen Congyang 442c6a8c328SWen Congyang /* Same as hbitmap_alloc() except for memset() instead of malloc() */ 443c6a8c328SWen Congyang for (i = HBITMAP_LEVELS; --i >= 1; ) { 444c6a8c328SWen Congyang memset(hb->levels[i], 0, hb->sizes[i] * sizeof(unsigned long)); 445c6a8c328SWen Congyang } 446c6a8c328SWen Congyang 447c6a8c328SWen Congyang hb->levels[0][0] = 1UL << (BITS_PER_LONG - 1); 448c6a8c328SWen Congyang hb->count = 0; 449c6a8c328SWen Congyang } 450c6a8c328SWen Congyang 45120a579deSMax Reitz bool hbitmap_is_serializable(const HBitmap *hb) 45220a579deSMax Reitz { 45320a579deSMax Reitz /* Every serialized chunk must be aligned to 64 bits so that endianness 45420a579deSMax Reitz * requirements can be fulfilled on both 64 bit and 32 bit hosts. 455ecbfa281SEric Blake * We have hbitmap_serialization_align() which converts this 45620a579deSMax Reitz * alignment requirement from bitmap bits to items covered (e.g. sectors). 45720a579deSMax Reitz * That value is: 45820a579deSMax Reitz * 64 << hb->granularity 45920a579deSMax Reitz * Since this value must not exceed UINT64_MAX, hb->granularity must be 46020a579deSMax Reitz * less than 58 (== 64 - 6, where 6 is ld(64), i.e. 1 << 6 == 64). 46120a579deSMax Reitz * 462ecbfa281SEric Blake * In order for hbitmap_serialization_align() to always return a 46320a579deSMax Reitz * meaningful value, bitmaps that are to be serialized must have a 46420a579deSMax Reitz * granularity of less than 58. */ 46520a579deSMax Reitz 46620a579deSMax Reitz return hb->granularity < 58; 46720a579deSMax Reitz } 46820a579deSMax Reitz 469e7c033c3SPaolo Bonzini bool hbitmap_get(const HBitmap *hb, uint64_t item) 470e7c033c3SPaolo Bonzini { 471e7c033c3SPaolo Bonzini /* Compute position and bit in the last layer. */ 472e7c033c3SPaolo Bonzini uint64_t pos = item >> hb->granularity; 473e7c033c3SPaolo Bonzini unsigned long bit = 1UL << (pos & (BITS_PER_LONG - 1)); 4740e321191SVladimir Sementsov-Ogievskiy assert(pos < hb->size); 475e7c033c3SPaolo Bonzini 476e7c033c3SPaolo Bonzini return (hb->levels[HBITMAP_LEVELS - 1][pos >> BITS_PER_LEVEL] & bit) != 0; 477e7c033c3SPaolo Bonzini } 478e7c033c3SPaolo Bonzini 479ecbfa281SEric Blake uint64_t hbitmap_serialization_align(const HBitmap *hb) 4808258888eSVladimir Sementsov-Ogievskiy { 48120a579deSMax Reitz assert(hbitmap_is_serializable(hb)); 4826725f887SMax Reitz 4838258888eSVladimir Sementsov-Ogievskiy /* Require at least 64 bit granularity to be safe on both 64 bit and 32 bit 4848258888eSVladimir Sementsov-Ogievskiy * hosts. */ 4856725f887SMax Reitz return UINT64_C(64) << hb->granularity; 4868258888eSVladimir Sementsov-Ogievskiy } 4878258888eSVladimir Sementsov-Ogievskiy 4888258888eSVladimir Sementsov-Ogievskiy /* Start should be aligned to serialization granularity, chunk size should be 4898258888eSVladimir Sementsov-Ogievskiy * aligned to serialization granularity too, except for last chunk. 4908258888eSVladimir Sementsov-Ogievskiy */ 4918258888eSVladimir Sementsov-Ogievskiy static void serialization_chunk(const HBitmap *hb, 4928258888eSVladimir Sementsov-Ogievskiy uint64_t start, uint64_t count, 4938258888eSVladimir Sementsov-Ogievskiy unsigned long **first_el, uint64_t *el_count) 4948258888eSVladimir Sementsov-Ogievskiy { 4958258888eSVladimir Sementsov-Ogievskiy uint64_t last = start + count - 1; 496ecbfa281SEric Blake uint64_t gran = hbitmap_serialization_align(hb); 4978258888eSVladimir Sementsov-Ogievskiy 4988258888eSVladimir Sementsov-Ogievskiy assert((start & (gran - 1)) == 0); 4998258888eSVladimir Sementsov-Ogievskiy assert((last >> hb->granularity) < hb->size); 5008258888eSVladimir Sementsov-Ogievskiy if ((last >> hb->granularity) != hb->size - 1) { 5018258888eSVladimir Sementsov-Ogievskiy assert((count & (gran - 1)) == 0); 5028258888eSVladimir Sementsov-Ogievskiy } 5038258888eSVladimir Sementsov-Ogievskiy 5048258888eSVladimir Sementsov-Ogievskiy start = (start >> hb->granularity) >> BITS_PER_LEVEL; 5058258888eSVladimir Sementsov-Ogievskiy last = (last >> hb->granularity) >> BITS_PER_LEVEL; 5068258888eSVladimir Sementsov-Ogievskiy 5078258888eSVladimir Sementsov-Ogievskiy *first_el = &hb->levels[HBITMAP_LEVELS - 1][start]; 5088258888eSVladimir Sementsov-Ogievskiy *el_count = last - start + 1; 5098258888eSVladimir Sementsov-Ogievskiy } 5108258888eSVladimir Sementsov-Ogievskiy 5118258888eSVladimir Sementsov-Ogievskiy uint64_t hbitmap_serialization_size(const HBitmap *hb, 5128258888eSVladimir Sementsov-Ogievskiy uint64_t start, uint64_t count) 5138258888eSVladimir Sementsov-Ogievskiy { 5148258888eSVladimir Sementsov-Ogievskiy uint64_t el_count; 5158258888eSVladimir Sementsov-Ogievskiy unsigned long *cur; 5168258888eSVladimir Sementsov-Ogievskiy 5178258888eSVladimir Sementsov-Ogievskiy if (!count) { 5188258888eSVladimir Sementsov-Ogievskiy return 0; 5198258888eSVladimir Sementsov-Ogievskiy } 5208258888eSVladimir Sementsov-Ogievskiy serialization_chunk(hb, start, count, &cur, &el_count); 5218258888eSVladimir Sementsov-Ogievskiy 5228258888eSVladimir Sementsov-Ogievskiy return el_count * sizeof(unsigned long); 5238258888eSVladimir Sementsov-Ogievskiy } 5248258888eSVladimir Sementsov-Ogievskiy 5258258888eSVladimir Sementsov-Ogievskiy void hbitmap_serialize_part(const HBitmap *hb, uint8_t *buf, 5268258888eSVladimir Sementsov-Ogievskiy uint64_t start, uint64_t count) 5278258888eSVladimir Sementsov-Ogievskiy { 5288258888eSVladimir Sementsov-Ogievskiy uint64_t el_count; 5298258888eSVladimir Sementsov-Ogievskiy unsigned long *cur, *end; 5308258888eSVladimir Sementsov-Ogievskiy 5318258888eSVladimir Sementsov-Ogievskiy if (!count) { 5328258888eSVladimir Sementsov-Ogievskiy return; 5338258888eSVladimir Sementsov-Ogievskiy } 5348258888eSVladimir Sementsov-Ogievskiy serialization_chunk(hb, start, count, &cur, &el_count); 5358258888eSVladimir Sementsov-Ogievskiy end = cur + el_count; 5368258888eSVladimir Sementsov-Ogievskiy 5378258888eSVladimir Sementsov-Ogievskiy while (cur != end) { 5388258888eSVladimir Sementsov-Ogievskiy unsigned long el = 5398258888eSVladimir Sementsov-Ogievskiy (BITS_PER_LONG == 32 ? cpu_to_le32(*cur) : cpu_to_le64(*cur)); 5408258888eSVladimir Sementsov-Ogievskiy 5418258888eSVladimir Sementsov-Ogievskiy memcpy(buf, &el, sizeof(el)); 5428258888eSVladimir Sementsov-Ogievskiy buf += sizeof(el); 5438258888eSVladimir Sementsov-Ogievskiy cur++; 5448258888eSVladimir Sementsov-Ogievskiy } 5458258888eSVladimir Sementsov-Ogievskiy } 5468258888eSVladimir Sementsov-Ogievskiy 5478258888eSVladimir Sementsov-Ogievskiy void hbitmap_deserialize_part(HBitmap *hb, uint8_t *buf, 5488258888eSVladimir Sementsov-Ogievskiy uint64_t start, uint64_t count, 5498258888eSVladimir Sementsov-Ogievskiy bool finish) 5508258888eSVladimir Sementsov-Ogievskiy { 5518258888eSVladimir Sementsov-Ogievskiy uint64_t el_count; 5528258888eSVladimir Sementsov-Ogievskiy unsigned long *cur, *end; 5538258888eSVladimir Sementsov-Ogievskiy 5548258888eSVladimir Sementsov-Ogievskiy if (!count) { 5558258888eSVladimir Sementsov-Ogievskiy return; 5568258888eSVladimir Sementsov-Ogievskiy } 5578258888eSVladimir Sementsov-Ogievskiy serialization_chunk(hb, start, count, &cur, &el_count); 5588258888eSVladimir Sementsov-Ogievskiy end = cur + el_count; 5598258888eSVladimir Sementsov-Ogievskiy 5608258888eSVladimir Sementsov-Ogievskiy while (cur != end) { 5618258888eSVladimir Sementsov-Ogievskiy memcpy(cur, buf, sizeof(*cur)); 5628258888eSVladimir Sementsov-Ogievskiy 5638258888eSVladimir Sementsov-Ogievskiy if (BITS_PER_LONG == 32) { 5648258888eSVladimir Sementsov-Ogievskiy le32_to_cpus((uint32_t *)cur); 5658258888eSVladimir Sementsov-Ogievskiy } else { 5668258888eSVladimir Sementsov-Ogievskiy le64_to_cpus((uint64_t *)cur); 5678258888eSVladimir Sementsov-Ogievskiy } 5688258888eSVladimir Sementsov-Ogievskiy 5698258888eSVladimir Sementsov-Ogievskiy buf += sizeof(unsigned long); 5708258888eSVladimir Sementsov-Ogievskiy cur++; 5718258888eSVladimir Sementsov-Ogievskiy } 5728258888eSVladimir Sementsov-Ogievskiy if (finish) { 5738258888eSVladimir Sementsov-Ogievskiy hbitmap_deserialize_finish(hb); 5748258888eSVladimir Sementsov-Ogievskiy } 5758258888eSVladimir Sementsov-Ogievskiy } 5768258888eSVladimir Sementsov-Ogievskiy 5778258888eSVladimir Sementsov-Ogievskiy void hbitmap_deserialize_zeroes(HBitmap *hb, uint64_t start, uint64_t count, 5788258888eSVladimir Sementsov-Ogievskiy bool finish) 5798258888eSVladimir Sementsov-Ogievskiy { 5808258888eSVladimir Sementsov-Ogievskiy uint64_t el_count; 5818258888eSVladimir Sementsov-Ogievskiy unsigned long *first; 5828258888eSVladimir Sementsov-Ogievskiy 5838258888eSVladimir Sementsov-Ogievskiy if (!count) { 5848258888eSVladimir Sementsov-Ogievskiy return; 5858258888eSVladimir Sementsov-Ogievskiy } 5868258888eSVladimir Sementsov-Ogievskiy serialization_chunk(hb, start, count, &first, &el_count); 5878258888eSVladimir Sementsov-Ogievskiy 5888258888eSVladimir Sementsov-Ogievskiy memset(first, 0, el_count * sizeof(unsigned long)); 5898258888eSVladimir Sementsov-Ogievskiy if (finish) { 5908258888eSVladimir Sementsov-Ogievskiy hbitmap_deserialize_finish(hb); 5918258888eSVladimir Sementsov-Ogievskiy } 5928258888eSVladimir Sementsov-Ogievskiy } 5938258888eSVladimir Sementsov-Ogievskiy 5946bdc8b71SVladimir Sementsov-Ogievskiy void hbitmap_deserialize_ones(HBitmap *hb, uint64_t start, uint64_t count, 5956bdc8b71SVladimir Sementsov-Ogievskiy bool finish) 5966bdc8b71SVladimir Sementsov-Ogievskiy { 5976bdc8b71SVladimir Sementsov-Ogievskiy uint64_t el_count; 5986bdc8b71SVladimir Sementsov-Ogievskiy unsigned long *first; 5996bdc8b71SVladimir Sementsov-Ogievskiy 6006bdc8b71SVladimir Sementsov-Ogievskiy if (!count) { 6016bdc8b71SVladimir Sementsov-Ogievskiy return; 6026bdc8b71SVladimir Sementsov-Ogievskiy } 6036bdc8b71SVladimir Sementsov-Ogievskiy serialization_chunk(hb, start, count, &first, &el_count); 6046bdc8b71SVladimir Sementsov-Ogievskiy 6056bdc8b71SVladimir Sementsov-Ogievskiy memset(first, 0xff, el_count * sizeof(unsigned long)); 6066bdc8b71SVladimir Sementsov-Ogievskiy if (finish) { 6076bdc8b71SVladimir Sementsov-Ogievskiy hbitmap_deserialize_finish(hb); 6086bdc8b71SVladimir Sementsov-Ogievskiy } 6096bdc8b71SVladimir Sementsov-Ogievskiy } 6106bdc8b71SVladimir Sementsov-Ogievskiy 6118258888eSVladimir Sementsov-Ogievskiy void hbitmap_deserialize_finish(HBitmap *bitmap) 6128258888eSVladimir Sementsov-Ogievskiy { 6138258888eSVladimir Sementsov-Ogievskiy int64_t i, size, prev_size; 6148258888eSVladimir Sementsov-Ogievskiy int lev; 6158258888eSVladimir Sementsov-Ogievskiy 6168258888eSVladimir Sementsov-Ogievskiy /* restore levels starting from penultimate to zero level, assuming 6178258888eSVladimir Sementsov-Ogievskiy * that the last level is ok */ 6188258888eSVladimir Sementsov-Ogievskiy size = MAX((bitmap->size + BITS_PER_LONG - 1) >> BITS_PER_LEVEL, 1); 6198258888eSVladimir Sementsov-Ogievskiy for (lev = HBITMAP_LEVELS - 1; lev-- > 0; ) { 6208258888eSVladimir Sementsov-Ogievskiy prev_size = size; 6218258888eSVladimir Sementsov-Ogievskiy size = MAX((size + BITS_PER_LONG - 1) >> BITS_PER_LEVEL, 1); 6228258888eSVladimir Sementsov-Ogievskiy memset(bitmap->levels[lev], 0, size * sizeof(unsigned long)); 6238258888eSVladimir Sementsov-Ogievskiy 6248258888eSVladimir Sementsov-Ogievskiy for (i = 0; i < prev_size; ++i) { 6258258888eSVladimir Sementsov-Ogievskiy if (bitmap->levels[lev + 1][i]) { 6268258888eSVladimir Sementsov-Ogievskiy bitmap->levels[lev][i >> BITS_PER_LEVEL] |= 6278258888eSVladimir Sementsov-Ogievskiy 1UL << (i & (BITS_PER_LONG - 1)); 6288258888eSVladimir Sementsov-Ogievskiy } 6298258888eSVladimir Sementsov-Ogievskiy } 6308258888eSVladimir Sementsov-Ogievskiy } 6318258888eSVladimir Sementsov-Ogievskiy 6328258888eSVladimir Sementsov-Ogievskiy bitmap->levels[0][0] |= 1UL << (BITS_PER_LONG - 1); 633*3260cdffSLiang Li bitmap->count = hb_count_between(bitmap, 0, bitmap->size - 1); 6348258888eSVladimir Sementsov-Ogievskiy } 6358258888eSVladimir Sementsov-Ogievskiy 636e7c033c3SPaolo Bonzini void hbitmap_free(HBitmap *hb) 637e7c033c3SPaolo Bonzini { 638e7c033c3SPaolo Bonzini unsigned i; 63907ac4cdbSFam Zheng assert(!hb->meta); 640e7c033c3SPaolo Bonzini for (i = HBITMAP_LEVELS; i-- > 0; ) { 641e7c033c3SPaolo Bonzini g_free(hb->levels[i]); 642e7c033c3SPaolo Bonzini } 643e7c033c3SPaolo Bonzini g_free(hb); 644e7c033c3SPaolo Bonzini } 645e7c033c3SPaolo Bonzini 646e7c033c3SPaolo Bonzini HBitmap *hbitmap_alloc(uint64_t size, int granularity) 647e7c033c3SPaolo Bonzini { 648e1cf5582SMarkus Armbruster HBitmap *hb = g_new0(struct HBitmap, 1); 649e7c033c3SPaolo Bonzini unsigned i; 650e7c033c3SPaolo Bonzini 651e7c033c3SPaolo Bonzini assert(granularity >= 0 && granularity < 64); 652e7c033c3SPaolo Bonzini size = (size + (1ULL << granularity) - 1) >> granularity; 653e7c033c3SPaolo Bonzini assert(size <= ((uint64_t)1 << HBITMAP_LOG_MAX_SIZE)); 654e7c033c3SPaolo Bonzini 655e7c033c3SPaolo Bonzini hb->size = size; 656e7c033c3SPaolo Bonzini hb->granularity = granularity; 657e7c033c3SPaolo Bonzini for (i = HBITMAP_LEVELS; i-- > 0; ) { 658e7c033c3SPaolo Bonzini size = MAX((size + BITS_PER_LONG - 1) >> BITS_PER_LEVEL, 1); 6598515efbeSJohn Snow hb->sizes[i] = size; 660e1cf5582SMarkus Armbruster hb->levels[i] = g_new0(unsigned long, size); 661e7c033c3SPaolo Bonzini } 662e7c033c3SPaolo Bonzini 663e7c033c3SPaolo Bonzini /* We necessarily have free bits in level 0 due to the definition 664e7c033c3SPaolo Bonzini * of HBITMAP_LEVELS, so use one for a sentinel. This speeds up 665e7c033c3SPaolo Bonzini * hbitmap_iter_skip_words. 666e7c033c3SPaolo Bonzini */ 667e7c033c3SPaolo Bonzini assert(size == 1); 668e7c033c3SPaolo Bonzini hb->levels[0][0] |= 1UL << (BITS_PER_LONG - 1); 669e7c033c3SPaolo Bonzini return hb; 670e7c033c3SPaolo Bonzini } 671be58721dSJohn Snow 672ce1ffea8SJohn Snow void hbitmap_truncate(HBitmap *hb, uint64_t size) 673ce1ffea8SJohn Snow { 674ce1ffea8SJohn Snow bool shrink; 675ce1ffea8SJohn Snow unsigned i; 676ce1ffea8SJohn Snow uint64_t num_elements = size; 677ce1ffea8SJohn Snow uint64_t old; 678ce1ffea8SJohn Snow 679ce1ffea8SJohn Snow /* Size comes in as logical elements, adjust for granularity. */ 680ce1ffea8SJohn Snow size = (size + (1ULL << hb->granularity) - 1) >> hb->granularity; 681ce1ffea8SJohn Snow assert(size <= ((uint64_t)1 << HBITMAP_LOG_MAX_SIZE)); 682ce1ffea8SJohn Snow shrink = size < hb->size; 683ce1ffea8SJohn Snow 684ce1ffea8SJohn Snow /* bit sizes are identical; nothing to do. */ 685ce1ffea8SJohn Snow if (size == hb->size) { 686ce1ffea8SJohn Snow return; 687ce1ffea8SJohn Snow } 688ce1ffea8SJohn Snow 689ce1ffea8SJohn Snow /* If we're losing bits, let's clear those bits before we invalidate all of 690ce1ffea8SJohn Snow * our invariants. This helps keep the bitcount consistent, and will prevent 691ce1ffea8SJohn Snow * us from carrying around garbage bits beyond the end of the map. 692ce1ffea8SJohn Snow */ 693ce1ffea8SJohn Snow if (shrink) { 694ce1ffea8SJohn Snow /* Don't clear partial granularity groups; 695ce1ffea8SJohn Snow * start at the first full one. */ 6966725f887SMax Reitz uint64_t start = ROUND_UP(num_elements, UINT64_C(1) << hb->granularity); 697ce1ffea8SJohn Snow uint64_t fix_count = (hb->size << hb->granularity) - start; 698ce1ffea8SJohn Snow 699ce1ffea8SJohn Snow assert(fix_count); 700ce1ffea8SJohn Snow hbitmap_reset(hb, start, fix_count); 701ce1ffea8SJohn Snow } 702ce1ffea8SJohn Snow 703ce1ffea8SJohn Snow hb->size = size; 704ce1ffea8SJohn Snow for (i = HBITMAP_LEVELS; i-- > 0; ) { 705ce1ffea8SJohn Snow size = MAX(BITS_TO_LONGS(size), 1); 706ce1ffea8SJohn Snow if (hb->sizes[i] == size) { 707ce1ffea8SJohn Snow break; 708ce1ffea8SJohn Snow } 709ce1ffea8SJohn Snow old = hb->sizes[i]; 710ce1ffea8SJohn Snow hb->sizes[i] = size; 711ce1ffea8SJohn Snow hb->levels[i] = g_realloc(hb->levels[i], size * sizeof(unsigned long)); 712ce1ffea8SJohn Snow if (!shrink) { 713ce1ffea8SJohn Snow memset(&hb->levels[i][old], 0x00, 714ce1ffea8SJohn Snow (size - old) * sizeof(*hb->levels[i])); 715ce1ffea8SJohn Snow } 716ce1ffea8SJohn Snow } 71707ac4cdbSFam Zheng if (hb->meta) { 71807ac4cdbSFam Zheng hbitmap_truncate(hb->meta, hb->size << hb->granularity); 71907ac4cdbSFam Zheng } 720ce1ffea8SJohn Snow } 721ce1ffea8SJohn Snow 722ce1ffea8SJohn Snow 723be58721dSJohn Snow /** 724be58721dSJohn Snow * Given HBitmaps A and B, let A := A (BITOR) B. 725be58721dSJohn Snow * Bitmap B will not be modified. 726be58721dSJohn Snow * 727be58721dSJohn Snow * @return true if the merge was successful, 728be58721dSJohn Snow * false if it was not attempted. 729be58721dSJohn Snow */ 730be58721dSJohn Snow bool hbitmap_merge(HBitmap *a, const HBitmap *b) 731be58721dSJohn Snow { 732be58721dSJohn Snow int i; 733be58721dSJohn Snow uint64_t j; 734be58721dSJohn Snow 735be58721dSJohn Snow if ((a->size != b->size) || (a->granularity != b->granularity)) { 736be58721dSJohn Snow return false; 737be58721dSJohn Snow } 738be58721dSJohn Snow 739be58721dSJohn Snow if (hbitmap_count(b) == 0) { 740be58721dSJohn Snow return true; 741be58721dSJohn Snow } 742be58721dSJohn Snow 743be58721dSJohn Snow /* This merge is O(size), as BITS_PER_LONG and HBITMAP_LEVELS are constant. 744be58721dSJohn Snow * It may be possible to improve running times for sparsely populated maps 745be58721dSJohn Snow * by using hbitmap_iter_next, but this is suboptimal for dense maps. 746be58721dSJohn Snow */ 747be58721dSJohn Snow for (i = HBITMAP_LEVELS - 1; i >= 0; i--) { 748be58721dSJohn Snow for (j = 0; j < a->sizes[i]; j++) { 749be58721dSJohn Snow a->levels[i][j] |= b->levels[i][j]; 750be58721dSJohn Snow } 751be58721dSJohn Snow } 752be58721dSJohn Snow 753be58721dSJohn Snow return true; 754be58721dSJohn Snow } 75507ac4cdbSFam Zheng 75607ac4cdbSFam Zheng HBitmap *hbitmap_create_meta(HBitmap *hb, int chunk_size) 75707ac4cdbSFam Zheng { 75807ac4cdbSFam Zheng assert(!(chunk_size & (chunk_size - 1))); 75907ac4cdbSFam Zheng assert(!hb->meta); 76007ac4cdbSFam Zheng hb->meta = hbitmap_alloc(hb->size << hb->granularity, 76107ac4cdbSFam Zheng hb->granularity + ctz32(chunk_size)); 76207ac4cdbSFam Zheng return hb->meta; 76307ac4cdbSFam Zheng } 76407ac4cdbSFam Zheng 76507ac4cdbSFam Zheng void hbitmap_free_meta(HBitmap *hb) 76607ac4cdbSFam Zheng { 76707ac4cdbSFam Zheng assert(hb->meta); 76807ac4cdbSFam Zheng hbitmap_free(hb->meta); 76907ac4cdbSFam Zheng hb->meta = NULL; 77007ac4cdbSFam Zheng } 771a3b52535SVladimir Sementsov-Ogievskiy 772a3b52535SVladimir Sementsov-Ogievskiy char *hbitmap_sha256(const HBitmap *bitmap, Error **errp) 773a3b52535SVladimir Sementsov-Ogievskiy { 774a3b52535SVladimir Sementsov-Ogievskiy size_t size = bitmap->sizes[HBITMAP_LEVELS - 1] * sizeof(unsigned long); 775a3b52535SVladimir Sementsov-Ogievskiy char *data = (char *)bitmap->levels[HBITMAP_LEVELS - 1]; 776a3b52535SVladimir Sementsov-Ogievskiy char *hash = NULL; 777a3b52535SVladimir Sementsov-Ogievskiy qcrypto_hash_digest(QCRYPTO_HASH_ALG_SHA256, data, size, &hash, errp); 778a3b52535SVladimir Sementsov-Ogievskiy 779a3b52535SVladimir Sementsov-Ogievskiy return hash; 780a3b52535SVladimir Sementsov-Ogievskiy } 781