xref: /qemu/util/hbitmap.c (revision 3260cdff)
1e7c033c3SPaolo Bonzini /*
2e7c033c3SPaolo Bonzini  * Hierarchical Bitmap Data Type
3e7c033c3SPaolo Bonzini  *
4e7c033c3SPaolo Bonzini  * Copyright Red Hat, Inc., 2012
5e7c033c3SPaolo Bonzini  *
6e7c033c3SPaolo Bonzini  * Author: Paolo Bonzini <pbonzini@redhat.com>
7e7c033c3SPaolo Bonzini  *
8e7c033c3SPaolo Bonzini  * This work is licensed under the terms of the GNU GPL, version 2 or
9e7c033c3SPaolo Bonzini  * later.  See the COPYING file in the top-level directory.
10e7c033c3SPaolo Bonzini  */
11e7c033c3SPaolo Bonzini 
12e7c033c3SPaolo Bonzini #include "qemu/osdep.h"
13e7c033c3SPaolo Bonzini #include "qemu/hbitmap.h"
14e7c033c3SPaolo Bonzini #include "qemu/host-utils.h"
15e7c033c3SPaolo Bonzini #include "trace.h"
16a3b52535SVladimir Sementsov-Ogievskiy #include "crypto/hash.h"
17e7c033c3SPaolo Bonzini 
18e7c033c3SPaolo Bonzini /* HBitmaps provides an array of bits.  The bits are stored as usual in an
19e7c033c3SPaolo Bonzini  * array of unsigned longs, but HBitmap is also optimized to provide fast
20e7c033c3SPaolo Bonzini  * iteration over set bits; going from one bit to the next is O(logB n)
21e7c033c3SPaolo Bonzini  * worst case, with B = sizeof(long) * CHAR_BIT: the result is low enough
22e7c033c3SPaolo Bonzini  * that the number of levels is in fact fixed.
23e7c033c3SPaolo Bonzini  *
24e7c033c3SPaolo Bonzini  * In order to do this, it stacks multiple bitmaps with progressively coarser
25e7c033c3SPaolo Bonzini  * granularity; in all levels except the last, bit N is set iff the N-th
26e7c033c3SPaolo Bonzini  * unsigned long is nonzero in the immediately next level.  When iteration
27e7c033c3SPaolo Bonzini  * completes on the last level it can examine the 2nd-last level to quickly
28e7c033c3SPaolo Bonzini  * skip entire words, and even do so recursively to skip blocks of 64 words or
29e7c033c3SPaolo Bonzini  * powers thereof (32 on 32-bit machines).
30e7c033c3SPaolo Bonzini  *
31e7c033c3SPaolo Bonzini  * Given an index in the bitmap, it can be split in group of bits like
32e7c033c3SPaolo Bonzini  * this (for the 64-bit case):
33e7c033c3SPaolo Bonzini  *
34e7c033c3SPaolo Bonzini  *   bits 0-57 => word in the last bitmap     | bits 58-63 => bit in the word
35e7c033c3SPaolo Bonzini  *   bits 0-51 => word in the 2nd-last bitmap | bits 52-57 => bit in the word
36e7c033c3SPaolo Bonzini  *   bits 0-45 => word in the 3rd-last bitmap | bits 46-51 => bit in the word
37e7c033c3SPaolo Bonzini  *
38e7c033c3SPaolo Bonzini  * So it is easy to move up simply by shifting the index right by
39e7c033c3SPaolo Bonzini  * log2(BITS_PER_LONG) bits.  To move down, you shift the index left
40e7c033c3SPaolo Bonzini  * similarly, and add the word index within the group.  Iteration uses
41e7c033c3SPaolo Bonzini  * ffs (find first set bit) to find the next word to examine; this
42e7c033c3SPaolo Bonzini  * operation can be done in constant time in most current architectures.
43e7c033c3SPaolo Bonzini  *
44e7c033c3SPaolo Bonzini  * Setting or clearing a range of m bits on all levels, the work to perform
45e7c033c3SPaolo Bonzini  * is O(m + m/W + m/W^2 + ...), which is O(m) like on a regular bitmap.
46e7c033c3SPaolo Bonzini  *
47e7c033c3SPaolo Bonzini  * When iterating on a bitmap, each bit (on any level) is only visited
48e7c033c3SPaolo Bonzini  * once.  Hence, The total cost of visiting a bitmap with m bits in it is
49e7c033c3SPaolo Bonzini  * the number of bits that are set in all bitmaps.  Unless the bitmap is
50e7c033c3SPaolo Bonzini  * extremely sparse, this is also O(m + m/W + m/W^2 + ...), so the amortized
51e7c033c3SPaolo Bonzini  * cost of advancing from one bit to the next is usually constant (worst case
52e7c033c3SPaolo Bonzini  * O(logB n) as in the non-amortized complexity).
53e7c033c3SPaolo Bonzini  */
54e7c033c3SPaolo Bonzini 
55e7c033c3SPaolo Bonzini struct HBitmap {
56e7c033c3SPaolo Bonzini     /* Number of total bits in the bottom level.  */
57e7c033c3SPaolo Bonzini     uint64_t size;
58e7c033c3SPaolo Bonzini 
59e7c033c3SPaolo Bonzini     /* Number of set bits in the bottom level.  */
60e7c033c3SPaolo Bonzini     uint64_t count;
61e7c033c3SPaolo Bonzini 
62e7c033c3SPaolo Bonzini     /* A scaling factor.  Given a granularity of G, each bit in the bitmap will
63e7c033c3SPaolo Bonzini      * will actually represent a group of 2^G elements.  Each operation on a
64e7c033c3SPaolo Bonzini      * range of bits first rounds the bits to determine which group they land
65e7c033c3SPaolo Bonzini      * in, and then affect the entire page; iteration will only visit the first
66e7c033c3SPaolo Bonzini      * bit of each group.  Here is an example of operations in a size-16,
67e7c033c3SPaolo Bonzini      * granularity-1 HBitmap:
68e7c033c3SPaolo Bonzini      *
69e7c033c3SPaolo Bonzini      *    initial state            00000000
70e7c033c3SPaolo Bonzini      *    set(start=0, count=9)    11111000 (iter: 0, 2, 4, 6, 8)
71e7c033c3SPaolo Bonzini      *    reset(start=1, count=3)  00111000 (iter: 4, 6, 8)
72e7c033c3SPaolo Bonzini      *    set(start=9, count=2)    00111100 (iter: 4, 6, 8, 10)
73e7c033c3SPaolo Bonzini      *    reset(start=5, count=5)  00000000
74e7c033c3SPaolo Bonzini      *
75e7c033c3SPaolo Bonzini      * From an implementation point of view, when setting or resetting bits,
76e7c033c3SPaolo Bonzini      * the bitmap will scale bit numbers right by this amount of bits.  When
77e7c033c3SPaolo Bonzini      * iterating, the bitmap will scale bit numbers left by this amount of
78e7c033c3SPaolo Bonzini      * bits.
79e7c033c3SPaolo Bonzini      */
80e7c033c3SPaolo Bonzini     int granularity;
81e7c033c3SPaolo Bonzini 
8207ac4cdbSFam Zheng     /* A meta dirty bitmap to track the dirtiness of bits in this HBitmap. */
8307ac4cdbSFam Zheng     HBitmap *meta;
8407ac4cdbSFam Zheng 
85e7c033c3SPaolo Bonzini     /* A number of progressively less coarse bitmaps (i.e. level 0 is the
86e7c033c3SPaolo Bonzini      * coarsest).  Each bit in level N represents a word in level N+1 that
87e7c033c3SPaolo Bonzini      * has a set bit, except the last level where each bit represents the
88e7c033c3SPaolo Bonzini      * actual bitmap.
89e7c033c3SPaolo Bonzini      *
90e7c033c3SPaolo Bonzini      * Note that all bitmaps have the same number of levels.  Even a 1-bit
91e7c033c3SPaolo Bonzini      * bitmap will still allocate HBITMAP_LEVELS arrays.
92e7c033c3SPaolo Bonzini      */
93e7c033c3SPaolo Bonzini     unsigned long *levels[HBITMAP_LEVELS];
948515efbeSJohn Snow 
958515efbeSJohn Snow     /* The length of each levels[] array. */
968515efbeSJohn Snow     uint64_t sizes[HBITMAP_LEVELS];
97e7c033c3SPaolo Bonzini };
98e7c033c3SPaolo Bonzini 
99e7c033c3SPaolo Bonzini /* Advance hbi to the next nonzero word and return it.  hbi->pos
100e7c033c3SPaolo Bonzini  * is updated.  Returns zero if we reach the end of the bitmap.
101e7c033c3SPaolo Bonzini  */
102e7c033c3SPaolo Bonzini unsigned long hbitmap_iter_skip_words(HBitmapIter *hbi)
103e7c033c3SPaolo Bonzini {
104e7c033c3SPaolo Bonzini     size_t pos = hbi->pos;
105e7c033c3SPaolo Bonzini     const HBitmap *hb = hbi->hb;
106e7c033c3SPaolo Bonzini     unsigned i = HBITMAP_LEVELS - 1;
107e7c033c3SPaolo Bonzini 
108e7c033c3SPaolo Bonzini     unsigned long cur;
109e7c033c3SPaolo Bonzini     do {
110f63ea4e9SVladimir Sementsov-Ogievskiy         i--;
111e7c033c3SPaolo Bonzini         pos >>= BITS_PER_LEVEL;
112f63ea4e9SVladimir Sementsov-Ogievskiy         cur = hbi->cur[i] & hb->levels[i][pos];
113e7c033c3SPaolo Bonzini     } while (cur == 0);
114e7c033c3SPaolo Bonzini 
115e7c033c3SPaolo Bonzini     /* Check for end of iteration.  We always use fewer than BITS_PER_LONG
116e7c033c3SPaolo Bonzini      * bits in the level 0 bitmap; thus we can repurpose the most significant
117e7c033c3SPaolo Bonzini      * bit as a sentinel.  The sentinel is set in hbitmap_alloc and ensures
118e7c033c3SPaolo Bonzini      * that the above loop ends even without an explicit check on i.
119e7c033c3SPaolo Bonzini      */
120e7c033c3SPaolo Bonzini 
121e7c033c3SPaolo Bonzini     if (i == 0 && cur == (1UL << (BITS_PER_LONG - 1))) {
122e7c033c3SPaolo Bonzini         return 0;
123e7c033c3SPaolo Bonzini     }
124e7c033c3SPaolo Bonzini     for (; i < HBITMAP_LEVELS - 1; i++) {
125e7c033c3SPaolo Bonzini         /* Shift back pos to the left, matching the right shifts above.
126e7c033c3SPaolo Bonzini          * The index of this word's least significant set bit provides
127e7c033c3SPaolo Bonzini          * the low-order bits.
128e7c033c3SPaolo Bonzini          */
12918331e7cSRichard Henderson         assert(cur);
13018331e7cSRichard Henderson         pos = (pos << BITS_PER_LEVEL) + ctzl(cur);
131e7c033c3SPaolo Bonzini         hbi->cur[i] = cur & (cur - 1);
132e7c033c3SPaolo Bonzini 
133e7c033c3SPaolo Bonzini         /* Set up next level for iteration.  */
134e7c033c3SPaolo Bonzini         cur = hb->levels[i + 1][pos];
135e7c033c3SPaolo Bonzini     }
136e7c033c3SPaolo Bonzini 
137e7c033c3SPaolo Bonzini     hbi->pos = pos;
138e7c033c3SPaolo Bonzini     trace_hbitmap_iter_skip_words(hbi->hb, hbi, pos, cur);
139e7c033c3SPaolo Bonzini 
140e7c033c3SPaolo Bonzini     assert(cur);
141e7c033c3SPaolo Bonzini     return cur;
142e7c033c3SPaolo Bonzini }
143e7c033c3SPaolo Bonzini 
144f63ea4e9SVladimir Sementsov-Ogievskiy int64_t hbitmap_iter_next(HBitmapIter *hbi)
145f63ea4e9SVladimir Sementsov-Ogievskiy {
146f63ea4e9SVladimir Sementsov-Ogievskiy     unsigned long cur = hbi->cur[HBITMAP_LEVELS - 1] &
147f63ea4e9SVladimir Sementsov-Ogievskiy             hbi->hb->levels[HBITMAP_LEVELS - 1][hbi->pos];
148f63ea4e9SVladimir Sementsov-Ogievskiy     int64_t item;
149f63ea4e9SVladimir Sementsov-Ogievskiy 
150f63ea4e9SVladimir Sementsov-Ogievskiy     if (cur == 0) {
151f63ea4e9SVladimir Sementsov-Ogievskiy         cur = hbitmap_iter_skip_words(hbi);
152f63ea4e9SVladimir Sementsov-Ogievskiy         if (cur == 0) {
153f63ea4e9SVladimir Sementsov-Ogievskiy             return -1;
154f63ea4e9SVladimir Sementsov-Ogievskiy         }
155f63ea4e9SVladimir Sementsov-Ogievskiy     }
156f63ea4e9SVladimir Sementsov-Ogievskiy 
157f63ea4e9SVladimir Sementsov-Ogievskiy     /* The next call will resume work from the next bit.  */
158f63ea4e9SVladimir Sementsov-Ogievskiy     hbi->cur[HBITMAP_LEVELS - 1] = cur & (cur - 1);
159f63ea4e9SVladimir Sementsov-Ogievskiy     item = ((uint64_t)hbi->pos << BITS_PER_LEVEL) + ctzl(cur);
160f63ea4e9SVladimir Sementsov-Ogievskiy 
161f63ea4e9SVladimir Sementsov-Ogievskiy     return item << hbi->granularity;
162f63ea4e9SVladimir Sementsov-Ogievskiy }
163f63ea4e9SVladimir Sementsov-Ogievskiy 
164e7c033c3SPaolo Bonzini void hbitmap_iter_init(HBitmapIter *hbi, const HBitmap *hb, uint64_t first)
165e7c033c3SPaolo Bonzini {
166e7c033c3SPaolo Bonzini     unsigned i, bit;
167e7c033c3SPaolo Bonzini     uint64_t pos;
168e7c033c3SPaolo Bonzini 
169e7c033c3SPaolo Bonzini     hbi->hb = hb;
170e7c033c3SPaolo Bonzini     pos = first >> hb->granularity;
1711b095244SPaolo Bonzini     assert(pos < hb->size);
172e7c033c3SPaolo Bonzini     hbi->pos = pos >> BITS_PER_LEVEL;
173e7c033c3SPaolo Bonzini     hbi->granularity = hb->granularity;
174e7c033c3SPaolo Bonzini 
175e7c033c3SPaolo Bonzini     for (i = HBITMAP_LEVELS; i-- > 0; ) {
176e7c033c3SPaolo Bonzini         bit = pos & (BITS_PER_LONG - 1);
177e7c033c3SPaolo Bonzini         pos >>= BITS_PER_LEVEL;
178e7c033c3SPaolo Bonzini 
179e7c033c3SPaolo Bonzini         /* Drop bits representing items before first.  */
180e7c033c3SPaolo Bonzini         hbi->cur[i] = hb->levels[i][pos] & ~((1UL << bit) - 1);
181e7c033c3SPaolo Bonzini 
182e7c033c3SPaolo Bonzini         /* We have already added level i+1, so the lowest set bit has
183e7c033c3SPaolo Bonzini          * been processed.  Clear it.
184e7c033c3SPaolo Bonzini          */
185e7c033c3SPaolo Bonzini         if (i != HBITMAP_LEVELS - 1) {
186e7c033c3SPaolo Bonzini             hbi->cur[i] &= ~(1UL << bit);
187e7c033c3SPaolo Bonzini         }
188e7c033c3SPaolo Bonzini     }
189e7c033c3SPaolo Bonzini }
190e7c033c3SPaolo Bonzini 
19156207df5SVladimir Sementsov-Ogievskiy int64_t hbitmap_next_zero(const HBitmap *hb, uint64_t start)
19256207df5SVladimir Sementsov-Ogievskiy {
19356207df5SVladimir Sementsov-Ogievskiy     size_t pos = (start >> hb->granularity) >> BITS_PER_LEVEL;
19456207df5SVladimir Sementsov-Ogievskiy     unsigned long *last_lev = hb->levels[HBITMAP_LEVELS - 1];
19556207df5SVladimir Sementsov-Ogievskiy     uint64_t sz = hb->sizes[HBITMAP_LEVELS - 1];
19656207df5SVladimir Sementsov-Ogievskiy     unsigned long cur = last_lev[pos];
19756207df5SVladimir Sementsov-Ogievskiy     unsigned start_bit_offset =
19856207df5SVladimir Sementsov-Ogievskiy             (start >> hb->granularity) & (BITS_PER_LONG - 1);
19956207df5SVladimir Sementsov-Ogievskiy     int64_t res;
20056207df5SVladimir Sementsov-Ogievskiy 
20156207df5SVladimir Sementsov-Ogievskiy     cur |= (1UL << start_bit_offset) - 1;
20256207df5SVladimir Sementsov-Ogievskiy     assert((start >> hb->granularity) < hb->size);
20356207df5SVladimir Sementsov-Ogievskiy 
20456207df5SVladimir Sementsov-Ogievskiy     if (cur == (unsigned long)-1) {
20556207df5SVladimir Sementsov-Ogievskiy         do {
20656207df5SVladimir Sementsov-Ogievskiy             pos++;
20756207df5SVladimir Sementsov-Ogievskiy         } while (pos < sz && last_lev[pos] == (unsigned long)-1);
20856207df5SVladimir Sementsov-Ogievskiy 
20956207df5SVladimir Sementsov-Ogievskiy         if (pos >= sz) {
21056207df5SVladimir Sementsov-Ogievskiy             return -1;
21156207df5SVladimir Sementsov-Ogievskiy         }
21256207df5SVladimir Sementsov-Ogievskiy 
21356207df5SVladimir Sementsov-Ogievskiy         cur = last_lev[pos];
21456207df5SVladimir Sementsov-Ogievskiy     }
21556207df5SVladimir Sementsov-Ogievskiy 
21656207df5SVladimir Sementsov-Ogievskiy     res = (pos << BITS_PER_LEVEL) + ctol(cur);
21756207df5SVladimir Sementsov-Ogievskiy     if (res >= hb->size) {
21856207df5SVladimir Sementsov-Ogievskiy         return -1;
21956207df5SVladimir Sementsov-Ogievskiy     }
22056207df5SVladimir Sementsov-Ogievskiy 
22156207df5SVladimir Sementsov-Ogievskiy     res = res << hb->granularity;
22256207df5SVladimir Sementsov-Ogievskiy     if (res < start) {
22356207df5SVladimir Sementsov-Ogievskiy         assert(((start - res) >> hb->granularity) == 0);
22456207df5SVladimir Sementsov-Ogievskiy         return start;
22556207df5SVladimir Sementsov-Ogievskiy     }
22656207df5SVladimir Sementsov-Ogievskiy 
22756207df5SVladimir Sementsov-Ogievskiy     return res;
22856207df5SVladimir Sementsov-Ogievskiy }
22956207df5SVladimir Sementsov-Ogievskiy 
230e7c033c3SPaolo Bonzini bool hbitmap_empty(const HBitmap *hb)
231e7c033c3SPaolo Bonzini {
232e7c033c3SPaolo Bonzini     return hb->count == 0;
233e7c033c3SPaolo Bonzini }
234e7c033c3SPaolo Bonzini 
235e7c033c3SPaolo Bonzini int hbitmap_granularity(const HBitmap *hb)
236e7c033c3SPaolo Bonzini {
237e7c033c3SPaolo Bonzini     return hb->granularity;
238e7c033c3SPaolo Bonzini }
239e7c033c3SPaolo Bonzini 
240e7c033c3SPaolo Bonzini uint64_t hbitmap_count(const HBitmap *hb)
241e7c033c3SPaolo Bonzini {
242e7c033c3SPaolo Bonzini     return hb->count << hb->granularity;
243e7c033c3SPaolo Bonzini }
244e7c033c3SPaolo Bonzini 
245e7c033c3SPaolo Bonzini /* Count the number of set bits between start and end, not accounting for
246e7c033c3SPaolo Bonzini  * the granularity.  Also an example of how to use hbitmap_iter_next_word.
247e7c033c3SPaolo Bonzini  */
248e7c033c3SPaolo Bonzini static uint64_t hb_count_between(HBitmap *hb, uint64_t start, uint64_t last)
249e7c033c3SPaolo Bonzini {
250e7c033c3SPaolo Bonzini     HBitmapIter hbi;
251e7c033c3SPaolo Bonzini     uint64_t count = 0;
252e7c033c3SPaolo Bonzini     uint64_t end = last + 1;
253e7c033c3SPaolo Bonzini     unsigned long cur;
254e7c033c3SPaolo Bonzini     size_t pos;
255e7c033c3SPaolo Bonzini 
256e7c033c3SPaolo Bonzini     hbitmap_iter_init(&hbi, hb, start << hb->granularity);
257e7c033c3SPaolo Bonzini     for (;;) {
258e7c033c3SPaolo Bonzini         pos = hbitmap_iter_next_word(&hbi, &cur);
259e7c033c3SPaolo Bonzini         if (pos >= (end >> BITS_PER_LEVEL)) {
260e7c033c3SPaolo Bonzini             break;
261e7c033c3SPaolo Bonzini         }
262591b320aSPeter Maydell         count += ctpopl(cur);
263e7c033c3SPaolo Bonzini     }
264e7c033c3SPaolo Bonzini 
265e7c033c3SPaolo Bonzini     if (pos == (end >> BITS_PER_LEVEL)) {
266e7c033c3SPaolo Bonzini         /* Drop bits representing the END-th and subsequent items.  */
267e7c033c3SPaolo Bonzini         int bit = end & (BITS_PER_LONG - 1);
268e7c033c3SPaolo Bonzini         cur &= (1UL << bit) - 1;
269591b320aSPeter Maydell         count += ctpopl(cur);
270e7c033c3SPaolo Bonzini     }
271e7c033c3SPaolo Bonzini 
272e7c033c3SPaolo Bonzini     return count;
273e7c033c3SPaolo Bonzini }
274e7c033c3SPaolo Bonzini 
275e7c033c3SPaolo Bonzini /* Setting starts at the last layer and propagates up if an element
27607ac4cdbSFam Zheng  * changes.
277e7c033c3SPaolo Bonzini  */
278e7c033c3SPaolo Bonzini static inline bool hb_set_elem(unsigned long *elem, uint64_t start, uint64_t last)
279e7c033c3SPaolo Bonzini {
280e7c033c3SPaolo Bonzini     unsigned long mask;
28107ac4cdbSFam Zheng     unsigned long old;
282e7c033c3SPaolo Bonzini 
283e7c033c3SPaolo Bonzini     assert((last >> BITS_PER_LEVEL) == (start >> BITS_PER_LEVEL));
284e7c033c3SPaolo Bonzini     assert(start <= last);
285e7c033c3SPaolo Bonzini 
286e7c033c3SPaolo Bonzini     mask = 2UL << (last & (BITS_PER_LONG - 1));
287e7c033c3SPaolo Bonzini     mask -= 1UL << (start & (BITS_PER_LONG - 1));
28807ac4cdbSFam Zheng     old = *elem;
289e7c033c3SPaolo Bonzini     *elem |= mask;
29007ac4cdbSFam Zheng     return old != *elem;
291e7c033c3SPaolo Bonzini }
292e7c033c3SPaolo Bonzini 
29307ac4cdbSFam Zheng /* The recursive workhorse (the depth is limited to HBITMAP_LEVELS)...
29407ac4cdbSFam Zheng  * Returns true if at least one bit is changed. */
29507ac4cdbSFam Zheng static bool hb_set_between(HBitmap *hb, int level, uint64_t start,
29607ac4cdbSFam Zheng                            uint64_t last)
297e7c033c3SPaolo Bonzini {
298e7c033c3SPaolo Bonzini     size_t pos = start >> BITS_PER_LEVEL;
299e7c033c3SPaolo Bonzini     size_t lastpos = last >> BITS_PER_LEVEL;
300e7c033c3SPaolo Bonzini     bool changed = false;
301e7c033c3SPaolo Bonzini     size_t i;
302e7c033c3SPaolo Bonzini 
303e7c033c3SPaolo Bonzini     i = pos;
304e7c033c3SPaolo Bonzini     if (i < lastpos) {
305e7c033c3SPaolo Bonzini         uint64_t next = (start | (BITS_PER_LONG - 1)) + 1;
306e7c033c3SPaolo Bonzini         changed |= hb_set_elem(&hb->levels[level][i], start, next - 1);
307e7c033c3SPaolo Bonzini         for (;;) {
308e7c033c3SPaolo Bonzini             start = next;
309e7c033c3SPaolo Bonzini             next += BITS_PER_LONG;
310e7c033c3SPaolo Bonzini             if (++i == lastpos) {
311e7c033c3SPaolo Bonzini                 break;
312e7c033c3SPaolo Bonzini             }
313e7c033c3SPaolo Bonzini             changed |= (hb->levels[level][i] == 0);
314e7c033c3SPaolo Bonzini             hb->levels[level][i] = ~0UL;
315e7c033c3SPaolo Bonzini         }
316e7c033c3SPaolo Bonzini     }
317e7c033c3SPaolo Bonzini     changed |= hb_set_elem(&hb->levels[level][i], start, last);
318e7c033c3SPaolo Bonzini 
319e7c033c3SPaolo Bonzini     /* If there was any change in this layer, we may have to update
320e7c033c3SPaolo Bonzini      * the one above.
321e7c033c3SPaolo Bonzini      */
322e7c033c3SPaolo Bonzini     if (level > 0 && changed) {
323e7c033c3SPaolo Bonzini         hb_set_between(hb, level - 1, pos, lastpos);
324e7c033c3SPaolo Bonzini     }
32507ac4cdbSFam Zheng     return changed;
326e7c033c3SPaolo Bonzini }
327e7c033c3SPaolo Bonzini 
328e7c033c3SPaolo Bonzini void hbitmap_set(HBitmap *hb, uint64_t start, uint64_t count)
329e7c033c3SPaolo Bonzini {
330e7c033c3SPaolo Bonzini     /* Compute range in the last layer.  */
33107ac4cdbSFam Zheng     uint64_t first, n;
332e7c033c3SPaolo Bonzini     uint64_t last = start + count - 1;
333e7c033c3SPaolo Bonzini 
334e7c033c3SPaolo Bonzini     trace_hbitmap_set(hb, start, count,
335e7c033c3SPaolo Bonzini                       start >> hb->granularity, last >> hb->granularity);
336e7c033c3SPaolo Bonzini 
33707ac4cdbSFam Zheng     first = start >> hb->granularity;
338e7c033c3SPaolo Bonzini     last >>= hb->granularity;
3390e321191SVladimir Sementsov-Ogievskiy     assert(last < hb->size);
34007ac4cdbSFam Zheng     n = last - first + 1;
341e7c033c3SPaolo Bonzini 
34207ac4cdbSFam Zheng     hb->count += n - hb_count_between(hb, first, last);
34307ac4cdbSFam Zheng     if (hb_set_between(hb, HBITMAP_LEVELS - 1, first, last) &&
34407ac4cdbSFam Zheng         hb->meta) {
34507ac4cdbSFam Zheng         hbitmap_set(hb->meta, start, count);
34607ac4cdbSFam Zheng     }
347e7c033c3SPaolo Bonzini }
348e7c033c3SPaolo Bonzini 
349e7c033c3SPaolo Bonzini /* Resetting works the other way round: propagate up if the new
350e7c033c3SPaolo Bonzini  * value is zero.
351e7c033c3SPaolo Bonzini  */
352e7c033c3SPaolo Bonzini static inline bool hb_reset_elem(unsigned long *elem, uint64_t start, uint64_t last)
353e7c033c3SPaolo Bonzini {
354e7c033c3SPaolo Bonzini     unsigned long mask;
355e7c033c3SPaolo Bonzini     bool blanked;
356e7c033c3SPaolo Bonzini 
357e7c033c3SPaolo Bonzini     assert((last >> BITS_PER_LEVEL) == (start >> BITS_PER_LEVEL));
358e7c033c3SPaolo Bonzini     assert(start <= last);
359e7c033c3SPaolo Bonzini 
360e7c033c3SPaolo Bonzini     mask = 2UL << (last & (BITS_PER_LONG - 1));
361e7c033c3SPaolo Bonzini     mask -= 1UL << (start & (BITS_PER_LONG - 1));
362e7c033c3SPaolo Bonzini     blanked = *elem != 0 && ((*elem & ~mask) == 0);
363e7c033c3SPaolo Bonzini     *elem &= ~mask;
364e7c033c3SPaolo Bonzini     return blanked;
365e7c033c3SPaolo Bonzini }
366e7c033c3SPaolo Bonzini 
36707ac4cdbSFam Zheng /* The recursive workhorse (the depth is limited to HBITMAP_LEVELS)...
36807ac4cdbSFam Zheng  * Returns true if at least one bit is changed. */
36907ac4cdbSFam Zheng static bool hb_reset_between(HBitmap *hb, int level, uint64_t start,
37007ac4cdbSFam Zheng                              uint64_t last)
371e7c033c3SPaolo Bonzini {
372e7c033c3SPaolo Bonzini     size_t pos = start >> BITS_PER_LEVEL;
373e7c033c3SPaolo Bonzini     size_t lastpos = last >> BITS_PER_LEVEL;
374e7c033c3SPaolo Bonzini     bool changed = false;
375e7c033c3SPaolo Bonzini     size_t i;
376e7c033c3SPaolo Bonzini 
377e7c033c3SPaolo Bonzini     i = pos;
378e7c033c3SPaolo Bonzini     if (i < lastpos) {
379e7c033c3SPaolo Bonzini         uint64_t next = (start | (BITS_PER_LONG - 1)) + 1;
380e7c033c3SPaolo Bonzini 
381e7c033c3SPaolo Bonzini         /* Here we need a more complex test than when setting bits.  Even if
382e7c033c3SPaolo Bonzini          * something was changed, we must not blank bits in the upper level
383e7c033c3SPaolo Bonzini          * unless the lower-level word became entirely zero.  So, remove pos
384e7c033c3SPaolo Bonzini          * from the upper-level range if bits remain set.
385e7c033c3SPaolo Bonzini          */
386e7c033c3SPaolo Bonzini         if (hb_reset_elem(&hb->levels[level][i], start, next - 1)) {
387e7c033c3SPaolo Bonzini             changed = true;
388e7c033c3SPaolo Bonzini         } else {
389e7c033c3SPaolo Bonzini             pos++;
390e7c033c3SPaolo Bonzini         }
391e7c033c3SPaolo Bonzini 
392e7c033c3SPaolo Bonzini         for (;;) {
393e7c033c3SPaolo Bonzini             start = next;
394e7c033c3SPaolo Bonzini             next += BITS_PER_LONG;
395e7c033c3SPaolo Bonzini             if (++i == lastpos) {
396e7c033c3SPaolo Bonzini                 break;
397e7c033c3SPaolo Bonzini             }
398e7c033c3SPaolo Bonzini             changed |= (hb->levels[level][i] != 0);
399e7c033c3SPaolo Bonzini             hb->levels[level][i] = 0UL;
400e7c033c3SPaolo Bonzini         }
401e7c033c3SPaolo Bonzini     }
402e7c033c3SPaolo Bonzini 
403e7c033c3SPaolo Bonzini     /* Same as above, this time for lastpos.  */
404e7c033c3SPaolo Bonzini     if (hb_reset_elem(&hb->levels[level][i], start, last)) {
405e7c033c3SPaolo Bonzini         changed = true;
406e7c033c3SPaolo Bonzini     } else {
407e7c033c3SPaolo Bonzini         lastpos--;
408e7c033c3SPaolo Bonzini     }
409e7c033c3SPaolo Bonzini 
410e7c033c3SPaolo Bonzini     if (level > 0 && changed) {
411e7c033c3SPaolo Bonzini         hb_reset_between(hb, level - 1, pos, lastpos);
412e7c033c3SPaolo Bonzini     }
41307ac4cdbSFam Zheng 
41407ac4cdbSFam Zheng     return changed;
41507ac4cdbSFam Zheng 
416e7c033c3SPaolo Bonzini }
417e7c033c3SPaolo Bonzini 
418e7c033c3SPaolo Bonzini void hbitmap_reset(HBitmap *hb, uint64_t start, uint64_t count)
419e7c033c3SPaolo Bonzini {
420e7c033c3SPaolo Bonzini     /* Compute range in the last layer.  */
42107ac4cdbSFam Zheng     uint64_t first;
422e7c033c3SPaolo Bonzini     uint64_t last = start + count - 1;
423e7c033c3SPaolo Bonzini 
424e7c033c3SPaolo Bonzini     trace_hbitmap_reset(hb, start, count,
425e7c033c3SPaolo Bonzini                         start >> hb->granularity, last >> hb->granularity);
426e7c033c3SPaolo Bonzini 
42707ac4cdbSFam Zheng     first = start >> hb->granularity;
428e7c033c3SPaolo Bonzini     last >>= hb->granularity;
4290e321191SVladimir Sementsov-Ogievskiy     assert(last < hb->size);
430e7c033c3SPaolo Bonzini 
43107ac4cdbSFam Zheng     hb->count -= hb_count_between(hb, first, last);
43207ac4cdbSFam Zheng     if (hb_reset_between(hb, HBITMAP_LEVELS - 1, first, last) &&
43307ac4cdbSFam Zheng         hb->meta) {
43407ac4cdbSFam Zheng         hbitmap_set(hb->meta, start, count);
43507ac4cdbSFam Zheng     }
436e7c033c3SPaolo Bonzini }
437e7c033c3SPaolo Bonzini 
438c6a8c328SWen Congyang void hbitmap_reset_all(HBitmap *hb)
439c6a8c328SWen Congyang {
440c6a8c328SWen Congyang     unsigned int i;
441c6a8c328SWen Congyang 
442c6a8c328SWen Congyang     /* Same as hbitmap_alloc() except for memset() instead of malloc() */
443c6a8c328SWen Congyang     for (i = HBITMAP_LEVELS; --i >= 1; ) {
444c6a8c328SWen Congyang         memset(hb->levels[i], 0, hb->sizes[i] * sizeof(unsigned long));
445c6a8c328SWen Congyang     }
446c6a8c328SWen Congyang 
447c6a8c328SWen Congyang     hb->levels[0][0] = 1UL << (BITS_PER_LONG - 1);
448c6a8c328SWen Congyang     hb->count = 0;
449c6a8c328SWen Congyang }
450c6a8c328SWen Congyang 
45120a579deSMax Reitz bool hbitmap_is_serializable(const HBitmap *hb)
45220a579deSMax Reitz {
45320a579deSMax Reitz     /* Every serialized chunk must be aligned to 64 bits so that endianness
45420a579deSMax Reitz      * requirements can be fulfilled on both 64 bit and 32 bit hosts.
455ecbfa281SEric Blake      * We have hbitmap_serialization_align() which converts this
45620a579deSMax Reitz      * alignment requirement from bitmap bits to items covered (e.g. sectors).
45720a579deSMax Reitz      * That value is:
45820a579deSMax Reitz      *    64 << hb->granularity
45920a579deSMax Reitz      * Since this value must not exceed UINT64_MAX, hb->granularity must be
46020a579deSMax Reitz      * less than 58 (== 64 - 6, where 6 is ld(64), i.e. 1 << 6 == 64).
46120a579deSMax Reitz      *
462ecbfa281SEric Blake      * In order for hbitmap_serialization_align() to always return a
46320a579deSMax Reitz      * meaningful value, bitmaps that are to be serialized must have a
46420a579deSMax Reitz      * granularity of less than 58. */
46520a579deSMax Reitz 
46620a579deSMax Reitz     return hb->granularity < 58;
46720a579deSMax Reitz }
46820a579deSMax Reitz 
469e7c033c3SPaolo Bonzini bool hbitmap_get(const HBitmap *hb, uint64_t item)
470e7c033c3SPaolo Bonzini {
471e7c033c3SPaolo Bonzini     /* Compute position and bit in the last layer.  */
472e7c033c3SPaolo Bonzini     uint64_t pos = item >> hb->granularity;
473e7c033c3SPaolo Bonzini     unsigned long bit = 1UL << (pos & (BITS_PER_LONG - 1));
4740e321191SVladimir Sementsov-Ogievskiy     assert(pos < hb->size);
475e7c033c3SPaolo Bonzini 
476e7c033c3SPaolo Bonzini     return (hb->levels[HBITMAP_LEVELS - 1][pos >> BITS_PER_LEVEL] & bit) != 0;
477e7c033c3SPaolo Bonzini }
478e7c033c3SPaolo Bonzini 
479ecbfa281SEric Blake uint64_t hbitmap_serialization_align(const HBitmap *hb)
4808258888eSVladimir Sementsov-Ogievskiy {
48120a579deSMax Reitz     assert(hbitmap_is_serializable(hb));
4826725f887SMax Reitz 
4838258888eSVladimir Sementsov-Ogievskiy     /* Require at least 64 bit granularity to be safe on both 64 bit and 32 bit
4848258888eSVladimir Sementsov-Ogievskiy      * hosts. */
4856725f887SMax Reitz     return UINT64_C(64) << hb->granularity;
4868258888eSVladimir Sementsov-Ogievskiy }
4878258888eSVladimir Sementsov-Ogievskiy 
4888258888eSVladimir Sementsov-Ogievskiy /* Start should be aligned to serialization granularity, chunk size should be
4898258888eSVladimir Sementsov-Ogievskiy  * aligned to serialization granularity too, except for last chunk.
4908258888eSVladimir Sementsov-Ogievskiy  */
4918258888eSVladimir Sementsov-Ogievskiy static void serialization_chunk(const HBitmap *hb,
4928258888eSVladimir Sementsov-Ogievskiy                                 uint64_t start, uint64_t count,
4938258888eSVladimir Sementsov-Ogievskiy                                 unsigned long **first_el, uint64_t *el_count)
4948258888eSVladimir Sementsov-Ogievskiy {
4958258888eSVladimir Sementsov-Ogievskiy     uint64_t last = start + count - 1;
496ecbfa281SEric Blake     uint64_t gran = hbitmap_serialization_align(hb);
4978258888eSVladimir Sementsov-Ogievskiy 
4988258888eSVladimir Sementsov-Ogievskiy     assert((start & (gran - 1)) == 0);
4998258888eSVladimir Sementsov-Ogievskiy     assert((last >> hb->granularity) < hb->size);
5008258888eSVladimir Sementsov-Ogievskiy     if ((last >> hb->granularity) != hb->size - 1) {
5018258888eSVladimir Sementsov-Ogievskiy         assert((count & (gran - 1)) == 0);
5028258888eSVladimir Sementsov-Ogievskiy     }
5038258888eSVladimir Sementsov-Ogievskiy 
5048258888eSVladimir Sementsov-Ogievskiy     start = (start >> hb->granularity) >> BITS_PER_LEVEL;
5058258888eSVladimir Sementsov-Ogievskiy     last = (last >> hb->granularity) >> BITS_PER_LEVEL;
5068258888eSVladimir Sementsov-Ogievskiy 
5078258888eSVladimir Sementsov-Ogievskiy     *first_el = &hb->levels[HBITMAP_LEVELS - 1][start];
5088258888eSVladimir Sementsov-Ogievskiy     *el_count = last - start + 1;
5098258888eSVladimir Sementsov-Ogievskiy }
5108258888eSVladimir Sementsov-Ogievskiy 
5118258888eSVladimir Sementsov-Ogievskiy uint64_t hbitmap_serialization_size(const HBitmap *hb,
5128258888eSVladimir Sementsov-Ogievskiy                                     uint64_t start, uint64_t count)
5138258888eSVladimir Sementsov-Ogievskiy {
5148258888eSVladimir Sementsov-Ogievskiy     uint64_t el_count;
5158258888eSVladimir Sementsov-Ogievskiy     unsigned long *cur;
5168258888eSVladimir Sementsov-Ogievskiy 
5178258888eSVladimir Sementsov-Ogievskiy     if (!count) {
5188258888eSVladimir Sementsov-Ogievskiy         return 0;
5198258888eSVladimir Sementsov-Ogievskiy     }
5208258888eSVladimir Sementsov-Ogievskiy     serialization_chunk(hb, start, count, &cur, &el_count);
5218258888eSVladimir Sementsov-Ogievskiy 
5228258888eSVladimir Sementsov-Ogievskiy     return el_count * sizeof(unsigned long);
5238258888eSVladimir Sementsov-Ogievskiy }
5248258888eSVladimir Sementsov-Ogievskiy 
5258258888eSVladimir Sementsov-Ogievskiy void hbitmap_serialize_part(const HBitmap *hb, uint8_t *buf,
5268258888eSVladimir Sementsov-Ogievskiy                             uint64_t start, uint64_t count)
5278258888eSVladimir Sementsov-Ogievskiy {
5288258888eSVladimir Sementsov-Ogievskiy     uint64_t el_count;
5298258888eSVladimir Sementsov-Ogievskiy     unsigned long *cur, *end;
5308258888eSVladimir Sementsov-Ogievskiy 
5318258888eSVladimir Sementsov-Ogievskiy     if (!count) {
5328258888eSVladimir Sementsov-Ogievskiy         return;
5338258888eSVladimir Sementsov-Ogievskiy     }
5348258888eSVladimir Sementsov-Ogievskiy     serialization_chunk(hb, start, count, &cur, &el_count);
5358258888eSVladimir Sementsov-Ogievskiy     end = cur + el_count;
5368258888eSVladimir Sementsov-Ogievskiy 
5378258888eSVladimir Sementsov-Ogievskiy     while (cur != end) {
5388258888eSVladimir Sementsov-Ogievskiy         unsigned long el =
5398258888eSVladimir Sementsov-Ogievskiy             (BITS_PER_LONG == 32 ? cpu_to_le32(*cur) : cpu_to_le64(*cur));
5408258888eSVladimir Sementsov-Ogievskiy 
5418258888eSVladimir Sementsov-Ogievskiy         memcpy(buf, &el, sizeof(el));
5428258888eSVladimir Sementsov-Ogievskiy         buf += sizeof(el);
5438258888eSVladimir Sementsov-Ogievskiy         cur++;
5448258888eSVladimir Sementsov-Ogievskiy     }
5458258888eSVladimir Sementsov-Ogievskiy }
5468258888eSVladimir Sementsov-Ogievskiy 
5478258888eSVladimir Sementsov-Ogievskiy void hbitmap_deserialize_part(HBitmap *hb, uint8_t *buf,
5488258888eSVladimir Sementsov-Ogievskiy                               uint64_t start, uint64_t count,
5498258888eSVladimir Sementsov-Ogievskiy                               bool finish)
5508258888eSVladimir Sementsov-Ogievskiy {
5518258888eSVladimir Sementsov-Ogievskiy     uint64_t el_count;
5528258888eSVladimir Sementsov-Ogievskiy     unsigned long *cur, *end;
5538258888eSVladimir Sementsov-Ogievskiy 
5548258888eSVladimir Sementsov-Ogievskiy     if (!count) {
5558258888eSVladimir Sementsov-Ogievskiy         return;
5568258888eSVladimir Sementsov-Ogievskiy     }
5578258888eSVladimir Sementsov-Ogievskiy     serialization_chunk(hb, start, count, &cur, &el_count);
5588258888eSVladimir Sementsov-Ogievskiy     end = cur + el_count;
5598258888eSVladimir Sementsov-Ogievskiy 
5608258888eSVladimir Sementsov-Ogievskiy     while (cur != end) {
5618258888eSVladimir Sementsov-Ogievskiy         memcpy(cur, buf, sizeof(*cur));
5628258888eSVladimir Sementsov-Ogievskiy 
5638258888eSVladimir Sementsov-Ogievskiy         if (BITS_PER_LONG == 32) {
5648258888eSVladimir Sementsov-Ogievskiy             le32_to_cpus((uint32_t *)cur);
5658258888eSVladimir Sementsov-Ogievskiy         } else {
5668258888eSVladimir Sementsov-Ogievskiy             le64_to_cpus((uint64_t *)cur);
5678258888eSVladimir Sementsov-Ogievskiy         }
5688258888eSVladimir Sementsov-Ogievskiy 
5698258888eSVladimir Sementsov-Ogievskiy         buf += sizeof(unsigned long);
5708258888eSVladimir Sementsov-Ogievskiy         cur++;
5718258888eSVladimir Sementsov-Ogievskiy     }
5728258888eSVladimir Sementsov-Ogievskiy     if (finish) {
5738258888eSVladimir Sementsov-Ogievskiy         hbitmap_deserialize_finish(hb);
5748258888eSVladimir Sementsov-Ogievskiy     }
5758258888eSVladimir Sementsov-Ogievskiy }
5768258888eSVladimir Sementsov-Ogievskiy 
5778258888eSVladimir Sementsov-Ogievskiy void hbitmap_deserialize_zeroes(HBitmap *hb, uint64_t start, uint64_t count,
5788258888eSVladimir Sementsov-Ogievskiy                                 bool finish)
5798258888eSVladimir Sementsov-Ogievskiy {
5808258888eSVladimir Sementsov-Ogievskiy     uint64_t el_count;
5818258888eSVladimir Sementsov-Ogievskiy     unsigned long *first;
5828258888eSVladimir Sementsov-Ogievskiy 
5838258888eSVladimir Sementsov-Ogievskiy     if (!count) {
5848258888eSVladimir Sementsov-Ogievskiy         return;
5858258888eSVladimir Sementsov-Ogievskiy     }
5868258888eSVladimir Sementsov-Ogievskiy     serialization_chunk(hb, start, count, &first, &el_count);
5878258888eSVladimir Sementsov-Ogievskiy 
5888258888eSVladimir Sementsov-Ogievskiy     memset(first, 0, el_count * sizeof(unsigned long));
5898258888eSVladimir Sementsov-Ogievskiy     if (finish) {
5908258888eSVladimir Sementsov-Ogievskiy         hbitmap_deserialize_finish(hb);
5918258888eSVladimir Sementsov-Ogievskiy     }
5928258888eSVladimir Sementsov-Ogievskiy }
5938258888eSVladimir Sementsov-Ogievskiy 
5946bdc8b71SVladimir Sementsov-Ogievskiy void hbitmap_deserialize_ones(HBitmap *hb, uint64_t start, uint64_t count,
5956bdc8b71SVladimir Sementsov-Ogievskiy                               bool finish)
5966bdc8b71SVladimir Sementsov-Ogievskiy {
5976bdc8b71SVladimir Sementsov-Ogievskiy     uint64_t el_count;
5986bdc8b71SVladimir Sementsov-Ogievskiy     unsigned long *first;
5996bdc8b71SVladimir Sementsov-Ogievskiy 
6006bdc8b71SVladimir Sementsov-Ogievskiy     if (!count) {
6016bdc8b71SVladimir Sementsov-Ogievskiy         return;
6026bdc8b71SVladimir Sementsov-Ogievskiy     }
6036bdc8b71SVladimir Sementsov-Ogievskiy     serialization_chunk(hb, start, count, &first, &el_count);
6046bdc8b71SVladimir Sementsov-Ogievskiy 
6056bdc8b71SVladimir Sementsov-Ogievskiy     memset(first, 0xff, el_count * sizeof(unsigned long));
6066bdc8b71SVladimir Sementsov-Ogievskiy     if (finish) {
6076bdc8b71SVladimir Sementsov-Ogievskiy         hbitmap_deserialize_finish(hb);
6086bdc8b71SVladimir Sementsov-Ogievskiy     }
6096bdc8b71SVladimir Sementsov-Ogievskiy }
6106bdc8b71SVladimir Sementsov-Ogievskiy 
6118258888eSVladimir Sementsov-Ogievskiy void hbitmap_deserialize_finish(HBitmap *bitmap)
6128258888eSVladimir Sementsov-Ogievskiy {
6138258888eSVladimir Sementsov-Ogievskiy     int64_t i, size, prev_size;
6148258888eSVladimir Sementsov-Ogievskiy     int lev;
6158258888eSVladimir Sementsov-Ogievskiy 
6168258888eSVladimir Sementsov-Ogievskiy     /* restore levels starting from penultimate to zero level, assuming
6178258888eSVladimir Sementsov-Ogievskiy      * that the last level is ok */
6188258888eSVladimir Sementsov-Ogievskiy     size = MAX((bitmap->size + BITS_PER_LONG - 1) >> BITS_PER_LEVEL, 1);
6198258888eSVladimir Sementsov-Ogievskiy     for (lev = HBITMAP_LEVELS - 1; lev-- > 0; ) {
6208258888eSVladimir Sementsov-Ogievskiy         prev_size = size;
6218258888eSVladimir Sementsov-Ogievskiy         size = MAX((size + BITS_PER_LONG - 1) >> BITS_PER_LEVEL, 1);
6228258888eSVladimir Sementsov-Ogievskiy         memset(bitmap->levels[lev], 0, size * sizeof(unsigned long));
6238258888eSVladimir Sementsov-Ogievskiy 
6248258888eSVladimir Sementsov-Ogievskiy         for (i = 0; i < prev_size; ++i) {
6258258888eSVladimir Sementsov-Ogievskiy             if (bitmap->levels[lev + 1][i]) {
6268258888eSVladimir Sementsov-Ogievskiy                 bitmap->levels[lev][i >> BITS_PER_LEVEL] |=
6278258888eSVladimir Sementsov-Ogievskiy                     1UL << (i & (BITS_PER_LONG - 1));
6288258888eSVladimir Sementsov-Ogievskiy             }
6298258888eSVladimir Sementsov-Ogievskiy         }
6308258888eSVladimir Sementsov-Ogievskiy     }
6318258888eSVladimir Sementsov-Ogievskiy 
6328258888eSVladimir Sementsov-Ogievskiy     bitmap->levels[0][0] |= 1UL << (BITS_PER_LONG - 1);
633*3260cdffSLiang Li     bitmap->count = hb_count_between(bitmap, 0, bitmap->size - 1);
6348258888eSVladimir Sementsov-Ogievskiy }
6358258888eSVladimir Sementsov-Ogievskiy 
636e7c033c3SPaolo Bonzini void hbitmap_free(HBitmap *hb)
637e7c033c3SPaolo Bonzini {
638e7c033c3SPaolo Bonzini     unsigned i;
63907ac4cdbSFam Zheng     assert(!hb->meta);
640e7c033c3SPaolo Bonzini     for (i = HBITMAP_LEVELS; i-- > 0; ) {
641e7c033c3SPaolo Bonzini         g_free(hb->levels[i]);
642e7c033c3SPaolo Bonzini     }
643e7c033c3SPaolo Bonzini     g_free(hb);
644e7c033c3SPaolo Bonzini }
645e7c033c3SPaolo Bonzini 
646e7c033c3SPaolo Bonzini HBitmap *hbitmap_alloc(uint64_t size, int granularity)
647e7c033c3SPaolo Bonzini {
648e1cf5582SMarkus Armbruster     HBitmap *hb = g_new0(struct HBitmap, 1);
649e7c033c3SPaolo Bonzini     unsigned i;
650e7c033c3SPaolo Bonzini 
651e7c033c3SPaolo Bonzini     assert(granularity >= 0 && granularity < 64);
652e7c033c3SPaolo Bonzini     size = (size + (1ULL << granularity) - 1) >> granularity;
653e7c033c3SPaolo Bonzini     assert(size <= ((uint64_t)1 << HBITMAP_LOG_MAX_SIZE));
654e7c033c3SPaolo Bonzini 
655e7c033c3SPaolo Bonzini     hb->size = size;
656e7c033c3SPaolo Bonzini     hb->granularity = granularity;
657e7c033c3SPaolo Bonzini     for (i = HBITMAP_LEVELS; i-- > 0; ) {
658e7c033c3SPaolo Bonzini         size = MAX((size + BITS_PER_LONG - 1) >> BITS_PER_LEVEL, 1);
6598515efbeSJohn Snow         hb->sizes[i] = size;
660e1cf5582SMarkus Armbruster         hb->levels[i] = g_new0(unsigned long, size);
661e7c033c3SPaolo Bonzini     }
662e7c033c3SPaolo Bonzini 
663e7c033c3SPaolo Bonzini     /* We necessarily have free bits in level 0 due to the definition
664e7c033c3SPaolo Bonzini      * of HBITMAP_LEVELS, so use one for a sentinel.  This speeds up
665e7c033c3SPaolo Bonzini      * hbitmap_iter_skip_words.
666e7c033c3SPaolo Bonzini      */
667e7c033c3SPaolo Bonzini     assert(size == 1);
668e7c033c3SPaolo Bonzini     hb->levels[0][0] |= 1UL << (BITS_PER_LONG - 1);
669e7c033c3SPaolo Bonzini     return hb;
670e7c033c3SPaolo Bonzini }
671be58721dSJohn Snow 
672ce1ffea8SJohn Snow void hbitmap_truncate(HBitmap *hb, uint64_t size)
673ce1ffea8SJohn Snow {
674ce1ffea8SJohn Snow     bool shrink;
675ce1ffea8SJohn Snow     unsigned i;
676ce1ffea8SJohn Snow     uint64_t num_elements = size;
677ce1ffea8SJohn Snow     uint64_t old;
678ce1ffea8SJohn Snow 
679ce1ffea8SJohn Snow     /* Size comes in as logical elements, adjust for granularity. */
680ce1ffea8SJohn Snow     size = (size + (1ULL << hb->granularity) - 1) >> hb->granularity;
681ce1ffea8SJohn Snow     assert(size <= ((uint64_t)1 << HBITMAP_LOG_MAX_SIZE));
682ce1ffea8SJohn Snow     shrink = size < hb->size;
683ce1ffea8SJohn Snow 
684ce1ffea8SJohn Snow     /* bit sizes are identical; nothing to do. */
685ce1ffea8SJohn Snow     if (size == hb->size) {
686ce1ffea8SJohn Snow         return;
687ce1ffea8SJohn Snow     }
688ce1ffea8SJohn Snow 
689ce1ffea8SJohn Snow     /* If we're losing bits, let's clear those bits before we invalidate all of
690ce1ffea8SJohn Snow      * our invariants. This helps keep the bitcount consistent, and will prevent
691ce1ffea8SJohn Snow      * us from carrying around garbage bits beyond the end of the map.
692ce1ffea8SJohn Snow      */
693ce1ffea8SJohn Snow     if (shrink) {
694ce1ffea8SJohn Snow         /* Don't clear partial granularity groups;
695ce1ffea8SJohn Snow          * start at the first full one. */
6966725f887SMax Reitz         uint64_t start = ROUND_UP(num_elements, UINT64_C(1) << hb->granularity);
697ce1ffea8SJohn Snow         uint64_t fix_count = (hb->size << hb->granularity) - start;
698ce1ffea8SJohn Snow 
699ce1ffea8SJohn Snow         assert(fix_count);
700ce1ffea8SJohn Snow         hbitmap_reset(hb, start, fix_count);
701ce1ffea8SJohn Snow     }
702ce1ffea8SJohn Snow 
703ce1ffea8SJohn Snow     hb->size = size;
704ce1ffea8SJohn Snow     for (i = HBITMAP_LEVELS; i-- > 0; ) {
705ce1ffea8SJohn Snow         size = MAX(BITS_TO_LONGS(size), 1);
706ce1ffea8SJohn Snow         if (hb->sizes[i] == size) {
707ce1ffea8SJohn Snow             break;
708ce1ffea8SJohn Snow         }
709ce1ffea8SJohn Snow         old = hb->sizes[i];
710ce1ffea8SJohn Snow         hb->sizes[i] = size;
711ce1ffea8SJohn Snow         hb->levels[i] = g_realloc(hb->levels[i], size * sizeof(unsigned long));
712ce1ffea8SJohn Snow         if (!shrink) {
713ce1ffea8SJohn Snow             memset(&hb->levels[i][old], 0x00,
714ce1ffea8SJohn Snow                    (size - old) * sizeof(*hb->levels[i]));
715ce1ffea8SJohn Snow         }
716ce1ffea8SJohn Snow     }
71707ac4cdbSFam Zheng     if (hb->meta) {
71807ac4cdbSFam Zheng         hbitmap_truncate(hb->meta, hb->size << hb->granularity);
71907ac4cdbSFam Zheng     }
720ce1ffea8SJohn Snow }
721ce1ffea8SJohn Snow 
722ce1ffea8SJohn Snow 
723be58721dSJohn Snow /**
724be58721dSJohn Snow  * Given HBitmaps A and B, let A := A (BITOR) B.
725be58721dSJohn Snow  * Bitmap B will not be modified.
726be58721dSJohn Snow  *
727be58721dSJohn Snow  * @return true if the merge was successful,
728be58721dSJohn Snow  *         false if it was not attempted.
729be58721dSJohn Snow  */
730be58721dSJohn Snow bool hbitmap_merge(HBitmap *a, const HBitmap *b)
731be58721dSJohn Snow {
732be58721dSJohn Snow     int i;
733be58721dSJohn Snow     uint64_t j;
734be58721dSJohn Snow 
735be58721dSJohn Snow     if ((a->size != b->size) || (a->granularity != b->granularity)) {
736be58721dSJohn Snow         return false;
737be58721dSJohn Snow     }
738be58721dSJohn Snow 
739be58721dSJohn Snow     if (hbitmap_count(b) == 0) {
740be58721dSJohn Snow         return true;
741be58721dSJohn Snow     }
742be58721dSJohn Snow 
743be58721dSJohn Snow     /* This merge is O(size), as BITS_PER_LONG and HBITMAP_LEVELS are constant.
744be58721dSJohn Snow      * It may be possible to improve running times for sparsely populated maps
745be58721dSJohn Snow      * by using hbitmap_iter_next, but this is suboptimal for dense maps.
746be58721dSJohn Snow      */
747be58721dSJohn Snow     for (i = HBITMAP_LEVELS - 1; i >= 0; i--) {
748be58721dSJohn Snow         for (j = 0; j < a->sizes[i]; j++) {
749be58721dSJohn Snow             a->levels[i][j] |= b->levels[i][j];
750be58721dSJohn Snow         }
751be58721dSJohn Snow     }
752be58721dSJohn Snow 
753be58721dSJohn Snow     return true;
754be58721dSJohn Snow }
75507ac4cdbSFam Zheng 
75607ac4cdbSFam Zheng HBitmap *hbitmap_create_meta(HBitmap *hb, int chunk_size)
75707ac4cdbSFam Zheng {
75807ac4cdbSFam Zheng     assert(!(chunk_size & (chunk_size - 1)));
75907ac4cdbSFam Zheng     assert(!hb->meta);
76007ac4cdbSFam Zheng     hb->meta = hbitmap_alloc(hb->size << hb->granularity,
76107ac4cdbSFam Zheng                              hb->granularity + ctz32(chunk_size));
76207ac4cdbSFam Zheng     return hb->meta;
76307ac4cdbSFam Zheng }
76407ac4cdbSFam Zheng 
76507ac4cdbSFam Zheng void hbitmap_free_meta(HBitmap *hb)
76607ac4cdbSFam Zheng {
76707ac4cdbSFam Zheng     assert(hb->meta);
76807ac4cdbSFam Zheng     hbitmap_free(hb->meta);
76907ac4cdbSFam Zheng     hb->meta = NULL;
77007ac4cdbSFam Zheng }
771a3b52535SVladimir Sementsov-Ogievskiy 
772a3b52535SVladimir Sementsov-Ogievskiy char *hbitmap_sha256(const HBitmap *bitmap, Error **errp)
773a3b52535SVladimir Sementsov-Ogievskiy {
774a3b52535SVladimir Sementsov-Ogievskiy     size_t size = bitmap->sizes[HBITMAP_LEVELS - 1] * sizeof(unsigned long);
775a3b52535SVladimir Sementsov-Ogievskiy     char *data = (char *)bitmap->levels[HBITMAP_LEVELS - 1];
776a3b52535SVladimir Sementsov-Ogievskiy     char *hash = NULL;
777a3b52535SVladimir Sementsov-Ogievskiy     qcrypto_hash_digest(QCRYPTO_HASH_ALG_SHA256, data, size, &hash, errp);
778a3b52535SVladimir Sementsov-Ogievskiy 
779a3b52535SVladimir Sementsov-Ogievskiy     return hash;
780a3b52535SVladimir Sementsov-Ogievskiy }
781