1 /*******************************************************************************
2 Copyright (c) 2015-2019 NVIDIA Corporation
3
4 Permission is hereby granted, free of charge, to any person obtaining a copy
5 of this software and associated documentation files (the "Software"), to
6 deal in the Software without restriction, including without limitation the
7 rights to use, copy, modify, merge, publish, distribute, sublicense, and/or
8 sell copies of the Software, and to permit persons to whom the Software is
9 furnished to do so, subject to the following conditions:
10
11 The above copyright notice and this permission notice shall be
12 included in all copies or substantial portions of the Software.
13
14 THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
15 IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
16 FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
17 THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
18 LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
19 FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
20 DEALINGS IN THE SOFTWARE.
21
22 *******************************************************************************/
23
24 #include "uvm_common.h"
25 #include "uvm_range_tree.h"
26
get_range_node(struct rb_node * rb_node)27 static uvm_range_tree_node_t *get_range_node(struct rb_node *rb_node)
28 {
29 return rb_entry(rb_node, uvm_range_tree_node_t, rb_node);
30 }
31
range_nodes_overlap(uvm_range_tree_node_t * a,uvm_range_tree_node_t * b)32 static bool range_nodes_overlap(uvm_range_tree_node_t *a, uvm_range_tree_node_t *b)
33 {
34 return uvm_ranges_overlap(a->start, a->end, b->start, b->end);
35 }
36
37 // Workhorse tree walking function.
38 //
39 // The parent and next pointers may be NULL if the caller doesn't need them.
40 // They facilitate node addition and range-based searches.
41 //
42 // If a node contains addr:
43 // - That node is returned
44 // - The parent pointer is set to node's parent, or to NULL if the node is the
45 // root.
46 // - The next pointer is set to the next node in address order in the tree, or
47 // to NULL if node is the last node in the tree.
48 //
49 // If no node contains addr:
50 // - NULL is returned
51 // - The parent pointer is set to the node under which a new node containing
52 // addr should be inserted. This will be NULL if the tree is empty.
53 // - The next pointer is set to the first node containing an address > addr, or
54 // NULL if there are no such nodes in the tree.
range_node_find(uvm_range_tree_t * tree,NvU64 addr,uvm_range_tree_node_t ** parent,uvm_range_tree_node_t ** next)55 static uvm_range_tree_node_t *range_node_find(uvm_range_tree_t *tree,
56 NvU64 addr,
57 uvm_range_tree_node_t **parent,
58 uvm_range_tree_node_t **next)
59 {
60 struct rb_node *rb_node = tree->rb_root.rb_node;
61 uvm_range_tree_node_t *node = NULL;
62 uvm_range_tree_node_t *_parent = NULL;
63
64 while (rb_node) {
65 node = get_range_node(rb_node);
66
67 if (addr < node->start)
68 rb_node = rb_node->rb_left;
69 else if (addr > node->end)
70 rb_node = rb_node->rb_right;
71 else // node contains addr
72 break;
73
74 _parent = node;
75 }
76
77 if (!rb_node)
78 node = NULL;
79
80 if (parent)
81 *parent = _parent;
82 if (next) {
83 *next = NULL; // Handles the empty tree case
84 if (node) {
85 *next = uvm_range_tree_next(tree, node);
86 }
87 else if (_parent) {
88 if (_parent->start > addr)
89 *next = _parent;
90 else
91 *next = uvm_range_tree_next(tree, _parent);
92 }
93 }
94
95 return node;
96 }
97
uvm_range_tree_init(uvm_range_tree_t * tree)98 void uvm_range_tree_init(uvm_range_tree_t *tree)
99 {
100 memset(tree, 0, sizeof(*tree));
101 tree->rb_root = RB_ROOT;
102 INIT_LIST_HEAD(&tree->head);
103 }
104
uvm_range_tree_add(uvm_range_tree_t * tree,uvm_range_tree_node_t * node)105 NV_STATUS uvm_range_tree_add(uvm_range_tree_t *tree, uvm_range_tree_node_t *node)
106 {
107 uvm_range_tree_node_t *match, *parent, *prev, *next;
108
109 UVM_ASSERT(node->start <= node->end);
110
111 match = range_node_find(tree, node->start, &parent, NULL);
112 if (match)
113 return NV_ERR_UVM_ADDRESS_IN_USE;
114
115 // If no match we know that the new start isn't contained in any existing
116 // node, but we still have to check for overlap on the rest of the new range.
117
118 // If there's no parent and we didn't match on the root node, the tree is
119 // empty.
120 if (!parent) {
121 rb_link_node(&node->rb_node, NULL, &tree->rb_root.rb_node);
122 rb_insert_color(&node->rb_node, &tree->rb_root);
123 list_add(&node->list, &tree->head);
124 return NV_OK;
125 }
126
127 // We know that start isn't contained in parent, but the rest of the new
128 // range might be.
129 if (range_nodes_overlap(node, parent))
130 return NV_ERR_UVM_ADDRESS_IN_USE;
131
132 // Verify that the new node doesn't overlap with its neighbor and insert
133 if (node->start < parent->start) {
134 // parent's prev can't overlap with node, otherwise it must overlap with
135 // start and would've been found by range_node_find above.
136 prev = uvm_range_tree_prev(tree, parent);
137 if (prev)
138 UVM_ASSERT(!range_nodes_overlap(node, prev));
139
140 rb_link_node(&node->rb_node, &parent->rb_node, &parent->rb_node.rb_left);
141 list_add_tail(&node->list, &parent->list);
142 }
143 else {
144 next = uvm_range_tree_next(tree, parent);
145 if (next && range_nodes_overlap(node, next))
146 return NV_ERR_UVM_ADDRESS_IN_USE;
147
148 rb_link_node(&node->rb_node, &parent->rb_node, &parent->rb_node.rb_right);
149 list_add(&node->list, &parent->list);
150 }
151
152 rb_insert_color(&node->rb_node, &tree->rb_root);
153 return NV_OK;
154 }
155
uvm_range_tree_shrink_node(uvm_range_tree_t * tree,uvm_range_tree_node_t * node,NvU64 new_start,NvU64 new_end)156 void uvm_range_tree_shrink_node(uvm_range_tree_t *tree, uvm_range_tree_node_t *node, NvU64 new_start, NvU64 new_end)
157 {
158 UVM_ASSERT_MSG(new_start <= new_end, "new_start 0x%llx new_end 0x%llx\n", new_start, new_end);
159 UVM_ASSERT_MSG(node->start <= new_start, "start 0x%llx new_start 0x%llx\n", node->start, new_start);
160 UVM_ASSERT_MSG(node->end >= new_end, "end 0x%llx new_end 0x%llx\n", node->end, new_end);
161
162 // The tree is not needed currently, but might be in the future.
163 (void)tree;
164
165 node->start = new_start;
166 node->end = new_end;
167 }
168
uvm_range_tree_split(uvm_range_tree_t * tree,uvm_range_tree_node_t * existing,uvm_range_tree_node_t * new)169 void uvm_range_tree_split(uvm_range_tree_t *tree,
170 uvm_range_tree_node_t *existing,
171 uvm_range_tree_node_t *new)
172 {
173 NV_STATUS status;
174
175 UVM_ASSERT(new->start > existing->start);
176 UVM_ASSERT(new->start <= existing->end);
177
178 // existing doesn't have to move anywhere, we just need to adjust its
179 // ranges. new will need to be inserted into the tree.
180 //
181 // Future optimization: insertion could walk down the tree starting from
182 // existing rather than from the root.
183 new->end = existing->end;
184 existing->end = new->start - 1;
185 status = uvm_range_tree_add(tree, new);
186 UVM_ASSERT(status == NV_OK); // There shouldn't be any collisions
187 }
188
uvm_range_tree_merge_prev(uvm_range_tree_t * tree,uvm_range_tree_node_t * node)189 uvm_range_tree_node_t *uvm_range_tree_merge_prev(uvm_range_tree_t *tree, uvm_range_tree_node_t *node)
190 {
191 uvm_range_tree_node_t *prev = uvm_range_tree_prev(tree, node);
192 if (!prev || prev->end != node->start - 1)
193 return NULL;
194
195 uvm_range_tree_remove(tree, prev);
196 node->start = prev->start;
197 return prev;
198 }
199
uvm_range_tree_merge_next(uvm_range_tree_t * tree,uvm_range_tree_node_t * node)200 uvm_range_tree_node_t *uvm_range_tree_merge_next(uvm_range_tree_t *tree, uvm_range_tree_node_t *node)
201 {
202 uvm_range_tree_node_t *next = uvm_range_tree_next(tree, node);
203 if (!next || next->start != node->end + 1)
204 return NULL;
205
206 uvm_range_tree_remove(tree, next);
207 node->end = next->end;
208 return next;
209 }
210
uvm_range_tree_find(uvm_range_tree_t * tree,NvU64 addr)211 uvm_range_tree_node_t *uvm_range_tree_find(uvm_range_tree_t *tree, NvU64 addr)
212 {
213 return range_node_find(tree, addr, NULL, NULL);
214 }
215
uvm_range_tree_iter_first(uvm_range_tree_t * tree,NvU64 start,NvU64 end)216 uvm_range_tree_node_t *uvm_range_tree_iter_first(uvm_range_tree_t *tree, NvU64 start, NvU64 end)
217 {
218 uvm_range_tree_node_t *node, *next;
219
220 UVM_ASSERT(start <= end);
221
222 node = range_node_find(tree, start, NULL, &next);
223 if (node)
224 return node;
225
226 // We didn't find a node containing start itself. Check if the target range
227 // overlaps with the next node after start.
228 if (next) {
229 // Sanity checks
230 UVM_ASSERT(start < next->start);
231 if (uvm_range_tree_prev(tree, next))
232 UVM_ASSERT(uvm_range_tree_prev(tree, next)->end < start);
233
234 if (next->start <= end)
235 return next;
236 }
237
238 return NULL;
239 }
240
uvm_range_tree_find_hole(uvm_range_tree_t * tree,NvU64 addr,NvU64 * start,NvU64 * end)241 NV_STATUS uvm_range_tree_find_hole(uvm_range_tree_t *tree, NvU64 addr, NvU64 *start, NvU64 *end)
242 {
243 uvm_range_tree_node_t *node;
244
245 // Find the first node on or after addr, if any
246 node = uvm_range_tree_iter_first(tree, addr, ULLONG_MAX);
247 if (node) {
248 if (node->start <= addr)
249 return NV_ERR_UVM_ADDRESS_IN_USE;
250
251 // node->start can't be 0, otherwise it would contain addr
252 if (end)
253 *end = node->start - 1;
254
255 node = uvm_range_tree_prev(tree, node);
256 }
257 else {
258 // All nodes in the tree must come before addr, if any exist
259 node = uvm_range_tree_last(tree);
260 if (end)
261 *end = ULLONG_MAX;
262 }
263
264 if (start) {
265 if (node)
266 *start = node->end + 1;
267 else
268 *start = 0;
269 }
270
271 return NV_OK;
272 }
273
uvm_range_tree_find_hole_in(uvm_range_tree_t * tree,NvU64 addr,NvU64 * start,NvU64 * end)274 NV_STATUS uvm_range_tree_find_hole_in(uvm_range_tree_t *tree, NvU64 addr, NvU64 *start, NvU64 *end)
275 {
276 NvU64 temp_start, temp_end;
277 NV_STATUS status;
278
279 UVM_ASSERT(start);
280 UVM_ASSERT(end);
281 UVM_ASSERT(*start <= addr);
282 UVM_ASSERT(*end >= addr);
283
284 status = uvm_range_tree_find_hole(tree, addr, &temp_start, &temp_end);
285 if (status == NV_OK) {
286 *start = max(temp_start, *start);
287 *end = min(temp_end, *end);
288 }
289
290 return status;
291 }
292