1 /* 2 * Copyright (C) 2012-2014 Canonical Ltd (Maarten Lankhorst) 3 * 4 * Based on bo.c which bears the following copyright notice, 5 * but is dual licensed: 6 * 7 * Copyright (c) 2006-2009 VMware, Inc., Palo Alto, CA., USA 8 * All Rights Reserved. 9 * 10 * Permission is hereby granted, free of charge, to any person obtaining a 11 * copy of this software and associated documentation files (the 12 * "Software"), to deal in the Software without restriction, including 13 * without limitation the rights to use, copy, modify, merge, publish, 14 * distribute, sub license, and/or sell copies of the Software, and to 15 * permit persons to whom the Software is furnished to do so, subject to 16 * the following conditions: 17 * 18 * The above copyright notice and this permission notice (including the 19 * next paragraph) shall be included in all copies or substantial portions 20 * of the Software. 21 * 22 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 23 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 24 * FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL 25 * THE COPYRIGHT HOLDERS, AUTHORS AND/OR ITS SUPPLIERS BE LIABLE FOR ANY CLAIM, 26 * DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR 27 * OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE 28 * USE OR OTHER DEALINGS IN THE SOFTWARE. 29 * 30 **************************************************************************/ 31 /* 32 * Authors: Thomas Hellstrom <thellstrom-at-vmware-dot-com> 33 */ 34 35 #include <linux/reservation.h> 36 #include <linux/export.h> 37 38 /** 39 * DOC: Reservation Object Overview 40 * 41 * The reservation object provides a mechanism to manage shared and 42 * exclusive fences associated with a buffer. A reservation object 43 * can have attached one exclusive fence (normally associated with 44 * write operations) or N shared fences (read operations). The RCU 45 * mechanism is used to protect read access to fences from locked 46 * write-side updates. 47 */ 48 49 DEFINE_WW_CLASS(reservation_ww_class); 50 EXPORT_SYMBOL(reservation_ww_class); 51 52 struct lock_class_key reservation_seqcount_class; 53 EXPORT_SYMBOL(reservation_seqcount_class); 54 55 const char reservation_seqcount_string[] = "reservation_seqcount"; 56 EXPORT_SYMBOL(reservation_seqcount_string); 57 58 /** 59 * reservation_object_reserve_shared - Reserve space to add a shared 60 * fence to a reservation_object. 61 * @obj: reservation object 62 * 63 * Should be called before reservation_object_add_shared_fence(). Must 64 * be called with obj->lock held. 65 * 66 * RETURNS 67 * Zero for success, or -errno 68 */ 69 int reservation_object_reserve_shared(struct reservation_object *obj) 70 { 71 struct reservation_object_list *fobj, *old; 72 u32 max; 73 74 old = reservation_object_get_list(obj); 75 76 if (old && old->shared_max) { 77 if (old->shared_count < old->shared_max) { 78 /* perform an in-place update */ 79 kfree(obj->staged); 80 obj->staged = NULL; 81 return 0; 82 } else 83 max = old->shared_max * 2; 84 } else 85 max = 4; 86 87 /* 88 * resize obj->staged or allocate if it doesn't exist, 89 * noop if already correct size 90 */ 91 fobj = krealloc(obj->staged, offsetof(typeof(*fobj), shared[max]), 92 M_DRM, GFP_KERNEL); 93 if (!fobj) 94 return -ENOMEM; 95 96 obj->staged = fobj; 97 fobj->shared_max = max; 98 return 0; 99 } 100 EXPORT_SYMBOL(reservation_object_reserve_shared); 101 102 static void 103 reservation_object_add_shared_inplace(struct reservation_object *obj, 104 struct reservation_object_list *fobj, 105 struct dma_fence *fence) 106 { 107 u32 i; 108 109 dma_fence_get(fence); 110 111 preempt_disable(); 112 write_seqcount_begin(&obj->seq); 113 114 for (i = 0; i < fobj->shared_count; ++i) { 115 struct dma_fence *old_fence; 116 117 old_fence = rcu_dereference_protected(fobj->shared[i], 118 reservation_object_held(obj)); 119 120 if (old_fence->context == fence->context) { 121 /* memory barrier is added by write_seqcount_begin */ 122 RCU_INIT_POINTER(fobj->shared[i], fence); 123 write_seqcount_end(&obj->seq); 124 preempt_enable(); 125 126 dma_fence_put(old_fence); 127 return; 128 } 129 } 130 131 /* 132 * memory barrier is added by write_seqcount_begin, 133 * fobj->shared_count is protected by this lock too 134 */ 135 RCU_INIT_POINTER(fobj->shared[fobj->shared_count], fence); 136 fobj->shared_count++; 137 138 write_seqcount_end(&obj->seq); 139 preempt_enable(); 140 } 141 142 static void 143 reservation_object_add_shared_replace(struct reservation_object *obj, 144 struct reservation_object_list *old, 145 struct reservation_object_list *fobj, 146 struct dma_fence *fence) 147 { 148 unsigned i; 149 struct dma_fence *old_fence = NULL; 150 151 dma_fence_get(fence); 152 153 if (!old) { 154 RCU_INIT_POINTER(fobj->shared[0], fence); 155 fobj->shared_count = 1; 156 goto done; 157 } 158 159 /* 160 * no need to bump fence refcounts, rcu_read access 161 * requires the use of kref_get_unless_zero, and the 162 * references from the old struct are carried over to 163 * the new. 164 */ 165 fobj->shared_count = old->shared_count; 166 167 for (i = 0; i < old->shared_count; ++i) { 168 struct dma_fence *check; 169 170 check = rcu_dereference_protected(old->shared[i], 171 reservation_object_held(obj)); 172 173 if (!old_fence && check->context == fence->context) { 174 old_fence = check; 175 RCU_INIT_POINTER(fobj->shared[i], fence); 176 } else 177 RCU_INIT_POINTER(fobj->shared[i], check); 178 } 179 if (!old_fence) { 180 RCU_INIT_POINTER(fobj->shared[fobj->shared_count], fence); 181 fobj->shared_count++; 182 } 183 184 done: 185 preempt_disable(); 186 write_seqcount_begin(&obj->seq); 187 /* 188 * RCU_INIT_POINTER can be used here, 189 * seqcount provides the necessary barriers 190 */ 191 RCU_INIT_POINTER(obj->fence, fobj); 192 write_seqcount_end(&obj->seq); 193 preempt_enable(); 194 195 if (old) 196 kfree_rcu(old, NULL); 197 198 dma_fence_put(old_fence); 199 } 200 201 /** 202 * reservation_object_add_shared_fence - Add a fence to a shared slot 203 * @obj: the reservation object 204 * @fence: the shared fence to add 205 * 206 * Add a fence to a shared slot, obj->lock must be held, and 207 * reservation_object_reserve_shared() has been called. 208 */ 209 void reservation_object_add_shared_fence(struct reservation_object *obj, 210 struct dma_fence *fence) 211 { 212 struct reservation_object_list *old, *fobj = obj->staged; 213 214 old = reservation_object_get_list(obj); 215 obj->staged = NULL; 216 217 if (!fobj) { 218 BUG_ON(old->shared_count >= old->shared_max); 219 reservation_object_add_shared_inplace(obj, old, fence); 220 } else 221 reservation_object_add_shared_replace(obj, old, fobj, fence); 222 } 223 EXPORT_SYMBOL(reservation_object_add_shared_fence); 224 225 /** 226 * reservation_object_add_excl_fence - Add an exclusive fence. 227 * @obj: the reservation object 228 * @fence: the shared fence to add 229 * 230 * Add a fence to the exclusive slot. The obj->lock must be held. 231 */ 232 void reservation_object_add_excl_fence(struct reservation_object *obj, 233 struct dma_fence *fence) 234 { 235 struct dma_fence *old_fence = reservation_object_get_excl(obj); 236 struct reservation_object_list *old; 237 u32 i = 0; 238 239 old = reservation_object_get_list(obj); 240 if (old) 241 i = old->shared_count; 242 243 if (fence) 244 dma_fence_get(fence); 245 246 preempt_disable(); 247 write_seqcount_begin(&obj->seq); 248 /* write_seqcount_begin provides the necessary memory barrier */ 249 RCU_INIT_POINTER(obj->fence_excl, fence); 250 if (old) 251 old->shared_count = 0; 252 write_seqcount_end(&obj->seq); 253 preempt_enable(); 254 255 /* inplace update, no shared fences */ 256 while (i--) 257 dma_fence_put(rcu_dereference_protected(old->shared[i], 258 reservation_object_held(obj))); 259 260 dma_fence_put(old_fence); 261 } 262 EXPORT_SYMBOL(reservation_object_add_excl_fence); 263 264 /** 265 * reservation_object_copy_fences - Copy all fences from src to dst. 266 * @dst: the destination reservation object 267 * @src: the source reservation object 268 * 269 * Copy all fences from src to dst. Both src->lock as well as dst-lock must be 270 * held. 271 */ 272 int reservation_object_copy_fences(struct reservation_object *dst, 273 struct reservation_object *src) 274 { 275 struct reservation_object_list *src_list, *dst_list; 276 struct dma_fence *old, *new; 277 size_t size; 278 unsigned i; 279 280 src_list = reservation_object_get_list(src); 281 282 if (src_list) { 283 size = offsetof(typeof(*src_list), 284 shared[src_list->shared_count]); 285 dst_list = kmalloc(size, M_DRM, GFP_KERNEL); 286 if (!dst_list) 287 return -ENOMEM; 288 289 dst_list->shared_count = src_list->shared_count; 290 dst_list->shared_max = src_list->shared_count; 291 for (i = 0; i < src_list->shared_count; ++i) 292 dst_list->shared[i] = 293 dma_fence_get(src_list->shared[i]); 294 } else { 295 dst_list = NULL; 296 } 297 298 kfree(dst->staged); 299 dst->staged = NULL; 300 301 src_list = reservation_object_get_list(dst); 302 303 old = reservation_object_get_excl(dst); 304 new = reservation_object_get_excl(src); 305 306 dma_fence_get(new); 307 308 preempt_disable(); 309 write_seqcount_begin(&dst->seq); 310 /* write_seqcount_begin provides the necessary memory barrier */ 311 RCU_INIT_POINTER(dst->fence_excl, new); 312 RCU_INIT_POINTER(dst->fence, dst_list); 313 write_seqcount_end(&dst->seq); 314 preempt_enable(); 315 316 if (src_list) 317 kfree_rcu(src_list, rcu); 318 dma_fence_put(old); 319 320 return 0; 321 } 322 EXPORT_SYMBOL(reservation_object_copy_fences); 323 324 /** 325 * reservation_object_get_fences_rcu - Get an object's shared and exclusive 326 * fences without update side lock held 327 * @obj: the reservation object 328 * @pfence_excl: the returned exclusive fence (or NULL) 329 * @pshared_count: the number of shared fences returned 330 * @pshared: the array of shared fence ptrs returned (array is krealloc'd to 331 * the required size, and must be freed by caller) 332 * 333 * RETURNS 334 * Zero or -errno 335 */ 336 int reservation_object_get_fences_rcu(struct reservation_object *obj, 337 struct dma_fence **pfence_excl, 338 unsigned *pshared_count, 339 struct dma_fence ***pshared) 340 { 341 struct dma_fence **shared = NULL; 342 struct dma_fence *fence_excl; 343 unsigned int shared_count; 344 int ret = 1; 345 346 do { 347 struct reservation_object_list *fobj; 348 unsigned seq; 349 unsigned int i; 350 351 shared_count = i = 0; 352 353 rcu_read_lock(); 354 seq = read_seqcount_begin(&obj->seq); 355 356 fence_excl = rcu_dereference(obj->fence_excl); 357 if (fence_excl && !dma_fence_get_rcu(fence_excl)) 358 goto unlock; 359 360 fobj = rcu_dereference(obj->fence); 361 if (fobj) { 362 struct dma_fence **nshared; 363 size_t sz = sizeof(*shared) * fobj->shared_max; 364 365 nshared = krealloc(shared, sz, M_DRM, 366 GFP_NOWAIT | __GFP_NOWARN); 367 if (!nshared) { 368 rcu_read_unlock(); 369 nshared = krealloc(shared, sz, M_DRM, 370 GFP_KERNEL); 371 if (nshared) { 372 shared = nshared; 373 continue; 374 } 375 376 ret = -ENOMEM; 377 break; 378 } 379 shared = nshared; 380 shared_count = fobj->shared_count; 381 382 for (i = 0; i < shared_count; ++i) { 383 shared[i] = rcu_dereference(fobj->shared[i]); 384 if (!dma_fence_get_rcu(shared[i])) 385 break; 386 } 387 } 388 389 if (i != shared_count || read_seqcount_retry(&obj->seq, seq)) { 390 while (i--) 391 dma_fence_put(shared[i]); 392 dma_fence_put(fence_excl); 393 goto unlock; 394 } 395 396 ret = 0; 397 unlock: 398 rcu_read_unlock(); 399 } while (ret); 400 401 if (!shared_count) { 402 kfree(shared); 403 shared = NULL; 404 } 405 406 if (pshared_count) 407 *pshared_count = shared_count; 408 if (pshared) 409 *pshared = shared; 410 if (pfence_excl) 411 *pfence_excl = fence_excl; 412 413 return ret; 414 } 415 EXPORT_SYMBOL_GPL(reservation_object_get_fences_rcu); 416 417 /** 418 * reservation_object_wait_timeout_rcu - Wait on reservation's objects 419 * shared and/or exclusive fences. 420 * @obj: the reservation object 421 * @wait_all: if true, wait on all fences, else wait on just exclusive fence 422 * @intr: if true, do interruptible wait 423 * @timeout: timeout value in jiffies or zero to return immediately 424 * 425 * RETURNS 426 * Returns -ERESTARTSYS if interrupted, 0 if the wait timed out, or 427 * greater than zer on success. 428 */ 429 long reservation_object_wait_timeout_rcu(struct reservation_object *obj, 430 bool wait_all, bool intr, 431 unsigned long timeout) 432 { 433 struct dma_fence *fence; 434 unsigned seq, shared_count, i = 0; 435 long ret = timeout ? timeout : 1; 436 437 retry: 438 shared_count = 0; 439 seq = read_seqcount_begin(&obj->seq); 440 rcu_read_lock(); 441 442 fence = rcu_dereference(obj->fence_excl); 443 if (fence && !test_bit(DMA_FENCE_FLAG_SIGNALED_BIT, &fence->flags)) { 444 if (!dma_fence_get_rcu(fence)) 445 goto unlock_retry; 446 447 if (dma_fence_is_signaled(fence)) { 448 dma_fence_put(fence); 449 fence = NULL; 450 } 451 452 } else { 453 fence = NULL; 454 } 455 456 if (!fence && wait_all) { 457 struct reservation_object_list *fobj = 458 rcu_dereference(obj->fence); 459 460 if (fobj) 461 shared_count = fobj->shared_count; 462 463 for (i = 0; i < shared_count; ++i) { 464 struct dma_fence *lfence = rcu_dereference(fobj->shared[i]); 465 466 if (test_bit(DMA_FENCE_FLAG_SIGNALED_BIT, 467 &lfence->flags)) 468 continue; 469 470 if (!dma_fence_get_rcu(lfence)) 471 goto unlock_retry; 472 473 if (dma_fence_is_signaled(lfence)) { 474 dma_fence_put(lfence); 475 continue; 476 } 477 478 fence = lfence; 479 break; 480 } 481 } 482 483 rcu_read_unlock(); 484 if (fence) { 485 if (read_seqcount_retry(&obj->seq, seq)) { 486 dma_fence_put(fence); 487 goto retry; 488 } 489 490 ret = dma_fence_wait_timeout(fence, intr, ret); 491 dma_fence_put(fence); 492 if (ret > 0 && wait_all && (i + 1 < shared_count)) 493 goto retry; 494 } 495 return ret; 496 497 unlock_retry: 498 rcu_read_unlock(); 499 goto retry; 500 } 501 EXPORT_SYMBOL_GPL(reservation_object_wait_timeout_rcu); 502 503 504 static inline int 505 reservation_object_test_signaled_single(struct dma_fence *passed_fence) 506 { 507 struct dma_fence *fence, *lfence = passed_fence; 508 int ret = 1; 509 510 if (!test_bit(DMA_FENCE_FLAG_SIGNALED_BIT, &lfence->flags)) { 511 fence = dma_fence_get_rcu(lfence); 512 if (!fence) 513 return -1; 514 515 ret = !!dma_fence_is_signaled(fence); 516 dma_fence_put(fence); 517 } 518 return ret; 519 } 520 521 /** 522 * reservation_object_test_signaled_rcu - Test if a reservation object's 523 * fences have been signaled. 524 * @obj: the reservation object 525 * @test_all: if true, test all fences, otherwise only test the exclusive 526 * fence 527 * 528 * RETURNS 529 * true if all fences signaled, else false 530 */ 531 bool reservation_object_test_signaled_rcu(struct reservation_object *obj, 532 bool test_all) 533 { 534 unsigned seq, shared_count; 535 int ret; 536 537 rcu_read_lock(); 538 retry: 539 ret = true; 540 shared_count = 0; 541 seq = read_seqcount_begin(&obj->seq); 542 543 if (test_all) { 544 unsigned i; 545 546 struct reservation_object_list *fobj = 547 rcu_dereference(obj->fence); 548 549 if (fobj) 550 shared_count = fobj->shared_count; 551 552 for (i = 0; i < shared_count; ++i) { 553 struct dma_fence *fence = rcu_dereference(fobj->shared[i]); 554 555 ret = reservation_object_test_signaled_single(fence); 556 if (ret < 0) 557 goto retry; 558 else if (!ret) 559 break; 560 } 561 562 if (read_seqcount_retry(&obj->seq, seq)) 563 goto retry; 564 } 565 566 if (!shared_count) { 567 struct dma_fence *fence_excl = rcu_dereference(obj->fence_excl); 568 569 if (fence_excl) { 570 ret = reservation_object_test_signaled_single( 571 fence_excl); 572 if (ret < 0) 573 goto retry; 574 575 if (read_seqcount_retry(&obj->seq, seq)) 576 goto retry; 577 } 578 } 579 580 rcu_read_unlock(); 581 return ret; 582 } 583 EXPORT_SYMBOL_GPL(reservation_object_test_signaled_rcu); 584