1 /* 2 * Copyright (C) 2012-2014 Canonical Ltd (Maarten Lankhorst) 3 * 4 * Based on bo.c which bears the following copyright notice, 5 * but is dual licensed: 6 * 7 * Copyright (c) 2006-2009 VMware, Inc., Palo Alto, CA., USA 8 * All Rights Reserved. 9 * 10 * Permission is hereby granted, free of charge, to any person obtaining a 11 * copy of this software and associated documentation files (the 12 * "Software"), to deal in the Software without restriction, including 13 * without limitation the rights to use, copy, modify, merge, publish, 14 * distribute, sub license, and/or sell copies of the Software, and to 15 * permit persons to whom the Software is furnished to do so, subject to 16 * the following conditions: 17 * 18 * The above copyright notice and this permission notice (including the 19 * next paragraph) shall be included in all copies or substantial portions 20 * of the Software. 21 * 22 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 23 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 24 * FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL 25 * THE COPYRIGHT HOLDERS, AUTHORS AND/OR ITS SUPPLIERS BE LIABLE FOR ANY CLAIM, 26 * DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR 27 * OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE 28 * USE OR OTHER DEALINGS IN THE SOFTWARE. 29 * 30 **************************************************************************/ 31 /* 32 * Authors: Thomas Hellstrom <thellstrom-at-vmware-dot-com> 33 */ 34 35 #include <linux/reservation.h> 36 #include <linux/export.h> 37 38 /** 39 * DOC: Reservation Object Overview 40 * 41 * The reservation object provides a mechanism to manage shared and 42 * exclusive fences associated with a buffer. A reservation object 43 * can have attached one exclusive fence (normally associated with 44 * write operations) or N shared fences (read operations). The RCU 45 * mechanism is used to protect read access to fences from locked 46 * write-side updates. 47 */ 48 49 DEFINE_WW_CLASS(reservation_ww_class); 50 EXPORT_SYMBOL(reservation_ww_class); 51 52 struct lock_class_key reservation_seqcount_class; 53 EXPORT_SYMBOL(reservation_seqcount_class); 54 55 const char reservation_seqcount_string[] = "reservation_seqcount"; 56 EXPORT_SYMBOL(reservation_seqcount_string); 57 58 /** 59 * reservation_object_reserve_shared - Reserve space to add a shared 60 * fence to a reservation_object. 61 * @obj: reservation object 62 * 63 * Should be called before reservation_object_add_shared_fence(). Must 64 * be called with obj->lock held. 65 * 66 * RETURNS 67 * Zero for success, or -errno 68 */ 69 int reservation_object_reserve_shared(struct reservation_object *obj) 70 { 71 struct reservation_object_list *fobj, *old; 72 u32 max; 73 74 old = reservation_object_get_list(obj); 75 76 if (old && old->shared_max) { 77 if (old->shared_count < old->shared_max) { 78 /* perform an in-place update */ 79 kfree(obj->staged); 80 obj->staged = NULL; 81 return 0; 82 } else 83 max = old->shared_max * 2; 84 } else 85 max = 4; 86 87 /* 88 * resize obj->staged or allocate if it doesn't exist, 89 * noop if already correct size 90 */ 91 fobj = krealloc(obj->staged, offsetof(typeof(*fobj), shared[max]), 92 M_DRM, GFP_KERNEL); 93 if (!fobj) 94 return -ENOMEM; 95 96 obj->staged = fobj; 97 fobj->shared_max = max; 98 return 0; 99 } 100 EXPORT_SYMBOL(reservation_object_reserve_shared); 101 102 static void 103 reservation_object_add_shared_inplace(struct reservation_object *obj, 104 struct reservation_object_list *fobj, 105 struct dma_fence *fence) 106 { 107 u32 i; 108 109 dma_fence_get(fence); 110 111 preempt_disable(); 112 write_seqcount_begin(&obj->seq); 113 114 for (i = 0; i < fobj->shared_count; ++i) { 115 struct dma_fence *old_fence; 116 117 old_fence = rcu_dereference_protected(fobj->shared[i], 118 reservation_object_held(obj)); 119 120 if (old_fence->context == fence->context) { 121 /* memory barrier is added by write_seqcount_begin */ 122 RCU_INIT_POINTER(fobj->shared[i], fence); 123 write_seqcount_end(&obj->seq); 124 preempt_enable(); 125 126 dma_fence_put(old_fence); 127 return; 128 } 129 } 130 131 /* 132 * memory barrier is added by write_seqcount_begin, 133 * fobj->shared_count is protected by this lock too 134 */ 135 RCU_INIT_POINTER(fobj->shared[fobj->shared_count], fence); 136 fobj->shared_count++; 137 138 write_seqcount_end(&obj->seq); 139 preempt_enable(); 140 } 141 142 static void 143 reservation_object_add_shared_replace(struct reservation_object *obj, 144 struct reservation_object_list *old, 145 struct reservation_object_list *fobj, 146 struct dma_fence *fence) 147 { 148 unsigned i; 149 struct dma_fence *old_fence = NULL; 150 151 dma_fence_get(fence); 152 153 if (!old) { 154 RCU_INIT_POINTER(fobj->shared[0], fence); 155 fobj->shared_count = 1; 156 goto done; 157 } 158 159 /* 160 * no need to bump fence refcounts, rcu_read access 161 * requires the use of kref_get_unless_zero, and the 162 * references from the old struct are carried over to 163 * the new. 164 */ 165 fobj->shared_count = old->shared_count; 166 167 for (i = 0; i < old->shared_count; ++i) { 168 struct dma_fence *check; 169 170 check = rcu_dereference_protected(old->shared[i], 171 reservation_object_held(obj)); 172 173 if (!old_fence && check->context == fence->context) { 174 old_fence = check; 175 RCU_INIT_POINTER(fobj->shared[i], fence); 176 } else 177 RCU_INIT_POINTER(fobj->shared[i], check); 178 } 179 if (!old_fence) { 180 RCU_INIT_POINTER(fobj->shared[fobj->shared_count], fence); 181 fobj->shared_count++; 182 } 183 184 done: 185 preempt_disable(); 186 write_seqcount_begin(&obj->seq); 187 /* 188 * RCU_INIT_POINTER can be used here, 189 * seqcount provides the necessary barriers 190 */ 191 RCU_INIT_POINTER(obj->fence, fobj); 192 write_seqcount_end(&obj->seq); 193 preempt_enable(); 194 195 if (old) 196 kfree_rcu(old, NULL); 197 198 dma_fence_put(old_fence); 199 } 200 201 /** 202 * reservation_object_add_shared_fence - Add a fence to a shared slot 203 * @obj: the reservation object 204 * @fence: the shared fence to add 205 * 206 * Add a fence to a shared slot, obj->lock must be held, and 207 * reservation_object_reserve_shared() has been called. 208 */ 209 void reservation_object_add_shared_fence(struct reservation_object *obj, 210 struct dma_fence *fence) 211 { 212 struct reservation_object_list *old, *fobj = obj->staged; 213 214 old = reservation_object_get_list(obj); 215 obj->staged = NULL; 216 217 if (!fobj) { 218 BUG_ON(old->shared_count >= old->shared_max); 219 reservation_object_add_shared_inplace(obj, old, fence); 220 } else 221 reservation_object_add_shared_replace(obj, old, fobj, fence); 222 } 223 EXPORT_SYMBOL(reservation_object_add_shared_fence); 224 225 /** 226 * reservation_object_add_excl_fence - Add an exclusive fence. 227 * @obj: the reservation object 228 * @fence: the shared fence to add 229 * 230 * Add a fence to the exclusive slot. The obj->lock must be held. 231 */ 232 void reservation_object_add_excl_fence(struct reservation_object *obj, 233 struct dma_fence *fence) 234 { 235 struct dma_fence *old_fence = reservation_object_get_excl(obj); 236 struct reservation_object_list *old; 237 u32 i = 0; 238 239 old = reservation_object_get_list(obj); 240 if (old) 241 i = old->shared_count; 242 243 if (fence) 244 dma_fence_get(fence); 245 246 preempt_disable(); 247 write_seqcount_begin(&obj->seq); 248 /* write_seqcount_begin provides the necessary memory barrier */ 249 RCU_INIT_POINTER(obj->fence_excl, fence); 250 if (old) 251 old->shared_count = 0; 252 write_seqcount_end(&obj->seq); 253 preempt_enable(); 254 255 /* inplace update, no shared fences */ 256 while (i--) 257 dma_fence_put(rcu_dereference_protected(old->shared[i], 258 reservation_object_held(obj))); 259 260 dma_fence_put(old_fence); 261 } 262 EXPORT_SYMBOL(reservation_object_add_excl_fence); 263 264 /** 265 * reservation_object_copy_fences - Copy all fences from src to dst. 266 * @dst: the destination reservation object 267 * @src: the source reservation object 268 * 269 * Copy all fences from src to dst. Both src->lock as well as dst-lock must be 270 * held. 271 */ 272 int reservation_object_copy_fences(struct reservation_object *dst, 273 struct reservation_object *src) 274 { 275 struct reservation_object_list *src_list, *dst_list; 276 struct dma_fence *old, *new; 277 size_t size; 278 unsigned i; 279 280 src_list = reservation_object_get_list(src); 281 282 if (src_list) { 283 size = offsetof(typeof(*src_list), 284 shared[src_list->shared_count]); 285 dst_list = kmalloc(size, M_DRM, GFP_KERNEL); 286 if (!dst_list) 287 return -ENOMEM; 288 289 dst_list->shared_count = src_list->shared_count; 290 dst_list->shared_max = src_list->shared_count; 291 for (i = 0; i < src_list->shared_count; ++i) 292 dst_list->shared[i] = 293 dma_fence_get(src_list->shared[i]); 294 } else { 295 dst_list = NULL; 296 } 297 298 kfree(dst->staged); 299 dst->staged = NULL; 300 301 src_list = reservation_object_get_list(dst); 302 303 old = reservation_object_get_excl(dst); 304 new = reservation_object_get_excl(src); 305 306 dma_fence_get(new); 307 308 preempt_disable(); 309 write_seqcount_begin(&dst->seq); 310 /* write_seqcount_begin provides the necessary memory barrier */ 311 RCU_INIT_POINTER(dst->fence_excl, new); 312 RCU_INIT_POINTER(dst->fence, dst_list); 313 write_seqcount_end(&dst->seq); 314 preempt_enable(); 315 316 if (src_list) 317 kfree_rcu(src_list, rcu); 318 dma_fence_put(old); 319 320 return 0; 321 } 322 EXPORT_SYMBOL(reservation_object_copy_fences); 323 324 /** 325 * reservation_object_get_fences_rcu - Get an object's shared and exclusive 326 * fences without update side lock held 327 * @obj: the reservation object 328 * @pfence_excl: the returned exclusive fence (or NULL) 329 * @pshared_count: the number of shared fences returned 330 * @pshared: the array of shared fence ptrs returned (array is krealloc'd to 331 * the required size, and must be freed by caller) 332 * 333 * RETURNS 334 * Zero or -errno 335 */ 336 int reservation_object_get_fences_rcu(struct reservation_object *obj, 337 struct dma_fence **pfence_excl, 338 unsigned *pshared_count, 339 struct dma_fence ***pshared) 340 { 341 struct dma_fence **shared = NULL; 342 struct dma_fence *fence_excl; 343 unsigned int shared_count; 344 int ret = 1; 345 346 do { 347 struct reservation_object_list *fobj; 348 unsigned seq; 349 unsigned int i; 350 351 shared_count = i = 0; 352 353 rcu_read_lock(); 354 seq = read_seqcount_begin(&obj->seq); 355 356 fence_excl = rcu_dereference(obj->fence_excl); 357 if (fence_excl && !dma_fence_get_rcu(fence_excl)) 358 goto unlock; 359 360 fobj = rcu_dereference(obj->fence); 361 if (fobj) { 362 struct dma_fence **nshared; 363 size_t sz = sizeof(*shared) * fobj->shared_max; 364 365 nshared = krealloc(shared, sz, M_DRM, 366 GFP_NOWAIT | __GFP_NOWARN); 367 if (!nshared) { 368 rcu_read_unlock(); 369 nshared = krealloc(shared, sz, M_DRM, 370 GFP_KERNEL); 371 if (nshared) { 372 shared = nshared; 373 continue; 374 } 375 376 ret = -ENOMEM; 377 break; 378 } 379 shared = nshared; 380 shared_count = fobj->shared_count; 381 382 for (i = 0; i < shared_count; ++i) { 383 shared[i] = rcu_dereference(fobj->shared[i]); 384 if (!dma_fence_get_rcu(shared[i])) 385 break; 386 } 387 } 388 389 if (i != shared_count || read_seqcount_retry(&obj->seq, seq)) { 390 while (i--) 391 dma_fence_put(shared[i]); 392 dma_fence_put(fence_excl); 393 goto unlock; 394 } 395 396 ret = 0; 397 unlock: 398 rcu_read_unlock(); 399 } while (ret); 400 401 if (!shared_count) { 402 kfree(shared); 403 shared = NULL; 404 } 405 406 *pshared_count = shared_count; 407 *pshared = shared; 408 *pfence_excl = fence_excl; 409 410 return ret; 411 } 412 EXPORT_SYMBOL_GPL(reservation_object_get_fences_rcu); 413 414 /** 415 * reservation_object_wait_timeout_rcu - Wait on reservation's objects 416 * shared and/or exclusive fences. 417 * @obj: the reservation object 418 * @wait_all: if true, wait on all fences, else wait on just exclusive fence 419 * @intr: if true, do interruptible wait 420 * @timeout: timeout value in jiffies or zero to return immediately 421 * 422 * RETURNS 423 * Returns -ERESTARTSYS if interrupted, 0 if the wait timed out, or 424 * greater than zer on success. 425 */ 426 long reservation_object_wait_timeout_rcu(struct reservation_object *obj, 427 bool wait_all, bool intr, 428 unsigned long timeout) 429 { 430 struct dma_fence *fence; 431 unsigned seq, shared_count, i = 0; 432 long ret = timeout ? timeout : 1; 433 434 retry: 435 shared_count = 0; 436 seq = read_seqcount_begin(&obj->seq); 437 rcu_read_lock(); 438 439 fence = rcu_dereference(obj->fence_excl); 440 if (fence && !test_bit(DMA_FENCE_FLAG_SIGNALED_BIT, &fence->flags)) { 441 if (!dma_fence_get_rcu(fence)) 442 goto unlock_retry; 443 444 if (dma_fence_is_signaled(fence)) { 445 dma_fence_put(fence); 446 fence = NULL; 447 } 448 449 } else { 450 fence = NULL; 451 } 452 453 if (!fence && wait_all) { 454 struct reservation_object_list *fobj = 455 rcu_dereference(obj->fence); 456 457 if (fobj) 458 shared_count = fobj->shared_count; 459 460 for (i = 0; i < shared_count; ++i) { 461 struct dma_fence *lfence = rcu_dereference(fobj->shared[i]); 462 463 if (test_bit(DMA_FENCE_FLAG_SIGNALED_BIT, 464 &lfence->flags)) 465 continue; 466 467 if (!dma_fence_get_rcu(lfence)) 468 goto unlock_retry; 469 470 if (dma_fence_is_signaled(lfence)) { 471 dma_fence_put(lfence); 472 continue; 473 } 474 475 fence = lfence; 476 break; 477 } 478 } 479 480 rcu_read_unlock(); 481 if (fence) { 482 if (read_seqcount_retry(&obj->seq, seq)) { 483 dma_fence_put(fence); 484 goto retry; 485 } 486 487 ret = dma_fence_wait_timeout(fence, intr, ret); 488 dma_fence_put(fence); 489 if (ret > 0 && wait_all && (i + 1 < shared_count)) 490 goto retry; 491 } 492 return ret; 493 494 unlock_retry: 495 rcu_read_unlock(); 496 goto retry; 497 } 498 EXPORT_SYMBOL_GPL(reservation_object_wait_timeout_rcu); 499 500 501 static inline int 502 reservation_object_test_signaled_single(struct dma_fence *passed_fence) 503 { 504 struct dma_fence *fence, *lfence = passed_fence; 505 int ret = 1; 506 507 if (!test_bit(DMA_FENCE_FLAG_SIGNALED_BIT, &lfence->flags)) { 508 fence = dma_fence_get_rcu(lfence); 509 if (!fence) 510 return -1; 511 512 ret = !!dma_fence_is_signaled(fence); 513 dma_fence_put(fence); 514 } 515 return ret; 516 } 517 518 /** 519 * reservation_object_test_signaled_rcu - Test if a reservation object's 520 * fences have been signaled. 521 * @obj: the reservation object 522 * @test_all: if true, test all fences, otherwise only test the exclusive 523 * fence 524 * 525 * RETURNS 526 * true if all fences signaled, else false 527 */ 528 bool reservation_object_test_signaled_rcu(struct reservation_object *obj, 529 bool test_all) 530 { 531 unsigned seq, shared_count; 532 int ret; 533 534 rcu_read_lock(); 535 retry: 536 ret = true; 537 shared_count = 0; 538 seq = read_seqcount_begin(&obj->seq); 539 540 if (test_all) { 541 unsigned i; 542 543 struct reservation_object_list *fobj = 544 rcu_dereference(obj->fence); 545 546 if (fobj) 547 shared_count = fobj->shared_count; 548 549 for (i = 0; i < shared_count; ++i) { 550 struct dma_fence *fence = rcu_dereference(fobj->shared[i]); 551 552 ret = reservation_object_test_signaled_single(fence); 553 if (ret < 0) 554 goto retry; 555 else if (!ret) 556 break; 557 } 558 559 if (read_seqcount_retry(&obj->seq, seq)) 560 goto retry; 561 } 562 563 if (!shared_count) { 564 struct dma_fence *fence_excl = rcu_dereference(obj->fence_excl); 565 566 if (fence_excl) { 567 ret = reservation_object_test_signaled_single( 568 fence_excl); 569 if (ret < 0) 570 goto retry; 571 572 if (read_seqcount_retry(&obj->seq, seq)) 573 goto retry; 574 } 575 } 576 577 rcu_read_unlock(); 578 return ret; 579 } 580 EXPORT_SYMBOL_GPL(reservation_object_test_signaled_rcu); 581