1 /* 2 * Copyright (C) 2012-2014 Canonical Ltd (Maarten Lankhorst) 3 * 4 * Based on bo.c which bears the following copyright notice, 5 * but is dual licensed: 6 * 7 * Copyright (c) 2006-2009 VMware, Inc., Palo Alto, CA., USA 8 * All Rights Reserved. 9 * 10 * Permission is hereby granted, free of charge, to any person obtaining a 11 * copy of this software and associated documentation files (the 12 * "Software"), to deal in the Software without restriction, including 13 * without limitation the rights to use, copy, modify, merge, publish, 14 * distribute, sub license, and/or sell copies of the Software, and to 15 * permit persons to whom the Software is furnished to do so, subject to 16 * the following conditions: 17 * 18 * The above copyright notice and this permission notice (including the 19 * next paragraph) shall be included in all copies or substantial portions 20 * of the Software. 21 * 22 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 23 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 24 * FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL 25 * THE COPYRIGHT HOLDERS, AUTHORS AND/OR ITS SUPPLIERS BE LIABLE FOR ANY CLAIM, 26 * DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR 27 * OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE 28 * USE OR OTHER DEALINGS IN THE SOFTWARE. 29 * 30 **************************************************************************/ 31 /* 32 * Authors: Thomas Hellstrom <thellstrom-at-vmware-dot-com> 33 */ 34 35 #include <linux/reservation.h> 36 #include <linux/export.h> 37 38 /** 39 * DOC: Reservation Object Overview 40 * 41 * The reservation object provides a mechanism to manage shared and 42 * exclusive fences associated with a buffer. A reservation object 43 * can have attached one exclusive fence (normally associated with 44 * write operations) or N shared fences (read operations). The RCU 45 * mechanism is used to protect read access to fences from locked 46 * write-side updates. 47 */ 48 49 DEFINE_WW_CLASS(reservation_ww_class); 50 EXPORT_SYMBOL(reservation_ww_class); 51 52 struct lock_class_key reservation_seqcount_class; 53 EXPORT_SYMBOL(reservation_seqcount_class); 54 55 const char reservation_seqcount_string[] = "reservation_seqcount"; 56 EXPORT_SYMBOL(reservation_seqcount_string); 57 58 /** 59 * reservation_object_reserve_shared - Reserve space to add a shared 60 * fence to a reservation_object. 61 * @obj: reservation object 62 * 63 * Should be called before reservation_object_add_shared_fence(). Must 64 * be called with obj->lock held. 65 * 66 * RETURNS 67 * Zero for success, or -errno 68 */ 69 int reservation_object_reserve_shared(struct reservation_object *obj) 70 { 71 struct reservation_object_list *fobj, *old; 72 u32 max; 73 74 old = reservation_object_get_list(obj); 75 76 if (old && old->shared_max) { 77 if (old->shared_count < old->shared_max) { 78 /* perform an in-place update */ 79 kfree(obj->staged); 80 obj->staged = NULL; 81 return 0; 82 } else 83 max = old->shared_max * 2; 84 } else 85 max = 4; 86 87 /* 88 * resize obj->staged or allocate if it doesn't exist, 89 * noop if already correct size 90 */ 91 fobj = krealloc(obj->staged, offsetof(typeof(*fobj), shared[max]), 92 M_DRM, GFP_KERNEL); 93 if (!fobj) 94 return -ENOMEM; 95 96 obj->staged = fobj; 97 fobj->shared_max = max; 98 return 0; 99 } 100 EXPORT_SYMBOL(reservation_object_reserve_shared); 101 102 static void 103 reservation_object_add_shared_inplace(struct reservation_object *obj, 104 struct reservation_object_list *fobj, 105 struct dma_fence *fence) 106 { 107 u32 i; 108 109 dma_fence_get(fence); 110 111 preempt_disable(); 112 write_seqcount_begin(&obj->seq); 113 114 for (i = 0; i < fobj->shared_count; ++i) { 115 struct dma_fence *old_fence; 116 117 old_fence = rcu_dereference_protected(fobj->shared[i], 118 reservation_object_held(obj)); 119 120 if (old_fence->context == fence->context) { 121 /* memory barrier is added by write_seqcount_begin */ 122 RCU_INIT_POINTER(fobj->shared[i], fence); 123 write_seqcount_end(&obj->seq); 124 preempt_enable(); 125 126 dma_fence_put(old_fence); 127 return; 128 } 129 } 130 131 /* 132 * memory barrier is added by write_seqcount_begin, 133 * fobj->shared_count is protected by this lock too 134 */ 135 RCU_INIT_POINTER(fobj->shared[fobj->shared_count], fence); 136 fobj->shared_count++; 137 138 write_seqcount_end(&obj->seq); 139 preempt_enable(); 140 } 141 142 static void 143 reservation_object_add_shared_replace(struct reservation_object *obj, 144 struct reservation_object_list *old, 145 struct reservation_object_list *fobj, 146 struct dma_fence *fence) 147 { 148 unsigned i; 149 struct dma_fence *old_fence = NULL; 150 151 dma_fence_get(fence); 152 153 if (!old) { 154 RCU_INIT_POINTER(fobj->shared[0], fence); 155 fobj->shared_count = 1; 156 goto done; 157 } 158 159 /* 160 * no need to bump fence refcounts, rcu_read access 161 * requires the use of kref_get_unless_zero, and the 162 * references from the old struct are carried over to 163 * the new. 164 */ 165 fobj->shared_count = old->shared_count; 166 167 for (i = 0; i < old->shared_count; ++i) { 168 struct dma_fence *check; 169 170 check = rcu_dereference_protected(old->shared[i], 171 reservation_object_held(obj)); 172 173 if (!old_fence && check->context == fence->context) { 174 old_fence = check; 175 RCU_INIT_POINTER(fobj->shared[i], fence); 176 } else 177 RCU_INIT_POINTER(fobj->shared[i], check); 178 } 179 if (!old_fence) { 180 RCU_INIT_POINTER(fobj->shared[fobj->shared_count], fence); 181 fobj->shared_count++; 182 } 183 184 done: 185 preempt_disable(); 186 write_seqcount_begin(&obj->seq); 187 /* 188 * RCU_INIT_POINTER can be used here, 189 * seqcount provides the necessary barriers 190 */ 191 RCU_INIT_POINTER(obj->fence, fobj); 192 write_seqcount_end(&obj->seq); 193 preempt_enable(); 194 195 if (old) 196 kfree_rcu(old, NULL); 197 198 if (old_fence) 199 dma_fence_put(old_fence); 200 } 201 202 /** 203 * reservation_object_add_shared_fence - Add a fence to a shared slot 204 * @obj: the reservation object 205 * @fence: the shared fence to add 206 * 207 * Add a fence to a shared slot, obj->lock must be held, and 208 * reservation_object_reserve_shared() has been called. 209 */ 210 void reservation_object_add_shared_fence(struct reservation_object *obj, 211 struct dma_fence *fence) 212 { 213 struct reservation_object_list *old, *fobj = obj->staged; 214 215 old = reservation_object_get_list(obj); 216 obj->staged = NULL; 217 218 if (!fobj) { 219 BUG_ON(old->shared_count >= old->shared_max); 220 reservation_object_add_shared_inplace(obj, old, fence); 221 } else 222 reservation_object_add_shared_replace(obj, old, fobj, fence); 223 } 224 EXPORT_SYMBOL(reservation_object_add_shared_fence); 225 226 /** 227 * reservation_object_add_excl_fence - Add an exclusive fence. 228 * @obj: the reservation object 229 * @fence: the shared fence to add 230 * 231 * Add a fence to the exclusive slot. The obj->lock must be held. 232 */ 233 void reservation_object_add_excl_fence(struct reservation_object *obj, 234 struct dma_fence *fence) 235 { 236 struct dma_fence *old_fence = reservation_object_get_excl(obj); 237 struct reservation_object_list *old; 238 u32 i = 0; 239 240 old = reservation_object_get_list(obj); 241 if (old) 242 i = old->shared_count; 243 244 if (fence) 245 dma_fence_get(fence); 246 247 preempt_disable(); 248 write_seqcount_begin(&obj->seq); 249 /* write_seqcount_begin provides the necessary memory barrier */ 250 RCU_INIT_POINTER(obj->fence_excl, fence); 251 if (old) 252 old->shared_count = 0; 253 write_seqcount_end(&obj->seq); 254 preempt_enable(); 255 256 /* inplace update, no shared fences */ 257 while (i--) 258 dma_fence_put(rcu_dereference_protected(old->shared[i], 259 reservation_object_held(obj))); 260 261 if (old_fence) 262 dma_fence_put(old_fence); 263 } 264 EXPORT_SYMBOL(reservation_object_add_excl_fence); 265 266 /** 267 * reservation_object_get_fences_rcu - Get an object's shared and exclusive 268 * fences without update side lock held 269 * @obj: the reservation object 270 * @pfence_excl: the returned exclusive fence (or NULL) 271 * @pshared_count: the number of shared fences returned 272 * @pshared: the array of shared fence ptrs returned (array is krealloc'd to 273 * the required size, and must be freed by caller) 274 * 275 * RETURNS 276 * Zero or -errno 277 */ 278 int reservation_object_get_fences_rcu(struct reservation_object *obj, 279 struct dma_fence **pfence_excl, 280 unsigned *pshared_count, 281 struct dma_fence ***pshared) 282 { 283 struct dma_fence **shared = NULL; 284 struct dma_fence *fence_excl; 285 unsigned int shared_count; 286 int ret = 1; 287 288 do { 289 struct reservation_object_list *fobj; 290 unsigned seq; 291 unsigned int i; 292 293 shared_count = i = 0; 294 295 rcu_read_lock(); 296 seq = read_seqcount_begin(&obj->seq); 297 298 fence_excl = rcu_dereference(obj->fence_excl); 299 if (fence_excl && !dma_fence_get_rcu(fence_excl)) 300 goto unlock; 301 302 fobj = rcu_dereference(obj->fence); 303 if (fobj) { 304 struct dma_fence **nshared; 305 size_t sz = sizeof(*shared) * fobj->shared_max; 306 307 nshared = krealloc(shared, sz, M_DRM, GFP_NOWAIT); 308 if (!nshared) { 309 rcu_read_unlock(); 310 nshared = krealloc(shared, sz, M_DRM, GFP_KERNEL); 311 if (nshared) { 312 shared = nshared; 313 continue; 314 } 315 316 ret = -ENOMEM; 317 break; 318 } 319 shared = nshared; 320 shared_count = fobj->shared_count; 321 322 for (i = 0; i < shared_count; ++i) { 323 shared[i] = rcu_dereference(fobj->shared[i]); 324 if (!dma_fence_get_rcu(shared[i])) 325 break; 326 } 327 } 328 329 if (i != shared_count || read_seqcount_retry(&obj->seq, seq)) { 330 while (i--) 331 dma_fence_put(shared[i]); 332 dma_fence_put(fence_excl); 333 goto unlock; 334 } 335 336 ret = 0; 337 unlock: 338 rcu_read_unlock(); 339 } while (ret); 340 341 if (!shared_count) { 342 kfree(shared); 343 shared = NULL; 344 } 345 346 *pshared_count = shared_count; 347 *pshared = shared; 348 *pfence_excl = fence_excl; 349 350 return ret; 351 } 352 EXPORT_SYMBOL_GPL(reservation_object_get_fences_rcu); 353 354 /** 355 * reservation_object_wait_timeout_rcu - Wait on reservation's objects 356 * shared and/or exclusive fences. 357 * @obj: the reservation object 358 * @wait_all: if true, wait on all fences, else wait on just exclusive fence 359 * @intr: if true, do interruptible wait 360 * @timeout: timeout value in jiffies or zero to return immediately 361 * 362 * RETURNS 363 * Returns -ERESTARTSYS if interrupted, 0 if the wait timed out, or 364 * greater than zer on success. 365 */ 366 long reservation_object_wait_timeout_rcu(struct reservation_object *obj, 367 bool wait_all, bool intr, 368 unsigned long timeout) 369 { 370 struct dma_fence *fence; 371 unsigned seq, shared_count, i = 0; 372 long ret = timeout; 373 374 if (!timeout) 375 return reservation_object_test_signaled_rcu(obj, wait_all); 376 377 retry: 378 fence = NULL; 379 shared_count = 0; 380 seq = read_seqcount_begin(&obj->seq); 381 rcu_read_lock(); 382 383 if (wait_all) { 384 struct reservation_object_list *fobj = 385 rcu_dereference(obj->fence); 386 387 if (fobj) 388 shared_count = fobj->shared_count; 389 390 for (i = 0; i < shared_count; ++i) { 391 struct dma_fence *lfence = rcu_dereference(fobj->shared[i]); 392 393 if (test_bit(DMA_FENCE_FLAG_SIGNALED_BIT, 394 &lfence->flags)) 395 continue; 396 397 if (!dma_fence_get_rcu(lfence)) 398 goto unlock_retry; 399 400 if (dma_fence_is_signaled(lfence)) { 401 dma_fence_put(lfence); 402 continue; 403 } 404 405 fence = lfence; 406 break; 407 } 408 } 409 410 if (!shared_count) { 411 struct dma_fence *fence_excl = rcu_dereference(obj->fence_excl); 412 413 if (fence_excl && 414 !test_bit(DMA_FENCE_FLAG_SIGNALED_BIT, 415 &fence_excl->flags)) { 416 if (!dma_fence_get_rcu(fence_excl)) 417 goto unlock_retry; 418 419 if (dma_fence_is_signaled(fence_excl)) 420 dma_fence_put(fence_excl); 421 else 422 fence = fence_excl; 423 } 424 } 425 426 rcu_read_unlock(); 427 if (fence) { 428 if (read_seqcount_retry(&obj->seq, seq)) { 429 dma_fence_put(fence); 430 goto retry; 431 } 432 433 ret = dma_fence_wait_timeout(fence, intr, ret); 434 dma_fence_put(fence); 435 if (ret > 0 && wait_all && (i + 1 < shared_count)) 436 goto retry; 437 } 438 return ret; 439 440 unlock_retry: 441 rcu_read_unlock(); 442 goto retry; 443 } 444 EXPORT_SYMBOL_GPL(reservation_object_wait_timeout_rcu); 445 446 447 static inline int 448 reservation_object_test_signaled_single(struct dma_fence *passed_fence) 449 { 450 struct dma_fence *fence, *lfence = passed_fence; 451 int ret = 1; 452 453 if (!test_bit(DMA_FENCE_FLAG_SIGNALED_BIT, &lfence->flags)) { 454 fence = dma_fence_get_rcu(lfence); 455 if (!fence) 456 return -1; 457 458 ret = !!dma_fence_is_signaled(fence); 459 dma_fence_put(fence); 460 } 461 return ret; 462 } 463 464 /** 465 * reservation_object_test_signaled_rcu - Test if a reservation object's 466 * fences have been signaled. 467 * @obj: the reservation object 468 * @test_all: if true, test all fences, otherwise only test the exclusive 469 * fence 470 * 471 * RETURNS 472 * true if all fences signaled, else false 473 */ 474 bool reservation_object_test_signaled_rcu(struct reservation_object *obj, 475 bool test_all) 476 { 477 unsigned seq, shared_count; 478 int ret; 479 480 rcu_read_lock(); 481 retry: 482 ret = true; 483 shared_count = 0; 484 seq = read_seqcount_begin(&obj->seq); 485 486 if (test_all) { 487 unsigned i; 488 489 struct reservation_object_list *fobj = 490 rcu_dereference(obj->fence); 491 492 if (fobj) 493 shared_count = fobj->shared_count; 494 495 for (i = 0; i < shared_count; ++i) { 496 struct dma_fence *fence = rcu_dereference(fobj->shared[i]); 497 498 ret = reservation_object_test_signaled_single(fence); 499 if (ret < 0) 500 goto retry; 501 else if (!ret) 502 break; 503 } 504 505 if (read_seqcount_retry(&obj->seq, seq)) 506 goto retry; 507 } 508 509 if (!shared_count) { 510 struct dma_fence *fence_excl = rcu_dereference(obj->fence_excl); 511 512 if (fence_excl) { 513 ret = reservation_object_test_signaled_single( 514 fence_excl); 515 if (ret < 0) 516 goto retry; 517 518 if (read_seqcount_retry(&obj->seq, seq)) 519 goto retry; 520 } 521 } 522 523 rcu_read_unlock(); 524 return ret; 525 } 526 EXPORT_SYMBOL_GPL(reservation_object_test_signaled_rcu); 527