1 // SPDX-License-Identifier: GPL-2.0-only 2 /* 3 * dma-fence-util: misc functions for dma_fence objects 4 * 5 * Copyright (C) 2022 Advanced Micro Devices, Inc. 6 * Authors: 7 * Christian König <christian.koenig@amd.com> 8 */ 9 10 #include <linux/dma-fence.h> 11 #include <linux/dma-fence-array.h> 12 #include <linux/dma-fence-chain.h> 13 #include <linux/dma-fence-unwrap.h> 14 #include <linux/slab.h> 15 16 /* Internal helper to start new array iteration, don't use directly */ 17 static struct dma_fence * 18 __dma_fence_unwrap_array(struct dma_fence_unwrap *cursor) 19 { 20 cursor->array = dma_fence_chain_contained(cursor->chain); 21 cursor->index = 0; 22 return dma_fence_array_first(cursor->array); 23 } 24 25 /** 26 * dma_fence_unwrap_first - return the first fence from fence containers 27 * @head: the entrypoint into the containers 28 * @cursor: current position inside the containers 29 * 30 * Unwraps potential dma_fence_chain/dma_fence_array containers and return the 31 * first fence. 32 */ 33 struct dma_fence *dma_fence_unwrap_first(struct dma_fence *head, 34 struct dma_fence_unwrap *cursor) 35 { 36 cursor->chain = dma_fence_get(head); 37 return __dma_fence_unwrap_array(cursor); 38 } 39 EXPORT_SYMBOL_GPL(dma_fence_unwrap_first); 40 41 /** 42 * dma_fence_unwrap_next - return the next fence from a fence containers 43 * @cursor: current position inside the containers 44 * 45 * Continue unwrapping the dma_fence_chain/dma_fence_array containers and return 46 * the next fence from them. 47 */ 48 struct dma_fence *dma_fence_unwrap_next(struct dma_fence_unwrap *cursor) 49 { 50 struct dma_fence *tmp; 51 52 ++cursor->index; 53 tmp = dma_fence_array_next(cursor->array, cursor->index); 54 if (tmp) 55 return tmp; 56 57 cursor->chain = dma_fence_chain_walk(cursor->chain); 58 return __dma_fence_unwrap_array(cursor); 59 } 60 EXPORT_SYMBOL_GPL(dma_fence_unwrap_next); 61 62 /* Implementation for the dma_fence_merge() marco, don't use directly */ 63 struct dma_fence *__dma_fence_unwrap_merge(unsigned int num_fences, 64 struct dma_fence **fences, 65 struct dma_fence_unwrap *iter) 66 { 67 struct dma_fence_array *result; 68 struct dma_fence *tmp, **array; 69 ktime_t timestamp; 70 unsigned int i; 71 size_t count; 72 73 count = 0; 74 timestamp = ns_to_ktime(0); 75 for (i = 0; i < num_fences; ++i) { 76 dma_fence_unwrap_for_each(tmp, &iter[i], fences[i]) { 77 if (!dma_fence_is_signaled(tmp)) { 78 ++count; 79 } else if (test_bit(DMA_FENCE_FLAG_TIMESTAMP_BIT, 80 &tmp->flags)) { 81 if (ktime_after(tmp->timestamp, timestamp)) 82 timestamp = tmp->timestamp; 83 } else { 84 /* 85 * Use the current time if the fence is 86 * currently signaling. 87 */ 88 timestamp = ktime_get(); 89 } 90 } 91 } 92 93 /* 94 * If we couldn't find a pending fence just return a private signaled 95 * fence with the timestamp of the last signaled one. 96 */ 97 if (count == 0) 98 return dma_fence_allocate_private_stub(timestamp); 99 100 array = kmalloc_array(count, sizeof(*array), GFP_KERNEL); 101 if (!array) 102 return NULL; 103 104 /* 105 * This trashes the input fence array and uses it as position for the 106 * following merge loop. This works because the dma_fence_merge() 107 * wrapper macro is creating this temporary array on the stack together 108 * with the iterators. 109 */ 110 for (i = 0; i < num_fences; ++i) 111 fences[i] = dma_fence_unwrap_first(fences[i], &iter[i]); 112 113 count = 0; 114 do { 115 unsigned int sel; 116 117 restart: 118 tmp = NULL; 119 for (i = 0; i < num_fences; ++i) { 120 struct dma_fence *next; 121 122 while (fences[i] && dma_fence_is_signaled(fences[i])) 123 fences[i] = dma_fence_unwrap_next(&iter[i]); 124 125 next = fences[i]; 126 if (!next) 127 continue; 128 129 /* 130 * We can't guarantee that inpute fences are ordered by 131 * context, but it is still quite likely when this 132 * function is used multiple times. So attempt to order 133 * the fences by context as we pass over them and merge 134 * fences with the same context. 135 */ 136 if (!tmp || tmp->context > next->context) { 137 tmp = next; 138 sel = i; 139 140 } else if (tmp->context < next->context) { 141 continue; 142 143 } else if (dma_fence_is_later(tmp, next)) { 144 fences[i] = dma_fence_unwrap_next(&iter[i]); 145 goto restart; 146 } else { 147 fences[sel] = dma_fence_unwrap_next(&iter[sel]); 148 goto restart; 149 } 150 } 151 152 if (tmp) { 153 array[count++] = dma_fence_get(tmp); 154 fences[sel] = dma_fence_unwrap_next(&iter[sel]); 155 } 156 } while (tmp); 157 158 if (count == 0) { 159 tmp = dma_fence_allocate_private_stub(ktime_get()); 160 goto return_tmp; 161 } 162 163 if (count == 1) { 164 tmp = array[0]; 165 goto return_tmp; 166 } 167 168 result = dma_fence_array_create(count, array, 169 dma_fence_context_alloc(1), 170 1, false); 171 if (!result) { 172 tmp = NULL; 173 goto return_tmp; 174 } 175 return &result->base; 176 177 return_tmp: 178 kfree(array); 179 return tmp; 180 } 181 EXPORT_SYMBOL_GPL(__dma_fence_unwrap_merge); 182