1 /******************************************************************************* 2 Copyright (c) 2022 NVIDIA Corporation 3 4 Permission is hereby granted, free of charge, to any person obtaining a copy 5 of this software and associated documentation files (the "Software"), to 6 deal in the Software without restriction, including without limitation the 7 rights to use, copy, modify, merge, publish, distribute, sublicense, and/or 8 sell copies of the Software, and to permit persons to whom the Software is 9 furnished to do so, subject to the following conditions: 10 11 The above copyright notice and this permission notice shall be 12 included in all copies or substantial portions of the Software. 13 14 THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 15 IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 16 FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL 17 THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 18 LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING 19 FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER 20 DEALINGS IN THE SOFTWARE. 21 22 *******************************************************************************/ 23 24 #ifndef __UVM_VA_POLICY_H__ 25 #define __UVM_VA_POLICY_H__ 26 27 #include "uvm_linux.h" 28 #include "uvm_forward_decl.h" 29 #include "uvm_processors.h" 30 #include "uvm_range_tree.h" 31 #include "uvm_va_block_types.h" 32 33 // This enum must be kept in sync with UVM_TEST_READ_DUPLICATION_POLICY in 34 // uvm_test_ioctl.h 35 typedef enum 36 { 37 UVM_READ_DUPLICATION_UNSET = 0, 38 UVM_READ_DUPLICATION_ENABLED, 39 UVM_READ_DUPLICATION_DISABLED, 40 UVM_READ_DUPLICATION_MAX 41 } uvm_read_duplication_policy_t; 42 43 typedef enum 44 { 45 UVM_VA_POLICY_PREFERRED_LOCATION = 0, 46 UVM_VA_POLICY_ACCESSED_BY, 47 UVM_VA_POLICY_READ_DUPLICATION, 48 } uvm_va_policy_type_t; 49 50 // 51 // A policy covers one or more contiguous Linux VMAs or portion of a VMA and 52 // does not cover non-existant VMAs. 53 // The VA range is determined from either the uvm_va_range_t for managed 54 // allocations or the uvm_va_policy_node_t for HMM allocations. 55 // 56 typedef struct uvm_va_policy_struct 57 { 58 // Read duplication policy for this VA range (unset, enabled, or disabled). 59 uvm_read_duplication_policy_t read_duplication; 60 61 // Processor ID of the preferred location for this VA range. 62 // This is set to UVM_ID_INVALID if no preferred location is set. 63 uvm_processor_id_t preferred_location; 64 65 // Mask of processors that are accessing this VA range and should have 66 // their page tables updated to access the (possibly remote) pages. 67 uvm_processor_mask_t accessed_by; 68 69 } uvm_va_policy_t; 70 71 // Policy nodes are used for storing policies in HMM va_blocks. 72 // The va_block lock protects the tree so that invalidation callbacks can 73 // update the VA policy tree. 74 typedef struct uvm_va_policy_node_struct 75 { 76 // Storage for the policy tree node. It also contains the range start and 77 // end. Start and end + 1 have to be PAGE_SIZED aligned. 78 uvm_range_tree_node_t node; 79 80 uvm_va_policy_t policy; 81 82 } uvm_va_policy_node_t; 83 84 // Function pointer prototype for uvm_hmm_split_as_needed() callback. 85 typedef bool (*uvm_va_policy_is_split_needed_t)(const uvm_va_policy_t *policy, void *data); 86 87 // Default policy to save uvm_va_policy_node_t space in HMM va_blocks. 88 extern const uvm_va_policy_t uvm_va_policy_default; 89 90 // Return true if policy is the default policy. 91 static bool uvm_va_policy_is_default(const uvm_va_policy_t *policy) 92 { 93 return policy == &uvm_va_policy_default; 94 } 95 96 bool uvm_va_policy_is_read_duplicate(const uvm_va_policy_t *policy, uvm_va_space_t *va_space); 97 98 // Returns the uvm_va_policy_t containing addr or default policy if not found. 99 // The va_block can be either a UVM or HMM va_block. 100 // Locking: The va_block lock must be held. 101 const uvm_va_policy_t *uvm_va_policy_get(uvm_va_block_t *va_block, NvU64 addr); 102 103 // Return a uvm_va_policy_node_t given a uvm_va_policy_t pointer. 104 static const uvm_va_policy_node_t *uvm_va_policy_node_from_policy(const uvm_va_policy_t *policy) 105 { 106 return container_of(policy, uvm_va_policy_node_t, policy); 107 } 108 109 #if UVM_IS_CONFIG_HMM() 110 111 // Module load/exit 112 NV_STATUS uvm_va_policy_init(void); 113 void uvm_va_policy_exit(void); 114 115 // Returns the uvm_va_policy_node_t containing addr or NULL if not found. 116 // The va_block must be a HMM va_block. 117 // Locking: The va_block lock must be held. 118 uvm_va_policy_node_t *uvm_va_policy_node_find(uvm_va_block_t *va_block, NvU64 addr); 119 120 // Split the old node. The old node will end at 'new_end' and the new node will 121 // start at 'new_end' + 1 and end at old end. 122 // The va_block must be a HMM va_block. 123 // Locking: The va_block lock must be held. 124 NV_STATUS uvm_va_policy_node_split(uvm_va_block_t *va_block, 125 uvm_va_policy_node_t *old, 126 NvU64 new_end, 127 uvm_va_policy_node_t **new_ptr); 128 129 // Move hints from 'old' to 'new' which must both be HMM va_blocks. 130 // The old va_block policies should have been pre-split and since we don't 131 // merge policy ranges, they should still be split after locking/unlocking 132 // 'old'. This should be called after splitting a block. 133 // TODO: Bug 1707562: Add support for merging policy ranges. 134 // Locking: The va_block lock must be held for both old and new. 135 void uvm_va_policy_node_split_move(uvm_va_block_t *old_va_block, 136 uvm_va_block_t *new_va_block); 137 138 // Remove all policy in the given range start/end where 'end' is inclusive. 139 // This function may clear a range larger than start/end if clearing the range 140 // requires memory allocation and the memory allocation fails. 141 // The va_block must be a HMM va_block. 142 // Locking: The va_block lock must be held. 143 void uvm_va_policy_clear(uvm_va_block_t *va_block, NvU64 start, NvU64 end); 144 145 // Fill in any missing policy nodes for the given range and set the policy 146 // to the given value. The caller is expected to split any policy nodes 147 // before calling this function so the range being set does not. 148 // The va_block must be a HMM va_block. 149 // Note that start and end + 1 must be page aligned, 'end' is inclusive. 150 // TODO: Bug 1707562: Add support for merging policy ranges. 151 // Locking: The va_block lock must be held. 152 NV_STATUS uvm_va_policy_set_range(uvm_va_block_t *va_block, 153 NvU64 start, 154 NvU64 end, 155 uvm_va_policy_type_t which, 156 bool is_default, 157 uvm_processor_id_t processor_id, 158 uvm_read_duplication_policy_t new_policy); 159 160 // This is an optimized version of uvm_va_policy_set_range() where the caller 161 // guarantees that the the processor_id is not the same as the existing 162 // policy for the given region and that the region doesn't require splitting 163 // the existing policy node 'old_policy'. 164 // Returns the updated policy or NULL if memory could not be allocated. 165 // Locking: The va_block lock must be held. 166 const uvm_va_policy_t *uvm_va_policy_set_preferred_location(uvm_va_block_t *va_block, 167 uvm_va_block_region_t region, 168 uvm_processor_id_t processor_id, 169 const uvm_va_policy_t *old_policy); 170 171 // Iterators for specific VA policy ranges. 172 173 // Returns the first policy node in the range [start, end], if any. 174 // The va_block must be a HMM va_block. 175 // Locking: The va_block lock must be held. 176 uvm_va_policy_node_t *uvm_va_policy_node_iter_first(uvm_va_block_t *va_block, NvU64 start, NvU64 end); 177 178 // Returns the next VA policy following the provided policy in address order, 179 // if that policy's start <= the provided end. 180 // The va_block must be a HMM va_block. 181 // Locking: The va_block lock must be held. 182 uvm_va_policy_node_t *uvm_va_policy_node_iter_next(uvm_va_block_t *va_block, uvm_va_policy_node_t *node, NvU64 end); 183 184 #define uvm_for_each_va_policy_node_in(node, va_block, start, end) \ 185 for ((node) = uvm_va_policy_node_iter_first((va_block), (start), (end)); \ 186 (node); \ 187 (node) = uvm_va_policy_node_iter_next((va_block), (node), (end))) 188 189 #define uvm_for_each_va_policy_node_in_safe(node, next, va_block, start, end) \ 190 for ((node) = uvm_va_policy_node_iter_first((va_block), (start), (end)), \ 191 (next) = uvm_va_policy_node_iter_next((va_block), (node), (end)); \ 192 (node); \ 193 (node) = (next)) 194 195 // Returns the first policy in the range [start, end], if any. 196 // Locking: The va_block lock must be held. 197 const uvm_va_policy_t *uvm_va_policy_iter_first(uvm_va_block_t *va_block, 198 NvU64 start, 199 NvU64 end, 200 uvm_va_policy_node_t **out_node, 201 uvm_va_block_region_t *out_region); 202 203 // Returns the next VA policy following the provided policy in address order, 204 // if that policy's start <= the provided end. 205 // Locking: The va_block lock must be held. 206 const uvm_va_policy_t *uvm_va_policy_iter_next(uvm_va_block_t *va_block, 207 const uvm_va_policy_t *policy, 208 NvU64 end, 209 uvm_va_policy_node_t **inout_node, 210 uvm_va_block_region_t *inout_region); 211 212 // Note that policy and region are set and usable in the loop body. 213 // The 'node' variable is used to retain loop state and 'policy' doesn't 214 // necessarily match &node->policy. 215 #define uvm_for_each_va_policy_in(policy, va_block, start, end, node, region) \ 216 for ((policy) = uvm_va_policy_iter_first((va_block), (start), (end), &(node), &(region)); \ 217 (policy); \ 218 (policy) = uvm_va_policy_iter_next((va_block), (policy), (end), &(node), &(region))) 219 220 #else // UVM_IS_CONFIG_HMM() 221 222 static NV_STATUS uvm_va_policy_init(void) 223 { 224 return NV_OK; 225 } 226 227 static void uvm_va_policy_exit(void) 228 { 229 } 230 231 static uvm_va_policy_node_t *uvm_va_policy_node_find(uvm_va_block_t *va_block, NvU64 addr) 232 { 233 UVM_ASSERT(0); 234 return NULL; 235 } 236 237 static NV_STATUS uvm_va_policy_node_split(uvm_va_block_t *va_block, 238 uvm_va_policy_node_t *old, 239 NvU64 new_end, 240 uvm_va_policy_node_t **new_ptr) 241 { 242 return NV_OK; 243 } 244 245 static void uvm_va_policy_node_split_move(uvm_va_block_t *old_va_block, 246 uvm_va_block_t *new_va_block) 247 { 248 } 249 250 static void uvm_va_policy_clear(uvm_va_block_t *va_block, NvU64 start, NvU64 end) 251 { 252 } 253 254 static NV_STATUS uvm_va_policy_set_range(uvm_va_block_t *va_block, 255 NvU64 start, 256 NvU64 end, 257 uvm_va_policy_type_t which, 258 bool is_default, 259 uvm_processor_id_t processor_id, 260 uvm_read_duplication_policy_t new_policy) 261 { 262 return NV_OK; 263 } 264 265 static uvm_va_policy_node_t *uvm_va_policy_node_iter_first(uvm_va_block_t *va_block, NvU64 start, NvU64 end) 266 { 267 return NULL; 268 } 269 270 #endif // UVM_IS_CONFIG_HMM() 271 272 #endif // __UVM_VA_POLICY_H__ 273