1 /* Copyright (c) 2015-2020 The Khronos Group Inc. 2 * Copyright (c) 2015-2020 Valve Corporation 3 * Copyright (c) 2015-2020 LunarG, Inc. 4 * Copyright (C) 2015-2020 Google Inc. 5 * 6 * Licensed under the Apache License, Version 2.0 (the "License"); 7 * you may not use this file except in compliance with the License. 8 * You may obtain a copy of the License at 9 * 10 * http://www.apache.org/licenses/LICENSE-2.0 11 * 12 * Unless required by applicable law or agreed to in writing, software 13 * distributed under the License is distributed on an "AS IS" BASIS, 14 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 * See the License for the specific language governing permissions and 16 * limitations under the License. 17 * 18 * Author: Chris Forbes <chrisf@ijw.co.nz> 19 */ 20 #ifndef VULKAN_SHADER_VALIDATION_H 21 #define VULKAN_SHADER_VALIDATION_H 22 23 #include <cassert> 24 #include <cstdlib> 25 #include <cstring> 26 #include <unordered_map> 27 #include <unordered_set> 28 #include <vector> 29 30 #include "vulkan/vulkan.h" 31 #include <spirv/unified1/spirv.hpp> 32 #include <generated/spirv_tools_commit_id.h> 33 #include "spirv-tools/optimizer.hpp" 34 #include "core_validation_types.h" 35 36 // A forward iterator over spirv instructions. Provides easy access to len, opcode, and content words 37 // without the caller needing to care too much about the physical SPIRV module layout. 38 struct spirv_inst_iter { 39 std::vector<uint32_t>::const_iterator zero; 40 std::vector<uint32_t>::const_iterator it; 41 lenspirv_inst_iter42 uint32_t len() const { 43 auto result = *it >> 16; 44 assert(result > 0); 45 return result; 46 } 47 opcodespirv_inst_iter48 uint32_t opcode() { return *it & 0x0ffffu; } 49 wordspirv_inst_iter50 uint32_t const &word(unsigned n) const { 51 assert(n < len()); 52 return it[n]; 53 } 54 offsetspirv_inst_iter55 uint32_t offset() { return (uint32_t)(it - zero); } 56 spirv_inst_iterspirv_inst_iter57 spirv_inst_iter() {} 58 spirv_inst_iterspirv_inst_iter59 spirv_inst_iter(std::vector<uint32_t>::const_iterator zero, std::vector<uint32_t>::const_iterator it) : zero(zero), it(it) {} 60 61 bool operator==(spirv_inst_iter const &other) const { return it == other.it; } 62 63 bool operator!=(spirv_inst_iter const &other) const { return it != other.it; } 64 65 spirv_inst_iter operator++(int) { // x++ 66 spirv_inst_iter ii = *this; 67 it += len(); 68 return ii; 69 } 70 71 spirv_inst_iter operator++() { // ++x; 72 it += len(); 73 return *this; 74 } 75 76 // The iterator and the value are the same thing. 77 spirv_inst_iter &operator*() { return *this; } 78 spirv_inst_iter const &operator*() const { return *this; } 79 }; 80 81 struct shader_stage_attributes { 82 char const *const name; 83 bool arrayed_input; 84 bool arrayed_output; 85 VkShaderStageFlags stage; 86 }; 87 88 struct decoration_set { 89 enum { 90 location_bit = 1 << 0, 91 patch_bit = 1 << 1, 92 relaxed_precision_bit = 1 << 2, 93 block_bit = 1 << 3, 94 buffer_block_bit = 1 << 4, 95 component_bit = 1 << 5, 96 input_attachment_index_bit = 1 << 6, 97 descriptor_set_bit = 1 << 7, 98 binding_bit = 1 << 8, 99 nonwritable_bit = 1 << 9, 100 builtin_bit = 1 << 10, 101 }; 102 uint32_t flags = 0; 103 uint32_t location = static_cast<uint32_t>(-1); 104 uint32_t component = 0; 105 uint32_t input_attachment_index = 0; 106 uint32_t descriptor_set = 0; 107 uint32_t binding = 0; 108 uint32_t builtin = static_cast<uint32_t>(-1); 109 mergedecoration_set110 void merge(decoration_set const &other) { 111 if (other.flags & location_bit) location = other.location; 112 if (other.flags & component_bit) component = other.component; 113 if (other.flags & input_attachment_index_bit) input_attachment_index = other.input_attachment_index; 114 if (other.flags & descriptor_set_bit) descriptor_set = other.descriptor_set; 115 if (other.flags & binding_bit) binding = other.binding; 116 if (other.flags & builtin_bit) builtin = other.builtin; 117 flags |= other.flags; 118 } 119 120 void add(uint32_t decoration, uint32_t value); 121 }; 122 123 struct function_set { 124 unsigned id; 125 unsigned offset; 126 unsigned length; 127 std::unordered_multimap<uint32_t, uint32_t> op_lists; // key: spv::Op, value: offset 128 function_setfunction_set129 function_set() : id(0), offset(0), length(0) {} 130 }; 131 132 struct shader_struct_member { 133 uint32_t offset; 134 uint32_t size; // A scalar size or a struct size. Not consider array 135 std::vector<uint32_t> array_length_hierarchy; // multi-dimensional array, mat, vec. mat is combined with 2 array. 136 // e.g :array[2] -> {2}, array[2][3][4] -> {2,3,4}, mat4[2] ->{2,4,4}, 137 std::vector<uint32_t> array_block_size; // When index increases, how many data increases. 138 // e.g : array[2][3][4] -> {12,4,1}, it means if the first index increases one, the 139 // array gets 12 data. If the second index increases one, the array gets 4 data. 140 std::vector<shader_struct_member> struct_members; // If the data is not a struct, it's empty. 141 shader_struct_member *root; 142 shader_struct_membershader_struct_member143 shader_struct_member() : offset(0), size(0), root(nullptr) {} 144 IsUsedshader_struct_member145 bool IsUsed() const { 146 if (!root) return false; 147 return root->used_bytes.size() ? true : false; 148 } 149 GetUsedbytesshader_struct_member150 std::vector<uint8_t> *GetUsedbytes() const { 151 if (!root) return nullptr; 152 return &root->used_bytes; 153 } 154 GetLocationDescshader_struct_member155 std::string GetLocationDesc(uint32_t index_used_bytes) const { 156 std::string desc = ""; 157 if (array_length_hierarchy.size() > 0) { 158 desc += " index:"; 159 for (const auto block_size : array_block_size) { 160 desc += "["; 161 desc += std::to_string(index_used_bytes / (block_size * size)); 162 desc += "]"; 163 index_used_bytes = index_used_bytes % (block_size * size); 164 } 165 } 166 const int struct_members_size = static_cast<int>(struct_members.size()); 167 if (struct_members_size > 0) { 168 desc += " member:"; 169 for (int i = struct_members_size - 1; i >= 0; --i) { 170 if (index_used_bytes > struct_members[i].offset) { 171 desc += std::to_string(i); 172 desc += struct_members[i].GetLocationDesc(index_used_bytes - struct_members[i].offset); 173 break; 174 } 175 } 176 } else { 177 desc += " offset:"; 178 desc += std::to_string(index_used_bytes); 179 } 180 return desc; 181 } 182 183 private: 184 std::vector<uint8_t> used_bytes; // This only works for root. 0: not used. 1: used. The totally array * size. 185 }; 186 187 struct SHADER_MODULE_STATE : public BASE_NODE { 188 // The spirv image itself 189 std::vector<uint32_t> words; 190 // A mapping of <id> to the first word of its def. this is useful because walking type 191 // trees, constant expressions, etc requires jumping all over the instruction stream. 192 std::unordered_map<unsigned, unsigned> def_index; 193 std::unordered_map<unsigned, decoration_set> decorations; 194 struct EntryPoint { 195 uint32_t offset; 196 VkShaderStageFlagBits stage; 197 std::unordered_multimap<unsigned, unsigned> decorate_list; // key: spv::Op, value: offset 198 std::vector<function_set> function_set_list; 199 shader_struct_member push_constant_used_in_shader; 200 }; 201 std::unordered_multimap<std::string, EntryPoint> entry_points; 202 bool has_valid_spirv; 203 bool has_specialization_constants{false}; 204 VkShaderModule vk_shader_module; 205 uint32_t gpu_validation_shader_id; 206 PreprocessShaderBinarySHADER_MODULE_STATE207 std::vector<uint32_t> PreprocessShaderBinary(uint32_t *src_binary, size_t binary_size, spv_target_env env) { 208 std::vector<uint32_t> src(src_binary, src_binary + binary_size / sizeof(uint32_t)); 209 210 // Check if there are any group decoration instructions, and flatten them if found. 211 bool has_group_decoration = false; 212 bool done = false; 213 214 // Walk through the first part of the SPIR-V module, looking for group decoration and specialization constant instructions. 215 // Skip the header (5 words). 216 auto itr = spirv_inst_iter(src.begin(), src.begin() + 5); 217 auto itrend = spirv_inst_iter(src.begin(), src.end()); 218 while (itr != itrend && !done) { 219 spv::Op opcode = (spv::Op)itr.opcode(); 220 switch (opcode) { 221 case spv::OpDecorationGroup: 222 case spv::OpGroupDecorate: 223 case spv::OpGroupMemberDecorate: 224 has_group_decoration = true; 225 break; 226 case spv::OpSpecConstantTrue: 227 case spv::OpSpecConstantFalse: 228 case spv::OpSpecConstant: 229 case spv::OpSpecConstantComposite: 230 case spv::OpSpecConstantOp: 231 has_specialization_constants = true; 232 break; 233 case spv::OpFunction: 234 // An OpFunction indicates there are no more decorations 235 done = true; 236 break; 237 default: 238 break; 239 } 240 itr++; 241 } 242 243 if (has_group_decoration) { 244 spvtools::Optimizer optimizer(env); 245 optimizer.RegisterPass(spvtools::CreateFlattenDecorationPass()); 246 std::vector<uint32_t> optimized_binary; 247 // Run optimizer to flatten decorations only, set skip_validation so as to not re-run validator 248 auto result = 249 optimizer.Run(src_binary, binary_size / sizeof(uint32_t), &optimized_binary, spvtools::ValidatorOptions(), true); 250 if (result) { 251 return optimized_binary; 252 } 253 } 254 // Return the original module. 255 return src; 256 } 257 SHADER_MODULE_STATESHADER_MODULE_STATE258 SHADER_MODULE_STATE(VkShaderModuleCreateInfo const *pCreateInfo, VkShaderModule shaderModule, spv_target_env env, 259 uint32_t unique_shader_id) 260 : words(), def_index(), has_valid_spirv(true), vk_shader_module(shaderModule), gpu_validation_shader_id(unique_shader_id) { 261 words = PreprocessShaderBinary((uint32_t *)pCreateInfo->pCode, pCreateInfo->codeSize, env); 262 BuildDefIndex(); 263 } 264 SHADER_MODULE_STATESHADER_MODULE_STATE265 SHADER_MODULE_STATE() : has_valid_spirv(false), vk_shader_module(VK_NULL_HANDLE), gpu_validation_shader_id(UINT32_MAX) {} 266 get_decorationsSHADER_MODULE_STATE267 decoration_set get_decorations(unsigned id) const { 268 // return the actual decorations for this id, or a default set. 269 auto it = decorations.find(id); 270 if (it != decorations.end()) return it->second; 271 return decoration_set(); 272 } 273 274 // Expose begin() / end() to enable range-based for beginSHADER_MODULE_STATE275 spirv_inst_iter begin() const { return spirv_inst_iter(words.begin(), words.begin() + 5); } // First insn endSHADER_MODULE_STATE276 spirv_inst_iter end() const { return spirv_inst_iter(words.begin(), words.end()); } // Just past last insn 277 // Given an offset into the module, produce an iterator there. atSHADER_MODULE_STATE278 spirv_inst_iter at(unsigned offset) const { return spirv_inst_iter(words.begin(), words.begin() + offset); } 279 280 // Gets an iterator to the definition of an id get_defSHADER_MODULE_STATE281 spirv_inst_iter get_def(unsigned id) const { 282 auto it = def_index.find(id); 283 if (it == def_index.end()) { 284 return end(); 285 } 286 return at(it->second); 287 } 288 289 void BuildDefIndex(); 290 }; 291 292 class ValidationCache { 293 // hashes of shaders that have passed validation before, and can be skipped. 294 // we don't store negative results, as we would have to also store what was 295 // wrong with them; also, we expect they will get fixed, so we're less 296 // likely to see them again. 297 std::unordered_set<uint32_t> good_shader_hashes; ValidationCache()298 ValidationCache() {} 299 300 public: Create(VkValidationCacheCreateInfoEXT const * pCreateInfo)301 static VkValidationCacheEXT Create(VkValidationCacheCreateInfoEXT const *pCreateInfo) { 302 auto cache = new ValidationCache(); 303 cache->Load(pCreateInfo); 304 return VkValidationCacheEXT(cache); 305 } 306 Load(VkValidationCacheCreateInfoEXT const * pCreateInfo)307 void Load(VkValidationCacheCreateInfoEXT const *pCreateInfo) { 308 const auto headerSize = 2 * sizeof(uint32_t) + VK_UUID_SIZE; 309 auto size = headerSize; 310 if (!pCreateInfo->pInitialData || pCreateInfo->initialDataSize < size) return; 311 312 uint32_t const *data = (uint32_t const *)pCreateInfo->pInitialData; 313 if (data[0] != size) return; 314 if (data[1] != VK_VALIDATION_CACHE_HEADER_VERSION_ONE_EXT) return; 315 uint8_t expected_uuid[VK_UUID_SIZE]; 316 Sha1ToVkUuid(SPIRV_TOOLS_COMMIT_ID, expected_uuid); 317 if (memcmp(&data[2], expected_uuid, VK_UUID_SIZE) != 0) return; // different version 318 319 data = (uint32_t const *)(reinterpret_cast<uint8_t const *>(data) + headerSize); 320 321 for (; size < pCreateInfo->initialDataSize; data++, size += sizeof(uint32_t)) { 322 good_shader_hashes.insert(*data); 323 } 324 } 325 Write(size_t * pDataSize,void * pData)326 void Write(size_t *pDataSize, void *pData) { 327 const auto headerSize = 2 * sizeof(uint32_t) + VK_UUID_SIZE; // 4 bytes for header size + 4 bytes for version number + UUID 328 if (!pData) { 329 *pDataSize = headerSize + good_shader_hashes.size() * sizeof(uint32_t); 330 return; 331 } 332 333 if (*pDataSize < headerSize) { 334 *pDataSize = 0; 335 return; // Too small for even the header! 336 } 337 338 uint32_t *out = (uint32_t *)pData; 339 size_t actualSize = headerSize; 340 341 // Write the header 342 *out++ = headerSize; 343 *out++ = VK_VALIDATION_CACHE_HEADER_VERSION_ONE_EXT; 344 Sha1ToVkUuid(SPIRV_TOOLS_COMMIT_ID, reinterpret_cast<uint8_t *>(out)); 345 out = (uint32_t *)(reinterpret_cast<uint8_t *>(out) + VK_UUID_SIZE); 346 347 for (auto it = good_shader_hashes.begin(); it != good_shader_hashes.end() && actualSize < *pDataSize; 348 it++, out++, actualSize += sizeof(uint32_t)) { 349 *out = *it; 350 } 351 352 *pDataSize = actualSize; 353 } 354 Merge(ValidationCache const * other)355 void Merge(ValidationCache const *other) { 356 good_shader_hashes.reserve(good_shader_hashes.size() + other->good_shader_hashes.size()); 357 for (auto h : other->good_shader_hashes) good_shader_hashes.insert(h); 358 } 359 360 static uint32_t MakeShaderHash(VkShaderModuleCreateInfo const *smci); 361 Contains(uint32_t hash)362 bool Contains(uint32_t hash) { return good_shader_hashes.count(hash) != 0; } 363 Insert(uint32_t hash)364 void Insert(uint32_t hash) { good_shader_hashes.insert(hash); } 365 366 private: Sha1ToVkUuid(const char * sha1_str,uint8_t * uuid)367 void Sha1ToVkUuid(const char *sha1_str, uint8_t *uuid) { 368 // Convert sha1_str from a hex string to binary. We only need VK_UUID_SIZE bytes of 369 // output, so pad with zeroes if the input string is shorter than that, and truncate 370 // if it's longer. 371 #if defined(__GNUC__) && (__GNUC__ > 8) 372 #pragma GCC diagnostic push 373 #pragma GCC diagnostic ignored "-Wstringop-truncation" 374 #endif 375 char padded_sha1_str[2 * VK_UUID_SIZE + 1] = {}; // 2 hex digits == 1 byte 376 std::strncpy(padded_sha1_str, sha1_str, 2 * VK_UUID_SIZE); 377 #if defined(__GNUC__) && (__GNUC__ > 8) 378 #pragma GCC diagnostic pop 379 #endif 380 for (uint32_t i = 0; i < VK_UUID_SIZE; ++i) { 381 const char byte_str[] = {padded_sha1_str[2 * i + 0], padded_sha1_str[2 * i + 1], '\0'}; 382 uuid[i] = static_cast<uint8_t>(std::strtoul(byte_str, nullptr, 16)); 383 } 384 } 385 }; 386 387 const SHADER_MODULE_STATE::EntryPoint *FindEntrypointStruct(SHADER_MODULE_STATE const *src, char const *name, 388 VkShaderStageFlagBits stageBits); 389 spirv_inst_iter FindEntrypoint(SHADER_MODULE_STATE const *src, char const *name, VkShaderStageFlagBits stageBits); 390 391 // For some analyses, we need to know about all ids referenced by the static call tree of a particular entrypoint. This is 392 // important for identifying the set of shader resources actually used by an entrypoint, for example. 393 // Note: we only explore parts of the image which might actually contain ids we care about for the above analyses. 394 // - NOT the shader input/output interfaces. 395 // 396 // TODO: The set of interesting opcodes here was determined by eyeballing the SPIRV spec. It might be worth 397 // converting parts of this to be generated from the machine-readable spec instead. 398 std::unordered_set<uint32_t> MarkAccessibleIds(SHADER_MODULE_STATE const *src, spirv_inst_iter entrypoint); 399 400 // Returns an int32_t corresponding to the spv::Dim of the given resource, when positive, and corresponding to an unknown type, when 401 // negative. 402 int32_t GetShaderResourceDimensionality(const SHADER_MODULE_STATE *module, const interface_var &resource); 403 404 bool FindLocalSize(SHADER_MODULE_STATE const *src, uint32_t &local_size_x, uint32_t &local_size_y, uint32_t &local_size_z); 405 406 void ProcessExecutionModes(SHADER_MODULE_STATE const *src, const spirv_inst_iter &entrypoint, PIPELINE_STATE *pipeline); 407 408 std::vector<std::pair<descriptor_slot_t, interface_var>> CollectInterfaceByDescriptorSlot( 409 SHADER_MODULE_STATE const *src, std::unordered_set<uint32_t> const &accessible_ids, bool *has_writable_descriptor, 410 bool *has_atomic_descriptor); 411 412 void SetPushConstantUsedInShader(SHADER_MODULE_STATE &src); 413 414 std::unordered_set<uint32_t> CollectWritableOutputLocationinFS(const SHADER_MODULE_STATE &module, 415 const VkPipelineShaderStageCreateInfo &stage_info); 416 417 uint32_t DescriptorTypeToReqs(SHADER_MODULE_STATE const *module, uint32_t type_id); 418 419 spv_target_env PickSpirvEnv(uint32_t api_version, bool spirv_1_4); 420 421 void AdjustValidatorOptions(const DeviceExtensions device_extensions, const DeviceFeatures enabled_features, 422 spvtools::ValidatorOptions &options); 423 424 void RunUsedStruct(const SHADER_MODULE_STATE &src, uint32_t offset, uint32_t access_chain_word_index, 425 spirv_inst_iter &access_chain_it, const shader_struct_member &data); 426 427 #endif // VULKAN_SHADER_VALIDATION_H 428