1 // Tencent is pleased to support the open source community by making ncnn available.
2 //
3 // Copyright (C) 2020 THL A29 Limited, a Tencent company. All rights reserved.
4 //
5 // Licensed under the BSD 3-Clause License (the "License"); you may not use this file except
6 // in compliance with the License. You may obtain a copy of the License at
7 //
8 // https://opensource.org/licenses/BSD-3-Clause
9 //
10 // Unless required by applicable law or agreed to in writing, software distributed
11 // under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR
12 // CONDITIONS OF ANY KIND, either express or implied. See the License for the
13 // specific language governing permissions and limitations under the License.
14
15 #include "pipelinecache.h"
16
17 #include "gpu.h"
18
19 namespace ncnn {
20
21 #if NCNN_VULKAN
22 // https://en.wikipedia.org/wiki/MurmurHash
murmur3_32(const uint32_t * data,int size)23 static uint32_t murmur3_32(const uint32_t* data, int size)
24 {
25 uint32_t h = 0;
26
27 for (int i = 0; i < size; i++)
28 {
29 uint32_t k = *data++;
30
31 k *= 0xcc9e2d51;
32 k = (k << 15) | (k >> (32 - 15));
33 k *= 0x1b873593;
34
35 h ^= k;
36 h = (h << 13) | (h >> (32 - 13));
37 h = (h * 5) + 0xe6546b64;
38 }
39
40 h ^= size * 4;
41
42 h ^= h >> 16;
43 h *= 0x85ebca6b;
44 h ^= h >> 13;
45 h *= 0xc2b2ae35;
46 h ^= h >> 16;
47
48 return h;
49 }
50
51 // https://en.wikipedia.org/wiki/Fowler%E2%80%93Noll%E2%80%93Vo_hash_function#FNV-1a_hash
fnv1a_32(const uint8_t * data,int size)52 static uint32_t fnv1a_32(const uint8_t* data, int size)
53 {
54 uint32_t h = 0x811c9dc5;
55
56 for (int i = 0; i < size; i++)
57 {
58 h ^= (uint32_t)*data++;
59 h *= 0x01000193;
60 }
61
62 return h;
63 }
64
65 class PipelineCachePrivate
66 {
67 public:
68 // digest -> artifact
69 struct pipeline_cache_digest
70 {
71 pipeline_cache_digest(const uint32_t* spv_data, size_t spv_data_size, const std::vector<vk_specialization_type>& specializations,
72 uint32_t local_size_x, uint32_t local_size_y, uint32_t local_size_z);
73 pipeline_cache_digest(int shader_type_index, const Option& opt, const std::vector<vk_specialization_type>& specializations,
74 uint32_t local_size_x, uint32_t local_size_y, uint32_t local_size_z);
75
operator ==ncnn::PipelineCachePrivate::pipeline_cache_digest76 bool operator==(const pipeline_cache_digest& rhs) const
77 {
78 return d0 == rhs.d0 && d1 == rhs.d1;
79 }
80
operator !=ncnn::PipelineCachePrivate::pipeline_cache_digest81 bool operator!=(const pipeline_cache_digest& rhs) const
82 {
83 return d0 != rhs.d0 || d1 != rhs.d1;
84 }
85
86 union
87 {
88 struct
89 {
90 union
91 {
92 uint32_t spv_data_murmur3;
93 int shader_type_index;
94 };
95 unsigned char opt_local_size_bits[4];
96 };
97
98 uint64_t d0;
99 };
100
101 union
102 {
103 struct
104 {
105 uint32_t specializations_murmur3;
106 uint32_t specializations_fnv1a;
107 };
108
109 uint64_t d1;
110 };
111 };
112
113 struct pipeline_cache_artifact
114 {
115 VkShaderModule shader_module;
116 VkDescriptorSetLayout descriptorset_layout;
117 VkPipelineLayout pipeline_layout;
118 VkPipeline pipeline;
119 VkDescriptorUpdateTemplateKHR descriptor_update_template;
120 ShaderInfo shader_info; // TODO use pointer ?
121 };
122
123 mutable std::vector<pipeline_cache_digest> cache_digests;
124 mutable std::vector<pipeline_cache_artifact> cache_artifacts;
125 mutable Mutex cache_lock;
126 };
127
pipeline_cache_digest(const uint32_t * spv_data,size_t spv_data_size,const std::vector<vk_specialization_type> & specializations,uint32_t local_size_x,uint32_t local_size_y,uint32_t local_size_z)128 PipelineCachePrivate::pipeline_cache_digest::pipeline_cache_digest(const uint32_t* spv_data, size_t spv_data_size, const std::vector<vk_specialization_type>& specializations,
129 uint32_t local_size_x, uint32_t local_size_y, uint32_t local_size_z)
130 {
131 spv_data_murmur3 = murmur3_32(spv_data, spv_data_size / 4);
132
133 // encode opt
134 opt_local_size_bits[0] = 0;
135
136 // encode local_size
137 opt_local_size_bits[1] = local_size_x;
138 opt_local_size_bits[2] = local_size_y;
139 opt_local_size_bits[3] = local_size_z;
140
141 // encode specializations
142 const int specialization_count = specializations.size();
143 specializations_murmur3 = murmur3_32((const uint32_t*)specializations.data(), specialization_count);
144 specializations_fnv1a = fnv1a_32((const uint8_t*)specializations.data(), specialization_count * sizeof(vk_specialization_type));
145 }
146
pipeline_cache_digest(int _shader_type_index,const Option & opt,const std::vector<vk_specialization_type> & specializations,uint32_t local_size_x,uint32_t local_size_y,uint32_t local_size_z)147 PipelineCachePrivate::pipeline_cache_digest::pipeline_cache_digest(int _shader_type_index, const Option& opt, const std::vector<vk_specialization_type>& specializations,
148 uint32_t local_size_x, uint32_t local_size_y, uint32_t local_size_z)
149 {
150 shader_type_index = _shader_type_index;
151
152 // encode opt
153 opt_local_size_bits[0] = opt.use_image_storage << 7
154 | opt.use_fp16_packed << 6
155 | opt.use_fp16_storage << 5
156 | opt.use_fp16_arithmetic << 4
157 | opt.use_int8_storage << 3
158 | opt.use_int8_arithmetic << 2;
159
160 // encode local_size
161 opt_local_size_bits[1] = local_size_x;
162 opt_local_size_bits[2] = local_size_y;
163 opt_local_size_bits[3] = local_size_z;
164
165 // encode specializations
166 const int specialization_count = specializations.size();
167 specializations_murmur3 = murmur3_32((const uint32_t*)specializations.data(), specialization_count);
168 specializations_fnv1a = fnv1a_32((const uint8_t*)specializations.data(), specialization_count * sizeof(vk_specialization_type));
169 }
170
PipelineCache(const VulkanDevice * _vkdev)171 PipelineCache::PipelineCache(const VulkanDevice* _vkdev)
172 : vkdev(_vkdev), d(new PipelineCachePrivate)
173 {
174 }
175
~PipelineCache()176 PipelineCache::~PipelineCache()
177 {
178 clear();
179
180 delete d;
181 }
182
PipelineCache(const PipelineCache &)183 PipelineCache::PipelineCache(const PipelineCache&)
184 : d(0)
185 {
186 }
187
operator =(const PipelineCache &)188 PipelineCache& PipelineCache::operator=(const PipelineCache&)
189 {
190 return *this;
191 }
192
clear()193 void PipelineCache::clear()
194 {
195 MutexLockGuard lock(d->cache_lock);
196
197 for (size_t i = 0; i < d->cache_artifacts.size(); i++)
198 {
199 const PipelineCachePrivate::pipeline_cache_artifact& cc = d->cache_artifacts[i];
200
201 if (vkdev->info.support_VK_KHR_descriptor_update_template())
202 {
203 if (cc.descriptor_update_template)
204 {
205 vkdev->vkDestroyDescriptorUpdateTemplateKHR(vkdev->vkdevice(), cc.descriptor_update_template, 0);
206 }
207 }
208
209 if (cc.pipeline)
210 {
211 vkDestroyPipeline(vkdev->vkdevice(), cc.pipeline, 0);
212 }
213
214 if (cc.pipeline_layout)
215 {
216 vkDestroyPipelineLayout(vkdev->vkdevice(), cc.pipeline_layout, 0);
217 }
218
219 if (cc.descriptorset_layout)
220 {
221 vkDestroyDescriptorSetLayout(vkdev->vkdevice(), cc.descriptorset_layout, 0);
222 }
223
224 if (cc.shader_module)
225 {
226 vkDestroyShaderModule(vkdev->vkdevice(), cc.shader_module, 0);
227 }
228 }
229
230 d->cache_digests.clear();
231 d->cache_artifacts.clear();
232 }
233
get_pipeline(const uint32_t * spv_data,size_t spv_data_size,const std::vector<vk_specialization_type> & specializations,uint32_t local_size_x,uint32_t local_size_y,uint32_t local_size_z,VkShaderModule * _shader_module,VkDescriptorSetLayout * descriptorset_layout,VkPipelineLayout * pipeline_layout,VkPipeline * pipeline,VkDescriptorUpdateTemplateKHR * descriptor_update_template,ShaderInfo & shader_info) const234 int PipelineCache::get_pipeline(const uint32_t* spv_data, size_t spv_data_size, const std::vector<vk_specialization_type>& specializations,
235 uint32_t local_size_x, uint32_t local_size_y, uint32_t local_size_z,
236 VkShaderModule* _shader_module,
237 VkDescriptorSetLayout* descriptorset_layout,
238 VkPipelineLayout* pipeline_layout,
239 VkPipeline* pipeline,
240 VkDescriptorUpdateTemplateKHR* descriptor_update_template,
241 ShaderInfo& shader_info) const
242 {
243 MutexLockGuard lock(d->cache_lock);
244
245 PipelineCachePrivate::pipeline_cache_digest key(spv_data, spv_data_size, specializations, local_size_x, local_size_y, local_size_z);
246
247 if (!vkdev->info.bug_corrupted_online_pipeline_cache())
248 {
249 // find cache
250 for (size_t i = 0; i < d->cache_digests.size(); i++)
251 {
252 if (d->cache_digests[i] != key)
253 continue;
254
255 // hit cache
256 const PipelineCachePrivate::pipeline_cache_artifact& cc = d->cache_artifacts[i];
257
258 *_shader_module = cc.shader_module;
259 *descriptorset_layout = cc.descriptorset_layout;
260 *pipeline_layout = cc.pipeline_layout;
261 *pipeline = cc.pipeline;
262 *descriptor_update_template = cc.descriptor_update_template;
263 shader_info = cc.shader_info;
264
265 // NCNN_LOGE("get_pipeline hit %d", last_digest_index);
266
267 return 0;
268 }
269 }
270
271 int ret = 0;
272
273 ret = resolve_shader_info(spv_data, spv_data_size, shader_info);
274 if (ret != 0)
275 {
276 NCNN_LOGE("resolve_shader_info failed %d", ret);
277 return -1;
278 }
279
280 VkShaderModule shader_module = vkdev->compile_shader_module(spv_data, spv_data_size, local_size_x, local_size_y, local_size_z);
281 if (!shader_module)
282 {
283 NCNN_LOGE("create_shader_module failed");
284 return -1;
285 }
286
287 ret = new_pipeline(shader_module, shader_info, specializations, descriptorset_layout, pipeline_layout, pipeline, descriptor_update_template);
288 if (ret != 0)
289 {
290 NCNN_LOGE("new_pipeline failed");
291 vkDestroyShaderModule(vkdev->vkdevice(), shader_module, 0);
292 return -1;
293 }
294
295 *_shader_module = shader_module;
296
297 // save to cache
298 {
299 PipelineCachePrivate::pipeline_cache_artifact cc;
300
301 cc.shader_module = *_shader_module;
302 cc.descriptorset_layout = *descriptorset_layout;
303 cc.pipeline_layout = *pipeline_layout;
304 cc.pipeline = *pipeline;
305 cc.descriptor_update_template = *descriptor_update_template;
306 cc.shader_info = shader_info;
307
308 d->cache_digests.push_back(key);
309 d->cache_artifacts.push_back(cc);
310 }
311
312 // NCNN_LOGE("new_pipeline %d", last_digest_index);
313
314 return 0;
315 }
316
get_pipeline(int shader_type_index,const Option & opt,const std::vector<vk_specialization_type> & specializations,uint32_t local_size_x,uint32_t local_size_y,uint32_t local_size_z,VkShaderModule * _shader_module,VkDescriptorSetLayout * descriptorset_layout,VkPipelineLayout * pipeline_layout,VkPipeline * pipeline,VkDescriptorUpdateTemplateKHR * descriptor_update_template,ShaderInfo & shader_info) const317 int PipelineCache::get_pipeline(int shader_type_index, const Option& opt, const std::vector<vk_specialization_type>& specializations,
318 uint32_t local_size_x, uint32_t local_size_y, uint32_t local_size_z,
319 VkShaderModule* _shader_module,
320 VkDescriptorSetLayout* descriptorset_layout,
321 VkPipelineLayout* pipeline_layout,
322 VkPipeline* pipeline,
323 VkDescriptorUpdateTemplateKHR* descriptor_update_template,
324 ShaderInfo& shader_info) const
325 {
326 MutexLockGuard lock(d->cache_lock);
327
328 PipelineCachePrivate::pipeline_cache_digest key(shader_type_index, opt, specializations, local_size_x, local_size_y, local_size_z);
329
330 if (!vkdev->info.bug_corrupted_online_pipeline_cache())
331 {
332 // find cache
333 for (size_t i = 0; i < d->cache_digests.size(); i++)
334 {
335 if (d->cache_digests[i] != key)
336 continue;
337
338 // hit cache
339 const PipelineCachePrivate::pipeline_cache_artifact& cc = d->cache_artifacts[i];
340
341 *_shader_module = cc.shader_module;
342 *descriptorset_layout = cc.descriptorset_layout;
343 *pipeline_layout = cc.pipeline_layout;
344 *pipeline = cc.pipeline;
345 *descriptor_update_template = cc.descriptor_update_template;
346 shader_info = cc.shader_info;
347
348 // NCNN_LOGE("get_pipeline hit %d", last_digest_index);
349
350 return 0;
351 }
352 }
353
354 int ret = 0;
355
356 // create new pipeline
357 VkShaderModule shader_module = 0;
358 ret = create_shader_module(shader_type_index, opt, local_size_x, local_size_y, local_size_z, &shader_module, shader_info);
359 if (ret != 0)
360 {
361 NCNN_LOGE("create_shader_module failed");
362 return -1;
363 }
364
365 ret = new_pipeline(shader_module, shader_info, specializations, descriptorset_layout, pipeline_layout, pipeline, descriptor_update_template);
366 if (ret != 0)
367 {
368 NCNN_LOGE("new_pipeline failed");
369 vkDestroyShaderModule(vkdev->vkdevice(), shader_module, 0);
370 return -1;
371 }
372
373 *_shader_module = shader_module;
374
375 // save to cache
376 {
377 PipelineCachePrivate::pipeline_cache_artifact cc;
378
379 cc.shader_module = *_shader_module;
380 cc.descriptorset_layout = *descriptorset_layout;
381 cc.pipeline_layout = *pipeline_layout;
382 cc.pipeline = *pipeline;
383 cc.descriptor_update_template = *descriptor_update_template;
384 cc.shader_info = shader_info;
385
386 d->cache_digests.push_back(key);
387 d->cache_artifacts.push_back(cc);
388 }
389
390 // NCNN_LOGE("new_pipeline %d", last_digest_index);
391
392 return 0;
393 }
394
create_shader_module(int shader_type_index,const Option & opt,uint32_t local_size_x,uint32_t local_size_y,uint32_t local_size_z,VkShaderModule * _shader_module,ShaderInfo & si) const395 int PipelineCache::create_shader_module(int shader_type_index, const Option& opt, uint32_t local_size_x, uint32_t local_size_y, uint32_t local_size_z,
396 VkShaderModule* _shader_module, ShaderInfo& si) const
397 {
398 std::vector<uint32_t> spirv;
399 int retc = compile_spirv_module(shader_type_index, opt, spirv);
400 if (retc != 0)
401 {
402 NCNN_LOGE("compile_spirv_module failed %d", retc);
403 return -1;
404 }
405
406 const uint32_t* spv_data = spirv.data();
407 size_t spv_data_size = spirv.size() * 4;
408
409 int ret = resolve_shader_info(spv_data, spv_data_size, si);
410 if (ret != 0)
411 {
412 NCNN_LOGE("resolve_shader_info failed %d", ret);
413 return -1;
414 }
415
416 VkShaderModule shader_module = vkdev->compile_shader_module(spv_data, spv_data_size, local_size_x, local_size_y, local_size_z);
417
418 if (!shader_module)
419 {
420 NCNN_LOGE("create_shader_module failed");
421 return -1;
422 }
423
424 *_shader_module = shader_module;
425
426 return 0;
427 }
428
new_pipeline(VkShaderModule shader_module,const ShaderInfo & shader_info,const std::vector<vk_specialization_type> & specializations,VkDescriptorSetLayout * _descriptorset_layout,VkPipelineLayout * _pipeline_layout,VkPipeline * _pipeline,VkDescriptorUpdateTemplateKHR * _descriptor_update_template) const429 int PipelineCache::new_pipeline(VkShaderModule shader_module, const ShaderInfo& shader_info, const std::vector<vk_specialization_type>& specializations,
430 VkDescriptorSetLayout* _descriptorset_layout,
431 VkPipelineLayout* _pipeline_layout,
432 VkPipeline* _pipeline,
433 VkDescriptorUpdateTemplateKHR* _descriptor_update_template) const
434 {
435 int ret = 0;
436
437 VkDescriptorSetLayout descriptorset_layout = 0;
438 VkPipelineLayout pipeline_layout = 0;
439 VkPipeline pipeline = 0;
440 VkDescriptorUpdateTemplateKHR descriptor_update_template = 0;
441
442 // create new pipeline
443 if ((int)specializations.size() != shader_info.specialization_count)
444 {
445 NCNN_LOGE("pipeline specialization count mismatch, expect %d but got %d", shader_info.specialization_count, (int)specializations.size());
446 goto ERROR_PipelineCache;
447 }
448
449 ret = vkdev->create_descriptorset_layout(shader_info.binding_count, shader_info.binding_types, &descriptorset_layout);
450 if (ret != 0)
451 goto ERROR_PipelineCache;
452
453 ret = vkdev->create_pipeline_layout(shader_info.push_constant_count, descriptorset_layout, &pipeline_layout);
454 if (ret != 0)
455 goto ERROR_PipelineCache;
456
457 ret = vkdev->create_pipeline(shader_module, pipeline_layout, specializations, &pipeline);
458 if (ret != 0)
459 goto ERROR_PipelineCache;
460
461 if (vkdev->info.support_VK_KHR_descriptor_update_template())
462 {
463 ret = vkdev->create_descriptor_update_template(shader_info.binding_count, shader_info.binding_types, descriptorset_layout, pipeline_layout, &descriptor_update_template);
464 if (ret != 0)
465 goto ERROR_PipelineCache;
466 }
467
468 *_descriptorset_layout = descriptorset_layout;
469 *_pipeline_layout = pipeline_layout;
470 *_pipeline = pipeline;
471 *_descriptor_update_template = descriptor_update_template;
472
473 return 0;
474
475 ERROR_PipelineCache:
476
477 if (vkdev->info.support_VK_KHR_descriptor_update_template())
478 {
479 if (descriptor_update_template)
480 {
481 vkdev->vkDestroyDescriptorUpdateTemplateKHR(vkdev->vkdevice(), descriptor_update_template, 0);
482 }
483 }
484
485 if (pipeline)
486 {
487 vkDestroyPipeline(vkdev->vkdevice(), pipeline, 0);
488 }
489
490 if (pipeline_layout)
491 {
492 vkDestroyPipelineLayout(vkdev->vkdevice(), pipeline_layout, 0);
493 }
494
495 if (descriptorset_layout)
496 {
497 vkDestroyDescriptorSetLayout(vkdev->vkdevice(), descriptorset_layout, 0);
498 }
499
500 return -1;
501 }
502
503 #endif // NCNN_VULKAN
504
505 } // namespace ncnn
506