1 /*
2  * Copyright © 2021 Collabora Ltd.
3  *
4  * derived from tu_private.h driver which is:
5  * Copyright © 2016 Red Hat.
6  * Copyright © 2016 Bas Nieuwenhuizen
7  * Copyright © 2015 Intel Corporation
8  *
9  * Permission is hereby granted, free of charge, to any person obtaining a
10  * copy of this software and associated documentation files (the "Software"),
11  * to deal in the Software without restriction, including without limitation
12  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
13  * and/or sell copies of the Software, and to permit persons to whom the
14  * Software is furnished to do so, subject to the following conditions:
15  *
16  * The above copyright notice and this permission notice (including the next
17  * paragraph) shall be included in all copies or substantial portions of the
18  * Software.
19  *
20  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
21  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
22  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
23  * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
24  * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
25  * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
26  * DEALINGS IN THE SOFTWARE.
27  */
28 
29 #ifndef PANVK_PRIVATE_H
30 #define PANVK_PRIVATE_H
31 
32 #include <assert.h>
33 #include <pthread.h>
34 #include <stdbool.h>
35 #include <stdint.h>
36 #include <stdio.h>
37 #include <stdlib.h>
38 #include <string.h>
39 #ifdef HAVE_VALGRIND
40 #include <memcheck.h>
41 #include <valgrind.h>
42 #define VG(x) x
43 #else
44 #define VG(x)
45 #endif
46 
47 #include "c11/threads.h"
48 #include "compiler/shader_enums.h"
49 #include "util/list.h"
50 #include "util/macros.h"
51 #include "vk_alloc.h"
52 #include "vk_command_buffer.h"
53 #include "vk_device.h"
54 #include "vk_instance.h"
55 #include "vk_log.h"
56 #include "vk_object.h"
57 #include "vk_physical_device.h"
58 #include "vk_queue.h"
59 #include "wsi_common.h"
60 
61 #include "drm-uapi/panfrost_drm.h"
62 
63 #include "pan_blend.h"
64 #include "pan_cs.h"
65 #include "pan_device.h"
66 #include "panvk_mempool.h"
67 #include "pan_texture.h"
68 #include "pan_scoreboard.h"
69 #include "vk_extensions.h"
70 #include "panvk_varyings.h"
71 
72 /* Pre-declarations needed for WSI entrypoints */
73 struct wl_surface;
74 struct wl_display;
75 typedef struct xcb_connection_t xcb_connection_t;
76 typedef uint32_t xcb_visualid_t;
77 typedef uint32_t xcb_window_t;
78 
79 #include <vulkan/vk_android_native_buffer.h>
80 #include <vulkan/vk_icd.h>
81 #include <vulkan/vulkan.h>
82 
83 #include "panvk_entrypoints.h"
84 
85 #define MAX_BIND_POINTS 2 /* compute + graphics */
86 #define MAX_VBS 16
87 #define MAX_VERTEX_ATTRIBS 16
88 #define MAX_RTS 8
89 #define MAX_VSC_PIPES 32
90 #define MAX_VIEWPORTS 1
91 #define MAX_SCISSORS 16
92 #define MAX_DISCARD_RECTANGLES 4
93 #define MAX_PUSH_CONSTANTS_SIZE 128
94 #define MAX_PUSH_DESCRIPTORS 32
95 #define MAX_DYNAMIC_UNIFORM_BUFFERS 16
96 #define MAX_DYNAMIC_STORAGE_BUFFERS 8
97 #define MAX_DYNAMIC_BUFFERS                                                  \
98    (MAX_DYNAMIC_UNIFORM_BUFFERS + MAX_DYNAMIC_STORAGE_BUFFERS)
99 #define MAX_SAMPLES_LOG2 4
100 #define NUM_META_FS_KEYS 13
101 #define PANVK_MAX_DRM_DEVICES 1
102 #define MAX_VIEWS 8
103 
104 #define NUM_DEPTH_CLEAR_PIPELINES 3
105 
106 #define panvk_printflike(a, b) __attribute__((__format__(__printf__, a, b)))
107 
108 void
109 panvk_logi(const char *format, ...) panvk_printflike(1, 2);
110 void
111 panvk_logi_v(const char *format, va_list va);
112 
113 #define panvk_stub() assert(!"stub")
114 
115 #define PANVK_META_COPY_BUF2IMG_NUM_FORMATS 12
116 #define PANVK_META_COPY_IMG2BUF_NUM_FORMATS 12
117 #define PANVK_META_COPY_IMG2IMG_NUM_FORMATS 14
118 #define PANVK_META_COPY_NUM_TEX_TYPES 5
119 #define PANVK_META_COPY_BUF2BUF_NUM_BLKSIZES 5
120 
121 static inline unsigned
panvk_meta_copy_tex_type(unsigned dim,bool isarray)122 panvk_meta_copy_tex_type(unsigned dim, bool isarray)
123 {
124    assert(dim > 0 && dim <= 3);
125    assert(dim < 3 || !isarray);
126    return (((dim - 1) << 1) | (isarray ? 1 : 0));
127 }
128 
129 struct panvk_meta {
130    struct panvk_pool bin_pool;
131    struct panvk_pool desc_pool;
132 
133    /* Access to the blitter pools are protected by the blitter
134     * shader/rsd locks. They can't be merged with other binary/desc
135     * pools unless we patch pan_blitter.c to external pool locks.
136     */
137    struct {
138       struct panvk_pool bin_pool;
139       struct panvk_pool desc_pool;
140    } blitter;
141 
142    struct {
143       struct {
144          mali_ptr shader;
145          struct pan_shader_info shader_info;
146       } color[MAX_RTS][3], zs, z, s; /* 3 base types */
147    } clear_attachment;
148 
149    struct {
150       struct {
151          mali_ptr rsd;
152          struct panfrost_ubo_push pushmap;
153       } buf2img[PANVK_META_COPY_BUF2IMG_NUM_FORMATS];
154       struct {
155          mali_ptr rsd;
156          struct panfrost_ubo_push pushmap;
157       } img2buf[PANVK_META_COPY_NUM_TEX_TYPES][PANVK_META_COPY_IMG2BUF_NUM_FORMATS];
158       struct {
159          mali_ptr rsd;
160       } img2img[2][PANVK_META_COPY_NUM_TEX_TYPES][PANVK_META_COPY_IMG2IMG_NUM_FORMATS];
161       struct {
162          mali_ptr rsd;
163          struct panfrost_ubo_push pushmap;
164       } buf2buf[PANVK_META_COPY_BUF2BUF_NUM_BLKSIZES];
165       struct {
166          mali_ptr rsd;
167          struct panfrost_ubo_push pushmap;
168       } fillbuf;
169    } copy;
170 };
171 
172 struct panvk_physical_device {
173    struct vk_physical_device vk;
174 
175    /* The API agnostic device object. */
176    struct panfrost_device pdev;
177 
178    struct panvk_instance *instance;
179 
180    char path[20];
181    char name[VK_MAX_PHYSICAL_DEVICE_NAME_SIZE];
182    uint8_t driver_uuid[VK_UUID_SIZE];
183    uint8_t device_uuid[VK_UUID_SIZE];
184    uint8_t cache_uuid[VK_UUID_SIZE];
185 
186    struct wsi_device wsi_device;
187    struct panvk_meta meta;
188 
189    int master_fd;
190 };
191 
192 enum panvk_debug_flags {
193    PANVK_DEBUG_STARTUP = 1 << 0,
194    PANVK_DEBUG_NIR = 1 << 1,
195    PANVK_DEBUG_TRACE = 1 << 2,
196    PANVK_DEBUG_SYNC = 1 << 3,
197    PANVK_DEBUG_AFBC = 1 << 4,
198    PANVK_DEBUG_LINEAR = 1 << 5,
199 };
200 
201 struct panvk_instance {
202    struct vk_instance vk;
203 
204    uint32_t api_version;
205    int physical_device_count;
206    struct panvk_physical_device physical_devices[PANVK_MAX_DRM_DEVICES];
207 
208    enum panvk_debug_flags debug_flags;
209 };
210 
211 VkResult
212 panvk_wsi_init(struct panvk_physical_device *physical_device);
213 void
214 panvk_wsi_finish(struct panvk_physical_device *physical_device);
215 
216 bool
217 panvk_instance_extension_supported(const char *name);
218 uint32_t
219 panvk_physical_device_api_version(struct panvk_physical_device *dev);
220 bool
221 panvk_physical_device_extension_supported(struct panvk_physical_device *dev,
222                                        const char *name);
223 
224 struct panvk_pipeline_cache {
225    struct vk_object_base base;
226    VkAllocationCallbacks alloc;
227 };
228 
229 /* queue types */
230 #define PANVK_QUEUE_GENERAL 0
231 
232 #define PANVK_MAX_QUEUE_FAMILIES 1
233 
234 struct panvk_queue {
235    struct vk_queue vk;
236    struct panvk_device *device;
237    uint32_t sync;
238 };
239 
240 struct panvk_device {
241    struct vk_device vk;
242 
243    struct panvk_instance *instance;
244 
245    struct panvk_queue *queues[PANVK_MAX_QUEUE_FAMILIES];
246    int queue_count[PANVK_MAX_QUEUE_FAMILIES];
247 
248    struct panvk_physical_device *physical_device;
249    int _lost;
250 };
251 
252 VkResult _panvk_device_set_lost(struct panvk_device *device,
253                                 const char *file, int line,
254                                 const char *msg, ...) PRINTFLIKE(4, 5);
255 #define panvk_device_set_lost(dev, ...) \
256    _panvk_device_set_lost(dev, __FILE__, __LINE__, __VA_ARGS__)
257 
258 static inline bool
panvk_device_is_lost(struct panvk_device * device)259 panvk_device_is_lost(struct panvk_device *device)
260 {
261    return unlikely(p_atomic_read(&device->_lost));
262 }
263 
264 #define TILER_DESC_WORDS 56
265 
266 struct panvk_batch {
267    struct list_head node;
268    struct util_dynarray jobs;
269    struct util_dynarray event_ops;
270    struct pan_scoreboard scoreboard;
271    struct {
272       const struct panvk_framebuffer *info;
273       struct panfrost_ptr desc;
274    } fb;
275    struct {
276       struct panfrost_bo *src, *dst;
277    } blit;
278    struct panfrost_ptr tls;
279    mali_ptr fragment_job;
280    struct {
281       struct pan_tiler_context ctx;
282       struct panfrost_ptr descs;
283       uint32_t templ[TILER_DESC_WORDS];
284    } tiler;
285    struct pan_tls_info tlsinfo;
286    unsigned wls_total_size;
287    bool issued;
288 };
289 
290 struct panvk_syncobj {
291    uint32_t permanent, temporary;
292 };
293 
294 enum panvk_event_op_type {
295    PANVK_EVENT_OP_SET,
296    PANVK_EVENT_OP_RESET,
297    PANVK_EVENT_OP_WAIT,
298 };
299 
300 struct panvk_event_op {
301    enum panvk_event_op_type type;
302    struct panvk_event *event;
303 };
304 
305 struct panvk_fence {
306    struct vk_object_base base;
307    struct panvk_syncobj syncobj;
308 };
309 
310 struct panvk_semaphore {
311    struct vk_object_base base;
312    struct panvk_syncobj syncobj;
313 };
314 
315 int
316 panvk_signal_syncobjs(struct panvk_device *device,
317                       struct panvk_syncobj *syncobj1,
318                       struct panvk_syncobj *syncobj2);
319 
320 int
321 panvk_syncobj_to_fd(struct panvk_device *device,
322                     struct panvk_syncobj *sync);
323 
324 struct panvk_device_memory {
325    struct vk_object_base base;
326    struct panfrost_bo *bo;
327 };
328 
329 struct panvk_descriptor {
330    union {
331       struct {
332          VkImageLayout layout;
333          struct panvk_image_view *view;
334          struct panvk_sampler *sampler;
335       } image;
336 
337       struct {
338          struct panvk_buffer *buffer;
339          uint64_t offset;
340          uint64_t range;
341       } buffer_info;
342 
343       struct panvk_buffer_view *buffer_view;
344    };
345 };
346 
347 struct panvk_descriptor_set {
348    struct vk_object_base base;
349    struct panvk_descriptor_pool *pool;
350    const struct panvk_descriptor_set_layout *layout;
351    struct panvk_descriptor *descs;
352    void *ubos;
353    void *samplers;
354    void *textures;
355 };
356 
357 #define MAX_SETS 4
358 
359 struct panvk_descriptor_set_binding_layout {
360    VkDescriptorType type;
361 
362    /* Number of array elements in this binding */
363    unsigned array_size;
364 
365    /* Indices in the desc arrays */
366    unsigned desc_idx;
367    union {
368       struct {
369          unsigned sampler_idx;
370          unsigned tex_idx;
371       };
372       struct {
373          union {
374             unsigned ssbo_idx;
375             unsigned ubo_idx;
376          };
377          unsigned dynoffset_idx;
378       };
379    };
380 
381    /* Shader stages affected by this set+binding */
382    uint16_t shader_stages;
383 
384    struct panvk_sampler **immutable_samplers;
385 };
386 
387 struct panvk_descriptor_set_layout {
388    struct vk_object_base base;
389 
390    /* The create flags for this descriptor set layout */
391    VkDescriptorSetLayoutCreateFlags flags;
392 
393    /* Shader stages affected by this descriptor set */
394    uint16_t shader_stages;
395 
396    unsigned num_descs;
397    unsigned num_samplers;
398    unsigned num_textures;
399    unsigned num_ubos;
400    unsigned num_ssbos;
401    unsigned num_dynoffsets;
402 
403    /* Number of bindings in this descriptor set */
404    uint32_t binding_count;
405 
406    /* Bindings in this descriptor set */
407    struct panvk_descriptor_set_binding_layout bindings[0];
408 };
409 
410 struct panvk_pipeline_layout {
411    struct vk_object_base base;
412    unsigned char sha1[20];
413 
414    unsigned num_samplers;
415    unsigned num_textures;
416    unsigned num_ubos;
417    unsigned num_ssbos;
418    unsigned num_dynoffsets;
419    uint32_t num_sets;
420 
421    struct {
422       struct panvk_descriptor_set_layout *layout;
423       unsigned sampler_offset;
424       unsigned tex_offset;
425       unsigned ubo_offset;
426       unsigned ssbo_offset;
427       unsigned dynoffset_offset;
428    } sets[MAX_SETS];
429 };
430 
431 struct panvk_desc_pool_counters {
432    unsigned samplers;
433    unsigned combined_image_samplers;
434    unsigned sampled_images;
435    unsigned storage_images;
436    unsigned uniform_texel_bufs;
437    unsigned storage_texel_bufs;
438    unsigned input_attachments;
439    unsigned uniform_bufs;
440    unsigned storage_bufs;
441    unsigned uniform_dyn_bufs;
442    unsigned storage_dyn_bufs;
443    unsigned sets;
444 };
445 
446 struct panvk_descriptor_pool {
447    struct vk_object_base base;
448    struct panvk_desc_pool_counters max;
449    struct panvk_desc_pool_counters cur;
450    struct panvk_descriptor_set *sets;
451 };
452 
453 struct panvk_buffer {
454    struct vk_object_base base;
455    VkDeviceSize size;
456 
457    VkBufferUsageFlags usage;
458    VkBufferCreateFlags flags;
459 
460    struct panfrost_bo *bo;
461    VkDeviceSize bo_offset;
462 };
463 
464 enum panvk_dynamic_state_bits {
465    PANVK_DYNAMIC_VIEWPORT = 1 << 0,
466    PANVK_DYNAMIC_SCISSOR = 1 << 1,
467    PANVK_DYNAMIC_LINE_WIDTH = 1 << 2,
468    PANVK_DYNAMIC_DEPTH_BIAS = 1 << 3,
469    PANVK_DYNAMIC_BLEND_CONSTANTS = 1 << 4,
470    PANVK_DYNAMIC_DEPTH_BOUNDS = 1 << 5,
471    PANVK_DYNAMIC_STENCIL_COMPARE_MASK = 1 << 6,
472    PANVK_DYNAMIC_STENCIL_WRITE_MASK = 1 << 7,
473    PANVK_DYNAMIC_STENCIL_REFERENCE = 1 << 8,
474    PANVK_DYNAMIC_DISCARD_RECTANGLE = 1 << 9,
475    PANVK_DYNAMIC_ALL = (1 << 10) - 1,
476 };
477 
478 struct panvk_descriptor_state {
479    struct {
480       const struct panvk_descriptor_set *set;
481       struct panfrost_ptr dynoffsets;
482    } sets[MAX_SETS];
483    mali_ptr sysvals[MESA_SHADER_STAGES];
484    mali_ptr ubos;
485    mali_ptr textures;
486    mali_ptr samplers;
487 };
488 
489 #define INVOCATION_DESC_WORDS 2
490 
491 struct panvk_draw_info {
492    unsigned first_index;
493    unsigned index_count;
494    unsigned first_vertex;
495    unsigned vertex_count;
496    unsigned padded_vertex_count;
497    unsigned first_instance;
498    unsigned instance_count;
499    int vertex_offset;
500    unsigned offset_start;
501    uint32_t invocation[INVOCATION_DESC_WORDS];
502    struct {
503       mali_ptr varyings;
504       mali_ptr attributes;
505       mali_ptr push_constants;
506    } stages[MESA_SHADER_STAGES];
507    mali_ptr varying_bufs;
508    mali_ptr attribute_bufs;
509    mali_ptr textures;
510    mali_ptr samplers;
511    mali_ptr ubos;
512    mali_ptr position;
513    union {
514       mali_ptr psiz;
515       float line_width;
516    };
517    mali_ptr tls;
518    mali_ptr fb;
519    const struct pan_tiler_context *tiler_ctx;
520    mali_ptr fs_rsd;
521    mali_ptr viewport;
522    struct {
523       struct panfrost_ptr vertex;
524       struct panfrost_ptr tiler;
525    } jobs;
526 };
527 
528 struct panvk_attrib_info {
529    unsigned buf;
530    unsigned offset;
531    enum pipe_format format;
532 };
533 
534 struct panvk_attrib_buf_info {
535    bool special;
536    union {
537       struct {
538          unsigned stride;
539          bool per_instance;
540       };
541       unsigned special_id;
542    };
543 };
544 
545 struct panvk_attribs_info {
546    struct panvk_attrib_info attrib[PAN_MAX_ATTRIBUTE];
547    unsigned attrib_count;
548    struct panvk_attrib_buf_info buf[PAN_MAX_ATTRIBUTE];
549    unsigned buf_count;
550 };
551 
552 struct panvk_attrib_buf {
553    mali_ptr address;
554    unsigned size;
555 };
556 
557 struct panvk_cmd_state {
558    uint32_t dirty;
559 
560    struct panvk_varyings_info varyings;
561    mali_ptr fs_rsd;
562 
563    struct {
564       float constants[4];
565    } blend;
566 
567    struct {
568       struct {
569          float constant_factor;
570          float clamp;
571          float slope_factor;
572       } depth_bias;
573       float line_width;
574    } rast;
575 
576    struct {
577       struct panvk_attrib_buf bufs[MAX_VBS];
578       unsigned count;
579       mali_ptr attribs;
580       mali_ptr attrib_bufs;
581    } vb;
582 
583    /* Index buffer */
584    struct {
585       struct panvk_buffer *buffer;
586       uint64_t offset;
587       uint32_t type;
588       uint32_t max_index_count;
589       uint8_t index_size;
590       uint64_t index_va;
591    } ib;
592 
593    struct {
594       struct {
595          uint8_t compare_mask;
596          uint8_t write_mask;
597          uint8_t ref;
598       } s_front, s_back;
599    } zs;
600 
601    struct {
602       struct pan_fb_info info;
603       bool crc_valid[MAX_RTS];
604    } fb;
605 
606    const struct panvk_render_pass *pass;
607    const struct panvk_subpass *subpass;
608    const struct panvk_framebuffer *framebuffer;
609    VkRect2D render_area;
610 
611    struct panvk_clear_value *clear;
612 
613    mali_ptr vpd;
614    VkViewport viewport;
615    VkRect2D scissor;
616 
617    struct panvk_batch *batch;
618 };
619 
620 struct panvk_cmd_pool {
621    struct vk_object_base base;
622    VkAllocationCallbacks alloc;
623    struct list_head active_cmd_buffers;
624    struct list_head free_cmd_buffers;
625    uint32_t queue_family_index;
626    struct panvk_bo_pool desc_bo_pool;
627    struct panvk_bo_pool varying_bo_pool;
628    struct panvk_bo_pool tls_bo_pool;
629 };
630 
631 enum panvk_cmd_buffer_status {
632    PANVK_CMD_BUFFER_STATUS_INVALID,
633    PANVK_CMD_BUFFER_STATUS_INITIAL,
634    PANVK_CMD_BUFFER_STATUS_RECORDING,
635    PANVK_CMD_BUFFER_STATUS_EXECUTABLE,
636    PANVK_CMD_BUFFER_STATUS_PENDING,
637 };
638 
639 struct panvk_cmd_bind_point_state {
640    struct panvk_descriptor_state desc_state;
641    const struct panvk_pipeline *pipeline;
642 };
643 
644 struct panvk_cmd_buffer {
645    struct vk_command_buffer vk;
646 
647    struct panvk_device *device;
648 
649    struct panvk_cmd_pool *pool;
650    struct list_head pool_link;
651    struct panvk_pool desc_pool;
652    struct panvk_pool varying_pool;
653    struct panvk_pool tls_pool;
654    struct list_head batches;
655 
656    VkCommandBufferUsageFlags usage_flags;
657    VkCommandBufferLevel level;
658    enum panvk_cmd_buffer_status status;
659 
660    struct panvk_cmd_state state;
661    uint32_t queue_family_index;
662 
663    uint8_t push_constants[MAX_PUSH_CONSTANTS_SIZE];
664    VkShaderStageFlags push_constant_stages;
665    struct panvk_descriptor_set meta_push_descriptors;
666 
667    struct panvk_cmd_bind_point_state bind_points[MAX_BIND_POINTS];
668 
669    VkResult record_result;
670 };
671 
672 #define panvk_cmd_get_bind_point_state(cmdbuf, bindpoint) \
673         &(cmdbuf)->bind_points[VK_PIPELINE_BIND_POINT_ ## bindpoint]
674 
675 #define panvk_cmd_get_pipeline(cmdbuf, bindpoint) \
676         (cmdbuf)->bind_points[VK_PIPELINE_BIND_POINT_ ## bindpoint].pipeline
677 
678 #define panvk_cmd_get_desc_state(cmdbuf, bindpoint) \
679         &(cmdbuf)->bind_points[VK_PIPELINE_BIND_POINT_ ## bindpoint].desc_state
680 
681 struct panvk_batch *
682 panvk_cmd_open_batch(struct panvk_cmd_buffer *cmdbuf);
683 
684 void
685 panvk_cmd_fb_info_set_subpass(struct panvk_cmd_buffer *cmdbuf);
686 
687 void
688 panvk_cmd_fb_info_init(struct panvk_cmd_buffer *cmdbuf);
689 
690 void
691 panvk_cmd_preload_fb_after_batch_split(struct panvk_cmd_buffer *cmdbuf);
692 
693 void
694 panvk_pack_color(struct panvk_clear_value *out,
695                  const VkClearColorValue *in,
696                  enum pipe_format format);
697 
698 struct panvk_event {
699    struct vk_object_base base;
700    uint32_t syncobj;
701 };
702 
703 struct panvk_shader_module {
704    struct vk_object_base base;
705    unsigned char sha1[20];
706 
707    uint32_t code_size;
708    const uint32_t *code[0];
709 };
710 
711 struct panvk_shader {
712    struct pan_shader_info info;
713    struct util_dynarray binary;
714    unsigned sysval_ubo;
715 };
716 
717 struct panvk_shader *
718 panvk_shader_create(struct panvk_device *dev,
719                     gl_shader_stage stage,
720                     const VkPipelineShaderStageCreateInfo *stage_info,
721                     const struct panvk_pipeline_layout *layout,
722                     unsigned sysval_ubo,
723                     struct pan_blend_state *blend_state,
724                     bool static_blend_constants,
725                     const VkAllocationCallbacks *alloc);
726 
727 void
728 panvk_shader_destroy(struct panvk_device *dev,
729                      struct panvk_shader *shader,
730                      const VkAllocationCallbacks *alloc);
731 
732 union panvk_sysval_data {
733    float f32[4];
734    double f64[2];
735    uint32_t u32[4];
736    uint64_t u64[2];
737 };
738 
739 #define RSD_WORDS 16
740 #define BLEND_DESC_WORDS 4
741 
742 struct panvk_pipeline {
743    struct vk_object_base base;
744 
745    struct panvk_varyings_info varyings;
746    struct panvk_attribs_info attribs;
747 
748    const struct panvk_pipeline_layout *layout;
749 
750    unsigned active_stages;
751 
752    uint32_t dynamic_state_mask;
753 
754    struct panfrost_bo *binary_bo;
755    struct panfrost_bo *state_bo;
756 
757    mali_ptr vpd;
758    mali_ptr rsds[MESA_SHADER_STAGES];
759 
760    unsigned num_ubos;
761    unsigned num_sysvals;
762 
763    struct {
764       unsigned ubo_idx;
765       mali_ptr ubo;
766       struct panfrost_sysvals ids;
767       uint32_t dirty_mask;
768    } sysvals[MESA_SHADER_STAGES];
769 
770    unsigned tls_size;
771    unsigned wls_size;
772 
773    struct {
774       mali_ptr address;
775       struct pan_shader_info info;
776       uint32_t rsd_template[RSD_WORDS];
777       bool required;
778       bool dynamic_rsd;
779       uint8_t rt_mask;
780    } fs;
781 
782    struct {
783       unsigned topology;
784       bool writes_point_size;
785       bool primitive_restart;
786    } ia;
787 
788    struct {
789       bool clamp_depth;
790       float line_width;
791       struct {
792          bool enable;
793          float constant_factor;
794          float clamp;
795          float slope_factor;
796       } depth_bias;
797       bool front_ccw;
798       bool cull_front_face;
799       bool cull_back_face;
800    } rast;
801 
802    struct {
803       bool z_test;
804       bool z_write;
805       unsigned z_compare_func;
806       bool s_test;
807       struct {
808          unsigned fail_op;
809          unsigned pass_op;
810          unsigned z_fail_op;
811          unsigned compare_func;
812          uint8_t compare_mask;
813          uint8_t write_mask;
814          uint8_t ref;
815       } s_front, s_back;
816    } zs;
817 
818    struct {
819       uint8_t rast_samples;
820       uint8_t min_samples;
821       uint16_t sample_mask;
822       bool alpha_to_coverage;
823       bool alpha_to_one;
824    } ms;
825 
826    struct {
827       struct pan_blend_state state;
828       uint32_t bd_template[8][BLEND_DESC_WORDS];
829       struct {
830          uint8_t index;
831          uint16_t bifrost_factor;
832       } constant[8];
833       bool reads_dest;
834    } blend;
835 
836    VkViewport viewport;
837    VkRect2D scissor;
838 };
839 
840 struct panvk_image_level {
841    VkDeviceSize offset;
842    VkDeviceSize size;
843    uint32_t pitch;
844 };
845 
846 struct panvk_slice_layout {
847    unsigned width;
848    unsigned height;
849    unsigned depth;
850    unsigned offset;
851    unsigned line_stride;
852    unsigned size;
853 
854    /* If there is a header preceding each slice, how big is
855     * that header? Used for AFBC.
856     */
857    unsigned afbc_header_size;
858 
859    /* If checksumming is enabled following the slice, what
860     * is its offset/stride?
861     */
862    struct {
863       unsigned offset;
864       unsigned stride;
865       unsigned size;
866    } checksum;
867 };
868 
869 #define PANVK_MAX_MIP_LEVELS 13
870 
871 struct panvk_plane_layout {
872    struct panvk_slice_layout slices[PANVK_MAX_MIP_LEVELS];
873    unsigned offset;
874    unsigned array_stride;
875    unsigned size;
876 };
877 
878 struct panvk_plane_memory {
879    const struct panfrost_bo *bo;
880    unsigned offset;
881 };
882 
883 #define PANVK_MAX_PLANES 1
884 
885 struct panvk_image {
886    struct vk_object_base base;
887    struct pan_image pimage;
888    VkImageType type;
889 
890    /* The original VkFormat provided by the client.  This may not match any
891     * of the actual surface formats.
892     */
893    VkFormat vk_format;
894    VkImageAspectFlags aspects;
895    VkImageUsageFlags usage;  /**< Superset of VkImageCreateInfo::usage. */
896    VkImageTiling tiling;     /** VkImageCreateInfo::tiling */
897    VkImageCreateFlags flags; /** VkImageCreateInfo::flags */
898    VkExtent3D extent;
899 
900    unsigned queue_family_mask;
901    bool exclusive;
902    bool shareable;
903 };
904 
905 unsigned
906 panvk_image_get_plane_size(const struct panvk_image *image, unsigned plane);
907 
908 unsigned
909 panvk_image_get_total_size(const struct panvk_image *image);
910 
911 #define TEXTURE_DESC_WORDS 8
912 
913 struct panvk_image_view {
914    struct vk_object_base base;
915    struct pan_image_view pview;
916 
917    VkFormat vk_format;
918    struct panfrost_bo *bo;
919    struct {
920       uint32_t tex[TEXTURE_DESC_WORDS];
921    } descs;
922 };
923 
924 #define SAMPLER_DESC_WORDS 8
925 
926 struct panvk_sampler {
927    struct vk_object_base base;
928    uint32_t desc[SAMPLER_DESC_WORDS];
929 };
930 
931 struct panvk_buffer_view {
932    struct vk_object_base base;
933 };
934 
935 struct panvk_attachment_info {
936    struct panvk_image_view *iview;
937 };
938 
939 struct panvk_framebuffer {
940    struct vk_object_base base;
941 
942    uint32_t width;
943    uint32_t height;
944    uint32_t layers;
945 
946    uint32_t attachment_count;
947    struct panvk_attachment_info attachments[0];
948 };
949 
950 struct panvk_clear_value {
951    union {
952       uint32_t color[4];
953       struct {
954          float depth;
955          uint8_t stencil;
956       };
957    };
958 };
959 
960 struct panvk_subpass_attachment {
961    uint32_t idx;
962    VkImageLayout layout;
963    bool clear;
964    bool preload;
965 };
966 
967 struct panvk_subpass {
968    uint32_t input_count;
969    uint32_t color_count;
970    struct panvk_subpass_attachment *input_attachments;
971    uint8_t active_color_attachments;
972    struct panvk_subpass_attachment *color_attachments;
973    struct panvk_subpass_attachment *resolve_attachments;
974    struct panvk_subpass_attachment zs_attachment;
975 
976    uint32_t view_mask;
977 };
978 
979 struct panvk_render_pass_attachment {
980    VkAttachmentDescriptionFlags flags;
981    enum pipe_format format;
982    unsigned samples;
983    VkAttachmentLoadOp load_op;
984    VkAttachmentStoreOp store_op;
985    VkAttachmentLoadOp stencil_load_op;
986    VkAttachmentStoreOp stencil_store_op;
987    VkImageLayout initial_layout;
988    VkImageLayout final_layout;
989    unsigned view_mask;
990    unsigned first_used_in_subpass;
991 };
992 
993 struct panvk_render_pass {
994    struct vk_object_base base;
995 
996    uint32_t attachment_count;
997    uint32_t subpass_count;
998    struct panvk_subpass_attachment *subpass_attachments;
999    struct panvk_render_pass_attachment *attachments;
1000    struct panvk_subpass subpasses[0];
1001 };
1002 
1003 VK_DEFINE_HANDLE_CASTS(panvk_cmd_buffer, vk.base, VkCommandBuffer, VK_OBJECT_TYPE_COMMAND_BUFFER)
1004 VK_DEFINE_HANDLE_CASTS(panvk_device, vk.base, VkDevice, VK_OBJECT_TYPE_DEVICE)
1005 VK_DEFINE_HANDLE_CASTS(panvk_instance, vk.base, VkInstance, VK_OBJECT_TYPE_INSTANCE)
1006 VK_DEFINE_HANDLE_CASTS(panvk_physical_device, vk.base, VkPhysicalDevice, VK_OBJECT_TYPE_PHYSICAL_DEVICE)
1007 VK_DEFINE_HANDLE_CASTS(panvk_queue, vk.base, VkQueue, VK_OBJECT_TYPE_QUEUE)
1008 
1009 VK_DEFINE_NONDISP_HANDLE_CASTS(panvk_cmd_pool, base, VkCommandPool, VK_OBJECT_TYPE_COMMAND_POOL)
1010 VK_DEFINE_NONDISP_HANDLE_CASTS(panvk_buffer, base, VkBuffer, VK_OBJECT_TYPE_BUFFER)
1011 VK_DEFINE_NONDISP_HANDLE_CASTS(panvk_buffer_view, base, VkBufferView, VK_OBJECT_TYPE_BUFFER_VIEW)
1012 VK_DEFINE_NONDISP_HANDLE_CASTS(panvk_descriptor_pool, base, VkDescriptorPool, VK_OBJECT_TYPE_DESCRIPTOR_POOL)
1013 VK_DEFINE_NONDISP_HANDLE_CASTS(panvk_descriptor_set, base, VkDescriptorSet, VK_OBJECT_TYPE_DESCRIPTOR_SET)
1014 VK_DEFINE_NONDISP_HANDLE_CASTS(panvk_descriptor_set_layout, base,
1015                                VkDescriptorSetLayout, VK_OBJECT_TYPE_DESCRIPTOR_SET_LAYOUT)
1016 VK_DEFINE_NONDISP_HANDLE_CASTS(panvk_device_memory, base, VkDeviceMemory, VK_OBJECT_TYPE_DEVICE_MEMORY)
1017 VK_DEFINE_NONDISP_HANDLE_CASTS(panvk_fence, base, VkFence, VK_OBJECT_TYPE_FENCE)
1018 VK_DEFINE_NONDISP_HANDLE_CASTS(panvk_event, base, VkEvent, VK_OBJECT_TYPE_EVENT)
1019 VK_DEFINE_NONDISP_HANDLE_CASTS(panvk_framebuffer, base, VkFramebuffer, VK_OBJECT_TYPE_FRAMEBUFFER)
1020 VK_DEFINE_NONDISP_HANDLE_CASTS(panvk_image, base, VkImage, VK_OBJECT_TYPE_IMAGE)
1021 VK_DEFINE_NONDISP_HANDLE_CASTS(panvk_image_view, base, VkImageView, VK_OBJECT_TYPE_IMAGE_VIEW);
1022 VK_DEFINE_NONDISP_HANDLE_CASTS(panvk_pipeline_cache, base, VkPipelineCache, VK_OBJECT_TYPE_PIPELINE_CACHE)
1023 VK_DEFINE_NONDISP_HANDLE_CASTS(panvk_pipeline, base, VkPipeline, VK_OBJECT_TYPE_PIPELINE)
1024 VK_DEFINE_NONDISP_HANDLE_CASTS(panvk_pipeline_layout, base, VkPipelineLayout, VK_OBJECT_TYPE_PIPELINE_LAYOUT)
1025 VK_DEFINE_NONDISP_HANDLE_CASTS(panvk_render_pass, base, VkRenderPass, VK_OBJECT_TYPE_RENDER_PASS)
1026 VK_DEFINE_NONDISP_HANDLE_CASTS(panvk_sampler, base, VkSampler, VK_OBJECT_TYPE_SAMPLER)
1027 VK_DEFINE_NONDISP_HANDLE_CASTS(panvk_shader_module, base, VkShaderModule, VK_OBJECT_TYPE_SHADER_MODULE)
1028 VK_DEFINE_NONDISP_HANDLE_CASTS(panvk_semaphore, base, VkSemaphore, VK_OBJECT_TYPE_SEMAPHORE)
1029 
1030 #define panvk_arch_name(name, version) panvk_## version ## _ ## name
1031 
1032 #define panvk_arch_dispatch(arch, name, ...) \
1033 do { \
1034    switch (arch) { \
1035    case 5: panvk_arch_name(name, v5)(__VA_ARGS__); break; \
1036    case 6: panvk_arch_name(name, v6)(__VA_ARGS__); break; \
1037    case 7: panvk_arch_name(name, v7)(__VA_ARGS__); break; \
1038    default: unreachable("Invalid arch"); \
1039    } \
1040 } while (0)
1041 
1042 #ifdef PAN_ARCH
1043 #if PAN_ARCH == 5
1044 #define panvk_per_arch(name) panvk_arch_name(name, v5)
1045 #elif PAN_ARCH == 6
1046 #define panvk_per_arch(name) panvk_arch_name(name, v6)
1047 #elif PAN_ARCH == 7
1048 #define panvk_per_arch(name) panvk_arch_name(name, v7)
1049 #endif
1050 #include "panvk_vX_cmd_buffer.h"
1051 #include "panvk_vX_cs.h"
1052 #include "panvk_vX_meta.h"
1053 #else
1054 #define PAN_ARCH 5
1055 #define panvk_per_arch(name) panvk_arch_name(name, v5)
1056 #include "panvk_vX_cmd_buffer.h"
1057 #include "panvk_vX_cs.h"
1058 #include "panvk_vX_meta.h"
1059 #undef PAN_ARCH
1060 #undef panvk_per_arch
1061 #define PAN_ARCH 6
1062 #define panvk_per_arch(name) panvk_arch_name(name, v6)
1063 #include "panvk_vX_cmd_buffer.h"
1064 #include "panvk_vX_cs.h"
1065 #include "panvk_vX_meta.h"
1066 #undef PAN_ARCH
1067 #undef panvk_per_arch
1068 #define PAN_ARCH 7
1069 #define panvk_per_arch(name) panvk_arch_name(name, v7)
1070 #include "panvk_vX_cmd_buffer.h"
1071 #include "panvk_vX_cs.h"
1072 #include "panvk_vX_meta.h"
1073 #undef PAN_ARCH
1074 #undef panvk_per_arch
1075 #endif
1076 
1077 #ifdef PAN_ARCH
1078 bool
1079 panvk_per_arch(blend_needs_lowering)(const struct panfrost_device *dev,
1080                                      const struct pan_blend_state *state,
1081                                      unsigned rt);
1082 
1083 struct panvk_shader *
1084 panvk_per_arch(shader_create)(struct panvk_device *dev,
1085                               gl_shader_stage stage,
1086                               const VkPipelineShaderStageCreateInfo *stage_info,
1087                               const struct panvk_pipeline_layout *layout,
1088                               unsigned sysval_ubo,
1089                               struct pan_blend_state *blend_state,
1090                               bool static_blend_constants,
1091                               const VkAllocationCallbacks *alloc);
1092 #endif
1093 
1094 #endif /* PANVK_PRIVATE_H */
1095