1 /*
2 * Copyright © 2014-2017 Broadcom
3 * Copyright (C) 2012 Rob Clark <robclark@freedesktop.org>
4 *
5 * Permission is hereby granted, free of charge, to any person obtaining a
6 * copy of this software and associated documentation files (the "Software"),
7 * to deal in the Software without restriction, including without limitation
8 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
9 * and/or sell copies of the Software, and to permit persons to whom the
10 * Software is furnished to do so, subject to the following conditions:
11 *
12 * The above copyright notice and this permission notice (including the next
13 * paragraph) shall be included in all copies or substantial portions of the
14 * Software.
15 *
16 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
17 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
18 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
19 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
20 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
21 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
22 * IN THE SOFTWARE.
23 */
24
25 #include "pipe/p_defines.h"
26 #include "util/u_memory.h"
27 #include "util/format/u_format.h"
28 #include "util/u_inlines.h"
29 #include "util/u_surface.h"
30 #include "util/u_transfer_helper.h"
31 #include "util/u_upload_mgr.h"
32 #include "util/format/u_format_zs.h"
33 #include "util/u_drm.h"
34
35 #include "drm-uapi/drm_fourcc.h"
36 #include "v3d_screen.h"
37 #include "v3d_context.h"
38 #include "v3d_resource.h"
39 #include "broadcom/cle/v3d_packet_v33_pack.h"
40
41 static void
v3d_debug_resource_layout(struct v3d_resource * rsc,const char * caller)42 v3d_debug_resource_layout(struct v3d_resource *rsc, const char *caller)
43 {
44 if (!(unlikely(V3D_DEBUG & V3D_DEBUG_SURFACE)))
45 return;
46
47 struct pipe_resource *prsc = &rsc->base;
48
49 if (prsc->target == PIPE_BUFFER) {
50 fprintf(stderr,
51 "rsc %s %p (format %s), %dx%d buffer @0x%08x-0x%08x\n",
52 caller, rsc,
53 util_format_short_name(prsc->format),
54 prsc->width0, prsc->height0,
55 rsc->bo->offset,
56 rsc->bo->offset + rsc->bo->size - 1);
57 return;
58 }
59
60 static const char *const tiling_descriptions[] = {
61 [V3D_TILING_RASTER] = "R",
62 [V3D_TILING_LINEARTILE] = "LT",
63 [V3D_TILING_UBLINEAR_1_COLUMN] = "UB1",
64 [V3D_TILING_UBLINEAR_2_COLUMN] = "UB2",
65 [V3D_TILING_UIF_NO_XOR] = "UIF",
66 [V3D_TILING_UIF_XOR] = "UIF^",
67 };
68
69 for (int i = 0; i <= prsc->last_level; i++) {
70 struct v3d_resource_slice *slice = &rsc->slices[i];
71
72 int level_width = slice->stride / rsc->cpp;
73 int level_height = slice->padded_height;
74 int level_depth =
75 u_minify(util_next_power_of_two(prsc->depth0), i);
76
77 fprintf(stderr,
78 "rsc %s %p (format %s), %dx%d: "
79 "level %d (%s) %dx%dx%d -> %dx%dx%d, stride %d@0x%08x\n",
80 caller, rsc,
81 util_format_short_name(prsc->format),
82 prsc->width0, prsc->height0,
83 i, tiling_descriptions[slice->tiling],
84 u_minify(prsc->width0, i),
85 u_minify(prsc->height0, i),
86 u_minify(prsc->depth0, i),
87 level_width,
88 level_height,
89 level_depth,
90 slice->stride,
91 rsc->bo->offset + slice->offset);
92 }
93 }
94
95 static bool
v3d_resource_bo_alloc(struct v3d_resource * rsc)96 v3d_resource_bo_alloc(struct v3d_resource *rsc)
97 {
98 struct pipe_resource *prsc = &rsc->base;
99 struct pipe_screen *pscreen = prsc->screen;
100 struct v3d_bo *bo;
101
102 bo = v3d_bo_alloc(v3d_screen(pscreen), rsc->size, "resource");
103 if (bo) {
104 v3d_bo_unreference(&rsc->bo);
105 rsc->bo = bo;
106 v3d_debug_resource_layout(rsc, "alloc");
107 return true;
108 } else {
109 return false;
110 }
111 }
112
113 static void
v3d_resource_transfer_unmap(struct pipe_context * pctx,struct pipe_transfer * ptrans)114 v3d_resource_transfer_unmap(struct pipe_context *pctx,
115 struct pipe_transfer *ptrans)
116 {
117 struct v3d_context *v3d = v3d_context(pctx);
118 struct v3d_transfer *trans = v3d_transfer(ptrans);
119
120 if (trans->map) {
121 struct v3d_resource *rsc = v3d_resource(ptrans->resource);
122 struct v3d_resource_slice *slice = &rsc->slices[ptrans->level];
123
124 if (ptrans->usage & PIPE_MAP_WRITE) {
125 for (int z = 0; z < ptrans->box.depth; z++) {
126 void *dst = rsc->bo->map +
127 v3d_layer_offset(&rsc->base,
128 ptrans->level,
129 ptrans->box.z + z);
130 v3d_store_tiled_image(dst,
131 slice->stride,
132 (trans->map +
133 ptrans->stride *
134 ptrans->box.height * z),
135 ptrans->stride,
136 slice->tiling, rsc->cpp,
137 slice->padded_height,
138 &ptrans->box);
139 }
140 }
141 free(trans->map);
142 }
143
144 pipe_resource_reference(&ptrans->resource, NULL);
145 slab_free(&v3d->transfer_pool, ptrans);
146 }
147
148 static void
rebind_sampler_views(struct v3d_context * v3d,struct v3d_resource * rsc)149 rebind_sampler_views(struct v3d_context *v3d,
150 struct v3d_resource *rsc)
151 {
152 for (int st = 0; st < PIPE_SHADER_TYPES; st++) {
153 struct v3d_texture_stateobj *tex = v3d->tex + st;
154
155 for (unsigned i = 0; i < tex->num_textures; i++) {
156 struct pipe_sampler_view *psview = tex->textures[i];
157
158 if (psview->texture != &rsc->base)
159 continue;
160
161 struct v3d_sampler_view *sview =
162 v3d_sampler_view(psview);
163
164 v3d_create_texture_shader_state_bo(v3d, sview);
165
166 v3d_flag_dirty_sampler_state(v3d, st);
167 }
168 }
169 }
170
171 static void
v3d_map_usage_prep(struct pipe_context * pctx,struct pipe_resource * prsc,unsigned usage)172 v3d_map_usage_prep(struct pipe_context *pctx,
173 struct pipe_resource *prsc,
174 unsigned usage)
175 {
176 struct v3d_context *v3d = v3d_context(pctx);
177 struct v3d_resource *rsc = v3d_resource(prsc);
178
179 if (usage & PIPE_MAP_DISCARD_WHOLE_RESOURCE) {
180 if (v3d_resource_bo_alloc(rsc)) {
181 /* If it might be bound as one of our vertex buffers
182 * or UBOs, make sure we re-emit vertex buffer state
183 * or uniforms.
184 */
185 if (prsc->bind & PIPE_BIND_VERTEX_BUFFER)
186 v3d->dirty |= V3D_DIRTY_VTXBUF;
187 if (prsc->bind & PIPE_BIND_CONSTANT_BUFFER)
188 v3d->dirty |= V3D_DIRTY_CONSTBUF;
189 if (prsc->bind & PIPE_BIND_SAMPLER_VIEW)
190 rebind_sampler_views(v3d, rsc);
191 } else {
192 /* If we failed to reallocate, flush users so that we
193 * don't violate any syncing requirements.
194 */
195 v3d_flush_jobs_reading_resource(v3d, prsc,
196 V3D_FLUSH_DEFAULT,
197 false);
198 }
199 } else if (!(usage & PIPE_MAP_UNSYNCHRONIZED)) {
200 /* If we're writing and the buffer is being used by the CL, we
201 * have to flush the CL first. If we're only reading, we need
202 * to flush if the CL has written our buffer.
203 */
204 if (usage & PIPE_MAP_WRITE) {
205 v3d_flush_jobs_reading_resource(v3d, prsc,
206 V3D_FLUSH_ALWAYS,
207 false);
208 } else {
209 v3d_flush_jobs_writing_resource(v3d, prsc,
210 V3D_FLUSH_ALWAYS,
211 false);
212 }
213 }
214
215 if (usage & PIPE_MAP_WRITE) {
216 rsc->writes++;
217 rsc->initialized_buffers = ~0;
218 }
219 }
220
221 static void *
v3d_resource_transfer_map(struct pipe_context * pctx,struct pipe_resource * prsc,unsigned level,unsigned usage,const struct pipe_box * box,struct pipe_transfer ** pptrans)222 v3d_resource_transfer_map(struct pipe_context *pctx,
223 struct pipe_resource *prsc,
224 unsigned level, unsigned usage,
225 const struct pipe_box *box,
226 struct pipe_transfer **pptrans)
227 {
228 struct v3d_context *v3d = v3d_context(pctx);
229 struct v3d_resource *rsc = v3d_resource(prsc);
230 struct v3d_transfer *trans;
231 struct pipe_transfer *ptrans;
232 enum pipe_format format = prsc->format;
233 char *buf;
234
235 /* MSAA maps should have been handled by u_transfer_helper. */
236 assert(prsc->nr_samples <= 1);
237
238 /* Upgrade DISCARD_RANGE to WHOLE_RESOURCE if the whole resource is
239 * being mapped.
240 */
241 if ((usage & PIPE_MAP_DISCARD_RANGE) &&
242 !(usage & PIPE_MAP_UNSYNCHRONIZED) &&
243 !(prsc->flags & PIPE_RESOURCE_FLAG_MAP_PERSISTENT) &&
244 prsc->last_level == 0 &&
245 prsc->width0 == box->width &&
246 prsc->height0 == box->height &&
247 prsc->depth0 == box->depth &&
248 prsc->array_size == 1 &&
249 rsc->bo->private) {
250 usage |= PIPE_MAP_DISCARD_WHOLE_RESOURCE;
251 }
252
253 v3d_map_usage_prep(pctx, prsc, usage);
254
255 trans = slab_alloc(&v3d->transfer_pool);
256 if (!trans)
257 return NULL;
258
259 /* XXX: Handle DONTBLOCK, DISCARD_RANGE, PERSISTENT, COHERENT. */
260
261 /* slab_alloc_st() doesn't zero: */
262 memset(trans, 0, sizeof(*trans));
263 ptrans = &trans->base;
264
265 pipe_resource_reference(&ptrans->resource, prsc);
266 ptrans->level = level;
267 ptrans->usage = usage;
268 ptrans->box = *box;
269
270 /* Note that the current kernel implementation is synchronous, so no
271 * need to do syncing stuff here yet.
272 */
273
274 if (usage & PIPE_MAP_UNSYNCHRONIZED)
275 buf = v3d_bo_map_unsynchronized(rsc->bo);
276 else
277 buf = v3d_bo_map(rsc->bo);
278 if (!buf) {
279 fprintf(stderr, "Failed to map bo\n");
280 goto fail;
281 }
282
283 *pptrans = ptrans;
284
285 /* Our load/store routines work on entire compressed blocks. */
286 ptrans->box.x /= util_format_get_blockwidth(format);
287 ptrans->box.y /= util_format_get_blockheight(format);
288 ptrans->box.width = DIV_ROUND_UP(ptrans->box.width,
289 util_format_get_blockwidth(format));
290 ptrans->box.height = DIV_ROUND_UP(ptrans->box.height,
291 util_format_get_blockheight(format));
292
293 struct v3d_resource_slice *slice = &rsc->slices[level];
294 if (rsc->tiled) {
295 /* No direct mappings of tiled, since we need to manually
296 * tile/untile.
297 */
298 if (usage & PIPE_MAP_DIRECTLY)
299 return NULL;
300
301 ptrans->stride = ptrans->box.width * rsc->cpp;
302 ptrans->layer_stride = ptrans->stride * ptrans->box.height;
303
304 trans->map = malloc(ptrans->layer_stride * ptrans->box.depth);
305
306 if (usage & PIPE_MAP_READ) {
307 for (int z = 0; z < ptrans->box.depth; z++) {
308 void *src = rsc->bo->map +
309 v3d_layer_offset(&rsc->base,
310 ptrans->level,
311 ptrans->box.z + z);
312 v3d_load_tiled_image((trans->map +
313 ptrans->stride *
314 ptrans->box.height * z),
315 ptrans->stride,
316 src,
317 slice->stride,
318 slice->tiling, rsc->cpp,
319 slice->padded_height,
320 &ptrans->box);
321 }
322 }
323 return trans->map;
324 } else {
325 ptrans->stride = slice->stride;
326 ptrans->layer_stride = rsc->cube_map_stride;
327
328 return buf + slice->offset +
329 ptrans->box.y * ptrans->stride +
330 ptrans->box.x * rsc->cpp +
331 ptrans->box.z * rsc->cube_map_stride;
332 }
333
334
335 fail:
336 v3d_resource_transfer_unmap(pctx, ptrans);
337 return NULL;
338 }
339
340 static void
v3d_texture_subdata(struct pipe_context * pctx,struct pipe_resource * prsc,unsigned level,unsigned usage,const struct pipe_box * box,const void * data,unsigned stride,unsigned layer_stride)341 v3d_texture_subdata(struct pipe_context *pctx,
342 struct pipe_resource *prsc,
343 unsigned level,
344 unsigned usage,
345 const struct pipe_box *box,
346 const void *data,
347 unsigned stride,
348 unsigned layer_stride)
349 {
350 struct v3d_resource *rsc = v3d_resource(prsc);
351 struct v3d_resource_slice *slice = &rsc->slices[level];
352
353 /* For a direct mapping, we can just take the u_transfer path. */
354 if (!rsc->tiled) {
355 return u_default_texture_subdata(pctx, prsc, level, usage, box,
356 data, stride, layer_stride);
357 }
358
359 /* Otherwise, map and store the texture data directly into the tiled
360 * texture. Note that gallium's texture_subdata may be called with
361 * obvious usage flags missing!
362 */
363 v3d_map_usage_prep(pctx, prsc, usage | (PIPE_MAP_WRITE |
364 PIPE_MAP_DISCARD_RANGE));
365
366 void *buf;
367 if (usage & PIPE_MAP_UNSYNCHRONIZED)
368 buf = v3d_bo_map_unsynchronized(rsc->bo);
369 else
370 buf = v3d_bo_map(rsc->bo);
371
372 for (int i = 0; i < box->depth; i++) {
373 v3d_store_tiled_image(buf +
374 v3d_layer_offset(&rsc->base,
375 level,
376 box->z + i),
377 slice->stride,
378 (void *)data + layer_stride * i,
379 stride,
380 slice->tiling, rsc->cpp, slice->padded_height,
381 box);
382 }
383 }
384
385 static void
v3d_resource_destroy(struct pipe_screen * pscreen,struct pipe_resource * prsc)386 v3d_resource_destroy(struct pipe_screen *pscreen,
387 struct pipe_resource *prsc)
388 {
389 struct v3d_screen *screen = v3d_screen(pscreen);
390 struct v3d_resource *rsc = v3d_resource(prsc);
391
392 if (rsc->scanout)
393 renderonly_scanout_destroy(rsc->scanout, screen->ro);
394
395 v3d_bo_unreference(&rsc->bo);
396 free(rsc);
397 }
398
399 static uint64_t
v3d_resource_modifier(struct v3d_resource * rsc)400 v3d_resource_modifier(struct v3d_resource *rsc)
401 {
402 if (rsc->tiled) {
403 /* A shared tiled buffer should always be allocated as UIF,
404 * not UBLINEAR or LT.
405 */
406 assert(rsc->slices[0].tiling == V3D_TILING_UIF_XOR ||
407 rsc->slices[0].tiling == V3D_TILING_UIF_NO_XOR);
408 return DRM_FORMAT_MOD_BROADCOM_UIF;
409 } else {
410 return DRM_FORMAT_MOD_LINEAR;
411 }
412 }
413
414 static bool
v3d_resource_get_handle(struct pipe_screen * pscreen,struct pipe_context * pctx,struct pipe_resource * prsc,struct winsys_handle * whandle,unsigned usage)415 v3d_resource_get_handle(struct pipe_screen *pscreen,
416 struct pipe_context *pctx,
417 struct pipe_resource *prsc,
418 struct winsys_handle *whandle,
419 unsigned usage)
420 {
421 struct v3d_screen *screen = v3d_screen(pscreen);
422 struct v3d_resource *rsc = v3d_resource(prsc);
423 struct v3d_bo *bo = rsc->bo;
424
425 whandle->stride = rsc->slices[0].stride;
426 whandle->offset = 0;
427 whandle->modifier = v3d_resource_modifier(rsc);
428
429 /* If we're passing some reference to our BO out to some other part of
430 * the system, then we can't do any optimizations about only us being
431 * the ones seeing it (like BO caching).
432 */
433 bo->private = false;
434
435 switch (whandle->type) {
436 case WINSYS_HANDLE_TYPE_SHARED:
437 return v3d_bo_flink(bo, &whandle->handle);
438 case WINSYS_HANDLE_TYPE_KMS:
439 if (screen->ro) {
440 if (renderonly_get_handle(rsc->scanout, whandle)) {
441 whandle->stride = rsc->slices[0].stride;
442 return true;
443 }
444 return false;
445 }
446 whandle->handle = bo->handle;
447 return true;
448 case WINSYS_HANDLE_TYPE_FD:
449 whandle->handle = v3d_bo_get_dmabuf(bo);
450 return whandle->handle != -1;
451 }
452
453 return false;
454 }
455
456 static bool
v3d_resource_get_param(struct pipe_screen * pscreen,struct pipe_context * pctx,struct pipe_resource * prsc,unsigned plane,unsigned layer,unsigned level,enum pipe_resource_param param,unsigned usage,uint64_t * value)457 v3d_resource_get_param(struct pipe_screen *pscreen,
458 struct pipe_context *pctx, struct pipe_resource *prsc,
459 unsigned plane, unsigned layer, unsigned level,
460 enum pipe_resource_param param,
461 unsigned usage, uint64_t *value)
462 {
463 struct v3d_resource *rsc = v3d_resource(prsc);
464
465 switch (param) {
466 case PIPE_RESOURCE_PARAM_STRIDE:
467 *value = rsc->slices[level].stride;
468 return true;
469 case PIPE_RESOURCE_PARAM_OFFSET:
470 *value = 0;
471 return true;
472 case PIPE_RESOURCE_PARAM_MODIFIER:
473 *value = v3d_resource_modifier(rsc);
474 return true;
475 default:
476 return false;
477 }
478 }
479
480 #define PAGE_UB_ROWS (V3D_UIFCFG_PAGE_SIZE / V3D_UIFBLOCK_ROW_SIZE)
481 #define PAGE_UB_ROWS_TIMES_1_5 ((PAGE_UB_ROWS * 3) >> 1)
482 #define PAGE_CACHE_UB_ROWS (V3D_PAGE_CACHE_SIZE / V3D_UIFBLOCK_ROW_SIZE)
483 #define PAGE_CACHE_MINUS_1_5_UB_ROWS (PAGE_CACHE_UB_ROWS - PAGE_UB_ROWS_TIMES_1_5)
484
485 /**
486 * Computes the HW's UIFblock padding for a given height/cpp.
487 *
488 * The goal of the padding is to keep pages of the same color (bank number) at
489 * least half a page away from each other vertically when crossing between
490 * between columns of UIF blocks.
491 */
492 static uint32_t
v3d_get_ub_pad(struct v3d_resource * rsc,uint32_t height)493 v3d_get_ub_pad(struct v3d_resource *rsc, uint32_t height)
494 {
495 uint32_t utile_h = v3d_utile_height(rsc->cpp);
496 uint32_t uif_block_h = utile_h * 2;
497 uint32_t height_ub = height / uif_block_h;
498
499 uint32_t height_offset_in_pc = height_ub % PAGE_CACHE_UB_ROWS;
500
501 /* For the perfectly-aligned-for-UIF-XOR case, don't add any pad. */
502 if (height_offset_in_pc == 0)
503 return 0;
504
505 /* Try padding up to where we're offset by at least half a page. */
506 if (height_offset_in_pc < PAGE_UB_ROWS_TIMES_1_5) {
507 /* If we fit entirely in the page cache, don't pad. */
508 if (height_ub < PAGE_CACHE_UB_ROWS)
509 return 0;
510 else
511 return PAGE_UB_ROWS_TIMES_1_5 - height_offset_in_pc;
512 }
513
514 /* If we're close to being aligned to page cache size, then round up
515 * and rely on XOR.
516 */
517 if (height_offset_in_pc > PAGE_CACHE_MINUS_1_5_UB_ROWS)
518 return PAGE_CACHE_UB_ROWS - height_offset_in_pc;
519
520 /* Otherwise, we're far enough away (top and bottom) to not need any
521 * padding.
522 */
523 return 0;
524 }
525
526 static void
v3d_setup_slices(struct v3d_resource * rsc,uint32_t winsys_stride,bool uif_top)527 v3d_setup_slices(struct v3d_resource *rsc, uint32_t winsys_stride,
528 bool uif_top)
529 {
530 struct pipe_resource *prsc = &rsc->base;
531 uint32_t width = prsc->width0;
532 uint32_t height = prsc->height0;
533 uint32_t depth = prsc->depth0;
534 /* Note that power-of-two padding is based on level 1. These are not
535 * equivalent to just util_next_power_of_two(dimension), because at a
536 * level 0 dimension of 9, the level 1 power-of-two padded value is 4,
537 * not 8.
538 */
539 uint32_t pot_width = 2 * util_next_power_of_two(u_minify(width, 1));
540 uint32_t pot_height = 2 * util_next_power_of_two(u_minify(height, 1));
541 uint32_t pot_depth = 2 * util_next_power_of_two(u_minify(depth, 1));
542 uint32_t offset = 0;
543 uint32_t utile_w = v3d_utile_width(rsc->cpp);
544 uint32_t utile_h = v3d_utile_height(rsc->cpp);
545 uint32_t uif_block_w = utile_w * 2;
546 uint32_t uif_block_h = utile_h * 2;
547 uint32_t block_width = util_format_get_blockwidth(prsc->format);
548 uint32_t block_height = util_format_get_blockheight(prsc->format);
549 bool msaa = prsc->nr_samples > 1;
550
551 /* MSAA textures/renderbuffers are always laid out as single-level
552 * UIF.
553 */
554 uif_top |= msaa;
555
556 /* Check some easy mistakes to make in a resource_create() call that
557 * will break our setup.
558 */
559 assert(prsc->array_size != 0);
560 assert(prsc->depth0 != 0);
561
562 for (int i = prsc->last_level; i >= 0; i--) {
563 struct v3d_resource_slice *slice = &rsc->slices[i];
564
565 uint32_t level_width, level_height, level_depth;
566 if (i < 2) {
567 level_width = u_minify(width, i);
568 level_height = u_minify(height, i);
569 } else {
570 level_width = u_minify(pot_width, i);
571 level_height = u_minify(pot_height, i);
572 }
573 if (i < 1)
574 level_depth = u_minify(depth, i);
575 else
576 level_depth = u_minify(pot_depth, i);
577
578 if (msaa) {
579 level_width *= 2;
580 level_height *= 2;
581 }
582
583 level_width = DIV_ROUND_UP(level_width, block_width);
584 level_height = DIV_ROUND_UP(level_height, block_height);
585
586 if (!rsc->tiled) {
587 slice->tiling = V3D_TILING_RASTER;
588 if (prsc->target == PIPE_TEXTURE_1D)
589 level_width = align(level_width, 64 / rsc->cpp);
590 } else {
591 if ((i != 0 || !uif_top) &&
592 (level_width <= utile_w ||
593 level_height <= utile_h)) {
594 slice->tiling = V3D_TILING_LINEARTILE;
595 level_width = align(level_width, utile_w);
596 level_height = align(level_height, utile_h);
597 } else if ((i != 0 || !uif_top) &&
598 level_width <= uif_block_w) {
599 slice->tiling = V3D_TILING_UBLINEAR_1_COLUMN;
600 level_width = align(level_width, uif_block_w);
601 level_height = align(level_height, uif_block_h);
602 } else if ((i != 0 || !uif_top) &&
603 level_width <= 2 * uif_block_w) {
604 slice->tiling = V3D_TILING_UBLINEAR_2_COLUMN;
605 level_width = align(level_width, 2 * uif_block_w);
606 level_height = align(level_height, uif_block_h);
607 } else {
608 /* We align the width to a 4-block column of
609 * UIF blocks, but we only align height to UIF
610 * blocks.
611 */
612 level_width = align(level_width,
613 4 * uif_block_w);
614 level_height = align(level_height,
615 uif_block_h);
616
617 slice->ub_pad = v3d_get_ub_pad(rsc,
618 level_height);
619 level_height += slice->ub_pad * uif_block_h;
620
621 /* If the padding set us to to be aligned to
622 * the page cache size, then the HW will use
623 * the XOR bit on odd columns to get us
624 * perfectly misaligned
625 */
626 if ((level_height / uif_block_h) %
627 (V3D_PAGE_CACHE_SIZE /
628 V3D_UIFBLOCK_ROW_SIZE) == 0) {
629 slice->tiling = V3D_TILING_UIF_XOR;
630 } else {
631 slice->tiling = V3D_TILING_UIF_NO_XOR;
632 }
633 }
634 }
635
636 slice->offset = offset;
637 if (winsys_stride)
638 slice->stride = winsys_stride;
639 else
640 slice->stride = level_width * rsc->cpp;
641 slice->padded_height = level_height;
642 slice->size = level_height * slice->stride;
643
644 uint32_t slice_total_size = slice->size * level_depth;
645
646 /* The HW aligns level 1's base to a page if any of level 1 or
647 * below could be UIF XOR. The lower levels then inherit the
648 * alignment for as long as necessary, thanks to being power of
649 * two aligned.
650 */
651 if (i == 1 &&
652 level_width > 4 * uif_block_w &&
653 level_height > PAGE_CACHE_MINUS_1_5_UB_ROWS * uif_block_h) {
654 slice_total_size = align(slice_total_size,
655 V3D_UIFCFG_PAGE_SIZE);
656 }
657
658 offset += slice_total_size;
659
660 }
661 rsc->size = offset;
662
663 /* UIF/UBLINEAR levels need to be aligned to UIF-blocks, and LT only
664 * needs to be aligned to utile boundaries. Since tiles are laid out
665 * from small to big in memory, we need to align the later UIF slices
666 * to UIF blocks, if they were preceded by non-UIF-block-aligned LT
667 * slices.
668 *
669 * We additionally align to 4k, which improves UIF XOR performance.
670 */
671 uint32_t page_align_offset = (align(rsc->slices[0].offset, 4096) -
672 rsc->slices[0].offset);
673 if (page_align_offset) {
674 rsc->size += page_align_offset;
675 for (int i = 0; i <= prsc->last_level; i++)
676 rsc->slices[i].offset += page_align_offset;
677 }
678
679 /* Arrays and cube textures have a stride which is the distance from
680 * one full mipmap tree to the next (64b aligned). For 3D textures,
681 * we need to program the stride between slices of miplevel 0.
682 */
683 if (prsc->target != PIPE_TEXTURE_3D) {
684 rsc->cube_map_stride = align(rsc->slices[0].offset +
685 rsc->slices[0].size, 64);
686 rsc->size += rsc->cube_map_stride * (prsc->array_size - 1);
687 } else {
688 rsc->cube_map_stride = rsc->slices[0].size;
689 }
690 }
691
692 uint32_t
v3d_layer_offset(struct pipe_resource * prsc,uint32_t level,uint32_t layer)693 v3d_layer_offset(struct pipe_resource *prsc, uint32_t level, uint32_t layer)
694 {
695 struct v3d_resource *rsc = v3d_resource(prsc);
696 struct v3d_resource_slice *slice = &rsc->slices[level];
697
698 if (prsc->target == PIPE_TEXTURE_3D)
699 return slice->offset + layer * slice->size;
700 else
701 return slice->offset + layer * rsc->cube_map_stride;
702 }
703
704 static struct v3d_resource *
v3d_resource_setup(struct pipe_screen * pscreen,const struct pipe_resource * tmpl)705 v3d_resource_setup(struct pipe_screen *pscreen,
706 const struct pipe_resource *tmpl)
707 {
708 struct v3d_screen *screen = v3d_screen(pscreen);
709 struct v3d_resource *rsc = CALLOC_STRUCT(v3d_resource);
710 if (!rsc)
711 return NULL;
712 struct pipe_resource *prsc = &rsc->base;
713
714 *prsc = *tmpl;
715
716 pipe_reference_init(&prsc->reference, 1);
717 prsc->screen = pscreen;
718
719 if (prsc->nr_samples <= 1 ||
720 screen->devinfo.ver >= 40 ||
721 util_format_is_depth_or_stencil(prsc->format)) {
722 rsc->cpp = util_format_get_blocksize(prsc->format);
723 if (screen->devinfo.ver < 40 && prsc->nr_samples > 1)
724 rsc->cpp *= prsc->nr_samples;
725 } else {
726 assert(v3d_rt_format_supported(&screen->devinfo, prsc->format));
727 uint32_t output_image_format =
728 v3d_get_rt_format(&screen->devinfo, prsc->format);
729 uint32_t internal_type;
730 uint32_t internal_bpp;
731 v3d_get_internal_type_bpp_for_output_format(&screen->devinfo,
732 output_image_format,
733 &internal_type,
734 &internal_bpp);
735 switch (internal_bpp) {
736 case V3D_INTERNAL_BPP_32:
737 rsc->cpp = 4;
738 break;
739 case V3D_INTERNAL_BPP_64:
740 rsc->cpp = 8;
741 break;
742 case V3D_INTERNAL_BPP_128:
743 rsc->cpp = 16;
744 break;
745 }
746 }
747
748 assert(rsc->cpp);
749
750 return rsc;
751 }
752
753 static struct pipe_resource *
v3d_resource_create_with_modifiers(struct pipe_screen * pscreen,const struct pipe_resource * tmpl,const uint64_t * modifiers,int count)754 v3d_resource_create_with_modifiers(struct pipe_screen *pscreen,
755 const struct pipe_resource *tmpl,
756 const uint64_t *modifiers,
757 int count)
758 {
759 struct v3d_screen *screen = v3d_screen(pscreen);
760
761 bool linear_ok = drm_find_modifier(DRM_FORMAT_MOD_LINEAR, modifiers, count);
762 struct v3d_resource *rsc = v3d_resource_setup(pscreen, tmpl);
763 struct pipe_resource *prsc = &rsc->base;
764 /* Use a tiled layout if we can, for better 3D performance. */
765 bool should_tile = true;
766
767 /* VBOs/PBOs are untiled (and 1 height). */
768 if (tmpl->target == PIPE_BUFFER)
769 should_tile = false;
770
771 /* Cursors are always linear, and the user can request linear as well.
772 */
773 if (tmpl->bind & (PIPE_BIND_LINEAR | PIPE_BIND_CURSOR))
774 should_tile = false;
775
776 /* 1D and 1D_ARRAY textures are always raster-order. */
777 if (tmpl->target == PIPE_TEXTURE_1D ||
778 tmpl->target == PIPE_TEXTURE_1D_ARRAY)
779 should_tile = false;
780
781 /* Scanout BOs for simulator need to be linear for interaction with
782 * i965.
783 */
784 if (using_v3d_simulator &&
785 tmpl->bind & (PIPE_BIND_SHARED | PIPE_BIND_SCANOUT))
786 should_tile = false;
787
788 /* If using the old-school SCANOUT flag, we don't know what the screen
789 * might support other than linear. Just force linear.
790 */
791 if (tmpl->bind & PIPE_BIND_SCANOUT)
792 should_tile = false;
793
794 /* No user-specified modifier; determine our own. */
795 if (count == 1 && modifiers[0] == DRM_FORMAT_MOD_INVALID) {
796 linear_ok = true;
797 rsc->tiled = should_tile;
798 } else if (should_tile &&
799 drm_find_modifier(DRM_FORMAT_MOD_BROADCOM_UIF,
800 modifiers, count)) {
801 rsc->tiled = true;
802 } else if (linear_ok) {
803 rsc->tiled = false;
804 } else {
805 fprintf(stderr, "Unsupported modifier requested\n");
806 goto fail;
807 }
808
809 rsc->internal_format = prsc->format;
810
811 v3d_setup_slices(rsc, 0, tmpl->bind & PIPE_BIND_SHARED);
812
813 /* If we're in a renderonly setup, use the other device to perform our
814 * allocation and just import it to v3d. The other device may be
815 * using CMA, and V3D can import from CMA but doesn't do CMA
816 * allocations on its own.
817 *
818 * We always allocate this way for SHARED, because get_handle will
819 * need a resource on the display fd.
820 */
821 if (screen->ro && (tmpl->bind & (PIPE_BIND_SCANOUT |
822 PIPE_BIND_SHARED))) {
823 struct winsys_handle handle;
824 struct pipe_resource scanout_tmpl = {
825 .target = prsc->target,
826 .format = PIPE_FORMAT_RGBA8888_UNORM,
827 .width0 = 1024, /* one page */
828 .height0 = align(rsc->size, 4096) / 4096,
829 .depth0 = 1,
830 .array_size = 1,
831 };
832
833 rsc->scanout =
834 renderonly_scanout_for_resource(&scanout_tmpl,
835 screen->ro,
836 &handle);
837
838 if (!rsc->scanout) {
839 fprintf(stderr, "Failed to create scanout resource\n");
840 goto fail;
841 }
842 assert(handle.type == WINSYS_HANDLE_TYPE_FD);
843 rsc->bo = v3d_bo_open_dmabuf(screen, handle.handle);
844 close(handle.handle);
845
846 if (!rsc->bo)
847 goto fail;
848
849 v3d_debug_resource_layout(rsc, "renderonly");
850
851 return prsc;
852 } else {
853 if (!v3d_resource_bo_alloc(rsc))
854 goto fail;
855 }
856
857 return prsc;
858 fail:
859 v3d_resource_destroy(pscreen, prsc);
860 return NULL;
861 }
862
863 struct pipe_resource *
v3d_resource_create(struct pipe_screen * pscreen,const struct pipe_resource * tmpl)864 v3d_resource_create(struct pipe_screen *pscreen,
865 const struct pipe_resource *tmpl)
866 {
867 const uint64_t mod = DRM_FORMAT_MOD_INVALID;
868 return v3d_resource_create_with_modifiers(pscreen, tmpl, &mod, 1);
869 }
870
871 static struct pipe_resource *
v3d_resource_from_handle(struct pipe_screen * pscreen,const struct pipe_resource * tmpl,struct winsys_handle * whandle,unsigned usage)872 v3d_resource_from_handle(struct pipe_screen *pscreen,
873 const struct pipe_resource *tmpl,
874 struct winsys_handle *whandle,
875 unsigned usage)
876 {
877 struct v3d_screen *screen = v3d_screen(pscreen);
878 struct v3d_resource *rsc = v3d_resource_setup(pscreen, tmpl);
879 struct pipe_resource *prsc = &rsc->base;
880 struct v3d_resource_slice *slice = &rsc->slices[0];
881
882 if (!rsc)
883 return NULL;
884
885 switch (whandle->modifier) {
886 case DRM_FORMAT_MOD_LINEAR:
887 rsc->tiled = false;
888 break;
889 case DRM_FORMAT_MOD_BROADCOM_UIF:
890 rsc->tiled = true;
891 break;
892 case DRM_FORMAT_MOD_INVALID:
893 rsc->tiled = screen->ro == NULL;
894 break;
895 default:
896 switch(fourcc_mod_broadcom_mod(whandle->modifier)) {
897 case DRM_FORMAT_MOD_BROADCOM_SAND128:
898 rsc->tiled = false;
899 rsc->sand_col128_stride =
900 fourcc_mod_broadcom_param(whandle->modifier);
901 break;
902 default:
903 fprintf(stderr,
904 "Attempt to import unsupported modifier 0x%llx\n",
905 (long long)whandle->modifier);
906 goto fail;
907 }
908 }
909
910 switch (whandle->type) {
911 case WINSYS_HANDLE_TYPE_SHARED:
912 rsc->bo = v3d_bo_open_name(screen, whandle->handle);
913 break;
914 case WINSYS_HANDLE_TYPE_FD:
915 rsc->bo = v3d_bo_open_dmabuf(screen, whandle->handle);
916 break;
917 default:
918 fprintf(stderr,
919 "Attempt to import unsupported handle type %d\n",
920 whandle->type);
921 goto fail;
922 }
923
924 if (!rsc->bo)
925 goto fail;
926
927 rsc->internal_format = prsc->format;
928
929 v3d_setup_slices(rsc, whandle->stride, true);
930 v3d_debug_resource_layout(rsc, "import");
931
932 if (whandle->offset != 0) {
933 if (rsc->tiled) {
934 fprintf(stderr,
935 "Attempt to import unsupported winsys offset %u\n",
936 whandle->offset);
937 goto fail;
938 }
939 rsc->slices[0].offset += whandle->offset;
940
941 if (rsc->slices[0].offset + rsc->slices[0].size >
942 rsc->bo->size) {
943 fprintf(stderr, "Attempt to import "
944 "with overflowing offset (%d + %d > %d)\n",
945 whandle->offset,
946 rsc->slices[0].size,
947 rsc->bo->size);
948 goto fail;
949 }
950 }
951
952 if (screen->ro) {
953 /* Make sure that renderonly has a handle to our buffer in the
954 * display's fd, so that a later renderonly_get_handle()
955 * returns correct handles or GEM names.
956 */
957 rsc->scanout =
958 renderonly_create_gpu_import_for_resource(prsc,
959 screen->ro,
960 NULL);
961 }
962
963 if (rsc->tiled && whandle->stride != slice->stride) {
964 static bool warned = false;
965 if (!warned) {
966 warned = true;
967 fprintf(stderr,
968 "Attempting to import %dx%d %s with "
969 "unsupported stride %d instead of %d\n",
970 prsc->width0, prsc->height0,
971 util_format_short_name(prsc->format),
972 whandle->stride,
973 slice->stride);
974 }
975 goto fail;
976 } else if (!rsc->tiled) {
977 slice->stride = whandle->stride;
978 }
979
980 return prsc;
981
982 fail:
983 v3d_resource_destroy(pscreen, prsc);
984 return NULL;
985 }
986
987 void
v3d_update_shadow_texture(struct pipe_context * pctx,struct pipe_sampler_view * pview)988 v3d_update_shadow_texture(struct pipe_context *pctx,
989 struct pipe_sampler_view *pview)
990 {
991 struct v3d_context *v3d = v3d_context(pctx);
992 struct v3d_sampler_view *view = v3d_sampler_view(pview);
993 struct v3d_resource *shadow = v3d_resource(view->texture);
994 struct v3d_resource *orig = v3d_resource(pview->texture);
995
996 assert(view->texture != pview->texture);
997
998 if (shadow->writes == orig->writes && orig->bo->private)
999 return;
1000
1001 perf_debug("Updating %dx%d@%d shadow for linear texture\n",
1002 orig->base.width0, orig->base.height0,
1003 pview->u.tex.first_level);
1004
1005 for (int i = 0; i <= shadow->base.last_level; i++) {
1006 unsigned width = u_minify(shadow->base.width0, i);
1007 unsigned height = u_minify(shadow->base.height0, i);
1008 struct pipe_blit_info info = {
1009 .dst = {
1010 .resource = &shadow->base,
1011 .level = i,
1012 .box = {
1013 .x = 0,
1014 .y = 0,
1015 .z = 0,
1016 .width = width,
1017 .height = height,
1018 .depth = 1,
1019 },
1020 .format = shadow->base.format,
1021 },
1022 .src = {
1023 .resource = &orig->base,
1024 .level = pview->u.tex.first_level + i,
1025 .box = {
1026 .x = 0,
1027 .y = 0,
1028 .z = 0,
1029 .width = width,
1030 .height = height,
1031 .depth = 1,
1032 },
1033 .format = orig->base.format,
1034 },
1035 .mask = util_format_get_mask(orig->base.format),
1036 };
1037 pctx->blit(pctx, &info);
1038 }
1039
1040 shadow->writes = orig->writes;
1041 }
1042
1043 static struct pipe_surface *
v3d_create_surface(struct pipe_context * pctx,struct pipe_resource * ptex,const struct pipe_surface * surf_tmpl)1044 v3d_create_surface(struct pipe_context *pctx,
1045 struct pipe_resource *ptex,
1046 const struct pipe_surface *surf_tmpl)
1047 {
1048 struct v3d_context *v3d = v3d_context(pctx);
1049 struct v3d_screen *screen = v3d->screen;
1050 struct v3d_surface *surface = CALLOC_STRUCT(v3d_surface);
1051 struct v3d_resource *rsc = v3d_resource(ptex);
1052
1053 if (!surface)
1054 return NULL;
1055
1056 struct pipe_surface *psurf = &surface->base;
1057 unsigned level = surf_tmpl->u.tex.level;
1058 struct v3d_resource_slice *slice = &rsc->slices[level];
1059
1060 pipe_reference_init(&psurf->reference, 1);
1061 pipe_resource_reference(&psurf->texture, ptex);
1062
1063 psurf->context = pctx;
1064 psurf->format = surf_tmpl->format;
1065 psurf->width = u_minify(ptex->width0, level);
1066 psurf->height = u_minify(ptex->height0, level);
1067 psurf->u.tex.level = level;
1068 psurf->u.tex.first_layer = surf_tmpl->u.tex.first_layer;
1069 psurf->u.tex.last_layer = surf_tmpl->u.tex.last_layer;
1070
1071 surface->offset = v3d_layer_offset(ptex, level,
1072 psurf->u.tex.first_layer);
1073 surface->tiling = slice->tiling;
1074
1075 surface->format = v3d_get_rt_format(&screen->devinfo, psurf->format);
1076
1077 const struct util_format_description *desc =
1078 util_format_description(psurf->format);
1079
1080 surface->swap_rb = (desc->swizzle[0] == PIPE_SWIZZLE_Z &&
1081 psurf->format != PIPE_FORMAT_B5G6R5_UNORM);
1082
1083 if (util_format_is_depth_or_stencil(psurf->format)) {
1084 switch (psurf->format) {
1085 case PIPE_FORMAT_Z16_UNORM:
1086 surface->internal_type = V3D_INTERNAL_TYPE_DEPTH_16;
1087 break;
1088 case PIPE_FORMAT_Z32_FLOAT:
1089 case PIPE_FORMAT_Z32_FLOAT_S8X24_UINT:
1090 surface->internal_type = V3D_INTERNAL_TYPE_DEPTH_32F;
1091 break;
1092 default:
1093 surface->internal_type = V3D_INTERNAL_TYPE_DEPTH_24;
1094 }
1095 } else {
1096 uint32_t bpp, type;
1097 v3d_get_internal_type_bpp_for_output_format(&screen->devinfo,
1098 surface->format,
1099 &type, &bpp);
1100 surface->internal_type = type;
1101 surface->internal_bpp = bpp;
1102 }
1103
1104 if (surface->tiling == V3D_TILING_UIF_NO_XOR ||
1105 surface->tiling == V3D_TILING_UIF_XOR) {
1106 surface->padded_height_of_output_image_in_uif_blocks =
1107 (slice->padded_height /
1108 (2 * v3d_utile_height(rsc->cpp)));
1109 }
1110
1111 if (rsc->separate_stencil) {
1112 surface->separate_stencil =
1113 v3d_create_surface(pctx, &rsc->separate_stencil->base,
1114 surf_tmpl);
1115 }
1116
1117 return &surface->base;
1118 }
1119
1120 static void
v3d_surface_destroy(struct pipe_context * pctx,struct pipe_surface * psurf)1121 v3d_surface_destroy(struct pipe_context *pctx, struct pipe_surface *psurf)
1122 {
1123 struct v3d_surface *surf = v3d_surface(psurf);
1124
1125 if (surf->separate_stencil)
1126 pipe_surface_reference(&surf->separate_stencil, NULL);
1127
1128 pipe_resource_reference(&psurf->texture, NULL);
1129 FREE(psurf);
1130 }
1131
1132 static void
v3d_flush_resource(struct pipe_context * pctx,struct pipe_resource * resource)1133 v3d_flush_resource(struct pipe_context *pctx, struct pipe_resource *resource)
1134 {
1135 /* All calls to flush_resource are followed by a flush of the context,
1136 * so there's nothing to do.
1137 */
1138 }
1139
1140 static enum pipe_format
v3d_resource_get_internal_format(struct pipe_resource * prsc)1141 v3d_resource_get_internal_format(struct pipe_resource *prsc)
1142 {
1143 return v3d_resource(prsc)->internal_format;
1144 }
1145
1146 static void
v3d_resource_set_stencil(struct pipe_resource * prsc,struct pipe_resource * stencil)1147 v3d_resource_set_stencil(struct pipe_resource *prsc,
1148 struct pipe_resource *stencil)
1149 {
1150 v3d_resource(prsc)->separate_stencil = v3d_resource(stencil);
1151 }
1152
1153 static struct pipe_resource *
v3d_resource_get_stencil(struct pipe_resource * prsc)1154 v3d_resource_get_stencil(struct pipe_resource *prsc)
1155 {
1156 struct v3d_resource *rsc = v3d_resource(prsc);
1157
1158 return &rsc->separate_stencil->base;
1159 }
1160
1161 static const struct u_transfer_vtbl transfer_vtbl = {
1162 .resource_create = v3d_resource_create,
1163 .resource_destroy = v3d_resource_destroy,
1164 .transfer_map = v3d_resource_transfer_map,
1165 .transfer_unmap = v3d_resource_transfer_unmap,
1166 .transfer_flush_region = u_default_transfer_flush_region,
1167 .get_internal_format = v3d_resource_get_internal_format,
1168 .set_stencil = v3d_resource_set_stencil,
1169 .get_stencil = v3d_resource_get_stencil,
1170 };
1171
1172 void
v3d_resource_screen_init(struct pipe_screen * pscreen)1173 v3d_resource_screen_init(struct pipe_screen *pscreen)
1174 {
1175 pscreen->resource_create_with_modifiers =
1176 v3d_resource_create_with_modifiers;
1177 pscreen->resource_create = u_transfer_helper_resource_create;
1178 pscreen->resource_from_handle = v3d_resource_from_handle;
1179 pscreen->resource_get_handle = v3d_resource_get_handle;
1180 pscreen->resource_get_param = v3d_resource_get_param;
1181 pscreen->resource_destroy = u_transfer_helper_resource_destroy;
1182 pscreen->transfer_helper = u_transfer_helper_create(&transfer_vtbl,
1183 true, false,
1184 true, true);
1185 }
1186
1187 void
v3d_resource_context_init(struct pipe_context * pctx)1188 v3d_resource_context_init(struct pipe_context *pctx)
1189 {
1190 pctx->buffer_map = u_transfer_helper_transfer_map;
1191 pctx->texture_map = u_transfer_helper_transfer_map;
1192 pctx->transfer_flush_region = u_transfer_helper_transfer_flush_region;
1193 pctx->buffer_unmap = u_transfer_helper_transfer_unmap;
1194 pctx->texture_unmap = u_transfer_helper_transfer_unmap;
1195 pctx->buffer_subdata = u_default_buffer_subdata;
1196 pctx->texture_subdata = v3d_texture_subdata;
1197 pctx->create_surface = v3d_create_surface;
1198 pctx->surface_destroy = v3d_surface_destroy;
1199 pctx->resource_copy_region = util_resource_copy_region;
1200 pctx->blit = v3d_blit;
1201 pctx->generate_mipmap = v3d_generate_mipmap;
1202 pctx->flush_resource = v3d_flush_resource;
1203 }
1204