50
create_pipeline(struct radv_device *device, VkShaderModule vs_module_h, VkFormat format,
50
create_pipeline(struct radv_device *device, VkShaderModule vs_module_h, VkFormat format, VkPipeline *pipeline)
54
53
VkDevice device_h = radv_device_to_handle(device);
71
70
if (!device->meta_state.resolve.p_layout) {
73
radv_CreatePipelineLayout(radv_device_to_handle(device), &pl_create_info,
74
&device->meta_state.alloc, &device->meta_state.resolve.p_layout);
71
result = radv_CreatePipelineLayout(radv_device_to_handle(device), &pl_create_info, &device->meta_state.alloc,
72
&device->meta_state.resolve.p_layout);
75
73
if (result != VK_SUCCESS)
79
VkFormat color_formats[2] = { format, format };
77
VkFormat color_formats[2] = {format, format};
80
78
const VkPipelineRenderingCreateInfo rendering_create_info = {
81
79
.sType = VK_STRUCTURE_TYPE_PIPELINE_RENDERING_CREATE_INFO,
82
80
.colorAttachmentCount = 2,
191
189
struct radv_meta_state *state = &device->meta_state;
193
191
for (uint32_t j = 0; j < NUM_META_FS_KEYS; j++) {
194
radv_DestroyPipeline(radv_device_to_handle(device), state->resolve.pipeline[j],
192
radv_DestroyPipeline(radv_device_to_handle(device), state->resolve.pipeline[j], &state->alloc);
197
radv_DestroyPipelineLayout(radv_device_to_handle(device), state->resolve.p_layout,
194
radv_DestroyPipelineLayout(radv_device_to_handle(device), state->resolve.p_layout, &state->alloc);
233
emit_resolve(struct radv_cmd_buffer *cmd_buffer, const struct radv_image *src_image,
234
const struct radv_image *dst_image, VkFormat vk_format)
229
emit_resolve(struct radv_cmd_buffer *cmd_buffer, const struct radv_image *src_image, const struct radv_image *dst_image,
236
232
struct radv_device *device = cmd_buffer->device;
237
233
VkCommandBuffer cmd_buffer_h = radv_cmd_buffer_to_handle(cmd_buffer);
242
238
radv_dst_access_flush(cmd_buffer, VK_ACCESS_2_COLOR_ATTACHMENT_READ_BIT, src_image) |
243
239
radv_dst_access_flush(cmd_buffer, VK_ACCESS_2_COLOR_ATTACHMENT_WRITE_BIT, dst_image);
245
radv_CmdBindPipeline(cmd_buffer_h, VK_PIPELINE_BIND_POINT_GRAPHICS,
246
device->meta_state.resolve.pipeline[fs_key]);
241
radv_CmdBindPipeline(cmd_buffer_h, VK_PIPELINE_BIND_POINT_GRAPHICS, device->meta_state.resolve.pipeline[fs_key]);
248
243
radv_CmdDraw(cmd_buffer_h, 3, 1, 0, 0);
249
cmd_buffer->state.flush_bits |=
250
radv_src_access_flush(cmd_buffer, VK_ACCESS_2_COLOR_ATTACHMENT_WRITE_BIT, dst_image);
244
cmd_buffer->state.flush_bits |= radv_src_access_flush(cmd_buffer, VK_ACCESS_2_COLOR_ATTACHMENT_WRITE_BIT, dst_image);
253
247
enum radv_resolve_method {
260
image_hw_resolve_compat(const struct radv_device *device, struct radv_image *src_image,
261
struct radv_image *dst_image)
254
image_hw_resolve_compat(const struct radv_device *device, struct radv_image *src_image, struct radv_image *dst_image)
263
256
if (device->physical_device->rad_info.gfx_level >= GFX9) {
264
return dst_image->planes[0].surface.u.gfx9.swizzle_mode ==
265
src_image->planes[0].surface.u.gfx9.swizzle_mode;
257
return dst_image->planes[0].surface.u.gfx9.swizzle_mode == src_image->planes[0].surface.u.gfx9.swizzle_mode;
267
return dst_image->planes[0].surface.micro_tile_mode ==
268
src_image->planes[0].surface.micro_tile_mode;
259
return dst_image->planes[0].surface.micro_tile_mode == src_image->planes[0].surface.micro_tile_mode;
273
radv_pick_resolve_method_images(struct radv_device *device, struct radv_image *src_image,
274
VkFormat src_format, struct radv_image *dest_image,
275
unsigned dest_level, VkImageLayout dest_image_layout,
276
struct radv_cmd_buffer *cmd_buffer,
277
enum radv_resolve_method *method)
264
radv_pick_resolve_method_images(struct radv_device *device, struct radv_image *src_image, VkFormat src_format,
265
struct radv_image *dst_image, unsigned dst_level, VkImageLayout dst_image_layout,
266
struct radv_cmd_buffer *cmd_buffer, enum radv_resolve_method *method)
280
uint32_t queue_mask = radv_image_queue_family_mask(dest_image, cmd_buffer->qf,
269
uint32_t queue_mask = radv_image_queue_family_mask(dst_image, cmd_buffer->qf, cmd_buffer->qf);
283
271
if (vk_format_is_color(src_format)) {
284
272
/* Using the fragment resolve path is currently a hint to
286
274
* re-initialize it after resolving using compute.
287
275
* TODO: Add support for layered and int to the fragment path.
289
if (radv_layout_dcc_compressed(device, dest_image, dest_level, dest_image_layout,
277
if (radv_layout_dcc_compressed(device, dst_image, dst_level, dst_image_layout, queue_mask)) {
291
278
*method = RESOLVE_FRAGMENT;
292
} else if (!image_hw_resolve_compat(device, src_image, dest_image)) {
279
} else if (!image_hw_resolve_compat(device, src_image, dst_image)) {
293
280
/* The micro tile mode only needs to match for the HW
294
281
* resolve path which is the default path for non-DCC
301
288
*method = RESOLVE_COMPUTE;
302
289
else if (vk_format_is_int(src_format))
303
290
*method = RESOLVE_COMPUTE;
304
else if (src_image->info.array_size > 1 || dest_image->info.array_size > 1)
291
else if (src_image->vk.array_layers > 1 || dst_image->vk.array_layers > 1)
305
292
*method = RESOLVE_COMPUTE;
307
if (src_image->info.array_size > 1 || dest_image->info.array_size > 1 ||
308
(dest_image->planes[0].surface.flags & RADEON_SURF_NO_RENDER_TARGET))
294
if (src_image->vk.array_layers > 1 || dst_image->vk.array_layers > 1 ||
295
(dst_image->planes[0].surface.flags & RADEON_SURF_NO_RENDER_TARGET))
309
296
*method = RESOLVE_COMPUTE;
311
298
*method = RESOLVE_FRAGMENT;
348
335
radv_meta_save(&saved_state, cmd_buffer, RADV_META_SAVE_GRAPHICS_PIPELINE);
350
assert(src_image->info.samples > 1);
351
assert(dst_image->info.samples == 1);
337
assert(src_image->vk.samples > 1);
338
assert(dst_image->vk.samples == 1);
353
340
unsigned fs_key = radv_format_meta_fs_key(device, dst_image->vk.format);
364
351
assert(region->dstSubresource.aspectMask == VK_IMAGE_ASPECT_COLOR_BIT);
365
352
assert(region->srcSubresource.layerCount == region->dstSubresource.layerCount);
367
const uint32_t src_base_layer =
368
radv_meta_get_iview_layer(src_image, ®ion->srcSubresource, ®ion->srcOffset);
354
const uint32_t src_base_layer = radv_meta_get_iview_layer(src_image, ®ion->srcSubresource, ®ion->srcOffset);
370
const uint32_t dst_base_layer =
371
radv_meta_get_iview_layer(dst_image, ®ion->dstSubresource, ®ion->dstOffset);
356
const uint32_t dst_base_layer = radv_meta_get_iview_layer(dst_image, ®ion->dstSubresource, ®ion->dstOffset);
374
359
* From Vulkan 1.0.6 spec: 18.6 Resolving Multisample Images
386
371
const struct VkExtent3D extent = vk_image_sanitize_extent(&src_image->vk, region->extent);
387
372
const struct VkOffset3D dstOffset = vk_image_sanitize_offset(&dst_image->vk, region->dstOffset);
389
uint32_t queue_mask = radv_image_queue_family_mask(dst_image, cmd_buffer->qf,
374
uint32_t queue_mask = radv_image_queue_family_mask(dst_image, cmd_buffer->qf, cmd_buffer->qf);
392
if (radv_layout_dcc_compressed(cmd_buffer->device, dst_image, region->dstSubresource.mipLevel,
393
dst_image_layout, queue_mask)) {
376
if (radv_layout_dcc_compressed(cmd_buffer->device, dst_image, region->dstSubresource.mipLevel, dst_image_layout,
394
378
VkImageSubresourceRange range = {
395
379
.aspectMask = VK_IMAGE_ASPECT_COLOR_BIT,
396
380
.baseMipLevel = region->dstSubresource.mipLevel,
405
389
VkRect2D resolve_area = {
406
.offset = { dstOffset.x, dstOffset.y },
407
.extent = { extent.width, extent.height },
390
.offset = {dstOffset.x, dstOffset.y},
391
.extent = {extent.width, extent.height},
410
394
radv_CmdSetViewport(radv_cmd_buffer_to_handle(cmd_buffer), 0, 1,
503
resolve_image(struct radv_cmd_buffer *cmd_buffer, struct radv_image *src_image,
504
VkImageLayout src_image_layout, struct radv_image *dst_image,
505
VkImageLayout dst_image_layout, const VkImageResolve2 *region,
487
resolve_image(struct radv_cmd_buffer *cmd_buffer, struct radv_image *src_image, VkImageLayout src_image_layout,
488
struct radv_image *dst_image, VkImageLayout dst_image_layout, const VkImageResolve2 *region,
506
489
enum radv_resolve_method resolve_method)
508
491
switch (resolve_method) {
510
radv_meta_resolve_hardware_image(cmd_buffer, src_image, src_image_layout, dst_image,
511
dst_image_layout, region);
493
radv_meta_resolve_hardware_image(cmd_buffer, src_image, src_image_layout, dst_image, dst_image_layout, region);
513
495
case RESOLVE_FRAGMENT:
514
496
radv_decompress_resolve_src(cmd_buffer, src_image, src_image_layout, region);
516
radv_meta_resolve_fragment_image(cmd_buffer, src_image, src_image_layout, dst_image,
517
dst_image_layout, region);
498
radv_meta_resolve_fragment_image(cmd_buffer, src_image, src_image_layout, dst_image, dst_image_layout, region);
519
500
case RESOLVE_COMPUTE:
520
501
radv_decompress_resolve_src(cmd_buffer, src_image, src_image_layout, region);
522
radv_meta_resolve_compute_image(cmd_buffer, src_image, src_image->vk.format, src_image_layout,
523
dst_image, dst_image->vk.format, dst_image_layout, region);
503
radv_meta_resolve_compute_image(cmd_buffer, src_image, src_image->vk.format, src_image_layout, dst_image,
504
dst_image->vk.format, dst_image_layout, region);
526
507
assert(!"Invalid resolve method selected");
530
511
VKAPI_ATTR void VKAPI_CALL
531
radv_CmdResolveImage2(VkCommandBuffer commandBuffer,
532
const VkResolveImageInfo2 *pResolveImageInfo)
512
radv_CmdResolveImage2(VkCommandBuffer commandBuffer, const VkResolveImageInfo2 *pResolveImageInfo)
534
514
RADV_FROM_HANDLE(radv_cmd_buffer, cmd_buffer, commandBuffer);
535
515
RADV_FROM_HANDLE(radv_image, src_image, pResolveImageInfo->srcImage);
537
517
VkImageLayout src_image_layout = pResolveImageInfo->srcImageLayout;
538
518
VkImageLayout dst_image_layout = pResolveImageInfo->dstImageLayout;
539
519
const struct radv_physical_device *pdevice = cmd_buffer->device->physical_device;
540
enum radv_resolve_method resolve_method =
541
pdevice->rad_info.gfx_level >= GFX11 ? RESOLVE_FRAGMENT : RESOLVE_HW;
520
enum radv_resolve_method resolve_method = pdevice->rad_info.gfx_level >= GFX11 ? RESOLVE_FRAGMENT : RESOLVE_HW;
543
522
/* we can use the hw resolve only for single full resolves */
544
523
if (pResolveImageInfo->regionCount == 1) {
545
if (pResolveImageInfo->pRegions[0].srcOffset.x ||
546
pResolveImageInfo->pRegions[0].srcOffset.y || pResolveImageInfo->pRegions[0].srcOffset.z)
524
if (pResolveImageInfo->pRegions[0].srcOffset.x || pResolveImageInfo->pRegions[0].srcOffset.y ||
525
pResolveImageInfo->pRegions[0].srcOffset.z)
547
526
resolve_method = RESOLVE_COMPUTE;
548
if (pResolveImageInfo->pRegions[0].dstOffset.x ||
549
pResolveImageInfo->pRegions[0].dstOffset.y || pResolveImageInfo->pRegions[0].dstOffset.z)
527
if (pResolveImageInfo->pRegions[0].dstOffset.x || pResolveImageInfo->pRegions[0].dstOffset.y ||
528
pResolveImageInfo->pRegions[0].dstOffset.z)
550
529
resolve_method = RESOLVE_COMPUTE;
552
if (pResolveImageInfo->pRegions[0].extent.width != src_image->info.width ||
553
pResolveImageInfo->pRegions[0].extent.height != src_image->info.height ||
554
pResolveImageInfo->pRegions[0].extent.depth != src_image->info.depth)
531
if (pResolveImageInfo->pRegions[0].extent.width != src_image->vk.extent.width ||
532
pResolveImageInfo->pRegions[0].extent.height != src_image->vk.extent.height ||
533
pResolveImageInfo->pRegions[0].extent.depth != src_image->vk.extent.depth)
555
534
resolve_method = RESOLVE_COMPUTE;
557
536
resolve_method = RESOLVE_COMPUTE;
560
539
const VkImageResolve2 *region = &pResolveImageInfo->pRegions[r];
562
541
radv_pick_resolve_method_images(cmd_buffer->device, src_image, src_image->vk.format, dst_image,
563
region->dstSubresource.mipLevel, dst_image_layout,
564
cmd_buffer, &resolve_method);
542
region->dstSubresource.mipLevel, dst_image_layout, cmd_buffer, &resolve_method);
566
resolve_image(cmd_buffer, src_image, src_image_layout, dst_image, dst_image_layout, region,
544
resolve_image(cmd_buffer, src_image, src_image_layout, dst_image, dst_image_layout, region, resolve_method);
574
551
struct radv_meta_saved_state saved_state;
576
radv_meta_save(&saved_state, cmd_buffer,
577
RADV_META_SAVE_GRAPHICS_PIPELINE | RADV_META_SAVE_RENDER);
553
radv_meta_save(&saved_state, cmd_buffer, RADV_META_SAVE_GRAPHICS_PIPELINE | RADV_META_SAVE_RENDER);
579
555
VkRect2D *resolve_area = &saved_state.render.area;
600
576
VkImageLayout dst_layout = saved_state.render.color_att[i].resolve_layout;
601
577
struct radv_image *dst_img = dst_iview->image;
603
uint32_t queue_mask = radv_image_queue_family_mask(dst_img, cmd_buffer->qf,
579
uint32_t queue_mask = radv_image_queue_family_mask(dst_img, cmd_buffer->qf, cmd_buffer->qf);
606
if (radv_layout_dcc_compressed(cmd_buffer->device, dst_img, dst_iview->vk.base_mip_level,
607
dst_layout, queue_mask)) {
581
if (radv_layout_dcc_compressed(cmd_buffer->device, dst_img, dst_iview->vk.base_mip_level, dst_layout,
608
583
VkImageSubresourceRange range = {
609
584
.aspectMask = dst_iview->vk.aspects,
610
585
.baseMipLevel = dst_iview->vk.base_mip_level,
645
620
radv_CmdBeginRendering(radv_cmd_buffer_to_handle(cmd_buffer), &rendering_info);
647
VkResult ret = build_resolve_pipeline(
648
cmd_buffer->device, radv_format_meta_fs_key(cmd_buffer->device, dst_iview->vk.format));
623
build_resolve_pipeline(cmd_buffer->device, radv_format_meta_fs_key(cmd_buffer->device, dst_iview->vk.format));
649
624
if (ret != VK_SUCCESS) {
650
625
vk_command_buffer_set_error(&cmd_buffer->vk, ret);
668
643
const struct radv_physical_device *pdevice = cmd_buffer->device->physical_device;
669
644
const struct radv_rendering_state *render = &cmd_buffer->state.render;
670
enum radv_resolve_method resolve_method =
671
pdevice->rad_info.gfx_level >= GFX11 ? RESOLVE_FRAGMENT : RESOLVE_HW;
645
enum radv_resolve_method resolve_method = pdevice->rad_info.gfx_level >= GFX11 ? RESOLVE_FRAGMENT : RESOLVE_HW;
673
647
bool has_color_resolve = false;
674
648
for (uint32_t i = 0; i < render->color_att_count; ++i) {
687
661
struct radv_image_view *dst_iview = render->ds_att.resolve_iview;
688
662
VkImageLayout dst_layout = render->ds_att.resolve_layout;
690
radv_pick_resolve_method_images(cmd_buffer->device, src_iview->image, src_iview->vk.format,
691
dst_iview->image, dst_iview->vk.base_mip_level, dst_layout,
692
cmd_buffer, &resolve_method);
664
radv_pick_resolve_method_images(cmd_buffer->device, src_iview->image, src_iview->vk.format, dst_iview->image,
665
dst_iview->vk.base_mip_level, dst_layout, cmd_buffer, &resolve_method);
694
if ((src_iview->vk.aspects & VK_IMAGE_ASPECT_DEPTH_BIT) &&
695
render->ds_att.resolve_mode != VK_RESOLVE_MODE_NONE) {
667
if ((src_iview->vk.aspects & VK_IMAGE_ASPECT_DEPTH_BIT) && render->ds_att.resolve_mode != VK_RESOLVE_MODE_NONE) {
696
668
if (resolve_method == RESOLVE_FRAGMENT) {
697
radv_depth_stencil_resolve_rendering_fs(cmd_buffer, VK_IMAGE_ASPECT_DEPTH_BIT,
698
render->ds_att.resolve_mode);
669
radv_depth_stencil_resolve_rendering_fs(cmd_buffer, VK_IMAGE_ASPECT_DEPTH_BIT, render->ds_att.resolve_mode);
700
671
assert(resolve_method == RESOLVE_COMPUTE);
701
radv_depth_stencil_resolve_rendering_cs(cmd_buffer, VK_IMAGE_ASPECT_DEPTH_BIT,
702
render->ds_att.resolve_mode);
672
radv_depth_stencil_resolve_rendering_cs(cmd_buffer, VK_IMAGE_ASPECT_DEPTH_BIT, render->ds_att.resolve_mode);
765
735
struct radv_image *dst_img = dst_iview->image;
767
737
radv_pick_resolve_method_images(cmd_buffer->device, src_img, src_iview->vk.format, dst_img,
768
dst_iview->vk.base_mip_level, dst_layout,
769
cmd_buffer, &resolve_method);
738
dst_iview->vk.base_mip_level, dst_layout, cmd_buffer, &resolve_method);
770
739
VkImageResolve2 region = {
771
740
.sType = VK_STRUCTURE_TYPE_IMAGE_RESOLVE_2,
773
.width = resolve_area.extent.width,
774
.height = resolve_area.extent.height,
743
.width = resolve_area.extent.width,
744
.height = resolve_area.extent.height,
777
747
.srcSubresource =
778
748
(VkImageSubresourceLayers){
779
749
.aspectMask = VK_IMAGE_ASPECT_COLOR_BIT,
788
758
.baseArrayLayer = dst_iview->vk.base_array_layer,
789
759
.layerCount = layer_count,
791
.srcOffset = { resolve_area.offset.x, resolve_area.offset.y, 0 },
792
.dstOffset = { resolve_area.offset.x, resolve_area.offset.y, 0 },
761
.srcOffset = {resolve_area.offset.x, resolve_area.offset.y, 0},
762
.dstOffset = {resolve_area.offset.x, resolve_area.offset.y, 0},
795
765
switch (resolve_method) {
799
769
case RESOLVE_COMPUTE:
800
770
radv_decompress_resolve_src(cmd_buffer, src_iview->image, src_layout, ®ion);
802
radv_cmd_buffer_resolve_rendering_cs(cmd_buffer, src_iview, src_layout, dst_iview,
803
dst_layout, ®ion);
772
radv_cmd_buffer_resolve_rendering_cs(cmd_buffer, src_iview, src_layout, dst_iview, dst_layout, ®ion);
805
774
case RESOLVE_FRAGMENT:
806
775
radv_decompress_resolve_src(cmd_buffer, src_iview->image, src_layout, ®ion);
808
radv_cmd_buffer_resolve_rendering_fs(cmd_buffer, src_iview, src_layout, dst_iview,
777
radv_cmd_buffer_resolve_rendering_fs(cmd_buffer, src_iview, src_layout, dst_iview, dst_layout);
812
780
unreachable("Invalid resolve method");
856
824
radv_decompress_resolve_src(struct radv_cmd_buffer *cmd_buffer, struct radv_image *src_image,
857
825
VkImageLayout src_image_layout, const VkImageResolve2 *region)
859
const uint32_t src_base_layer =
860
radv_meta_get_iview_layer(src_image, ®ion->srcSubresource, ®ion->srcOffset);
827
const uint32_t src_base_layer = radv_meta_get_iview_layer(src_image, ®ion->srcSubresource, ®ion->srcOffset);
862
829
VkImageMemoryBarrier2 barrier = {
863
830
.sType = VK_STRUCTURE_TYPE_IMAGE_MEMORY_BARRIER_2,
868
835
.oldLayout = src_image_layout,
869
836
.newLayout = VK_IMAGE_LAYOUT_TRANSFER_SRC_OPTIMAL,
870
837
.image = radv_image_to_handle(src_image),
871
.subresourceRange = (VkImageSubresourceRange){
872
.aspectMask = region->srcSubresource.aspectMask,
873
.baseMipLevel = region->srcSubresource.mipLevel,
875
.baseArrayLayer = src_base_layer,
876
.layerCount = region->srcSubresource.layerCount,
839
(VkImageSubresourceRange){
840
.aspectMask = region->srcSubresource.aspectMask,
841
.baseMipLevel = region->srcSubresource.mipLevel,
843
.baseArrayLayer = src_base_layer,
844
.layerCount = region->srcSubresource.layerCount,
880
848
VkSampleLocationsInfoEXT sample_loc_info;
882
850
/* If the depth/stencil image uses different sample
883
851
* locations, we need them during HTILE decompressions.
885
struct radv_sample_locations_state *sample_locs =
886
&cmd_buffer->state.render.sample_locations;
853
struct radv_sample_locations_state *sample_locs = &cmd_buffer->state.render.sample_locations;
888
855
sample_loc_info = (VkSampleLocationsInfoEXT){
889
856
.sType = VK_STRUCTURE_TYPE_SAMPLE_LOCATIONS_INFO_EXT,