← Back to branch summary

~mmach/netext73/mesa-ryzen

« back to all changes in this revision

Viewing changes to src/amd/vulkan/radv_private.h

  • Committer: mmach
  • Date: 2023-11-02 21:31:35 UTC
  • Revision ID: netbit73@gmail.com-20231102213135-18d4tzh7tj0uz752
2023-11-02 22:11:57

Show diffs side-by-side

added added

removed removed

Lines of Context:
src/amd/vulkan/radv_private.h
60
60
#include "vk_debug_report.h"
61
61
#include "vk_device.h"
62
62
#include "vk_format.h"
 
63
#include "vk_image.h"
63
64
#include "vk_instance.h"
64
65
#include "vk_log.h"
65
66
#include "vk_physical_device.h"
 
67
#include "vk_queue.h"
66
68
#include "vk_shader_module.h"
67
 
#include "vk_queue.h"
68
69
#include "vk_util.h"
69
 
#include "vk_image.h"
 
70
#include "vk_video.h"
70
71
#include "vk_ycbcr_conversion.h"
71
 
#include "vk_video.h"
72
72
 
73
73
#include "rmv/vk_rmv_common.h"
74
74
#include "rmv/vk_rmv_tokens.h"
79
79
#include "ac_spm.h"
80
80
#include "ac_sqtt.h"
81
81
#include "ac_surface.h"
 
82
#include "ac_vcn.h"
82
83
#include "radv_constants.h"
83
84
#include "radv_descriptor_set.h"
84
85
#include "radv_radeon_winsys.h"
105
106
#include "wsi_common.h"
106
107
 
107
108
#ifdef __cplusplus
108
 
extern "C"
109
 
{
 
109
extern "C" {
110
110
#endif
111
111
 
112
112
/* Helper to determine if we should compile
122
122
#define RADV_SUPPORT_ANDROID_HARDWARE_BUFFER 0
123
123
#endif
124
124
 
125
 
#if defined(VK_USE_PLATFORM_WAYLAND_KHR) || defined(VK_USE_PLATFORM_XCB_KHR) ||                    \
126
 
   defined(VK_USE_PLATFORM_XLIB_KHR) || defined(VK_USE_PLATFORM_DISPLAY_KHR)
 
125
#if defined(VK_USE_PLATFORM_WAYLAND_KHR) || defined(VK_USE_PLATFORM_XCB_KHR) || defined(VK_USE_PLATFORM_XLIB_KHR) ||   \
 
126
   defined(VK_USE_PLATFORM_DISPLAY_KHR)
127
127
#define RADV_USE_WSI_PLATFORM
128
128
#endif
129
129
 
248
248
 
249
249
struct radv_image_view;
250
250
struct radv_instance;
 
251
struct rvcn_decode_buffer_s;
251
252
 
252
253
/* A non-fatal assert.  Useful for debugging. */
253
254
#ifdef NDEBUG
254
 
#define radv_assert(x)                                                                             \
255
 
   do {                                                                                            \
 
255
#define radv_assert(x)                                                                                                 \
 
256
   do {                                                                                                                \
256
257
   } while (0)
257
258
#else
258
 
#define radv_assert(x)                                                                             \
259
 
   do {                                                                                            \
260
 
      if (unlikely(!(x)))                                                                          \
261
 
         fprintf(stderr, "%s:%d ASSERT: %s\n", __FILE__, __LINE__, #x);                            \
 
259
#define radv_assert(x)                                                                                                 \
 
260
   do {                                                                                                                \
 
261
      if (unlikely(!(x)))                                                                                              \
 
262
         fprintf(stderr, "%s:%d ASSERT: %s\n", __FILE__, __LINE__, #x);                                                \
262
263
   } while (0)
263
264
#endif
264
265
 
372
373
      unsigned cmd;
373
374
      unsigned cntl;
374
375
   } vid_dec_reg;
 
376
   enum amd_ip_type vid_decode_ip;
 
377
   uint32_t vid_addr_gfx_mode;
 
378
   uint32_t stream_handle_base;
 
379
   uint32_t stream_handle_counter;
375
380
};
376
381
 
377
 
uint32_t radv_find_memory_index(struct radv_physical_device *pdevice, VkMemoryPropertyFlags flags);
 
382
uint32_t radv_find_memory_index(const struct radv_physical_device *pdevice, VkMemoryPropertyFlags flags);
378
383
 
379
384
VkResult create_null_physical_device(struct vk_instance *vk_instance);
380
385
 
383
388
 
384
389
void radv_physical_device_destroy(struct vk_physical_device *vk_device);
385
390
 
386
 
bool radv_thread_trace_enabled(void);
 
391
enum radv_trace_mode {
 
392
   /** Radeon GPU Profiler */
 
393
   RADV_TRACE_MODE_RGP = 1 << VK_TRACE_MODE_COUNT,
 
394
 
 
395
   /** Radeon Raytracing Analyzer */
 
396
   RADV_TRACE_MODE_RRA = 1 << (VK_TRACE_MODE_COUNT + 1),
 
397
};
387
398
 
388
399
struct radv_instance {
389
400
   struct vk_instance vk;
410
421
   bool enable_unified_heap_on_apu;
411
422
   bool tex_non_uniform;
412
423
   bool flush_before_timestamp_write;
 
424
   bool force_rt_wave64;
 
425
   bool dual_color_blend_by_location;
413
426
   char *app_layer;
414
427
};
415
428
 
423
436
                                bool *found_in_application_cache);
424
437
 
425
438
void radv_pipeline_cache_insert(struct radv_device *device, struct vk_pipeline_cache *cache,
426
 
                                struct radv_pipeline *pipeline,
427
 
                                struct radv_shader_part_binary *ps_epilog_binary,
 
439
                                struct radv_pipeline *pipeline, struct radv_shader_part_binary *ps_epilog_binary,
428
440
                                const unsigned char *sha1);
429
441
 
 
442
struct radv_ray_tracing_pipeline;
 
443
bool radv_ray_tracing_pipeline_cache_search(struct radv_device *device, struct vk_pipeline_cache *cache,
 
444
                                            struct radv_ray_tracing_pipeline *pipeline,
 
445
                                            const VkRayTracingPipelineCreateInfoKHR *create_info);
 
446
 
 
447
void radv_ray_tracing_pipeline_cache_insert(struct radv_device *device, struct vk_pipeline_cache *cache,
 
448
                                            struct radv_ray_tracing_pipeline *pipeline, unsigned num_stages,
 
449
                                            const unsigned char *sha1);
 
450
 
 
451
struct vk_pipeline_cache_object *
 
452
radv_pipeline_cache_search_nir(struct radv_device *device, struct vk_pipeline_cache *cache, const unsigned char *sha1);
 
453
 
 
454
struct vk_pipeline_cache_object *radv_pipeline_cache_nir_to_handle(struct radv_device *device,
 
455
                                                                   struct vk_pipeline_cache *cache,
 
456
                                                                   struct nir_shader *nir, const unsigned char *sha1,
 
457
                                                                   bool cached);
 
458
 
 
459
struct nir_shader *radv_pipeline_cache_handle_to_nir(struct radv_device *device,
 
460
                                                     struct vk_pipeline_cache_object *object);
 
461
 
430
462
enum radv_blit_ds_layout {
431
463
   RADV_BLIT_DS_LAYOUT_TILE_ENABLE,
432
464
   RADV_BLIT_DS_LAYOUT_TILE_DISABLE,
436
468
static inline enum radv_blit_ds_layout
437
469
radv_meta_blit_ds_to_type(VkImageLayout layout)
438
470
{
439
 
   return (layout == VK_IMAGE_LAYOUT_GENERAL) ? RADV_BLIT_DS_LAYOUT_TILE_DISABLE
440
 
                                              : RADV_BLIT_DS_LAYOUT_TILE_ENABLE;
 
471
   return (layout == VK_IMAGE_LAYOUT_GENERAL) ? RADV_BLIT_DS_LAYOUT_TILE_DISABLE : RADV_BLIT_DS_LAYOUT_TILE_ENABLE;
441
472
}
442
473
 
443
474
static inline VkImageLayout
444
475
radv_meta_blit_ds_to_layout(enum radv_blit_ds_layout ds_layout)
445
476
{
446
 
   return ds_layout == RADV_BLIT_DS_LAYOUT_TILE_ENABLE ? VK_IMAGE_LAYOUT_TRANSFER_DST_OPTIMAL
447
 
                                                       : VK_IMAGE_LAYOUT_GENERAL;
 
477
   return ds_layout == RADV_BLIT_DS_LAYOUT_TILE_ENABLE ? VK_IMAGE_LAYOUT_TRANSFER_DST_OPTIMAL : VK_IMAGE_LAYOUT_GENERAL;
448
478
}
449
479
 
450
480
enum radv_meta_dst_layout {
456
486
static inline enum radv_meta_dst_layout
457
487
radv_meta_dst_layout_from_layout(VkImageLayout layout)
458
488
{
459
 
   return (layout == VK_IMAGE_LAYOUT_GENERAL) ? RADV_META_DST_LAYOUT_GENERAL
460
 
                                              : RADV_META_DST_LAYOUT_OPTIMAL;
 
489
   return (layout == VK_IMAGE_LAYOUT_GENERAL) ? RADV_META_DST_LAYOUT_GENERAL : RADV_META_DST_LAYOUT_OPTIMAL;
461
490
}
462
491
 
463
492
static inline VkImageLayout
464
493
radv_meta_dst_layout_to_layout(enum radv_meta_dst_layout layout)
465
494
{
466
 
   return layout == RADV_META_DST_LAYOUT_OPTIMAL ? VK_IMAGE_LAYOUT_TRANSFER_DST_OPTIMAL
467
 
                                                 : VK_IMAGE_LAYOUT_GENERAL;
 
495
   return layout == RADV_META_DST_LAYOUT_OPTIMAL ? VK_IMAGE_LAYOUT_TRANSFER_DST_OPTIMAL : VK_IMAGE_LAYOUT_GENERAL;
468
496
}
469
497
 
470
498
struct radv_meta_state {
739
767
static inline enum radv_queue_family
740
768
vk_queue_to_radv(const struct radv_physical_device *phys_dev, int queue_family_index)
741
769
{
742
 
   if (queue_family_index == VK_QUEUE_FAMILY_EXTERNAL ||
743
 
       queue_family_index == VK_QUEUE_FAMILY_FOREIGN_EXT)
 
770
   if (queue_family_index == VK_QUEUE_FAMILY_EXTERNAL || queue_family_index == VK_QUEUE_FAMILY_FOREIGN_EXT)
744
771
      return RADV_QUEUE_FOREIGN;
745
772
   if (queue_family_index == VK_QUEUE_FAMILY_IGNORED)
746
773
      return RADV_QUEUE_IGNORED;
749
776
   return phys_dev->vk_queue_to_radv[queue_family_index];
750
777
}
751
778
 
752
 
enum amd_ip_type radv_queue_family_to_ring(struct radv_physical_device *physical_device,
753
 
                                         enum radv_queue_family f);
 
779
enum amd_ip_type radv_queue_family_to_ring(const struct radv_physical_device *physical_device,
 
780
                                           enum radv_queue_family f);
754
781
 
755
782
static inline bool
756
783
radv_has_uvd(struct radv_physical_device *phys_dev)
817
844
   struct radeon_winsys_ctx *hw_ctx;
818
845
   enum radeon_ctx_priority priority;
819
846
   struct radv_queue_state state;
820
 
   struct radv_queue_state *ace_internal_state;
 
847
   struct radv_queue_state *follower_state;
821
848
   struct radeon_winsys_bo *gang_sem_bo;
822
849
 
823
850
   uint64_t last_shader_upload_seq;
829
856
 
830
857
void radv_queue_finish(struct radv_queue *queue);
831
858
 
832
 
enum radeon_ctx_priority
833
 
radv_get_queue_global_priority(const VkDeviceQueueGlobalPriorityCreateInfoKHR *pObj);
 
859
enum radeon_ctx_priority radv_get_queue_global_priority(const VkDeviceQueueGlobalPriorityCreateInfoKHR *pObj);
 
860
 
 
861
struct radv_shader_free_list {
 
862
   uint8_t size_mask;
 
863
   struct list_head free_lists[RADV_SHADER_ALLOC_NUM_FREE_LISTS];
 
864
};
834
865
 
835
866
struct radv_shader_dma_submission {
836
867
   struct list_head list;
893
924
void radv_destroy_rra_accel_struct_data(VkDevice device, struct radv_rra_accel_struct_data *data);
894
925
 
895
926
struct radv_rra_trace_data {
896
 
   int elapsed_frames;
897
 
   int trace_frame;
898
 
   char *trigger_file;
899
927
   struct hash_table *accel_structs;
900
928
   struct hash_table_u64 *accel_struct_vas;
901
929
   simple_mtx_t data_mtx;
920
948
   struct vk_device_dispatch_table rmv;
921
949
};
922
950
 
 
951
enum radv_buffer_robustness {
 
952
   RADV_BUFFER_ROBUSTNESS_DISABLED,
 
953
   RADV_BUFFER_ROBUSTNESS_1, /* robustBufferAccess */
 
954
   RADV_BUFFER_ROBUSTNESS_2, /* robustBufferAccess2 */
 
955
};
 
956
 
923
957
struct radv_device {
924
958
   struct vk_device vk;
925
959
 
970
1004
   uint32_t fmask_mrt_offset_counter;
971
1005
 
972
1006
   struct list_head shader_arenas;
 
1007
   struct hash_table_u64 *capture_replay_arena_vas;
973
1008
   unsigned shader_arena_shift;
974
1009
   uint8_t shader_free_list_mask;
975
 
   struct list_head shader_free_lists[RADV_SHADER_ALLOC_NUM_FREE_LISTS];
 
1010
   struct radv_shader_free_list shader_free_list;
 
1011
   struct radv_shader_free_list capture_replay_free_list;
976
1012
   struct list_head shader_block_obj_pool;
977
1013
   mtx_t shader_arena_mutex;
978
1014
 
991
1027
   uint64_t dmesg_timestamp;
992
1028
 
993
1029
   /* Whether the app has enabled the robustBufferAccess/robustBufferAccess2 features. */
994
 
   bool robust_buffer_access;
995
 
   bool robust_buffer_access2;
 
1030
   enum radv_buffer_robustness buffer_robustness;
996
1031
 
997
1032
   /* Whether to inline the compute dispatch size in user sgprs. */
998
1033
   bool load_grid_size_from_user_sgpr;
1018
1053
   struct radv_device_border_color_data border_color_data;
1019
1054
 
1020
1055
   /* Thread trace. */
1021
 
   struct ac_thread_trace_data thread_trace;
 
1056
   struct ac_sqtt sqtt;
 
1057
   bool sqtt_enabled;
 
1058
   bool sqtt_triggered;
1022
1059
 
1023
1060
   /* Memory trace. */
1024
1061
   struct radv_memory_trace_data memory_trace;
1025
1062
 
1026
1063
   /* SPM. */
1027
 
   struct ac_spm_trace_data spm_trace;
 
1064
   struct ac_spm spm;
1028
1065
 
1029
1066
   /* Radeon Raytracing Analyzer trace. */
1030
1067
   struct radv_rra_trace_data rra_trace;
1083
1120
 
1084
1121
   bool uses_device_generated_commands;
1085
1122
 
 
1123
   /* Whether smooth lines is enabled. */
 
1124
   bool smooth_lines;
 
1125
 
1086
1126
   bool uses_shadow_regs;
1087
1127
 
1088
1128
   struct hash_table *rt_handles;
1109
1149
#endif
1110
1150
};
1111
1151
 
1112
 
void radv_device_memory_init(struct radv_device_memory *mem, struct radv_device *device,
1113
 
                             struct radeon_winsys_bo *bo);
 
1152
void radv_device_memory_init(struct radv_device_memory *mem, struct radv_device *device, struct radeon_winsys_bo *bo);
1114
1153
void radv_device_memory_finish(struct radv_device_memory *mem);
1115
1154
 
1116
1155
struct radv_descriptor_range {
1209
1248
   VkDeviceSize offset;
1210
1249
};
1211
1250
 
1212
 
void radv_buffer_init(struct radv_buffer *buffer, struct radv_device *device,
1213
 
                      struct radeon_winsys_bo *bo, uint64_t size, uint64_t offset);
 
1251
void radv_buffer_init(struct radv_buffer *buffer, struct radv_device *device, struct radeon_winsys_bo *bo,
 
1252
                      uint64_t size, uint64_t offset);
1214
1253
void radv_buffer_finish(struct radv_buffer *buffer);
1215
1254
 
1216
1255
enum radv_dynamic_state_bits {
1262
1301
   RADV_DYNAMIC_COLOR_BLEND_EQUATION = 1ull << 45,
1263
1302
   RADV_DYNAMIC_DISCARD_RECTANGLE_ENABLE = 1ull << 46,
1264
1303
   RADV_DYNAMIC_DISCARD_RECTANGLE_MODE = 1ull << 47,
1265
 
   RADV_DYNAMIC_ALL = (1ull << 48) - 1,
 
1304
   RADV_DYNAMIC_ATTACHMENT_FEEDBACK_LOOP_ENABLE = 1ull << 48,
 
1305
   RADV_DYNAMIC_SAMPLE_LOCATIONS_ENABLE = 1ull << 49,
 
1306
   RADV_DYNAMIC_ALL = (1ull << 50) - 1,
1266
1307
};
1267
1308
 
1268
1309
enum radv_cmd_dirty_bits {
1316
1357
   RADV_CMD_DIRTY_DYNAMIC_COLOR_BLEND_EQUATION = 1ull << 45,
1317
1358
   RADV_CMD_DIRTY_DYNAMIC_DISCARD_RECTANGLE_ENABLE = 1ull << 46,
1318
1359
   RADV_CMD_DIRTY_DYNAMIC_DISCARD_RECTANGLE_MODE = 1ull << 47,
1319
 
   RADV_CMD_DIRTY_DYNAMIC_ALL = (1ull << 48) - 1,
1320
 
   RADV_CMD_DIRTY_PIPELINE = 1ull << 48,
1321
 
   RADV_CMD_DIRTY_INDEX_BUFFER = 1ull << 49,
1322
 
   RADV_CMD_DIRTY_FRAMEBUFFER = 1ull << 50,
1323
 
   RADV_CMD_DIRTY_VERTEX_BUFFER = 1ull << 51,
1324
 
   RADV_CMD_DIRTY_STREAMOUT_BUFFER = 1ull << 52,
1325
 
   RADV_CMD_DIRTY_GUARDBAND = 1ull << 53,
1326
 
   RADV_CMD_DIRTY_RBPLUS = 1ull << 54,
1327
 
   RADV_CMD_DIRTY_NGG_QUERY = 1ull << 55,
1328
 
   RADV_CMD_DIRTY_OCCLUSION_QUERY = 1ull << 56,
 
1360
   RADV_CMD_DIRTY_DYNAMIC_ATTACHMENT_FEEDBACK_LOOP_ENABLE = 1ull << 48,
 
1361
   RADV_CMD_DIRTY_DYNAMIC_SAMPLE_LOCATIONS_ENABLE = 1ull << 49,
 
1362
   RADV_CMD_DIRTY_DYNAMIC_ALL = (1ull << 50) - 1,
 
1363
   RADV_CMD_DIRTY_PIPELINE = 1ull << 50,
 
1364
   RADV_CMD_DIRTY_INDEX_BUFFER = 1ull << 51,
 
1365
   RADV_CMD_DIRTY_FRAMEBUFFER = 1ull << 52,
 
1366
   RADV_CMD_DIRTY_VERTEX_BUFFER = 1ull << 53,
 
1367
   RADV_CMD_DIRTY_STREAMOUT_BUFFER = 1ull << 54,
 
1368
   RADV_CMD_DIRTY_GUARDBAND = 1ull << 55,
 
1369
   RADV_CMD_DIRTY_RBPLUS = 1ull << 56,
 
1370
   RADV_CMD_DIRTY_NGG_QUERY = 1ull << 57,
 
1371
   RADV_CMD_DIRTY_OCCLUSION_QUERY = 1ull << 58,
 
1372
   RADV_CMD_DIRTY_DB_SHADER_CONTROL = 1ull << 59,
1329
1373
};
1330
1374
 
1331
1375
enum radv_cmd_flush_bits {
1360
1404
   RADV_CMD_FLAG_STOP_PIPELINE_STATS = 1 << 15,
1361
1405
   RADV_CMD_FLAG_VGT_STREAMOUT_SYNC = 1 << 16,
1362
1406
 
1363
 
   RADV_CMD_FLUSH_AND_INV_FRAMEBUFFER =
1364
 
      (RADV_CMD_FLAG_FLUSH_AND_INV_CB | RADV_CMD_FLAG_FLUSH_AND_INV_CB_META |
1365
 
       RADV_CMD_FLAG_FLUSH_AND_INV_DB | RADV_CMD_FLAG_FLUSH_AND_INV_DB_META),
 
1407
   RADV_CMD_FLUSH_AND_INV_FRAMEBUFFER = (RADV_CMD_FLAG_FLUSH_AND_INV_CB | RADV_CMD_FLAG_FLUSH_AND_INV_CB_META |
 
1408
                                         RADV_CMD_FLAG_FLUSH_AND_INV_DB | RADV_CMD_FLAG_FLUSH_AND_INV_DB_META),
1366
1409
 
1367
 
   RADV_CMD_FLUSH_ALL_COMPUTE =
1368
 
      (RADV_CMD_FLAG_INV_ICACHE | RADV_CMD_FLAG_INV_SCACHE | RADV_CMD_FLAG_INV_VCACHE |
1369
 
       RADV_CMD_FLAG_INV_L2 | RADV_CMD_FLAG_WB_L2 | RADV_CMD_FLAG_CS_PARTIAL_FLUSH),
 
1410
   RADV_CMD_FLUSH_ALL_COMPUTE = (RADV_CMD_FLAG_INV_ICACHE | RADV_CMD_FLAG_INV_SCACHE | RADV_CMD_FLAG_INV_VCACHE |
 
1411
                                 RADV_CMD_FLAG_INV_L2 | RADV_CMD_FLAG_WB_L2 | RADV_CMD_FLAG_CS_PARTIAL_FLUSH),
1370
1412
};
1371
1413
 
1372
1414
enum radv_nggc_settings {
1431
1473
   } hw_vp;
1432
1474
 
1433
1475
   struct radv_sample_locations_state sample_location;
 
1476
 
 
1477
   VkImageAspectFlags feedback_loop_aspects;
1434
1478
};
1435
1479
 
1436
1480
const char *radv_get_debug_option_name(int id);
1437
1481
 
1438
1482
const char *radv_get_perftest_option_name(int id);
1439
1483
 
1440
 
int radv_get_int_debug_option(const char *name, int default_value);
1441
 
 
1442
1484
struct radv_color_buffer_info {
1443
1485
   uint64_t cb_color_base;
1444
1486
   uint64_t cb_color_cmask;
1472
1514
   uint32_t db_depth_size;
1473
1515
   uint32_t db_depth_slice;
1474
1516
   uint32_t db_htile_surface;
1475
 
   uint32_t pa_su_poly_offset_db_fmt_cntl;
1476
1517
   uint32_t db_z_info2;       /* GFX9 only */
1477
1518
   uint32_t db_stencil_info2; /* GFX9 only */
1478
1519
   uint32_t db_render_override2;
 
1520
   uint32_t db_render_control;
1479
1521
};
1480
1522
 
 
1523
void radv_initialise_color_surface_va(struct radv_device *device, struct radv_color_buffer_info *cb,
 
1524
                                      struct radv_image_view *iview);
1481
1525
void radv_initialise_color_surface(struct radv_device *device, struct radv_color_buffer_info *cb,
1482
1526
                                   struct radv_image_view *iview);
1483
 
void radv_initialise_ds_surface(struct radv_device *device, struct radv_ds_buffer_info *ds,
 
1527
void radv_initialise_ds_surface_va(const struct radv_device *device, struct radv_ds_buffer_info *ds,
 
1528
                                   struct radv_image_view *iview);
 
1529
void radv_initialise_ds_surface(const struct radv_device *device, struct radv_ds_buffer_info *ds,
1484
1530
                                struct radv_image_view *iview);
1485
1531
void radv_initialise_vrs_surface(struct radv_image *image, struct radv_buffer *htile_buffer,
1486
1532
                                 struct radv_ds_buffer_info *ds);
1487
1533
 
 
1534
void radv_gfx11_set_db_render_control(const struct radv_device *device, unsigned num_samples,
 
1535
                                      unsigned *db_render_control);
1488
1536
/**
1489
1537
 * Attachment state when recording a renderpass instance.
1490
1538
 *
1561
1609
 
1562
1610
struct radv_multisample_state {
1563
1611
   bool sample_shading_enable;
1564
 
   bool uses_user_sample_locations;
1565
1612
   float min_sample_shading;
1566
1613
};
1567
1614
 
1605
1652
   uint64_t index_va;
1606
1653
   int32_t last_index_type;
1607
1654
 
1608
 
   uint32_t last_primitive_reset_index;
 
1655
   uint32_t last_primitive_reset_index; /* only relevant on GFX6-7 */
1609
1656
   enum radv_cmd_flush_bits flush_bits;
1610
1657
   unsigned active_occlusion_queries;
1611
1658
   bool perfect_occlusion_queries_enabled;
1631
1678
 
1632
1679
   uint32_t last_db_count_control;
1633
1680
 
 
1681
   uint32_t last_db_shader_control;
 
1682
 
1634
1683
   /* Whether CP DMA is busy/idle. */
1635
1684
   bool dma_is_busy;
1636
1685
 
1653
1702
   uint32_t current_event_type;
1654
1703
   uint32_t num_events;
1655
1704
   uint32_t num_layout_transitions;
 
1705
   bool in_barrier;
1656
1706
   bool pending_sqtt_barrier_end;
1657
1707
   enum rgp_flush_bits sqtt_flush_bits;
1658
1708
 
1664
1714
 
1665
1715
   uint8_t cb_mip[MAX_RTS];
1666
1716
 
1667
 
   /* Whether DRAW_{INDEX}_INDIRECT_MULTI is emitted. */
1668
 
   bool uses_draw_indirect_multi;
 
1717
   /* Whether DRAW_{INDEX}_INDIRECT_{MULTI} is emitted. */
 
1718
   bool uses_draw_indirect;
1669
1719
 
1670
1720
   uint32_t rt_stack_size;
1671
1721
 
1771
1821
   uint64_t mec_inv_pred_va;  /* For inverted predication when using MEC. */
1772
1822
   bool mec_inv_pred_emitted; /* To ensure we don't have to repeat inverting the VA. */
1773
1823
 
 
1824
   /**
 
1825
    * Gang state.
 
1826
    * Used when the command buffer needs work done on a different queue
 
1827
    * (eg. when a graphics command buffer needs compute work).
 
1828
    * Currently only one follower is possible per command buffer.
 
1829
    */
1774
1830
   struct {
1775
 
      /**
1776
 
       * Internal command stream that is used when some graphics work
1777
 
       * also requires a submission to the compute queue.
1778
 
       */
 
1831
      /** Follower command stream. */
1779
1832
      struct radeon_cmdbuf *cs;
1780
1833
 
1781
 
      /** Flush bits for the internal cmdbuf. */
 
1834
      /** Flush bits for the follower cmdbuf. */
1782
1835
      enum radv_cmd_flush_bits flush_bits;
1783
1836
 
1784
1837
      /**
1785
 
       * For synchronization between the ACE and GFX cmdbuf.
1786
 
       * The value of this semaphore is incremented whenever we
1787
 
       * encounter a barrier that affects ACE. At sync points,
1788
 
       * GFX writes the value to its address, and ACE waits until
1789
 
       * it detects that the value has been written.
 
1838
       * For synchronization between the follower and leader.
 
1839
       * The value of these semaphores are incremented whenever we
 
1840
       * encounter a barrier that affects the follower.
 
1841
       *
 
1842
       * DWORD 0: Leader to follower semaphore.
 
1843
       *          The leader writes the value and the follower waits.
 
1844
       * DWORD 1: Follower to leader semaphore.
 
1845
       *          The follower writes the value, and the leader waits.
1790
1846
       */
1791
1847
      struct {
1792
 
         uint64_t va;                    /* Virtual address of the semaphore. */
1793
 
         uint32_t gfx2ace_value;         /* Current value on GFX. */
1794
 
         uint32_t emitted_gfx2ace_value; /* Emitted value on GFX. */
 
1848
         uint64_t va;                   /* Virtual address of the semaphore. */
 
1849
         uint32_t leader_value;         /* Current value of the leader. */
 
1850
         uint32_t emitted_leader_value; /* Emitted value emitted by the leader. */
1795
1851
      } sem;
1796
 
   } ace_internal;
 
1852
   } gang;
1797
1853
 
1798
1854
   /**
1799
 
    * Whether a query pool has been resetted and we have to flush caches.
 
1855
    * Whether a query pool has been reset and we have to flush caches.
1800
1856
    */
1801
1857
   bool pending_reset_query;
1802
1858
 
1808
1864
   struct {
1809
1865
      struct radv_video_session *vid;
1810
1866
      struct radv_video_session_params *params;
 
1867
      struct rvcn_sq_var sq;
 
1868
      struct rvcn_decode_buffer_s *decode_buffer;
1811
1869
   } video;
1812
1870
 
1813
1871
   uint64_t shader_upload_seq;
 
1872
 
 
1873
   uint32_t sqtt_cb_id;
1814
1874
};
1815
1875
 
1816
1876
static inline bool
1850
1910
   uint64_t va;
1851
1911
};
1852
1912
 
1853
 
void radv_compute_dispatch(struct radv_cmd_buffer *cmd_buffer,
1854
 
                           const struct radv_dispatch_info *info);
 
1913
void radv_compute_dispatch(struct radv_cmd_buffer *cmd_buffer, const struct radv_dispatch_info *info);
1855
1914
 
1856
1915
struct radv_image;
1857
1916
struct radv_image_view;
1863
1922
void si_emit_graphics(struct radv_device *device, struct radeon_cmdbuf *cs);
1864
1923
void si_emit_compute(struct radv_device *device, struct radeon_cmdbuf *cs);
1865
1924
 
1866
 
void cik_create_gfx_config(struct radv_device *device);
1867
 
 
1868
 
void si_write_scissors(struct radeon_cmdbuf *cs, int count, const VkRect2D *scissors,
1869
 
                       const VkViewport *viewports);
1870
 
 
1871
 
void si_write_guardband(struct radeon_cmdbuf *cs, int count, const VkViewport *viewports,
1872
 
                        unsigned rast_prim, unsigned polygon_mode, float line_width);
1873
 
 
1874
 
VkResult radv_create_shadow_regs_preamble(const struct radv_device *device,
1875
 
                                          struct radv_queue_state *queue_state);
1876
 
void radv_destroy_shadow_regs_preamble(struct radv_queue_state *queue_state,
1877
 
                                       struct radeon_winsys *ws);
 
1925
void radv_create_gfx_config(struct radv_device *device);
 
1926
 
 
1927
void si_write_scissors(struct radeon_cmdbuf *cs, int count, const VkRect2D *scissors, const VkViewport *viewports);
 
1928
 
 
1929
void si_write_guardband(struct radeon_cmdbuf *cs, int count, const VkViewport *viewports, unsigned rast_prim,
 
1930
                        unsigned polygon_mode, float line_width);
 
1931
 
 
1932
VkResult radv_create_shadow_regs_preamble(const struct radv_device *device, struct radv_queue_state *queue_state);
 
1933
void radv_destroy_shadow_regs_preamble(struct radv_queue_state *queue_state, struct radeon_winsys *ws);
1878
1934
void radv_emit_shadow_regs_preamble(struct radeon_cmdbuf *cs, const struct radv_device *device,
1879
1935
                                    struct radv_queue_state *queue_state);
1880
 
VkResult radv_init_shadowed_regs_buffer_state(const struct radv_device *device,
1881
 
                                              struct radv_queue *queue);
1882
 
 
1883
 
uint32_t si_get_ia_multi_vgt_param(struct radv_cmd_buffer *cmd_buffer, bool instanced_draw,
1884
 
                                   bool indirect_draw, bool count_from_stream_output,
1885
 
                                   uint32_t draw_vertex_count, unsigned topology,
1886
 
                                   bool prim_restart_enable, unsigned patch_control_points,
1887
 
                                   unsigned num_tess_patches);
1888
 
void si_cs_emit_write_event_eop(struct radeon_cmdbuf *cs, enum amd_gfx_level gfx_level, bool is_mec,
1889
 
                                unsigned event, unsigned event_flags, unsigned dst_sel,
1890
 
                                unsigned data_sel, uint64_t va, uint32_t new_fence,
1891
 
                                uint64_t gfx9_eop_bug_va);
1892
 
 
1893
 
void radv_cp_wait_mem(struct radeon_cmdbuf *cs, uint32_t op, uint64_t va, uint32_t ref,
1894
 
                      uint32_t mask);
1895
 
void si_cs_emit_cache_flush(struct radeon_winsys *ws, struct radeon_cmdbuf *cs,
1896
 
                            enum amd_gfx_level gfx_level, uint32_t *flush_cnt, uint64_t flush_va,
1897
 
                            bool is_mec, enum radv_cmd_flush_bits flush_bits,
 
1936
VkResult radv_init_shadowed_regs_buffer_state(const struct radv_device *device, struct radv_queue *queue);
 
1937
 
 
1938
uint32_t si_get_ia_multi_vgt_param(struct radv_cmd_buffer *cmd_buffer, bool instanced_draw, bool indirect_draw,
 
1939
                                   bool count_from_stream_output, uint32_t draw_vertex_count, unsigned topology,
 
1940
                                   bool prim_restart_enable, unsigned patch_control_points, unsigned num_tess_patches);
 
1941
void si_cs_emit_write_event_eop(struct radeon_cmdbuf *cs, enum amd_gfx_level gfx_level, bool is_mec, unsigned event,
 
1942
                                unsigned event_flags, unsigned dst_sel, unsigned data_sel, uint64_t va,
 
1943
                                uint32_t new_fence, uint64_t gfx9_eop_bug_va);
 
1944
 
 
1945
struct radv_vgt_shader_key {
 
1946
   uint8_t tess : 1;
 
1947
   uint8_t gs : 1;
 
1948
   uint8_t mesh_scratch_ring : 1;
 
1949
   uint8_t mesh : 1;
 
1950
   uint8_t ngg_passthrough : 1;
 
1951
   uint8_t ngg : 1;       /* gfx10+ */
 
1952
   uint8_t streamout : 1; /* only used with NGG */
 
1953
   uint8_t hs_wave32 : 1;
 
1954
   uint8_t gs_wave32 : 1;
 
1955
   uint8_t vs_wave32 : 1;
 
1956
};
 
1957
 
 
1958
void radv_cp_wait_mem(struct radeon_cmdbuf *cs, uint32_t op, uint64_t va, uint32_t ref, uint32_t mask);
 
1959
void si_cs_emit_cache_flush(struct radeon_winsys *ws, struct radeon_cmdbuf *cs, enum amd_gfx_level gfx_level,
 
1960
                            uint32_t *flush_cnt, uint64_t flush_va, bool is_mec, enum radv_cmd_flush_bits flush_bits,
1898
1961
                            enum rgp_flush_bits *sqtt_flush_bits, uint64_t gfx9_eop_bug_va);
1899
1962
void si_emit_cache_flush(struct radv_cmd_buffer *cmd_buffer);
1900
 
void si_emit_set_predication_state(struct radv_cmd_buffer *cmd_buffer, bool draw_visible,
1901
 
                                   unsigned pred_op, uint64_t va);
1902
 
void si_cp_dma_buffer_copy(struct radv_cmd_buffer *cmd_buffer, uint64_t src_va, uint64_t dest_va,
1903
 
                           uint64_t size);
1904
 
void si_cs_cp_dma_prefetch(const struct radv_device *device, struct radeon_cmdbuf *cs, uint64_t va,
1905
 
                           unsigned size, bool predicating);
 
1963
void si_emit_set_predication_state(struct radv_cmd_buffer *cmd_buffer, bool draw_visible, unsigned pred_op,
 
1964
                                   uint64_t va);
 
1965
void si_cp_dma_buffer_copy(struct radv_cmd_buffer *cmd_buffer, uint64_t src_va, uint64_t dest_va, uint64_t size);
 
1966
void si_cs_cp_dma_prefetch(const struct radv_device *device, struct radeon_cmdbuf *cs, uint64_t va, unsigned size,
 
1967
                           bool predicating);
1906
1968
void si_cp_dma_prefetch(struct radv_cmd_buffer *cmd_buffer, uint64_t va, unsigned size);
1907
 
void si_cp_dma_clear_buffer(struct radv_cmd_buffer *cmd_buffer, uint64_t va, uint64_t size,
1908
 
                            unsigned value);
 
1969
void si_cp_dma_clear_buffer(struct radv_cmd_buffer *cmd_buffer, uint64_t va, uint64_t size, unsigned value);
1909
1970
void si_cp_dma_wait_for_idle(struct radv_cmd_buffer *cmd_buffer);
1910
1971
 
1911
1972
uint32_t radv_get_pa_su_sc_mode_cntl(const struct radv_cmd_buffer *cmd_buffer);
1918
1979
uint32_t radv_hash_ps_epilog(const void *key_);
1919
1980
bool radv_cmp_ps_epilog(const void *a_, const void *b_);
1920
1981
 
1921
 
struct radv_ps_epilog_state
1922
 
{
 
1982
struct radv_ps_epilog_state {
1923
1983
   uint8_t color_attachment_count;
1924
1984
   VkFormat color_attachment_formats[MAX_RTS];
1925
1985
 
1931
1991
};
1932
1992
 
1933
1993
struct radv_ps_epilog_key radv_generate_ps_epilog_key(const struct radv_device *device,
1934
 
                                                      const struct radv_graphics_pipeline *pipeline,
1935
1994
                                                      const struct radv_ps_epilog_state *state,
1936
1995
                                                      bool disable_mrt_compaction);
1937
1996
 
 
1997
bool radv_needs_null_export_workaround(const struct radv_device *device, const struct radv_shader *ps,
 
1998
                                       unsigned custom_blend_mode);
 
1999
 
1938
2000
void radv_cmd_buffer_reset_rendering(struct radv_cmd_buffer *cmd_buffer);
1939
 
bool radv_cmd_buffer_upload_alloc_aligned(struct radv_cmd_buffer *cmd_buffer, unsigned size,
1940
 
                                          unsigned alignment,
 
2001
bool radv_cmd_buffer_upload_alloc_aligned(struct radv_cmd_buffer *cmd_buffer, unsigned size, unsigned alignment,
1941
2002
                                          unsigned *out_offset, void **ptr);
1942
 
bool radv_cmd_buffer_upload_alloc(struct radv_cmd_buffer *cmd_buffer, unsigned size,
1943
 
                                  unsigned *out_offset, void **ptr);
1944
 
bool radv_cmd_buffer_upload_data(struct radv_cmd_buffer *cmd_buffer, unsigned size,
1945
 
                                 const void *data, unsigned *out_offset);
 
2003
bool radv_cmd_buffer_upload_alloc(struct radv_cmd_buffer *cmd_buffer, unsigned size, unsigned *out_offset, void **ptr);
 
2004
bool radv_cmd_buffer_upload_data(struct radv_cmd_buffer *cmd_buffer, unsigned size, const void *data,
 
2005
                                 unsigned *out_offset);
1946
2006
void radv_write_vertex_descriptors(const struct radv_cmd_buffer *cmd_buffer,
1947
 
                                   const struct radv_graphics_pipeline *pipeline,
1948
 
                                   bool full_null_descriptors, void *vb_ptr);
 
2007
                                   const struct radv_graphics_pipeline *pipeline, bool full_null_descriptors,
 
2008
                                   void *vb_ptr);
1949
2009
void radv_write_scissors(struct radv_cmd_buffer *cmd_buffer, struct radeon_cmdbuf *cs);
1950
2010
 
1951
 
void radv_cmd_buffer_clear_attachment(struct radv_cmd_buffer *cmd_buffer,
1952
 
                                      const VkClearAttachment *attachment);
1953
 
void radv_cmd_buffer_clear_rendering(struct radv_cmd_buffer *cmd_buffer,
1954
 
                                     const VkRenderingInfo *render_info);
 
2011
void radv_cmd_buffer_clear_attachment(struct radv_cmd_buffer *cmd_buffer, const VkClearAttachment *attachment);
 
2012
void radv_cmd_buffer_clear_rendering(struct radv_cmd_buffer *cmd_buffer, const VkRenderingInfo *render_info);
1955
2013
void radv_cmd_buffer_resolve_rendering(struct radv_cmd_buffer *cmd_buffer);
1956
 
void radv_cmd_buffer_resolve_rendering_cs(struct radv_cmd_buffer *cmd_buffer,
1957
 
                                          struct radv_image_view *src_iview,
1958
 
                                          VkImageLayout src_layout,
1959
 
                                          struct radv_image_view *dst_iview,
1960
 
                                          VkImageLayout dst_layout,
1961
 
                                          const VkImageResolve2 *region);
1962
 
void radv_depth_stencil_resolve_rendering_cs(struct radv_cmd_buffer *cmd_buffer,
1963
 
                                             VkImageAspectFlags aspects,
 
2014
void radv_cmd_buffer_resolve_rendering_cs(struct radv_cmd_buffer *cmd_buffer, struct radv_image_view *src_iview,
 
2015
                                          VkImageLayout src_layout, struct radv_image_view *dst_iview,
 
2016
                                          VkImageLayout dst_layout, const VkImageResolve2 *region);
 
2017
void radv_depth_stencil_resolve_rendering_cs(struct radv_cmd_buffer *cmd_buffer, VkImageAspectFlags aspects,
1964
2018
                                             VkResolveModeFlagBits resolve_mode);
1965
 
void  radv_cmd_buffer_resolve_rendering_fs(struct radv_cmd_buffer *cmd_buffer,
1966
 
                                           struct radv_image_view *src_iview,
1967
 
                                           VkImageLayout src_layout,
1968
 
                                           struct radv_image_view *dst_iview,
1969
 
                                           VkImageLayout dst_layout);
1970
 
void radv_depth_stencil_resolve_rendering_fs(struct radv_cmd_buffer *cmd_buffer,
1971
 
                                             VkImageAspectFlags aspects,
 
2019
void radv_cmd_buffer_resolve_rendering_fs(struct radv_cmd_buffer *cmd_buffer, struct radv_image_view *src_iview,
 
2020
                                          VkImageLayout src_layout, struct radv_image_view *dst_iview,
 
2021
                                          VkImageLayout dst_layout);
 
2022
void radv_depth_stencil_resolve_rendering_fs(struct radv_cmd_buffer *cmd_buffer, VkImageAspectFlags aspects,
1972
2023
                                             VkResolveModeFlagBits resolve_mode);
1973
2024
void radv_emit_default_sample_locations(struct radeon_cmdbuf *cs, int nr_samples);
1974
2025
unsigned radv_get_default_max_sample_dist(int log_samples);
1975
2026
void radv_device_init_msaa(struct radv_device *device);
1976
2027
VkResult radv_device_init_vrs_state(struct radv_device *device);
1977
2028
 
1978
 
void radv_emit_write_data_imm(struct radeon_cmdbuf *cs, unsigned engine_sel, uint64_t va,
1979
 
                              uint32_t imm);
1980
 
 
1981
 
void radv_update_ds_clear_metadata(struct radv_cmd_buffer *cmd_buffer,
1982
 
                                   const struct radv_image_view *iview,
1983
 
                                   VkClearDepthStencilValue ds_clear_value,
1984
 
                                   VkImageAspectFlags aspects);
1985
 
 
1986
 
void radv_update_color_clear_metadata(struct radv_cmd_buffer *cmd_buffer,
1987
 
                                      const struct radv_image_view *iview, int cb_idx,
1988
 
                                      uint32_t color_values[2]);
1989
 
 
1990
 
bool radv_image_use_dcc_image_stores(const struct radv_device *device,
1991
 
                                     const struct radv_image *image);
1992
 
bool radv_image_use_dcc_predication(const struct radv_device *device,
1993
 
                                    const struct radv_image *image);
 
2029
void radv_emit_write_data_imm(struct radeon_cmdbuf *cs, unsigned engine_sel, uint64_t va, uint32_t imm);
 
2030
 
 
2031
void radv_update_ds_clear_metadata(struct radv_cmd_buffer *cmd_buffer, const struct radv_image_view *iview,
 
2032
                                   VkClearDepthStencilValue ds_clear_value, VkImageAspectFlags aspects);
 
2033
 
 
2034
void radv_update_color_clear_metadata(struct radv_cmd_buffer *cmd_buffer, const struct radv_image_view *iview,
 
2035
                                      int cb_idx, uint32_t color_values[2]);
 
2036
 
 
2037
bool radv_image_use_dcc_image_stores(const struct radv_device *device, const struct radv_image *image);
 
2038
bool radv_image_use_dcc_predication(const struct radv_device *device, const struct radv_image *image);
1994
2039
 
1995
2040
void radv_update_fce_metadata(struct radv_cmd_buffer *cmd_buffer, struct radv_image *image,
1996
2041
                              const VkImageSubresourceRange *range, bool value);
1997
2042
 
1998
2043
void radv_update_dcc_metadata(struct radv_cmd_buffer *cmd_buffer, struct radv_image *image,
1999
2044
                              const VkImageSubresourceRange *range, bool value);
2000
 
enum radv_cmd_flush_bits radv_src_access_flush(struct radv_cmd_buffer *cmd_buffer,
2001
 
                                               VkAccessFlags2 src_flags,
 
2045
enum radv_cmd_flush_bits radv_src_access_flush(struct radv_cmd_buffer *cmd_buffer, VkAccessFlags2 src_flags,
2002
2046
                                               const struct radv_image *image);
2003
 
enum radv_cmd_flush_bits radv_dst_access_flush(struct radv_cmd_buffer *cmd_buffer,
2004
 
                                               VkAccessFlags2 dst_flags,
 
2047
enum radv_cmd_flush_bits radv_dst_access_flush(struct radv_cmd_buffer *cmd_buffer, VkAccessFlags2 dst_flags,
2005
2048
                                               const struct radv_image *image);
2006
2049
uint32_t radv_fill_buffer(struct radv_cmd_buffer *cmd_buffer, const struct radv_image *image,
2007
2050
                          struct radeon_winsys_bo *bo, uint64_t va, uint64_t size, uint32_t value);
2008
2051
void radv_copy_buffer(struct radv_cmd_buffer *cmd_buffer, struct radeon_winsys_bo *src_bo,
2009
 
                      struct radeon_winsys_bo *dst_bo, uint64_t src_offset, uint64_t dst_offset,
2010
 
                      uint64_t size);
 
2052
                      struct radeon_winsys_bo *dst_bo, uint64_t src_offset, uint64_t dst_offset, uint64_t size);
2011
2053
 
2012
2054
void radv_cmd_buffer_trace_emit(struct radv_cmd_buffer *cmd_buffer);
2013
2055
bool radv_get_memory_fd(struct radv_device *device, struct radv_device_memory *memory, int *pFD);
2036
2078
}
2037
2079
 
2038
2080
static inline void
2039
 
radv_emit_shader_pointer(struct radv_device *device, struct radeon_cmdbuf *cs, uint32_t sh_offset,
2040
 
                         uint64_t va, bool global)
 
2081
radv_emit_shader_pointer(struct radv_device *device, struct radeon_cmdbuf *cs, uint32_t sh_offset, uint64_t va,
 
2082
                         bool global)
2041
2083
{
2042
2084
   bool use_32bit_pointers = !global;
2043
2085
 
2058
2100
}
2059
2101
 
2060
2102
static inline const struct radv_push_constant_state *
2061
 
radv_get_push_constants_state(const struct radv_cmd_buffer *cmd_buffer,
2062
 
                              VkPipelineBindPoint bind_point)
 
2103
radv_get_push_constants_state(const struct radv_cmd_buffer *cmd_buffer, VkPipelineBindPoint bind_point)
2063
2104
{
2064
2105
   return &cmd_buffer->push_constant_state[vk_to_bind_point(bind_point)];
2065
2106
}
2066
2107
 
2067
 
void
2068
 
radv_get_viewport_xform(const VkViewport *viewport, float scale[3], float translate[3]);
 
2108
void radv_get_viewport_xform(const VkViewport *viewport, float scale[3], float translate[3]);
2069
2109
 
2070
2110
/*
2071
2111
 * Takes x,y,z as exact numbers of invocations, instead of blocks.
2073
2113
 * Limitations: Can't call normal dispatch functions without binding or rebinding
2074
2114
 *              the compute pipeline.
2075
2115
 */
2076
 
void radv_unaligned_dispatch(struct radv_cmd_buffer *cmd_buffer, uint32_t x, uint32_t y,
2077
 
                             uint32_t z);
 
2116
void radv_unaligned_dispatch(struct radv_cmd_buffer *cmd_buffer, uint32_t x, uint32_t y, uint32_t z);
2078
2117
 
2079
 
void radv_indirect_dispatch(struct radv_cmd_buffer *cmd_buffer, struct radeon_winsys_bo *bo,
2080
 
                            uint64_t va);
 
2118
void radv_indirect_dispatch(struct radv_cmd_buffer *cmd_buffer, struct radeon_winsys_bo *bo, uint64_t va);
2081
2119
 
2082
2120
struct radv_event {
2083
2121
   struct vk_object_base base;
2085
2123
   uint64_t *map;
2086
2124
};
2087
2125
 
2088
 
#define RADV_HASH_SHADER_CS_WAVE32         (1 << 1)
2089
 
#define RADV_HASH_SHADER_PS_WAVE32         (1 << 2)
2090
 
#define RADV_HASH_SHADER_GE_WAVE32         (1 << 3)
2091
 
#define RADV_HASH_SHADER_LLVM              (1 << 4)
2092
 
#define RADV_HASH_SHADER_KEEP_STATISTICS   (1 << 8)
2093
 
#define RADV_HASH_SHADER_USE_NGG_CULLING   (1 << 13)
2094
 
#define RADV_HASH_SHADER_ROBUST_BUFFER_ACCESS (1 << 14)
2095
 
#define RADV_HASH_SHADER_ROBUST_BUFFER_ACCESS2 (1 << 15)
2096
 
#define RADV_HASH_SHADER_EMULATE_RT            (1 << 16)
2097
 
#define RADV_HASH_SHADER_SPLIT_FMA             (1 << 17)
2098
 
#define RADV_HASH_SHADER_RT_WAVE64             (1 << 18)
2099
 
#define RADV_HASH_SHADER_NO_FMASK              (1 << 19)
2100
 
#define RADV_HASH_SHADER_NGG_STREAMOUT         (1 << 20)
 
2126
#define RADV_HASH_SHADER_CS_WAVE32       (1 << 1)
 
2127
#define RADV_HASH_SHADER_PS_WAVE32       (1 << 2)
 
2128
#define RADV_HASH_SHADER_GE_WAVE32       (1 << 3)
 
2129
#define RADV_HASH_SHADER_LLVM            (1 << 4)
 
2130
#define RADV_HASH_SHADER_KEEP_STATISTICS (1 << 8)
 
2131
#define RADV_HASH_SHADER_USE_NGG_CULLING (1 << 13)
 
2132
#define RADV_HASH_SHADER_EMULATE_RT      (1 << 16)
 
2133
#define RADV_HASH_SHADER_SPLIT_FMA       (1 << 17)
 
2134
#define RADV_HASH_SHADER_RT_WAVE64       (1 << 18)
 
2135
#define RADV_HASH_SHADER_NO_FMASK        (1 << 19)
 
2136
#define RADV_HASH_SHADER_NGG_STREAMOUT   (1 << 20)
 
2137
#define RADV_HASH_SHADER_NO_RT           (1 << 21)
 
2138
#define RADV_HASH_SHADER_DUAL_BLEND_MRT1 (1 << 22)
2101
2139
 
2102
 
struct radv_ray_tracing_module;
2103
2140
struct radv_pipeline_key;
2104
 
 
2105
 
void radv_pipeline_stage_init(const VkPipelineShaderStageCreateInfo *sinfo,
2106
 
                              struct radv_pipeline_stage *out_stage, gl_shader_stage stage);
2107
 
 
2108
 
void radv_hash_shaders(unsigned char *hash, const struct radv_pipeline_stage *stages,
2109
 
                       uint32_t stage_count, const struct radv_pipeline_layout *layout,
2110
 
                       const struct radv_pipeline_key *key, uint32_t flags);
2111
 
 
2112
 
void radv_hash_rt_stages(struct mesa_sha1 *ctx, const VkPipelineShaderStageCreateInfo *stages,
2113
 
                         unsigned stage_count);
 
2141
struct radv_ray_tracing_group;
 
2142
 
 
2143
void radv_pipeline_stage_init(const VkPipelineShaderStageCreateInfo *sinfo, struct radv_pipeline_stage *out_stage,
 
2144
                              gl_shader_stage stage);
 
2145
 
 
2146
void radv_hash_shaders(unsigned char *hash, const struct radv_pipeline_stage *stages, uint32_t stage_count,
 
2147
                       const struct radv_pipeline_layout *layout, const struct radv_pipeline_key *key, uint32_t flags);
 
2148
 
 
2149
void radv_hash_rt_stages(struct mesa_sha1 *ctx, const VkPipelineShaderStageCreateInfo *stages, unsigned stage_count);
2114
2150
 
2115
2151
void radv_hash_rt_shaders(unsigned char *hash, const VkRayTracingPipelineCreateInfoKHR *pCreateInfo,
2116
 
                          const struct radv_pipeline_key *key,
2117
 
                          const struct radv_ray_tracing_module *groups, uint32_t flags);
 
2152
                          const struct radv_pipeline_key *key, const struct radv_ray_tracing_group *groups,
 
2153
                          uint32_t flags);
2118
2154
 
2119
2155
uint32_t radv_get_hash_flags(const struct radv_device *device, bool stats);
2120
2156
 
2123
2159
bool radv_emulate_rt(const struct radv_physical_device *pdevice);
2124
2160
 
2125
2161
enum {
2126
 
   RADV_RT_STAGE_BITS = (VK_SHADER_STAGE_RAYGEN_BIT_KHR | VK_SHADER_STAGE_ANY_HIT_BIT_KHR |
2127
 
                         VK_SHADER_STAGE_CLOSEST_HIT_BIT_KHR | VK_SHADER_STAGE_MISS_BIT_KHR |
2128
 
                         VK_SHADER_STAGE_INTERSECTION_BIT_KHR | VK_SHADER_STAGE_CALLABLE_BIT_KHR)
 
2162
   RADV_RT_STAGE_BITS =
 
2163
      (VK_SHADER_STAGE_RAYGEN_BIT_KHR | VK_SHADER_STAGE_ANY_HIT_BIT_KHR | VK_SHADER_STAGE_CLOSEST_HIT_BIT_KHR |
 
2164
       VK_SHADER_STAGE_MISS_BIT_KHR | VK_SHADER_STAGE_INTERSECTION_BIT_KHR | VK_SHADER_STAGE_CALLABLE_BIT_KHR)
2129
2165
};
2130
2166
 
2131
2167
#define RADV_STAGE_MASK ((1 << MESA_VULKAN_SHADER_STAGES) - 1)
2132
2168
 
2133
 
#define radv_foreach_stage(stage, stage_bits)                                                      \
2134
 
   for (gl_shader_stage stage, __tmp = (gl_shader_stage)((stage_bits)&RADV_STAGE_MASK);            \
2135
 
        stage = ffs(__tmp) - 1, __tmp; __tmp &= ~(1 << (stage)))
 
2169
#define radv_foreach_stage(stage, stage_bits)                                                                          \
 
2170
   for (gl_shader_stage stage, __tmp = (gl_shader_stage)((stage_bits)&RADV_STAGE_MASK); stage = ffs(__tmp) - 1, __tmp; \
 
2171
        __tmp &= ~(1 << (stage)))
2136
2172
 
2137
2173
extern const VkFormat radv_fs_key_format_exemplars[NUM_META_FS_KEYS];
2138
2174
unsigned radv_format_meta_fs_key(struct radv_device *device, VkFormat format);
2139
2175
 
2140
 
struct radv_vrs_state {
2141
 
   uint32_t pa_cl_vrs_cntl;
2142
 
};
2143
 
 
2144
2176
struct radv_prim_vertex_count {
2145
2177
   uint8_t min;
2146
2178
   uint8_t incr;
2153
2185
   RADV_PIPELINE_GRAPHICS_LIB,
2154
2186
   /* Compute pipeline */
2155
2187
   RADV_PIPELINE_COMPUTE,
2156
 
   RADV_PIPELINE_RAY_TRACING_LIB,
2157
2188
   /* Raytracing pipeline */
2158
2189
   RADV_PIPELINE_RAY_TRACING,
2159
2190
};
2160
2191
 
2161
2192
struct radv_pipeline_group_handle {
 
2193
   uint64_t recursive_shader_ptr;
 
2194
 
2162
2195
   union {
2163
2196
      uint32_t general_index;
2164
2197
      uint32_t closest_hit_index;
2169
2202
   };
2170
2203
};
2171
2204
 
2172
 
struct radv_pipeline_shader_stack_size {
2173
 
   uint32_t recursive_size;
2174
 
   /* anyhit + intersection */
2175
 
   uint32_t non_recursive_size;
 
2205
struct radv_serialized_shader_arena_block {
 
2206
   uint32_t offset;
 
2207
   uint32_t size;
 
2208
   uint64_t arena_va;
 
2209
   uint32_t arena_size;
 
2210
};
 
2211
 
 
2212
struct radv_rt_capture_replay_handle {
 
2213
   struct radv_serialized_shader_arena_block recursive_shader_alloc;
 
2214
   uint32_t non_recursive_idx;
2176
2215
};
2177
2216
 
2178
2217
enum radv_depth_clamp_mode {
2179
 
   RADV_DEPTH_CLAMP_MODE_VIEWPORT = 0,       /* Clamp to the viewport min/max depth bounds */
2180
 
   RADV_DEPTH_CLAMP_MODE_ZERO_TO_ONE = 1,    /* Clamp between 0.0f and 1.0f */
2181
 
   RADV_DEPTH_CLAMP_MODE_DISABLED = 2,       /* Disable depth clamping */
 
2218
   RADV_DEPTH_CLAMP_MODE_VIEWPORT = 0,    /* Clamp to the viewport min/max depth bounds */
 
2219
   RADV_DEPTH_CLAMP_MODE_ZERO_TO_ONE = 1, /* Clamp between 0.0f and 1.0f */
 
2220
   RADV_DEPTH_CLAMP_MODE_DISABLED = 2,    /* Disable depth clamping */
2182
2221
};
2183
2222
 
2184
2223
struct radv_pipeline {
2185
2224
   struct vk_object_base base;
2186
2225
   enum radv_pipeline_type type;
2187
2226
 
 
2227
   struct vk_pipeline_cache_object *cache_object;
 
2228
 
2188
2229
   bool is_internal;
2189
2230
   bool need_indirect_descriptor_sets;
2190
2231
   struct radv_shader *shaders[MESA_VULKAN_SHADER_STAGES];
2221
2262
   bool uses_drawid;
2222
2263
   bool uses_baseinstance;
2223
2264
 
2224
 
   bool need_null_export_workaround;
2225
2265
   /* Whether the pipeline forces per-vertex VRS (GFX10.3+). */
2226
2266
   bool force_vrs_per_vertex;
2227
2267
 
2245
2285
   struct radv_vs_input_state vs_input_state;
2246
2286
 
2247
2287
   struct radv_multisample_state ms;
2248
 
   struct radv_vrs_state vrs;
2249
2288
   struct radv_ia_multi_vgt_param_helpers ia_multi_vgt_param;
2250
2289
   uint32_t binding_stride[MAX_VBS];
2251
2290
   uint8_t attrib_bindings[MAX_VERTEX_ATTRIBS];
2287
2326
   struct radv_pipeline base;
2288
2327
};
2289
2328
 
2290
 
struct radv_ray_tracing_module {
 
2329
struct radv_ray_tracing_group {
 
2330
   VkRayTracingShaderGroupTypeKHR type;
 
2331
   uint32_t recursive_shader; /* generalShader or closestHitShader */
 
2332
   uint32_t any_hit_shader;
 
2333
   uint32_t intersection_shader;
2291
2334
   struct radv_pipeline_group_handle handle;
2292
 
   struct radv_pipeline_shader_stack_size stack_size;
2293
 
};
2294
 
 
2295
 
struct radv_ray_tracing_lib_pipeline {
2296
 
   struct radv_pipeline base;
2297
 
 
2298
 
   /* ralloc context used for allocating pipeline library resources. */
2299
 
   void *ctx;
2300
 
 
 
2335
};
 
2336
 
 
2337
struct radv_ray_tracing_stage {
 
2338
   struct vk_pipeline_cache_object *shader;
 
2339
   gl_shader_stage stage;
 
2340
   uint32_t stack_size;
 
2341
 
 
2342
   uint8_t sha1[SHA1_DIGEST_LENGTH];
 
2343
};
 
2344
 
 
2345
static inline bool
 
2346
radv_ray_tracing_stage_is_compiled(struct radv_ray_tracing_stage *stage)
 
2347
{
 
2348
   return stage->stage == MESA_SHADER_RAYGEN || stage->stage == MESA_SHADER_CALLABLE ||
 
2349
          stage->stage == MESA_SHADER_CLOSEST_HIT || stage->stage == MESA_SHADER_MISS;
 
2350
}
 
2351
 
 
2352
struct radv_ray_tracing_pipeline {
 
2353
   struct radv_compute_pipeline base;
 
2354
 
 
2355
   struct radv_shader *prolog;
 
2356
 
 
2357
   struct radv_ray_tracing_stage *stages;
 
2358
   struct radv_ray_tracing_group *groups;
2301
2359
   unsigned stage_count;
2302
 
   VkPipelineShaderStageCreateInfo *stages;
2303
2360
   unsigned group_count;
2304
 
   VkRayTracingShaderGroupCreateInfoKHR *group_infos;
2305
 
   struct radv_ray_tracing_module groups[];
 
2361
 
 
2362
   uint8_t sha1[SHA1_DIGEST_LENGTH];
 
2363
   uint32_t stack_size;
 
2364
 
 
2365
   /* set if any shaders from this pipeline require robustness2 in the merged traversal shader */
 
2366
   bool traversal_storage_robustness2 : 1;
 
2367
   bool traversal_uniform_robustness2 : 1;
2306
2368
};
2307
2369
 
2308
2370
struct radv_graphics_lib_pipeline {
2326
2388
   VkPipelineShaderStageCreateInfo *stages;
2327
2389
};
2328
2390
 
2329
 
struct radv_ray_tracing_pipeline {
2330
 
   struct radv_compute_pipeline base;
2331
 
 
2332
 
   uint32_t group_count;
2333
 
   uint32_t stack_size;
2334
 
   struct radv_ray_tracing_module groups[];
2335
 
};
2336
 
 
2337
 
#define RADV_DECL_PIPELINE_DOWNCAST(pipe_type, pipe_enum)            \
2338
 
   static inline struct radv_##pipe_type##_pipeline *                \
2339
 
   radv_pipeline_to_##pipe_type(struct radv_pipeline *pipeline)      \
2340
 
   {                                                                 \
2341
 
      assert(pipeline->type == pipe_enum);                           \
2342
 
      return (struct radv_##pipe_type##_pipeline *) pipeline;        \
 
2391
#define RADV_DECL_PIPELINE_DOWNCAST(pipe_type, pipe_enum)                                                              \
 
2392
   static inline struct radv_##pipe_type##_pipeline *radv_pipeline_to_##pipe_type(struct radv_pipeline *pipeline)      \
 
2393
   {                                                                                                                   \
 
2394
      assert(pipeline->type == pipe_enum);                                                                             \
 
2395
      return (struct radv_##pipe_type##_pipeline *)pipeline;                                                           \
2343
2396
   }
2344
2397
 
2345
2398
RADV_DECL_PIPELINE_DOWNCAST(graphics, RADV_PIPELINE_GRAPHICS)
2346
2399
RADV_DECL_PIPELINE_DOWNCAST(graphics_lib, RADV_PIPELINE_GRAPHICS_LIB)
2347
2400
RADV_DECL_PIPELINE_DOWNCAST(compute, RADV_PIPELINE_COMPUTE)
2348
 
RADV_DECL_PIPELINE_DOWNCAST(ray_tracing_lib, RADV_PIPELINE_RAY_TRACING_LIB)
2349
2401
RADV_DECL_PIPELINE_DOWNCAST(ray_tracing, RADV_PIPELINE_RAY_TRACING)
2350
2402
 
2351
2403
struct radv_pipeline_stage {
2388
2440
void radv_pipeline_emit_hw_cs(const struct radv_physical_device *pdevice, struct radeon_cmdbuf *cs,
2389
2441
                              const struct radv_shader *shader);
2390
2442
 
2391
 
void radv_pipeline_emit_compute_state(const struct radv_physical_device *pdevice,
2392
 
                                      struct radeon_cmdbuf *cs, const struct radv_shader *shader);
2393
 
 
2394
 
bool radv_mem_vectorize_callback(unsigned align_mul, unsigned align_offset, unsigned bit_size,
2395
 
                                 unsigned num_components, nir_intrinsic_instr *low, nir_intrinsic_instr *high,
2396
 
                                 void *data);
2397
 
 
2398
 
void radv_compute_pipeline_init(const struct radv_device *device,
2399
 
                                struct radv_compute_pipeline *pipeline,
2400
 
                                const struct radv_pipeline_layout *layout);
 
2443
void radv_pipeline_emit_compute_state(const struct radv_physical_device *pdevice, struct radeon_cmdbuf *cs,
 
2444
                                      const struct radv_shader *shader);
 
2445
 
 
2446
bool radv_mem_vectorize_callback(unsigned align_mul, unsigned align_offset, unsigned bit_size, unsigned num_components,
 
2447
                                 nir_intrinsic_instr *low, nir_intrinsic_instr *high, void *data);
 
2448
 
 
2449
void radv_compute_pipeline_init(const struct radv_device *device, struct radv_compute_pipeline *pipeline,
 
2450
                                const struct radv_pipeline_layout *layout, struct radv_shader *shader);
2401
2451
 
2402
2452
struct radv_graphics_pipeline_create_info {
2403
2453
   bool use_rectlist;
2410
2460
};
2411
2461
 
2412
2462
struct radv_pipeline_key radv_generate_pipeline_key(const struct radv_device *device,
2413
 
                                                    const struct radv_pipeline *pipeline,
2414
 
                                                    VkPipelineCreateFlags flags);
 
2463
                                                    const VkPipelineShaderStageCreateInfo *stages,
 
2464
                                                    const unsigned num_stages, VkPipelineCreateFlags flags,
 
2465
                                                    const void *pNext);
2415
2466
 
2416
 
void radv_pipeline_init(struct radv_device *device, struct radv_pipeline *pipeline,
2417
 
                        enum radv_pipeline_type type);
 
2467
void radv_pipeline_init(struct radv_device *device, struct radv_pipeline *pipeline, enum radv_pipeline_type type);
2418
2468
 
2419
2469
VkResult radv_graphics_pipeline_create(VkDevice device, VkPipelineCache cache,
2420
2470
                                       const VkGraphicsPipelineCreateInfo *pCreateInfo,
2423
2473
 
2424
2474
VkResult radv_compute_pipeline_create(VkDevice _device, VkPipelineCache _cache,
2425
2475
                                      const VkComputePipelineCreateInfo *pCreateInfo,
2426
 
                                      const VkAllocationCallbacks *pAllocator,
2427
 
                                      VkPipeline *pPipeline);
 
2476
                                      const VkAllocationCallbacks *pAllocator, VkPipeline *pPipeline);
2428
2477
 
2429
2478
bool radv_pipeline_capture_shaders(const struct radv_device *device, VkPipelineCreateFlags flags);
2430
 
bool radv_pipeline_capture_shader_stats(const struct radv_device *device,
2431
 
                                        VkPipelineCreateFlags flags);
 
2479
bool radv_pipeline_capture_shader_stats(const struct radv_device *device, VkPipelineCreateFlags flags);
2432
2480
 
2433
 
VkPipelineShaderStageCreateInfo *
2434
 
radv_copy_shader_stage_create_info(struct radv_device *device, uint32_t stageCount,
2435
 
                                   const VkPipelineShaderStageCreateInfo *pStages, void *mem_ctx);
 
2481
VkPipelineShaderStageCreateInfo *radv_copy_shader_stage_create_info(struct radv_device *device, uint32_t stageCount,
 
2482
                                                                    const VkPipelineShaderStageCreateInfo *pStages,
 
2483
                                                                    void *mem_ctx);
2436
2484
 
2437
2485
bool radv_shader_need_indirect_descriptor_sets(const struct radv_shader *shader);
2438
2486
 
2439
 
void radv_pipeline_init_scratch(const struct radv_device *device, struct radv_pipeline *pipeline);
 
2487
void radv_pipeline_init_scratch(const struct radv_device *device, struct radv_pipeline *pipeline,
 
2488
                                struct radv_shader *shader);
2440
2489
 
2441
2490
bool radv_pipeline_has_ngg(const struct radv_graphics_pipeline *pipeline);
2442
2491
 
2444
2493
                           const VkAllocationCallbacks *allocator);
2445
2494
 
2446
2495
struct vk_format_description;
2447
 
uint32_t radv_translate_buffer_dataformat(const struct util_format_description *desc,
2448
 
                                          int first_non_void);
2449
 
uint32_t radv_translate_buffer_numformat(const struct util_format_description *desc,
2450
 
                                         int first_non_void);
 
2496
uint32_t radv_translate_buffer_dataformat(const struct util_format_description *desc, int first_non_void);
 
2497
uint32_t radv_translate_buffer_numformat(const struct util_format_description *desc, int first_non_void);
2451
2498
bool radv_is_buffer_format_supported(VkFormat format, bool *scaled);
2452
 
uint32_t radv_translate_colorformat(VkFormat format);
2453
 
uint32_t radv_translate_color_numformat(VkFormat format, const struct util_format_description *desc,
2454
 
                                        int first_non_void);
2455
2499
uint32_t radv_colorformat_endian_swap(uint32_t colorformat);
2456
2500
unsigned radv_translate_colorswap(VkFormat format, bool do_endian_swap);
2457
2501
uint32_t radv_translate_dbformat(VkFormat format);
2458
 
uint32_t radv_translate_tex_dataformat(VkFormat format, const struct util_format_description *desc,
2459
 
                                       int first_non_void);
2460
 
uint32_t radv_translate_tex_numformat(VkFormat format, const struct util_format_description *desc,
2461
 
                                      int first_non_void);
2462
 
bool radv_format_pack_clear_color(VkFormat format, uint32_t clear_vals[2],
2463
 
                                  VkClearColorValue *value);
2464
 
bool radv_is_storage_image_format_supported(struct radv_physical_device *physical_device,
2465
 
                                            VkFormat format);
2466
 
bool radv_is_colorbuffer_format_supported(const struct radv_physical_device *pdevice,
2467
 
                                          VkFormat format, bool *blendable);
 
2502
uint32_t radv_translate_tex_dataformat(VkFormat format, const struct util_format_description *desc, int first_non_void);
 
2503
uint32_t radv_translate_tex_numformat(VkFormat format, const struct util_format_description *desc, int first_non_void);
 
2504
bool radv_format_pack_clear_color(VkFormat format, uint32_t clear_vals[2], VkClearColorValue *value);
 
2505
bool radv_is_storage_image_format_supported(const struct radv_physical_device *physical_device, VkFormat format);
 
2506
bool radv_is_colorbuffer_format_supported(const struct radv_physical_device *pdevice, VkFormat format, bool *blendable);
2468
2507
bool radv_dcc_formats_compatible(enum amd_gfx_level gfx_level, VkFormat format1, VkFormat format2,
2469
2508
                                 bool *sign_reinterpret);
2470
2509
bool radv_is_atomic_format_supported(VkFormat format);
2471
 
bool radv_device_supports_etc(struct radv_physical_device *physical_device);
 
2510
bool radv_device_supports_etc(const struct radv_physical_device *physical_device);
2472
2511
 
2473
2512
static const VkImageUsageFlags RADV_IMAGE_USAGE_WRITE_BITS =
2474
 
   VK_IMAGE_USAGE_TRANSFER_DST_BIT | VK_IMAGE_USAGE_COLOR_ATTACHMENT_BIT |
2475
 
   VK_IMAGE_USAGE_DEPTH_STENCIL_ATTACHMENT_BIT | VK_IMAGE_USAGE_STORAGE_BIT;
 
2513
   VK_IMAGE_USAGE_TRANSFER_DST_BIT | VK_IMAGE_USAGE_COLOR_ATTACHMENT_BIT | VK_IMAGE_USAGE_DEPTH_STENCIL_ATTACHMENT_BIT |
 
2514
   VK_IMAGE_USAGE_STORAGE_BIT;
2476
2515
 
2477
2516
struct radv_image_plane {
2478
2517
   VkFormat format;
2488
2527
struct radv_image {
2489
2528
   struct vk_image vk;
2490
2529
 
2491
 
   struct ac_surf_info info;
2492
 
 
2493
2530
   VkDeviceSize size;
2494
2531
   uint32_t alignment;
2495
2532
 
2523
2560
   struct radv_image_plane planes[0];
2524
2561
};
2525
2562
 
 
2563
struct ac_surf_info radv_get_ac_surf_info(struct radv_device *device, const struct radv_image *image);
 
2564
 
2526
2565
/* Whether the image has a htile  that is known consistent with the contents of
2527
2566
 * the image and is allowed to be in compressed form.
2528
2567
 *
2529
2568
 * If this is false reads that don't use the htile should be able to return
2530
2569
 * correct results.
2531
2570
 */
2532
 
bool radv_layout_is_htile_compressed(const struct radv_device *device,
2533
 
                                     const struct radv_image *image, VkImageLayout layout,
2534
 
                                     unsigned queue_mask);
2535
 
 
2536
 
bool radv_layout_can_fast_clear(const struct radv_device *device, const struct radv_image *image,
2537
 
                                unsigned level, VkImageLayout layout, unsigned queue_mask);
2538
 
 
2539
 
bool radv_layout_dcc_compressed(const struct radv_device *device, const struct radv_image *image,
2540
 
                                unsigned level, VkImageLayout layout, unsigned queue_mask);
 
2571
bool radv_layout_is_htile_compressed(const struct radv_device *device, const struct radv_image *image,
 
2572
                                     VkImageLayout layout, unsigned queue_mask);
 
2573
 
 
2574
bool radv_layout_can_fast_clear(const struct radv_device *device, const struct radv_image *image, unsigned level,
 
2575
                                VkImageLayout layout, unsigned queue_mask);
 
2576
 
 
2577
bool radv_layout_dcc_compressed(const struct radv_device *device, const struct radv_image *image, unsigned level,
 
2578
                                VkImageLayout layout, unsigned queue_mask);
2541
2579
 
2542
2580
enum radv_fmask_compression {
2543
2581
   RADV_FMASK_COMPRESSION_NONE,
2545
2583
   RADV_FMASK_COMPRESSION_FULL,
2546
2584
};
2547
2585
 
2548
 
enum radv_fmask_compression radv_layout_fmask_compression(
2549
 
      const struct radv_device *device, const struct radv_image *image, VkImageLayout layout,
2550
 
      unsigned queue_mask);
 
2586
enum radv_fmask_compression radv_layout_fmask_compression(const struct radv_device *device,
 
2587
                                                          const struct radv_image *image, VkImageLayout layout,
 
2588
                                                          unsigned queue_mask);
2551
2589
 
2552
2590
/**
2553
2591
 * Return whether the image has CMASK metadata for color surfaces.
2573
2611
static inline bool
2574
2612
radv_image_has_dcc(const struct radv_image *image)
2575
2613
{
2576
 
   return !(image->planes[0].surface.flags & RADEON_SURF_Z_OR_SBUFFER) &&
2577
 
          image->planes[0].surface.meta_offset;
 
2614
   return !(image->planes[0].surface.flags & RADEON_SURF_Z_OR_SBUFFER) && image->planes[0].surface.meta_offset;
2578
2615
}
2579
2616
 
2580
2617
/**
2610
2647
static inline bool
2611
2648
radv_image_has_htile(const struct radv_image *image)
2612
2649
{
2613
 
   return image->planes[0].surface.flags & RADEON_SURF_Z_OR_SBUFFER &&
2614
 
          image->planes[0].surface.meta_size;
 
2650
   return image->planes[0].surface.flags & RADEON_SURF_Z_OR_SBUFFER && image->planes[0].surface.meta_size;
2615
2651
}
2616
2652
 
2617
2653
/**
2642
2678
static inline bool
2643
2679
radv_image_is_tc_compat_htile(const struct radv_image *image)
2644
2680
{
2645
 
   return radv_image_has_htile(image) &&
2646
 
          (image->planes[0].surface.flags & RADEON_SURF_TC_COMPATIBLE_HTILE);
 
2681
   return radv_image_has_htile(image) && (image->planes[0].surface.flags & RADEON_SURF_TC_COMPATIBLE_HTILE);
2647
2682
}
2648
2683
 
2649
2684
/**
2759
2794
}
2760
2795
 
2761
2796
static inline bool
2762
 
radv_image_get_iterate256(struct radv_device *device, struct radv_image *image)
 
2797
radv_image_get_iterate256(const struct radv_device *device, struct radv_image *image)
2763
2798
{
2764
2799
   /* ITERATE_256 is required for depth or stencil MSAA images that are TC-compatible HTILE. */
2765
2800
   return device->physical_device->rad_info.gfx_level >= GFX10 &&
2766
 
          (image->vk.usage &
2767
 
           (VK_IMAGE_USAGE_DEPTH_STENCIL_ATTACHMENT_BIT | VK_IMAGE_USAGE_TRANSFER_DST_BIT)) &&
2768
 
          radv_image_is_tc_compat_htile(image) && image->info.samples > 1;
 
2801
          (image->vk.usage & (VK_IMAGE_USAGE_DEPTH_STENCIL_ATTACHMENT_BIT | VK_IMAGE_USAGE_TRANSFER_DST_BIT)) &&
 
2802
          radv_image_is_tc_compat_htile(image) && image->vk.samples > 1;
2769
2803
}
2770
2804
 
2771
 
unsigned radv_image_queue_family_mask(const struct radv_image *image,
2772
 
                                      enum radv_queue_family family,
 
2805
unsigned radv_image_queue_family_mask(const struct radv_image *image, enum radv_queue_family family,
2773
2806
                                      enum radv_queue_family queue_family);
2774
2807
 
2775
 
static inline uint32_t
2776
 
radv_get_layerCount(const struct radv_image *image, const VkImageSubresourceRange *range)
2777
 
{
2778
 
   return range->layerCount == VK_REMAINING_ARRAY_LAYERS
2779
 
             ? image->info.array_size - range->baseArrayLayer
2780
 
             : range->layerCount;
2781
 
}
2782
 
 
2783
 
static inline uint32_t
2784
 
radv_get_levelCount(const struct radv_image *image, const VkImageSubresourceRange *range)
2785
 
{
2786
 
   return range->levelCount == VK_REMAINING_MIP_LEVELS ? image->info.levels - range->baseMipLevel
2787
 
                                                       : range->levelCount;
2788
 
}
2789
 
 
2790
 
bool radv_image_is_renderable(struct radv_device *device, struct radv_image *image);
 
2808
bool radv_image_is_renderable(const struct radv_device *device, const struct radv_image *image);
2791
2809
 
2792
2810
struct radeon_bo_metadata;
2793
 
void radv_init_metadata(struct radv_device *device, struct radv_image *image,
2794
 
                        struct radeon_bo_metadata *metadata);
 
2811
void radv_init_metadata(struct radv_device *device, struct radv_image *image, struct radeon_bo_metadata *metadata);
2795
2812
 
2796
 
void radv_image_override_offset_stride(struct radv_device *device, struct radv_image *image,
2797
 
                                       uint64_t offset, uint32_t stride);
 
2813
void radv_image_override_offset_stride(struct radv_device *device, struct radv_image *image, uint64_t offset,
 
2814
                                       uint32_t stride);
2798
2815
 
2799
2816
union radv_descriptor {
2800
2817
   struct {
2827
2844
 
2828
2845
   /* Block-compressed image views on GFX10+. */
2829
2846
   struct ac_surf_nbc_view nbc_view;
 
2847
 
 
2848
   union {
 
2849
      struct radv_color_buffer_info cb;
 
2850
      struct radv_ds_buffer_info ds;
 
2851
   };
2830
2852
};
2831
2853
 
2832
2854
struct radv_image_create_info {
2837
2859
   const struct radeon_bo_metadata *bo_metadata;
2838
2860
};
2839
2861
 
2840
 
VkResult
2841
 
radv_image_create_layout(struct radv_device *device, struct radv_image_create_info create_info,
2842
 
                         const struct VkImageDrmFormatModifierExplicitCreateInfoEXT *mod_info,
2843
 
                         struct radv_image *image);
 
2862
VkResult radv_image_create_layout(struct radv_device *device, struct radv_image_create_info create_info,
 
2863
                                  const struct VkImageDrmFormatModifierExplicitCreateInfoEXT *mod_info,
 
2864
                                  const struct VkVideoProfileListInfoKHR *profile_list, struct radv_image *image);
2844
2865
 
2845
2866
VkResult radv_image_create(VkDevice _device, const struct radv_image_create_info *info,
2846
2867
                           const VkAllocationCallbacks *alloc, VkImage *pImage, bool is_internal);
2847
2868
 
2848
 
bool radv_are_formats_dcc_compatible(const struct radv_physical_device *pdev, const void *pNext,
2849
 
                                     VkFormat format, VkImageCreateFlags flags,
2850
 
                                     bool *sign_reinterpret);
 
2869
bool radv_are_formats_dcc_compatible(const struct radv_physical_device *pdev, const void *pNext, VkFormat format,
 
2870
                                     VkImageCreateFlags flags, bool *sign_reinterpret);
2851
2871
 
2852
2872
bool vi_alpha_is_on_msb(struct radv_device *device, VkFormat format);
2853
2873
 
2854
2874
VkResult radv_image_from_gralloc(VkDevice device_h, const VkImageCreateInfo *base_info,
2855
 
                                 const VkNativeBufferANDROID *gralloc_info,
2856
 
                                 const VkAllocationCallbacks *alloc, VkImage *out_image_h);
2857
 
VkResult radv_import_ahb_memory(struct radv_device *device, struct radv_device_memory *mem,
2858
 
                                unsigned priority,
 
2875
                                 const VkNativeBufferANDROID *gralloc_info, const VkAllocationCallbacks *alloc,
 
2876
                                 VkImage *out_image_h);
 
2877
VkResult radv_import_ahb_memory(struct radv_device *device, struct radv_device_memory *mem, unsigned priority,
2859
2878
                                const VkImportAndroidHardwareBufferInfoANDROID *info);
2860
 
VkResult radv_create_ahb_memory(struct radv_device *device, struct radv_device_memory *mem,
2861
 
                                unsigned priority, const VkMemoryAllocateInfo *pAllocateInfo);
 
2879
VkResult radv_create_ahb_memory(struct radv_device *device, struct radv_device_memory *mem, unsigned priority,
 
2880
                                const VkMemoryAllocateInfo *pAllocateInfo);
2862
2881
 
2863
2882
unsigned radv_ahb_format_for_vk_format(VkFormat vk_format);
2864
2883
 
2874
2893
};
2875
2894
 
2876
2895
void radv_image_view_init(struct radv_image_view *view, struct radv_device *device,
2877
 
                          const VkImageViewCreateInfo *pCreateInfo,
2878
 
                          VkImageCreateFlags img_create_flags,
 
2896
                          const VkImageViewCreateInfo *pCreateInfo, VkImageCreateFlags img_create_flags,
2879
2897
                          const struct radv_image_view_extra_create_info *extra_create_info);
2880
2898
void radv_image_view_finish(struct radv_image_view *iview);
2881
2899
 
2894
2912
static inline bool
2895
2913
radv_image_extent_compare(const struct radv_image *image, const VkExtent3D *extent)
2896
2914
{
2897
 
   if (extent->width != image->info.width || extent->height != image->info.height ||
2898
 
       extent->depth != image->info.depth)
 
2915
   if (extent->width != image->vk.extent.width || extent->height != image->vk.extent.height ||
 
2916
       extent->depth != image->vk.extent.depth)
2899
2917
      return false;
2900
2918
   return true;
2901
2919
}
2914
2932
   VkAccessFlags2 dst_access_mask;
2915
2933
};
2916
2934
 
2917
 
void radv_emit_resolve_barrier(struct radv_cmd_buffer *cmd_buffer,
2918
 
                               const struct radv_resolve_barrier *barrier);
 
2935
void radv_emit_resolve_barrier(struct radv_cmd_buffer *cmd_buffer, const struct radv_resolve_barrier *barrier);
2919
2936
 
2920
2937
VkResult radv_device_init_meta(struct radv_device *device);
2921
2938
void radv_device_finish_meta(struct radv_device *device);
2947
2964
};
2948
2965
 
2949
2966
void radv_pc_deinit_query_pool(struct radv_pc_query_pool *pool);
2950
 
VkResult radv_pc_init_query_pool(struct radv_physical_device *pdevice,
2951
 
                                 const VkQueryPoolCreateInfo *pCreateInfo,
 
2967
VkResult radv_pc_init_query_pool(struct radv_physical_device *pdevice, const VkQueryPoolCreateInfo *pCreateInfo,
2952
2968
                                 struct radv_pc_query_pool *pool);
2953
 
void radv_pc_begin_query(struct radv_cmd_buffer *cmd_buffer, struct radv_pc_query_pool *pool,
2954
 
                         uint64_t va);
2955
 
void radv_pc_end_query(struct radv_cmd_buffer *cmd_buffer, struct radv_pc_query_pool *pool,
2956
 
                       uint64_t va);
 
2969
void radv_pc_begin_query(struct radv_cmd_buffer *cmd_buffer, struct radv_pc_query_pool *pool, uint64_t va);
 
2970
void radv_pc_end_query(struct radv_cmd_buffer *cmd_buffer, struct radv_pc_query_pool *pool, uint64_t va);
2957
2971
void radv_pc_get_results(const struct radv_pc_query_pool *pc_pool, const uint64_t *data, void *out);
2958
2972
 
2959
 
#define VL_MACROBLOCK_WIDTH 16
 
2973
#define VL_MACROBLOCK_WIDTH  16
2960
2974
#define VL_MACROBLOCK_HEIGHT 16
2961
2975
 
2962
2976
struct radv_vid_mem {
2963
2977
   struct radv_device_memory *mem;
2964
 
   VkDeviceSize       offset;
2965
 
   VkDeviceSize       size;
 
2978
   VkDeviceSize offset;
 
2979
   VkDeviceSize size;
2966
2980
};
2967
2981
 
2968
2982
struct radv_video_session {
2971
2985
   uint32_t stream_handle;
2972
2986
   unsigned stream_type;
2973
2987
   bool interlaced;
2974
 
   enum {
2975
 
      DPB_MAX_RES = 0,
2976
 
      DPB_DYNAMIC_TIER_1,
2977
 
      DPB_DYNAMIC_TIER_2
2978
 
   } dpb_type;
 
2988
   enum { DPB_MAX_RES = 0, DPB_DYNAMIC_TIER_1, DPB_DYNAMIC_TIER_2 } dpb_type;
2979
2989
   unsigned db_alignment;
2980
2990
 
2981
2991
   struct radv_vid_mem sessionctx;
2988
2998
   struct vk_video_session_parameters vk;
2989
2999
};
2990
3000
 
2991
 
/* needed for ac_gpu_info codecs */
2992
 
#define RADV_VIDEO_FORMAT_UNKNOWN 0
2993
 
#define RADV_VIDEO_FORMAT_MPEG12 1   /**< MPEG1, MPEG2 */
2994
 
#define RADV_VIDEO_FORMAT_MPEG4 2   /**< DIVX, XVID */
2995
 
#define RADV_VIDEO_FORMAT_VC1 3      /**< WMV */
2996
 
#define RADV_VIDEO_FORMAT_MPEG4_AVC 4/**< H.264 */
2997
 
#define RADV_VIDEO_FORMAT_HEVC 5     /**< H.265 */
2998
 
#define RADV_VIDEO_FORMAT_JPEG 6     /**< JPEG */
2999
 
#define RADV_VIDEO_FORMAT_VP9 7      /**< VP9 */
3000
 
#define RADV_VIDEO_FORMAT_AV1 8      /**< AV1 */
3001
 
 
3002
3001
bool radv_queue_internal_submit(struct radv_queue *queue, struct radeon_cmdbuf *cs);
3003
3002
 
3004
3003
int radv_queue_init(struct radv_device *device, struct radv_queue *queue, int idx,
3010
3009
 
3011
3010
void radv_cmd_update_descriptor_sets(struct radv_device *device, struct radv_cmd_buffer *cmd_buffer,
3012
3011
                                     VkDescriptorSet overrideSet, uint32_t descriptorWriteCount,
3013
 
                                     const VkWriteDescriptorSet *pDescriptorWrites,
3014
 
                                     uint32_t descriptorCopyCount,
 
3012
                                     const VkWriteDescriptorSet *pDescriptorWrites, uint32_t descriptorCopyCount,
3015
3013
                                     const VkCopyDescriptorSet *pDescriptorCopies);
3016
3014
 
3017
 
void radv_cmd_update_descriptor_set_with_template(struct radv_device *device,
3018
 
                                                  struct radv_cmd_buffer *cmd_buffer,
 
3015
void radv_cmd_update_descriptor_set_with_template(struct radv_device *device, struct radv_cmd_buffer *cmd_buffer,
3019
3016
                                                  struct radv_descriptor_set *set,
3020
3017
                                                  VkDescriptorUpdateTemplate descriptorUpdateTemplate,
3021
3018
                                                  const void *pData);
3022
3019
 
3023
 
void radv_meta_push_descriptor_set(struct radv_cmd_buffer *cmd_buffer,
3024
 
                                   VkPipelineBindPoint pipelineBindPoint, VkPipelineLayout _layout,
3025
 
                                   uint32_t set, uint32_t descriptorWriteCount,
 
3020
void radv_meta_push_descriptor_set(struct radv_cmd_buffer *cmd_buffer, VkPipelineBindPoint pipelineBindPoint,
 
3021
                                   VkPipelineLayout _layout, uint32_t set, uint32_t descriptorWriteCount,
3026
3022
                                   const VkWriteDescriptorSet *pDescriptorWrites);
3027
3023
 
3028
 
void radv_make_texel_buffer_descriptor(struct radv_device *device, uint64_t va,
3029
 
                                       VkFormat vk_format, unsigned offset, unsigned range,
3030
 
                                       uint32_t *state);
 
3024
void radv_make_texel_buffer_descriptor(struct radv_device *device, uint64_t va, VkFormat vk_format, unsigned offset,
 
3025
                                       unsigned range, uint32_t *state);
3031
3026
 
3032
3027
uint32_t radv_init_dcc(struct radv_cmd_buffer *cmd_buffer, struct radv_image *image,
3033
3028
                       const VkImageSubresourceRange *range, uint32_t value);
3040
3035
struct radv_nir_compiler_options;
3041
3036
struct radv_shader_info;
3042
3037
 
3043
 
void llvm_compile_shader(const struct radv_nir_compiler_options *options,
3044
 
                         const struct radv_shader_info *info, unsigned shader_count,
3045
 
                         struct nir_shader *const *shaders, struct radv_shader_binary **binary,
 
3038
void llvm_compile_shader(const struct radv_nir_compiler_options *options, const struct radv_shader_info *info,
 
3039
                         unsigned shader_count, struct nir_shader *const *shaders, struct radv_shader_binary **binary,
3046
3040
                         const struct radv_shader_args *args);
3047
3041
 
3048
3042
/* radv_shader_info.h */
3049
3043
struct radv_shader_info;
3050
3044
 
3051
 
void radv_nir_shader_info_pass(struct radv_device *device, const struct nir_shader *nir,
3052
 
                               gl_shader_stage next_stage,
3053
 
                               const struct radv_pipeline_layout *layout,
3054
 
                               const struct radv_pipeline_key *pipeline_key,
3055
 
                               const enum radv_pipeline_type pipeline_type,
3056
 
                               bool consider_force_vrs,
 
3045
void radv_nir_shader_info_pass(struct radv_device *device, const struct nir_shader *nir, gl_shader_stage next_stage,
 
3046
                               const struct radv_pipeline_layout *layout, const struct radv_pipeline_key *pipeline_key,
 
3047
                               const enum radv_pipeline_type pipeline_type, bool consider_force_vrs,
3057
3048
                               struct radv_shader_info *info);
3058
3049
 
3059
3050
void radv_nir_shader_info_init(struct radv_shader_info *info);
3060
3051
 
3061
 
void radv_nir_shader_info_link(struct radv_device *device,
3062
 
                               const struct radv_pipeline_key *pipeline_key,
 
3052
void radv_nir_shader_info_link(struct radv_device *device, const struct radv_pipeline_key *pipeline_key,
3063
3053
                               struct radv_pipeline_stage *stages);
3064
3054
 
3065
 
bool radv_thread_trace_init(struct radv_device *device);
3066
 
void radv_thread_trace_finish(struct radv_device *device);
3067
 
bool radv_begin_thread_trace(struct radv_queue *queue);
3068
 
bool radv_end_thread_trace(struct radv_queue *queue);
3069
 
bool radv_get_thread_trace(struct radv_queue *queue, struct ac_thread_trace *thread_trace);
3070
 
void radv_emit_thread_trace_userdata(struct radv_cmd_buffer *cmd_buffer, const void *data,
3071
 
                                     uint32_t num_dwords);
 
3055
bool radv_sqtt_init(struct radv_device *device);
 
3056
void radv_sqtt_finish(struct radv_device *device);
 
3057
bool radv_begin_sqtt(struct radv_queue *queue);
 
3058
bool radv_end_sqtt(struct radv_queue *queue);
 
3059
bool radv_get_sqtt_trace(struct radv_queue *queue, struct ac_sqtt_trace *sqtt_trace);
 
3060
void radv_reset_sqtt_trace(struct radv_device *device);
 
3061
void radv_emit_sqtt_userdata(const struct radv_cmd_buffer *cmd_buffer, const void *data, uint32_t num_dwords);
3072
3062
bool radv_is_instruction_timing_enabled(void);
3073
 
 
3074
 
void radv_emit_inhibit_clockgating(struct radv_device *device, struct radeon_cmdbuf *cs,
3075
 
                                   bool inhibit);
3076
 
void radv_emit_spi_config_cntl(struct radv_device *device, struct radeon_cmdbuf *cs, bool enable);
3077
 
 
3078
 
int radv_rra_trace_frame(void);
3079
 
char *radv_rra_trace_trigger_file(void);
3080
 
bool radv_rra_trace_enabled(void);
 
3063
bool radv_sqtt_sample_clocks(struct radv_device *device);
 
3064
 
 
3065
void radv_emit_inhibit_clockgating(const struct radv_device *device, struct radeon_cmdbuf *cs, bool inhibit);
 
3066
void radv_emit_spi_config_cntl(const struct radv_device *device, struct radeon_cmdbuf *cs, bool enable);
3081
3067
 
3082
3068
void radv_rra_trace_init(struct radv_device *device);
3083
3069
 
3084
3070
VkResult radv_rra_dump_trace(VkQueue vk_queue, char *filename);
3085
3071
void radv_rra_trace_finish(VkDevice vk_device, struct radv_rra_trace_data *data);
3086
3072
 
3087
 
bool radv_sdma_copy_image(struct radv_device *device, struct radeon_cmdbuf *cs,
3088
 
                          struct radv_image *image, struct radv_buffer *buffer,
3089
 
                          const VkBufferImageCopy2 *region);
3090
 
void radv_sdma_copy_buffer(struct radv_device *device, struct radeon_cmdbuf *cs, uint64_t src_va,
3091
 
                           uint64_t dst_va, uint64_t size);
 
3073
bool radv_sdma_copy_image(struct radv_device *device, struct radeon_cmdbuf *cs, struct radv_image *image,
 
3074
                          struct radv_buffer *buffer, const VkBufferImageCopy2 *region);
 
3075
void radv_sdma_copy_buffer(struct radv_device *device, struct radeon_cmdbuf *cs, uint64_t src_va, uint64_t dst_va,
 
3076
                           uint64_t size);
3092
3077
 
3093
3078
void radv_memory_trace_init(struct radv_device *device);
3094
 
void radv_rmv_log_bo_allocate(struct radv_device *device, struct radeon_winsys_bo *bo,
3095
 
                              uint32_t size, bool is_internal);
 
3079
void radv_rmv_log_bo_allocate(struct radv_device *device, struct radeon_winsys_bo *bo, uint32_t size, bool is_internal);
3096
3080
void radv_rmv_log_bo_destroy(struct radv_device *device, struct radeon_winsys_bo *bo);
3097
3081
void radv_rmv_log_heap_create(struct radv_device *device, VkDeviceMemory heap, bool is_internal,
3098
3082
                              VkMemoryAllocateFlags alloc_flags);
3099
3083
void radv_rmv_log_buffer_bind(struct radv_device *device, VkBuffer _buffer);
3100
 
void radv_rmv_log_image_create(struct radv_device *device, const VkImageCreateInfo *create_info,
3101
 
                               bool is_internal, VkImage _image);
 
3084
void radv_rmv_log_image_create(struct radv_device *device, const VkImageCreateInfo *create_info, bool is_internal,
 
3085
                               VkImage _image);
3102
3086
void radv_rmv_log_image_bind(struct radv_device *device, VkImage _image);
3103
3087
void radv_rmv_log_query_pool_create(struct radv_device *device, VkQueryPool pool, bool is_internal);
3104
3088
void radv_rmv_log_command_buffer_bo_create(struct radv_device *device, struct radeon_winsys_bo *bo,
3105
 
                                           uint32_t executable_size, uint32_t data_size,
3106
 
                                           uint32_t scratch_size);
3107
 
void radv_rmv_log_command_buffer_bo_destroy(struct radv_device *device,
3108
 
                                            struct radeon_winsys_bo *bo);
3109
 
void radv_rmv_log_border_color_palette_create(struct radv_device *device,
3110
 
                                              struct radeon_winsys_bo *bo);
3111
 
void radv_rmv_log_border_color_palette_destroy(struct radv_device *device,
3112
 
                                               struct radeon_winsys_bo *bo);
3113
 
void radv_rmv_log_sparse_add_residency(struct radv_device *device, struct radeon_winsys_bo *src_bo,
3114
 
                                       uint64_t offset);
3115
 
void radv_rmv_log_sparse_remove_residency(struct radv_device *device,
3116
 
                                          struct radeon_winsys_bo *src_bo, uint64_t offset);
3117
 
void radv_rmv_log_descriptor_pool_create(struct radv_device *device,
3118
 
                                         const VkDescriptorPoolCreateInfo *create_info,
 
3089
                                           uint32_t executable_size, uint32_t data_size, uint32_t scratch_size);
 
3090
void radv_rmv_log_command_buffer_bo_destroy(struct radv_device *device, struct radeon_winsys_bo *bo);
 
3091
void radv_rmv_log_border_color_palette_create(struct radv_device *device, struct radeon_winsys_bo *bo);
 
3092
void radv_rmv_log_border_color_palette_destroy(struct radv_device *device, struct radeon_winsys_bo *bo);
 
3093
void radv_rmv_log_sparse_add_residency(struct radv_device *device, struct radeon_winsys_bo *src_bo, uint64_t offset);
 
3094
void radv_rmv_log_sparse_remove_residency(struct radv_device *device, struct radeon_winsys_bo *src_bo, uint64_t offset);
 
3095
void radv_rmv_log_descriptor_pool_create(struct radv_device *device, const VkDescriptorPoolCreateInfo *create_info,
3119
3096
                                         VkDescriptorPool pool, bool is_internal);
3120
3097
void radv_rmv_log_graphics_pipeline_create(struct radv_device *device, VkPipelineCreateFlags flags,
3121
3098
                                           struct radv_pipeline *pipeline, bool is_internal);
3122
3099
void radv_rmv_log_compute_pipeline_create(struct radv_device *device, VkPipelineCreateFlags flags,
3123
3100
                                          struct radv_pipeline *pipeline, bool is_internal);
3124
 
void radv_rmv_log_event_create(struct radv_device *device, VkEvent event, VkEventCreateFlags flags,
3125
 
                               bool is_internal);
 
3101
void radv_rmv_log_event_create(struct radv_device *device, VkEvent event, VkEventCreateFlags flags, bool is_internal);
3126
3102
void radv_rmv_log_resource_destroy(struct radv_device *device, uint64_t handle);
3127
3103
void radv_rmv_log_submit(struct radv_device *device, enum amd_ip_type type);
3128
 
void radv_rmv_fill_device_info(struct radv_physical_device *device,
3129
 
                               struct vk_rmv_device_info *info);
 
3104
void radv_rmv_fill_device_info(const struct radv_physical_device *device, struct vk_rmv_device_info *info);
3130
3105
void radv_rmv_collect_trace_events(struct radv_device *device);
3131
3106
void radv_memory_trace_finish(struct radv_device *device);
3132
3107
 
3133
3108
VkResult radv_create_buffer(struct radv_device *device, const VkBufferCreateInfo *pCreateInfo,
3134
 
                            const VkAllocationCallbacks *pAllocator, VkBuffer *pBuffer,
3135
 
                            bool is_internal);
 
3109
                            const VkAllocationCallbacks *pAllocator, VkBuffer *pBuffer, bool is_internal);
3136
3110
VkResult radv_alloc_memory(struct radv_device *device, const VkMemoryAllocateInfo *pAllocateInfo,
3137
 
                           const VkAllocationCallbacks *pAllocator, VkDeviceMemory *pMem,
3138
 
                           bool is_internal);
3139
 
VkResult radv_create_query_pool(struct radv_device *device,
3140
 
                                const VkQueryPoolCreateInfo *pCreateInfo,
3141
 
                                const VkAllocationCallbacks *pAllocator, VkQueryPool *pQueryPool,
3142
 
                                bool is_internal);
3143
 
VkResult radv_create_descriptor_pool(struct radv_device *device,
3144
 
                                     const VkDescriptorPoolCreateInfo *pCreateInfo,
3145
 
                                     const VkAllocationCallbacks *pAllocator,
3146
 
                                     VkDescriptorPool *pDescriptorPool, bool is_internal);
 
3111
                           const VkAllocationCallbacks *pAllocator, VkDeviceMemory *pMem, bool is_internal);
 
3112
VkResult radv_create_query_pool(struct radv_device *device, const VkQueryPoolCreateInfo *pCreateInfo,
 
3113
                                const VkAllocationCallbacks *pAllocator, VkQueryPool *pQueryPool, bool is_internal);
 
3114
VkResult radv_create_descriptor_pool(struct radv_device *device, const VkDescriptorPoolCreateInfo *pCreateInfo,
 
3115
                                     const VkAllocationCallbacks *pAllocator, VkDescriptorPool *pDescriptorPool,
 
3116
                                     bool is_internal);
3147
3117
VkResult radv_create_event(struct radv_device *device, const VkEventCreateInfo *pCreateInfo,
3148
 
                           const VkAllocationCallbacks *pAllocator, VkEvent *pEvent,
3149
 
                           bool is_internal);
 
3118
                           const VkAllocationCallbacks *pAllocator, VkEvent *pEvent, bool is_internal);
3150
3119
 
3151
3120
/* radv_sqtt_layer_.c */
3152
3121
struct radv_barrier_data {
3194
3163
void radv_describe_end_cmd_buffer(struct radv_cmd_buffer *cmd_buffer);
3195
3164
void radv_describe_draw(struct radv_cmd_buffer *cmd_buffer);
3196
3165
void radv_describe_dispatch(struct radv_cmd_buffer *cmd_buffer, int x, int y, int z);
3197
 
void radv_describe_begin_render_pass_clear(struct radv_cmd_buffer *cmd_buffer,
3198
 
                                           VkImageAspectFlagBits aspects);
 
3166
void radv_describe_begin_render_pass_clear(struct radv_cmd_buffer *cmd_buffer, VkImageAspectFlagBits aspects);
3199
3167
void radv_describe_end_render_pass_clear(struct radv_cmd_buffer *cmd_buffer);
3200
3168
void radv_describe_begin_render_pass_resolve(struct radv_cmd_buffer *cmd_buffer);
3201
3169
void radv_describe_end_render_pass_resolve(struct radv_cmd_buffer *cmd_buffer);
3202
 
void radv_describe_barrier_start(struct radv_cmd_buffer *cmd_buffer,
3203
 
                                 enum rgp_barrier_reason reason);
 
3170
void radv_describe_barrier_start(struct radv_cmd_buffer *cmd_buffer, enum rgp_barrier_reason reason);
3204
3171
void radv_describe_barrier_end(struct radv_cmd_buffer *cmd_buffer);
3205
3172
void radv_describe_barrier_end_delayed(struct radv_cmd_buffer *cmd_buffer);
3206
 
void radv_describe_layout_transition(struct radv_cmd_buffer *cmd_buffer,
3207
 
                                     const struct radv_barrier_data *barrier);
 
3173
void radv_describe_layout_transition(struct radv_cmd_buffer *cmd_buffer, const struct radv_barrier_data *barrier);
3208
3174
 
3209
 
void radv_sqtt_emit_relocated_shaders(struct radv_cmd_buffer *cmd_buffer,
3210
 
                                      struct radv_graphics_pipeline *pipeline);
 
3175
void radv_sqtt_emit_relocated_shaders(struct radv_cmd_buffer *cmd_buffer, struct radv_graphics_pipeline *pipeline);
3211
3176
 
3212
3177
struct radv_indirect_command_layout {
3213
3178
   struct vk_object_base base;
3237
3202
 
3238
3203
uint32_t radv_get_indirect_cmdbuf_size(const VkGeneratedCommandsInfoNV *cmd_info);
3239
3204
 
3240
 
void radv_prepare_dgc(struct radv_cmd_buffer *cmd_buffer,
3241
 
                      const VkGeneratedCommandsInfoNV *pGeneratedCommandsInfo);
 
3205
void radv_prepare_dgc(struct radv_cmd_buffer *cmd_buffer, const VkGeneratedCommandsInfoNV *pGeneratedCommandsInfo);
3242
3206
 
3243
3207
static inline uint32_t
3244
3208
si_conv_prim_to_gs_out(uint32_t topology, bool is_ngg)
3490
3454
   case VK_BLEND_FACTOR_ONE_MINUS_DST_ALPHA:
3491
3455
      return V_028780_BLEND_ONE_MINUS_DST_ALPHA;
3492
3456
   case VK_BLEND_FACTOR_CONSTANT_COLOR:
3493
 
      return gfx_level >= GFX11 ? V_028780_BLEND_CONSTANT_COLOR_GFX11
3494
 
                                : V_028780_BLEND_CONSTANT_COLOR_GFX6;
 
3457
      return gfx_level >= GFX11 ? V_028780_BLEND_CONSTANT_COLOR_GFX11 : V_028780_BLEND_CONSTANT_COLOR_GFX6;
3495
3458
   case VK_BLEND_FACTOR_ONE_MINUS_CONSTANT_COLOR:
3496
3459
      return gfx_level >= GFX11 ? V_028780_BLEND_ONE_MINUS_CONSTANT_COLOR_GFX11
3497
 
                                 : V_028780_BLEND_ONE_MINUS_CONSTANT_COLOR_GFX6;
 
3460
                                : V_028780_BLEND_ONE_MINUS_CONSTANT_COLOR_GFX6;
3498
3461
   case VK_BLEND_FACTOR_CONSTANT_ALPHA:
3499
 
      return gfx_level >= GFX11 ? V_028780_BLEND_CONSTANT_ALPHA_GFX11
3500
 
                                 : V_028780_BLEND_CONSTANT_ALPHA_GFX6;
 
3462
      return gfx_level >= GFX11 ? V_028780_BLEND_CONSTANT_ALPHA_GFX11 : V_028780_BLEND_CONSTANT_ALPHA_GFX6;
3501
3463
   case VK_BLEND_FACTOR_ONE_MINUS_CONSTANT_ALPHA:
3502
3464
      return gfx_level >= GFX11 ? V_028780_BLEND_ONE_MINUS_CONSTANT_ALPHA_GFX11
3503
 
                                 : V_028780_BLEND_ONE_MINUS_CONSTANT_ALPHA_GFX6;
 
3465
                                : V_028780_BLEND_ONE_MINUS_CONSTANT_ALPHA_GFX6;
3504
3466
   case VK_BLEND_FACTOR_SRC_ALPHA_SATURATE:
3505
3467
      return V_028780_BLEND_SRC_ALPHA_SATURATE;
3506
3468
   case VK_BLEND_FACTOR_SRC1_COLOR:
3507
3469
      return gfx_level >= GFX11 ? V_028780_BLEND_SRC1_COLOR_GFX11 : V_028780_BLEND_SRC1_COLOR_GFX6;
3508
3470
   case VK_BLEND_FACTOR_ONE_MINUS_SRC1_COLOR:
3509
 
      return gfx_level >= GFX11 ? V_028780_BLEND_INV_SRC1_COLOR_GFX11
3510
 
                                 : V_028780_BLEND_INV_SRC1_COLOR_GFX6;
 
3471
      return gfx_level >= GFX11 ? V_028780_BLEND_INV_SRC1_COLOR_GFX11 : V_028780_BLEND_INV_SRC1_COLOR_GFX6;
3511
3472
   case VK_BLEND_FACTOR_SRC1_ALPHA:
3512
3473
      return gfx_level >= GFX11 ? V_028780_BLEND_SRC1_ALPHA_GFX11 : V_028780_BLEND_SRC1_ALPHA_GFX6;
3513
3474
   case VK_BLEND_FACTOR_ONE_MINUS_SRC1_ALPHA:
3514
 
      return gfx_level >= GFX11 ? V_028780_BLEND_INV_SRC1_ALPHA_GFX11
3515
 
                                 : V_028780_BLEND_INV_SRC1_ALPHA_GFX6;
 
3475
      return gfx_level >= GFX11 ? V_028780_BLEND_INV_SRC1_ALPHA_GFX11 : V_028780_BLEND_INV_SRC1_ALPHA_GFX6;
3516
3476
   default:
3517
3477
      return 0;
3518
3478
   }
3527
3487
   case VK_BLEND_FACTOR_ONE:
3528
3488
      return V_028760_BLEND_OPT_PRESERVE_ALL_IGNORE_NONE;
3529
3489
   case VK_BLEND_FACTOR_SRC_COLOR:
3530
 
      return is_alpha ? V_028760_BLEND_OPT_PRESERVE_A1_IGNORE_A0
3531
 
                      : V_028760_BLEND_OPT_PRESERVE_C1_IGNORE_C0;
 
3490
      return is_alpha ? V_028760_BLEND_OPT_PRESERVE_A1_IGNORE_A0 : V_028760_BLEND_OPT_PRESERVE_C1_IGNORE_C0;
3532
3491
   case VK_BLEND_FACTOR_ONE_MINUS_SRC_COLOR:
3533
 
      return is_alpha ? V_028760_BLEND_OPT_PRESERVE_A0_IGNORE_A1
3534
 
                      : V_028760_BLEND_OPT_PRESERVE_C0_IGNORE_C1;
 
3492
      return is_alpha ? V_028760_BLEND_OPT_PRESERVE_A0_IGNORE_A1 : V_028760_BLEND_OPT_PRESERVE_C0_IGNORE_C1;
3535
3493
   case VK_BLEND_FACTOR_SRC_ALPHA:
3536
3494
      return V_028760_BLEND_OPT_PRESERVE_A1_IGNORE_A0;
3537
3495
   case VK_BLEND_FACTOR_ONE_MINUS_SRC_ALPHA:
3538
3496
      return V_028760_BLEND_OPT_PRESERVE_A0_IGNORE_A1;
3539
3497
   case VK_BLEND_FACTOR_SRC_ALPHA_SATURATE:
3540
 
      return is_alpha ? V_028760_BLEND_OPT_PRESERVE_ALL_IGNORE_NONE
3541
 
                      : V_028760_BLEND_OPT_PRESERVE_NONE_IGNORE_A0;
 
3498
      return is_alpha ? V_028760_BLEND_OPT_PRESERVE_ALL_IGNORE_NONE : V_028760_BLEND_OPT_PRESERVE_NONE_IGNORE_A0;
3542
3499
   default:
3543
3500
      return V_028760_BLEND_OPT_PRESERVE_NONE_IGNORE_NONE;
3544
3501
   }
3567
3524
si_blend_factor_uses_dst(VkBlendFactor factor)
3568
3525
{
3569
3526
   return factor == VK_BLEND_FACTOR_DST_COLOR || factor == VK_BLEND_FACTOR_DST_ALPHA ||
3570
 
          factor == VK_BLEND_FACTOR_SRC_ALPHA_SATURATE ||
3571
 
          factor == VK_BLEND_FACTOR_ONE_MINUS_DST_ALPHA ||
 
3527
          factor == VK_BLEND_FACTOR_SRC_ALPHA_SATURATE || factor == VK_BLEND_FACTOR_ONE_MINUS_DST_ALPHA ||
3572
3528
          factor == VK_BLEND_FACTOR_ONE_MINUS_DST_COLOR;
3573
3529
}
3574
3530
 
3586
3542
   }
3587
3543
}
3588
3544
 
 
3545
static ALWAYS_INLINE bool
 
3546
radv_can_enable_dual_src(const struct vk_color_blend_attachment_state *att)
 
3547
{
 
3548
   VkBlendOp eqRGB = att->color_blend_op;
 
3549
   VkBlendFactor srcRGB = att->src_color_blend_factor;
 
3550
   VkBlendFactor dstRGB = att->dst_color_blend_factor;
 
3551
   VkBlendOp eqA = att->alpha_blend_op;
 
3552
   VkBlendFactor srcA = att->src_alpha_blend_factor;
 
3553
   VkBlendFactor dstA = att->dst_alpha_blend_factor;
 
3554
   bool eqRGB_minmax = eqRGB == VK_BLEND_OP_MIN || eqRGB == VK_BLEND_OP_MAX;
 
3555
   bool eqA_minmax = eqA == VK_BLEND_OP_MIN || eqA == VK_BLEND_OP_MAX;
 
3556
 
 
3557
   if (!eqRGB_minmax && (radv_is_dual_src(srcRGB) || radv_is_dual_src(dstRGB)))
 
3558
      return true;
 
3559
   if (!eqA_minmax && (radv_is_dual_src(srcA) || radv_is_dual_src(dstA)))
 
3560
      return true;
 
3561
   return false;
 
3562
}
 
3563
 
3589
3564
static inline void
3590
3565
radv_normalize_blend_factor(VkBlendOp op, VkBlendFactor *src_factor, VkBlendFactor *dst_factor)
3591
3566
{
3598
3573
void si_blend_remove_dst(VkBlendOp *func, VkBlendFactor *src_factor, VkBlendFactor *dst_factor,
3599
3574
                         VkBlendFactor expected_dst, VkBlendFactor replacement_src);
3600
3575
 
3601
 
bool radv_can_enable_dual_src(const struct vk_color_blend_attachment_state *att);
3602
 
 
3603
3576
uint32_t radv_get_tess_output_topology(const struct radv_graphics_pipeline *pipeline,
3604
3577
                                       VkTessellationDomainOrigin domain_origin);
3605
3578
 
3609
3582
   struct radv_streamout_state *so = &cmd_buffer->state.streamout;
3610
3583
 
3611
3584
   /* Streamout must be enabled for the PRIMITIVES_GENERATED query to work. */
3612
 
   return (so->streamout_enabled || cmd_buffer->state.active_prims_gen_queries) &&
3613
 
          !cmd_buffer->state.suspend_streamout;
 
3585
   return (so->streamout_enabled || cmd_buffer->state.active_prims_gen_queries) && !cmd_buffer->state.suspend_streamout;
3614
3586
}
3615
3587
 
3616
3588
/*
3618
3590
 * placed here as it needs queue + device structs.
3619
3591
 */
3620
3592
static inline enum amd_ip_type
3621
 
radv_queue_ring(struct radv_queue *queue)
 
3593
radv_queue_ring(const struct radv_queue *queue)
3622
3594
{
3623
3595
   return radv_queue_family_to_ring(queue->device->physical_device, queue->state.qf);
3624
3596
}
3625
3597
 
3626
3598
/* radv_video */
3627
3599
void radv_init_physical_device_decoder(struct radv_physical_device *pdevice);
3628
 
 
 
3600
void radv_video_get_profile_alignments(struct radv_physical_device *pdevice,
 
3601
                                       const VkVideoProfileListInfoKHR *profile_list, uint32_t *width_align_out,
 
3602
                                       uint32_t *height_align_out);
3629
3603
/**
3630
3604
 * Helper used for debugging compiler issues by enabling/disabling LLVM for a
3631
3605
 * specific shader stage (developers only).
3639
3613
static inline bool
3640
3614
radv_has_shader_buffer_float_minmax(const struct radv_physical_device *pdevice, unsigned bitsize)
3641
3615
{
3642
 
   return (pdevice->rad_info.gfx_level <= GFX7 && !pdevice->use_llvm) ||
3643
 
          pdevice->rad_info.gfx_level == GFX10 || pdevice->rad_info.gfx_level == GFX10_3 ||
3644
 
          (pdevice->rad_info.gfx_level == GFX11 && bitsize == 32);
 
3616
   return (pdevice->rad_info.gfx_level <= GFX7 && !pdevice->use_llvm) || pdevice->rad_info.gfx_level == GFX10 ||
 
3617
          pdevice->rad_info.gfx_level == GFX10_3 || (pdevice->rad_info.gfx_level == GFX11 && bitsize == 32);
 
3618
}
 
3619
 
 
3620
static inline bool
 
3621
radv_has_pops(const struct radv_physical_device *pdevice)
 
3622
{
 
3623
   return pdevice->rad_info.gfx_level >= GFX9 && !pdevice->use_llvm;
3645
3624
}
3646
3625
 
3647
3626
/* radv_perfcounter.c */
3648
3627
void radv_perfcounter_emit_shaders(struct radeon_cmdbuf *cs, unsigned shaders);
3649
3628
void radv_perfcounter_emit_spm_reset(struct radeon_cmdbuf *cs);
3650
 
void radv_perfcounter_emit_spm_start(struct radv_device *device, struct radeon_cmdbuf *cs,
3651
 
                                     int family);
3652
 
void radv_perfcounter_emit_spm_stop(struct radv_device *device, struct radeon_cmdbuf *cs,
3653
 
                                    int family);
 
3629
void radv_perfcounter_emit_spm_start(struct radv_device *device, struct radeon_cmdbuf *cs, int family);
 
3630
void radv_perfcounter_emit_spm_stop(struct radv_device *device, struct radeon_cmdbuf *cs, int family);
3654
3631
 
3655
3632
/* radv_spm.c */
3656
3633
bool radv_spm_init(struct radv_device *device);
3657
3634
void radv_spm_finish(struct radv_device *device);
3658
3635
void radv_emit_spm_setup(struct radv_device *device, struct radeon_cmdbuf *cs);
3659
3636
 
3660
 
void radv_destroy_graphics_pipeline(struct radv_device *device,
3661
 
                                    struct radv_graphics_pipeline *pipeline);
3662
 
void radv_destroy_graphics_lib_pipeline(struct radv_device *device,
3663
 
                                        struct radv_graphics_lib_pipeline *pipeline);
3664
 
void radv_destroy_compute_pipeline(struct radv_device *device,
3665
 
                                   struct radv_compute_pipeline *pipeline);
3666
 
void radv_destroy_ray_tracing_lib_pipeline(struct radv_device *device,
3667
 
                                           struct radv_ray_tracing_lib_pipeline *pipeline);
3668
 
void radv_destroy_ray_tracing_pipeline(struct radv_device *device,
3669
 
                                       struct radv_ray_tracing_pipeline *pipeline);
3670
 
 
3671
 
#define RADV_FROM_HANDLE(__radv_type, __name, __handle) \
3672
 
   VK_FROM_HANDLE(__radv_type, __name, __handle)
3673
 
 
3674
 
VK_DEFINE_HANDLE_CASTS(radv_cmd_buffer, vk.base, VkCommandBuffer,
3675
 
                       VK_OBJECT_TYPE_COMMAND_BUFFER)
 
3637
void radv_destroy_graphics_pipeline(struct radv_device *device, struct radv_graphics_pipeline *pipeline);
 
3638
void radv_destroy_graphics_lib_pipeline(struct radv_device *device, struct radv_graphics_lib_pipeline *pipeline);
 
3639
void radv_destroy_compute_pipeline(struct radv_device *device, struct radv_compute_pipeline *pipeline);
 
3640
void radv_destroy_ray_tracing_pipeline(struct radv_device *device, struct radv_ray_tracing_pipeline *pipeline);
 
3641
 
 
3642
#define RADV_FROM_HANDLE(__radv_type, __name, __handle) VK_FROM_HANDLE(__radv_type, __name, __handle)
 
3643
 
 
3644
VK_DEFINE_HANDLE_CASTS(radv_cmd_buffer, vk.base, VkCommandBuffer, VK_OBJECT_TYPE_COMMAND_BUFFER)
3676
3645
VK_DEFINE_HANDLE_CASTS(radv_device, vk.base, VkDevice, VK_OBJECT_TYPE_DEVICE)
3677
3646
VK_DEFINE_HANDLE_CASTS(radv_instance, vk.base, VkInstance, VK_OBJECT_TYPE_INSTANCE)
3678
 
VK_DEFINE_HANDLE_CASTS(radv_physical_device, vk.base, VkPhysicalDevice,
3679
 
                       VK_OBJECT_TYPE_PHYSICAL_DEVICE)
 
3647
VK_DEFINE_HANDLE_CASTS(radv_physical_device, vk.base, VkPhysicalDevice, VK_OBJECT_TYPE_PHYSICAL_DEVICE)
3680
3648
VK_DEFINE_HANDLE_CASTS(radv_queue, vk.base, VkQueue, VK_OBJECT_TYPE_QUEUE)
3681
3649
VK_DEFINE_NONDISP_HANDLE_CASTS(radv_buffer, vk.base, VkBuffer, VK_OBJECT_TYPE_BUFFER)
3682
 
VK_DEFINE_NONDISP_HANDLE_CASTS(radv_buffer_view, base, VkBufferView,
3683
 
                               VK_OBJECT_TYPE_BUFFER_VIEW)
3684
 
VK_DEFINE_NONDISP_HANDLE_CASTS(radv_descriptor_pool, base, VkDescriptorPool,
3685
 
                               VK_OBJECT_TYPE_DESCRIPTOR_POOL)
3686
 
VK_DEFINE_NONDISP_HANDLE_CASTS(radv_descriptor_set, header.base, VkDescriptorSet,
3687
 
                               VK_OBJECT_TYPE_DESCRIPTOR_SET)
 
3650
VK_DEFINE_NONDISP_HANDLE_CASTS(radv_buffer_view, base, VkBufferView, VK_OBJECT_TYPE_BUFFER_VIEW)
 
3651
VK_DEFINE_NONDISP_HANDLE_CASTS(radv_descriptor_pool, base, VkDescriptorPool, VK_OBJECT_TYPE_DESCRIPTOR_POOL)
 
3652
VK_DEFINE_NONDISP_HANDLE_CASTS(radv_descriptor_set, header.base, VkDescriptorSet, VK_OBJECT_TYPE_DESCRIPTOR_SET)
3688
3653
VK_DEFINE_NONDISP_HANDLE_CASTS(radv_descriptor_set_layout, vk.base, VkDescriptorSetLayout,
3689
3654
                               VK_OBJECT_TYPE_DESCRIPTOR_SET_LAYOUT)
3690
 
VK_DEFINE_NONDISP_HANDLE_CASTS(radv_descriptor_update_template, base,
3691
 
                               VkDescriptorUpdateTemplate,
 
3655
VK_DEFINE_NONDISP_HANDLE_CASTS(radv_descriptor_update_template, base, VkDescriptorUpdateTemplate,
3692
3656
                               VK_OBJECT_TYPE_DESCRIPTOR_UPDATE_TEMPLATE)
3693
 
VK_DEFINE_NONDISP_HANDLE_CASTS(radv_device_memory, base, VkDeviceMemory,
3694
 
                               VK_OBJECT_TYPE_DEVICE_MEMORY)
 
3657
VK_DEFINE_NONDISP_HANDLE_CASTS(radv_device_memory, base, VkDeviceMemory, VK_OBJECT_TYPE_DEVICE_MEMORY)
3695
3658
VK_DEFINE_NONDISP_HANDLE_CASTS(radv_event, base, VkEvent, VK_OBJECT_TYPE_EVENT)
3696
3659
VK_DEFINE_NONDISP_HANDLE_CASTS(radv_image, vk.base, VkImage, VK_OBJECT_TYPE_IMAGE)
3697
 
VK_DEFINE_NONDISP_HANDLE_CASTS(radv_image_view, vk.base, VkImageView,
3698
 
                               VK_OBJECT_TYPE_IMAGE_VIEW);
 
3660
VK_DEFINE_NONDISP_HANDLE_CASTS(radv_image_view, vk.base, VkImageView, VK_OBJECT_TYPE_IMAGE_VIEW);
3699
3661
VK_DEFINE_NONDISP_HANDLE_CASTS(radv_indirect_command_layout, base, VkIndirectCommandsLayoutNV,
3700
3662
                               VK_OBJECT_TYPE_INDIRECT_COMMANDS_LAYOUT_NV)
3701
 
VK_DEFINE_NONDISP_HANDLE_CASTS(radv_pipeline, base, VkPipeline,
3702
 
                               VK_OBJECT_TYPE_PIPELINE)
3703
 
VK_DEFINE_NONDISP_HANDLE_CASTS(radv_pipeline_layout, base, VkPipelineLayout,
3704
 
                               VK_OBJECT_TYPE_PIPELINE_LAYOUT)
3705
 
VK_DEFINE_NONDISP_HANDLE_CASTS(radv_query_pool, base, VkQueryPool,
3706
 
                               VK_OBJECT_TYPE_QUERY_POOL)
3707
 
VK_DEFINE_NONDISP_HANDLE_CASTS(radv_sampler, base, VkSampler,
3708
 
                               VK_OBJECT_TYPE_SAMPLER)
 
3663
VK_DEFINE_NONDISP_HANDLE_CASTS(radv_pipeline, base, VkPipeline, VK_OBJECT_TYPE_PIPELINE)
 
3664
VK_DEFINE_NONDISP_HANDLE_CASTS(radv_pipeline_layout, base, VkPipelineLayout, VK_OBJECT_TYPE_PIPELINE_LAYOUT)
 
3665
VK_DEFINE_NONDISP_HANDLE_CASTS(radv_query_pool, base, VkQueryPool, VK_OBJECT_TYPE_QUERY_POOL)
 
3666
VK_DEFINE_NONDISP_HANDLE_CASTS(radv_sampler, base, VkSampler, VK_OBJECT_TYPE_SAMPLER)
3709
3667
 
3710
3668
VK_DEFINE_NONDISP_HANDLE_CASTS(radv_video_session, vk.base, VkVideoSessionKHR, VK_OBJECT_TYPE_VIDEO_SESSION_KHR)
3711
 
VK_DEFINE_NONDISP_HANDLE_CASTS(radv_video_session_params, vk.base, VkVideoSessionParametersKHR, VK_OBJECT_TYPE_VIDEO_SESSION_PARAMETERS_KHR)
 
3669
VK_DEFINE_NONDISP_HANDLE_CASTS(radv_video_session_params, vk.base, VkVideoSessionParametersKHR,
 
3670
                               VK_OBJECT_TYPE_VIDEO_SESSION_PARAMETERS_KHR)
3712
3671
 
3713
3672
#ifdef __cplusplus
3714
3673
}