← Back to branch summary

~mmach/netext73/mesa-ryzen

« back to all changes in this revision

Viewing changes to src/gallium/drivers/freedreno/a6xx/fd6_draw.cc

  • Committer: mmach
  • Date: 2023-11-02 21:31:35 UTC
  • Revision ID: netbit73@gmail.com-20231102213135-18d4tzh7tj0uz752
2023-11-02 22:11:57

Show diffs side-by-side

added added

removed removed

Lines of Context:
src/gallium/drivers/freedreno/a6xx/fd6_draw.cc
32
32
#include "util/u_prim.h"
33
33
#include "util/u_string.h"
34
34
 
 
35
#include "freedreno_blitter.h"
35
36
#include "freedreno_resource.h"
36
37
#include "freedreno_state.h"
37
38
 
38
39
#include "fd6_barrier.h"
 
40
#include "fd6_blitter.h"
39
41
#include "fd6_context.h"
40
42
#include "fd6_draw.h"
41
43
#include "fd6_emit.h"
45
47
 
46
48
#include "fd6_pack.h"
47
49
 
 
50
enum draw_type {
 
51
   DRAW_DIRECT_OP_NORMAL,
 
52
   DRAW_DIRECT_OP_INDEXED,
 
53
   DRAW_INDIRECT_OP_XFB,
 
54
   DRAW_INDIRECT_OP_INDIRECT_COUNT_INDEXED,
 
55
   DRAW_INDIRECT_OP_INDIRECT_COUNT,
 
56
   DRAW_INDIRECT_OP_INDEXED,
 
57
   DRAW_INDIRECT_OP_NORMAL,
 
58
};
 
59
 
 
60
static inline bool
 
61
is_indirect(enum draw_type type)
 
62
{
 
63
   return type >= DRAW_INDIRECT_OP_XFB;
 
64
}
 
65
 
 
66
static inline bool
 
67
is_indexed(enum draw_type type)
 
68
{
 
69
   switch (type) {
 
70
   case DRAW_DIRECT_OP_INDEXED:
 
71
   case DRAW_INDIRECT_OP_INDIRECT_COUNT_INDEXED:
 
72
   case DRAW_INDIRECT_OP_INDEXED:
 
73
      return true;
 
74
   default:
 
75
      return false;
 
76
   }
 
77
}
 
78
 
48
79
static void
49
80
draw_emit_xfb(struct fd_ringbuffer *ring, struct CP_DRAW_INDX_OFFSET_0 *draw0,
50
81
              const struct pipe_draw_info *info,
54
85
      fd_stream_output_target(indirect->count_from_stream_output);
55
86
   struct fd_resource *offset = fd_resource(target->offset_buf);
56
87
 
57
 
   /* All known firmware versions do not wait for WFI's with CP_DRAW_AUTO.
58
 
    * Plus, for the common case where the counter buffer is written by
59
 
    * vkCmdEndTransformFeedback, we need to wait for the CP_WAIT_MEM_WRITES to
60
 
    * complete which means we need a WAIT_FOR_ME anyway.
61
 
    */
62
 
   OUT_PKT7(ring, CP_WAIT_FOR_ME, 0);
63
 
 
64
88
   OUT_PKT7(ring, CP_DRAW_AUTO, 6);
65
89
   OUT_RING(ring, pack_CP_DRAW_INDX_OFFSET_0(*draw0).value);
66
90
   OUT_RING(ring, info->instance_count);
71
95
   OUT_RING(ring, target->stride);
72
96
}
73
97
 
 
98
static inline unsigned
 
99
max_indices(const struct pipe_draw_info *info, unsigned index_offset)
 
100
{
 
101
   struct pipe_resource *idx = info->index.resource;
 
102
 
 
103
   assert((info->index_size == 1) ||
 
104
          (info->index_size == 2) ||
 
105
          (info->index_size == 4));
 
106
 
 
107
   /* Conceptually we divide by the index_size.  But if we had
 
108
    * log2(index_size) we could convert that into a right-shift
 
109
    * instead.  Conveniently the index_size will only be 1, 2,
 
110
    * or 4.  And dividing by two (right-shift by one) gives us
 
111
    * the same answer for those three values.  So instead of
 
112
    * divide we can do two right-shifts.
 
113
    */
 
114
   unsigned index_size_shift = info->index_size >> 1;
 
115
   return (idx->width0 - index_offset) >> index_size_shift;
 
116
}
 
117
 
 
118
template <draw_type DRAW>
74
119
static void
75
 
draw_emit_indirect(struct fd_ringbuffer *ring,
 
120
draw_emit_indirect(struct fd_context *ctx,
 
121
                   struct fd_ringbuffer *ring,
76
122
                   struct CP_DRAW_INDX_OFFSET_0 *draw0,
77
123
                   const struct pipe_draw_info *info,
78
124
                   const struct pipe_draw_indirect_info *indirect,
79
 
                   unsigned index_offset)
 
125
                   unsigned index_offset, uint32_t driver_param)
80
126
{
81
127
   struct fd_resource *ind = fd_resource(indirect->buffer);
82
128
 
83
 
   if (info->index_size) {
84
 
      struct pipe_resource *idx = info->index.resource;
85
 
      unsigned max_indices = (idx->width0 - index_offset) / info->index_size;
86
 
 
87
 
      OUT_PKT(ring, CP_DRAW_INDX_INDIRECT, pack_CP_DRAW_INDX_OFFSET_0(*draw0),
88
 
              A5XX_CP_DRAW_INDX_INDIRECT_INDX_BASE(fd_resource(idx)->bo,
89
 
                                                   index_offset),
90
 
              A5XX_CP_DRAW_INDX_INDIRECT_3(.max_indices = max_indices),
91
 
              A5XX_CP_DRAW_INDX_INDIRECT_INDIRECT(ind->bo, indirect->offset));
92
 
   } else {
93
 
      OUT_PKT(ring, CP_DRAW_INDIRECT, pack_CP_DRAW_INDX_OFFSET_0(*draw0),
94
 
              A5XX_CP_DRAW_INDIRECT_INDIRECT(ind->bo, indirect->offset));
 
129
   if (DRAW == DRAW_INDIRECT_OP_INDIRECT_COUNT_INDEXED) {
 
130
      OUT_PKT7(ring, CP_DRAW_INDIRECT_MULTI, 11);
 
131
      OUT_RING(ring, pack_CP_DRAW_INDX_OFFSET_0(*draw0).value);
 
132
      OUT_RING(ring,
 
133
         (A6XX_CP_DRAW_INDIRECT_MULTI_1_OPCODE(INDIRECT_OP_INDIRECT_COUNT_INDEXED)
 
134
         | A6XX_CP_DRAW_INDIRECT_MULTI_1_DST_OFF(driver_param)));
 
135
      struct fd_resource *count_buf = fd_resource(indirect->indirect_draw_count);
 
136
      struct pipe_resource *idx = info->index.resource;
 
137
      OUT_RING(ring, indirect->draw_count);
 
138
      OUT_RELOC(ring, fd_resource(idx)->bo, index_offset, 0, 0);
 
139
      OUT_RING(ring, max_indices(info, index_offset));
 
140
      OUT_RELOC(ring, ind->bo, indirect->offset, 0, 0);
 
141
      OUT_RELOC(ring, count_buf->bo, indirect->indirect_draw_count_offset, 0, 0);
 
142
      OUT_RING(ring, indirect->stride);
 
143
   } else if (DRAW == DRAW_INDIRECT_OP_INDEXED) {
 
144
      OUT_PKT7(ring, CP_DRAW_INDIRECT_MULTI, 9);
 
145
      OUT_RING(ring, pack_CP_DRAW_INDX_OFFSET_0(*draw0).value);
 
146
      OUT_RING(ring,
 
147
         (A6XX_CP_DRAW_INDIRECT_MULTI_1_OPCODE(INDIRECT_OP_INDEXED)
 
148
         | A6XX_CP_DRAW_INDIRECT_MULTI_1_DST_OFF(driver_param)));
 
149
      struct pipe_resource *idx = info->index.resource;
 
150
      OUT_RING(ring, indirect->draw_count);
 
151
      //index va
 
152
      OUT_RELOC(ring, fd_resource(idx)->bo, index_offset, 0, 0);
 
153
      //max indices
 
154
      OUT_RING(ring, max_indices(info, index_offset));
 
155
      OUT_RELOC(ring, ind->bo, indirect->offset, 0, 0);
 
156
      OUT_RING(ring, indirect->stride);
 
157
   }  else if(DRAW == DRAW_INDIRECT_OP_INDIRECT_COUNT) {
 
158
      OUT_PKT7(ring, CP_DRAW_INDIRECT_MULTI, 8);
 
159
      OUT_RING(ring, pack_CP_DRAW_INDX_OFFSET_0(*draw0).value);
 
160
      OUT_RING(ring,
 
161
         (A6XX_CP_DRAW_INDIRECT_MULTI_1_OPCODE(INDIRECT_OP_INDIRECT_COUNT)
 
162
         | A6XX_CP_DRAW_INDIRECT_MULTI_1_DST_OFF(driver_param)));
 
163
      struct fd_resource *count_buf = fd_resource(indirect->indirect_draw_count);
 
164
      OUT_RING(ring, indirect->draw_count);
 
165
      OUT_RELOC(ring, ind->bo, indirect->offset, 0, 0);
 
166
      OUT_RELOC(ring, count_buf->bo, indirect->indirect_draw_count_offset, 0, 0);
 
167
      OUT_RING(ring, indirect->stride);
 
168
   } else if (DRAW == DRAW_INDIRECT_OP_NORMAL) {
 
169
      OUT_PKT7(ring, CP_DRAW_INDIRECT_MULTI, 6);
 
170
      OUT_RING(ring, pack_CP_DRAW_INDX_OFFSET_0(*draw0).value);
 
171
      OUT_RING(ring,
 
172
         (A6XX_CP_DRAW_INDIRECT_MULTI_1_OPCODE(INDIRECT_OP_NORMAL)
 
173
         | A6XX_CP_DRAW_INDIRECT_MULTI_1_DST_OFF(driver_param)));
 
174
      OUT_RING(ring, indirect->draw_count);
 
175
      OUT_RELOC(ring, ind->bo, indirect->offset, 0, 0);
 
176
      OUT_RING(ring, indirect->stride);
95
177
   }
96
178
}
97
179
 
 
180
template <draw_type DRAW>
98
181
static void
99
182
draw_emit(struct fd_ringbuffer *ring, struct CP_DRAW_INDX_OFFSET_0 *draw0,
100
183
          const struct pipe_draw_info *info,
101
184
          const struct pipe_draw_start_count_bias *draw, unsigned index_offset)
102
185
{
103
 
   if (info->index_size) {
 
186
   if (DRAW == DRAW_DIRECT_OP_INDEXED) {
104
187
      assert(!info->has_user_indices);
105
188
 
106
189
      struct pipe_resource *idx_buffer = info->index.resource;
107
 
      unsigned max_indices =
108
 
         (idx_buffer->width0 - index_offset) / info->index_size;
109
190
 
110
191
      OUT_PKT(ring, CP_DRAW_INDX_OFFSET, pack_CP_DRAW_INDX_OFFSET_0(*draw0),
111
192
              CP_DRAW_INDX_OFFSET_1(.num_instances = info->instance_count),
113
194
              CP_DRAW_INDX_OFFSET_3(.first_indx = draw->start),
114
195
              A5XX_CP_DRAW_INDX_OFFSET_INDX_BASE(fd_resource(idx_buffer)->bo,
115
196
                                                 index_offset),
116
 
              A5XX_CP_DRAW_INDX_OFFSET_6(.max_indices = max_indices));
117
 
   } else {
 
197
              A5XX_CP_DRAW_INDX_OFFSET_6(.max_indices = max_indices(info, index_offset)));
 
198
   } else if (DRAW == DRAW_DIRECT_OP_NORMAL) {
118
199
      OUT_PKT(ring, CP_DRAW_INDX_OFFSET, pack_CP_DRAW_INDX_OFFSET_0(*draw0),
119
200
              CP_DRAW_INDX_OFFSET_1(.num_instances = info->instance_count),
120
201
              CP_DRAW_INDX_OFFSET_2(.num_indices = draw->count));
132
213
   }
133
214
}
134
215
 
 
216
template <fd6_pipeline_type PIPELINE>
135
217
static const struct fd6_program_state *
136
218
get_program_state(struct fd_context *ctx, const struct pipe_draw_info *info)
137
219
   assert_dt
153
235
   key.key.msaa = (ctx->framebuffer.samples > 1);
154
236
   key.key.rasterflat = ctx->rasterizer->flatshade;
155
237
 
156
 
   if (info->mode == PIPE_PRIM_PATCHES) {
157
 
      struct shader_info *gs_info =
158
 
            ir3_get_shader_info((struct ir3_shader_state *)ctx->prog.gs);
159
 
 
160
 
      key.hs = (struct ir3_shader_state *)ctx->prog.hs;
161
 
      key.ds = (struct ir3_shader_state *)ctx->prog.ds;
162
 
 
163
 
      struct shader_info *ds_info = ir3_get_shader_info(key.ds);
164
 
      key.key.tessellation = ir3_tess_mode(ds_info->tess._primitive_mode);
165
 
 
166
 
      struct shader_info *fs_info = ir3_get_shader_info(key.fs);
167
 
      key.key.tcs_store_primid =
168
 
         BITSET_TEST(ds_info->system_values_read, SYSTEM_VALUE_PRIMITIVE_ID) ||
169
 
         (gs_info && BITSET_TEST(gs_info->system_values_read, SYSTEM_VALUE_PRIMITIVE_ID)) ||
170
 
         (fs_info && (fs_info->inputs_read & (1ull << VARYING_SLOT_PRIMITIVE_ID)));
171
 
   }
172
 
 
173
 
   if (key.gs) {
174
 
      key.key.has_gs = true;
 
238
   if (PIPELINE == HAS_TESS_GS) {
 
239
      if (info->mode == MESA_PRIM_PATCHES) {
 
240
         struct shader_info *gs_info =
 
241
               ir3_get_shader_info((struct ir3_shader_state *)ctx->prog.gs);
 
242
 
 
243
         key.hs = (struct ir3_shader_state *)ctx->prog.hs;
 
244
         key.ds = (struct ir3_shader_state *)ctx->prog.ds;
 
245
 
 
246
         struct shader_info *ds_info = ir3_get_shader_info(key.ds);
 
247
         key.key.tessellation = ir3_tess_mode(ds_info->tess._primitive_mode);
 
248
 
 
249
         struct shader_info *fs_info = ir3_get_shader_info(key.fs);
 
250
         key.key.tcs_store_primid =
 
251
               BITSET_TEST(ds_info->system_values_read, SYSTEM_VALUE_PRIMITIVE_ID) ||
 
252
               (gs_info && BITSET_TEST(gs_info->system_values_read, SYSTEM_VALUE_PRIMITIVE_ID)) ||
 
253
               (fs_info && (fs_info->inputs_read & (1ull << VARYING_SLOT_PRIMITIVE_ID)));
 
254
      }
 
255
 
 
256
      if (key.gs) {
 
257
         key.key.has_gs = true;
 
258
      }
175
259
   }
176
260
 
177
261
   ir3_fixup_shader_state(&ctx->base, &key.key);
202
286
   }
203
287
}
204
288
 
 
289
template <chip CHIP, fd6_pipeline_type PIPELINE, draw_type DRAW>
205
290
static void
206
 
fd6_draw_vbos(struct fd_context *ctx, const struct pipe_draw_info *info,
207
 
              unsigned drawid_offset,
208
 
              const struct pipe_draw_indirect_info *indirect,
209
 
              const struct pipe_draw_start_count_bias *draws,
210
 
              unsigned num_draws,
211
 
              unsigned index_offset)
 
291
draw_vbos(struct fd_context *ctx, const struct pipe_draw_info *info,
 
292
          unsigned drawid_offset,
 
293
          const struct pipe_draw_indirect_info *indirect,
 
294
          const struct pipe_draw_start_count_bias *draws,
 
295
          unsigned num_draws,
 
296
          unsigned index_offset)
212
297
   assert_dt
213
298
{
214
299
   struct fd6_context *fd6_ctx = fd6_context(ctx);
221
306
   emit.rasterflat = ctx->rasterizer->flatshade;
222
307
   emit.sprite_coord_enable = ctx->rasterizer->sprite_coord_enable;
223
308
   emit.sprite_coord_mode = ctx->rasterizer->sprite_coord_mode;
224
 
   emit.primitive_restart = info->primitive_restart && info->index_size;
 
309
   emit.primitive_restart = info->primitive_restart && is_indexed(DRAW);
225
310
   emit.state.num_groups = 0;
226
311
   emit.streamout_mask = 0;
227
312
   emit.prog = NULL;
 
313
   emit.draw_id = 0;
228
314
 
229
315
   if (!(ctx->prog.vs && ctx->prog.fs))
230
316
      return;
231
317
 
232
 
   if ((info->mode == PIPE_PRIM_PATCHES) || ctx->prog.gs) {
233
 
      ctx->gen_dirty |= BIT(FD6_GROUP_PRIMITIVE_PARAMS);
234
 
   } else if (!indirect) {
 
318
   if (PIPELINE == HAS_TESS_GS) {
 
319
      if ((info->mode == MESA_PRIM_PATCHES) || ctx->prog.gs) {
 
320
         ctx->gen_dirty |= BIT(FD6_GROUP_PRIMITIVE_PARAMS);
 
321
      }
 
322
   }
 
323
 
 
324
   if ((PIPELINE == NO_TESS_GS) && !is_indirect(DRAW)) {
235
325
      fd6_vsc_update_sizes(ctx->batch, info, &draws[0]);
236
326
   }
237
327
 
241
331
    * Otherwise we can just use the previous prog state.
242
332
    */
243
333
   if (unlikely(ctx->gen_dirty & BIT(FD6_GROUP_PROG_KEY))) {
244
 
      emit.prog = get_program_state(ctx, info);
 
334
      emit.prog = get_program_state<PIPELINE>(ctx, info);
245
335
   } else {
246
336
      emit.prog = fd6_ctx->prog;
247
337
   }
256
346
   emit.dirty_groups = ctx->gen_dirty;
257
347
 
258
348
   emit.vs = fd6_emit_get_prog(&emit)->vs;
259
 
   emit.hs = fd6_emit_get_prog(&emit)->hs;
260
 
   emit.ds = fd6_emit_get_prog(&emit)->ds;
261
 
   emit.gs = fd6_emit_get_prog(&emit)->gs;
 
349
   if (PIPELINE == HAS_TESS_GS) {
 
350
      emit.hs = fd6_emit_get_prog(&emit)->hs;
 
351
      emit.ds = fd6_emit_get_prog(&emit)->ds;
 
352
      emit.gs = fd6_emit_get_prog(&emit)->gs;
 
353
   }
262
354
   emit.fs = fd6_emit_get_prog(&emit)->fs;
263
355
 
264
356
   if (emit.prog->num_driver_params || fd6_ctx->has_dp_state) {
286
378
      .gs_enable = !!ctx->prog.gs,
287
379
   };
288
380
 
289
 
   if (indirect && indirect->count_from_stream_output) {
 
381
   if (DRAW == DRAW_INDIRECT_OP_XFB) {
290
382
      draw0.source_select = DI_SRC_SEL_AUTO_XFB;
291
 
   } else if (info->index_size) {
 
383
   } else if (DRAW == DRAW_DIRECT_OP_INDEXED ||
 
384
              DRAW == DRAW_INDIRECT_OP_INDIRECT_COUNT_INDEXED ||
 
385
              DRAW == DRAW_INDIRECT_OP_INDEXED) {
292
386
      draw0.source_select = DI_SRC_SEL_DMA;
293
387
      draw0.index_size = fd4_size2indextype(info->index_size);
294
388
   } else {
295
389
      draw0.source_select = DI_SRC_SEL_AUTO_INDEX;
296
390
   }
297
391
 
298
 
   if (info->mode == PIPE_PRIM_PATCHES) {
 
392
   if ((PIPELINE == HAS_TESS_GS) && (info->mode == MESA_PRIM_PATCHES)) {
299
393
      struct shader_info *ds_info =
300
394
            ir3_get_shader_info((struct ir3_shader_state *)ctx->prog.ds);
301
395
      unsigned tessellation = ir3_tess_mode(ds_info->tess._primitive_mode);
322
416
      ctx->batch->tessellation = true;
323
417
   }
324
418
 
325
 
   uint32_t index_start = info->index_size ? draws[0].index_bias : draws[0].start;
 
419
   uint32_t index_start = is_indexed(DRAW) ? draws[0].index_bias : draws[0].start;
326
420
   if (ctx->last.dirty || (ctx->last.index_start != index_start)) {
327
421
      OUT_PKT4(ring, REG_A6XX_VFD_INDEX_OFFSET, 1);
328
422
      OUT_RING(ring, index_start); /* VFD_INDEX_OFFSET */
344
438
   }
345
439
 
346
440
   if (emit.dirty_groups)
347
 
      fd6_emit_3d_state(ring, &emit);
 
441
      fd6_emit_3d_state<CHIP, PIPELINE>(ring, &emit);
 
442
 
 
443
   /* All known firmware versions do not wait for WFI's with CP_DRAW_AUTO.
 
444
    * Plus, for the common case where the counter buffer is written by
 
445
    * vkCmdEndTransformFeedback, we need to wait for the CP_WAIT_MEM_WRITES to
 
446
    * complete which means we need a WAIT_FOR_ME anyway.
 
447
    *
 
448
    * Also, on some firmwares CP_DRAW_INDIRECT_MULTI waits for WFIs before
 
449
    * reading the draw parameters but after reading the count, so commands
 
450
    * that use indirect draw count need a WFM anyway.
 
451
    */
 
452
   if (DRAW == DRAW_INDIRECT_OP_XFB ||
 
453
       DRAW == DRAW_INDIRECT_OP_INDIRECT_COUNT_INDEXED ||
 
454
       DRAW == DRAW_INDIRECT_OP_INDIRECT_COUNT)
 
455
      ctx->batch->barrier |= FD6_WAIT_FOR_ME;
348
456
 
349
457
   if (ctx->batch->barrier)
350
458
      fd6_barrier_flush(ctx->batch);
357
465
    */
358
466
   emit_marker6(ring, 7);
359
467
 
360
 
   if (indirect) {
 
468
   if (is_indirect(DRAW)) {
361
469
      assert(num_draws == 1);  /* only >1 for direct draws */
362
 
      if (indirect->count_from_stream_output) {
 
470
      if (DRAW == DRAW_INDIRECT_OP_XFB) {
363
471
         draw_emit_xfb(ring, &draw0, info, indirect);
364
472
      } else {
365
 
         draw_emit_indirect(ring, &draw0, info, indirect, index_offset);
 
473
         const struct ir3_const_state *const_state = ir3_const_state(emit.vs);
 
474
         uint32_t dst_offset_dp = const_state->offsets.driver_param;
 
475
 
 
476
         /* If unused, pass 0 for DST_OFF: */
 
477
         if (dst_offset_dp > emit.vs->constlen)
 
478
            dst_offset_dp = 0;
 
479
 
 
480
         draw_emit_indirect<DRAW>(ctx, ring, &draw0, info, indirect, index_offset, dst_offset_dp);
366
481
      }
367
482
   } else {
368
 
      draw_emit(ring, &draw0, info, &draws[0], index_offset);
 
483
      draw_emit<DRAW>(ring, &draw0, info, &draws[0], index_offset);
369
484
 
370
485
      if (unlikely(num_draws > 1)) {
371
486
 
388
503
 
389
504
            fd6_vsc_update_sizes(ctx->batch, info, &draws[i]);
390
505
 
391
 
            uint32_t index_start = info->index_size ? draws[i].index_bias : draws[i].start;
 
506
            uint32_t index_start = is_indexed(DRAW) ? draws[i].index_bias : draws[i].start;
392
507
            if (last_index_start != index_start) {
393
508
               OUT_PKT4(ring, REG_A6XX_VFD_INDEX_OFFSET, 1);
394
509
               OUT_RING(ring, index_start); /* VFD_INDEX_OFFSET */
398
513
            if (emit.dirty_groups) {
399
514
               emit.state.num_groups = 0;
400
515
               emit.draw = &draws[i];
401
 
               fd6_emit_3d_state(ring, &emit);
 
516
               emit.draw_id = info->increment_draw_id ? i : 0;
 
517
               fd6_emit_3d_state<CHIP, PIPELINE>(ring, &emit);
402
518
            }
403
519
 
404
520
            assert(!index_offset); /* handled by util_draw_multi() */
405
521
 
406
 
            draw_emit(ring, &draw0, info, &draws[i], 0);
 
522
            draw_emit<DRAW>(ring, &draw0, info, &draws[i], 0);
407
523
         }
408
524
 
409
525
         ctx->last.index_start = last_index_start;
411
527
   }
412
528
 
413
529
   emit_marker6(ring, 7);
414
 
   fd_reset_wfi(ctx->batch);
415
530
 
416
531
   flush_streamout(ctx, &emit);
417
532
 
418
533
   fd_context_all_clean(ctx);
419
534
}
420
535
 
421
 
static void
422
 
fd6_clear_lrz(struct fd_batch *batch, struct fd_resource *zsbuf, double depth) assert_dt
423
 
{
424
 
   struct fd_ringbuffer *ring;
425
 
   struct fd_screen *screen = batch->ctx->screen;
426
 
 
427
 
   ring = fd_batch_get_prologue(batch);
428
 
 
429
 
   emit_marker6(ring, 7);
430
 
   OUT_PKT7(ring, CP_SET_MARKER, 1);
431
 
   OUT_RING(ring, A6XX_CP_SET_MARKER_0_MODE(RM6_BYPASS));
432
 
   emit_marker6(ring, 7);
433
 
 
434
 
   OUT_WFI5(ring);
435
 
 
436
 
   fd6_emit_ccu_cntl(ring, screen, false);
437
 
 
438
 
   OUT_REG(ring,
439
 
           A6XX_HLSQ_INVALIDATE_CMD(.vs_state = true, .hs_state = true,
440
 
                                    .ds_state = true, .gs_state = true,
441
 
                                    .fs_state = true, .cs_state = true,
442
 
                                    .cs_ibo = true, .gfx_ibo = true,
443
 
                                    .gfx_shared_const = true,
444
 
                                    .cs_bindless = 0x1f, .gfx_bindless = 0x1f));
445
 
 
446
 
   emit_marker6(ring, 7);
447
 
   OUT_PKT7(ring, CP_SET_MARKER, 1);
448
 
   OUT_RING(ring, A6XX_CP_SET_MARKER_0_MODE(RM6_BLIT2DSCALE));
449
 
   emit_marker6(ring, 7);
450
 
 
451
 
   OUT_PKT4(ring, REG_A6XX_RB_2D_UNKNOWN_8C01, 1);
452
 
   OUT_RING(ring, 0x0);
453
 
 
454
 
   OUT_PKT4(ring, REG_A6XX_SP_PS_2D_SRC_INFO, 13);
455
 
   OUT_RING(ring, 0x00000000);
456
 
   OUT_RING(ring, 0x00000000);
457
 
   OUT_RING(ring, 0x00000000);
458
 
   OUT_RING(ring, 0x00000000);
459
 
   OUT_RING(ring, 0x00000000);
460
 
   OUT_RING(ring, 0x00000000);
461
 
   OUT_RING(ring, 0x00000000);
462
 
   OUT_RING(ring, 0x00000000);
463
 
   OUT_RING(ring, 0x00000000);
464
 
   OUT_RING(ring, 0x00000000);
465
 
   OUT_RING(ring, 0x00000000);
466
 
   OUT_RING(ring, 0x00000000);
467
 
   OUT_RING(ring, 0x00000000);
468
 
 
469
 
   OUT_PKT4(ring, REG_A6XX_SP_2D_DST_FORMAT, 1);
470
 
   OUT_RING(ring, 0x0000f410);
471
 
 
472
 
   OUT_PKT4(ring, REG_A6XX_GRAS_2D_BLIT_CNTL, 1);
473
 
   OUT_RING(ring,
474
 
            A6XX_GRAS_2D_BLIT_CNTL_COLOR_FORMAT(FMT6_16_UNORM) | 0x4f00080);
475
 
 
476
 
   OUT_PKT4(ring, REG_A6XX_RB_2D_BLIT_CNTL, 1);
477
 
   OUT_RING(ring, A6XX_RB_2D_BLIT_CNTL_COLOR_FORMAT(FMT6_16_UNORM) | 0x4f00080);
478
 
 
479
 
   fd6_event_write(batch, ring, PC_CCU_FLUSH_COLOR_TS, true);
480
 
   fd6_event_write(batch, ring, PC_CCU_INVALIDATE_COLOR, false);
481
 
   fd_wfi(batch, ring);
482
 
 
483
 
   OUT_PKT4(ring, REG_A6XX_RB_2D_SRC_SOLID_C0, 4);
484
 
   OUT_RING(ring, fui(depth));
485
 
   OUT_RING(ring, 0x00000000);
486
 
   OUT_RING(ring, 0x00000000);
487
 
   OUT_RING(ring, 0x00000000);
488
 
 
489
 
   OUT_PKT4(ring, REG_A6XX_RB_2D_DST_INFO, 9);
490
 
   OUT_RING(ring, A6XX_RB_2D_DST_INFO_COLOR_FORMAT(FMT6_16_UNORM) |
491
 
                     A6XX_RB_2D_DST_INFO_TILE_MODE(TILE6_LINEAR) |
492
 
                     A6XX_RB_2D_DST_INFO_COLOR_SWAP(WZYX));
493
 
   OUT_RELOC(ring, zsbuf->lrz, 0, 0, 0);
494
 
   OUT_RING(ring, A6XX_RB_2D_DST_PITCH(zsbuf->lrz_pitch * 2).value);
495
 
   OUT_RING(ring, 0x00000000);
496
 
   OUT_RING(ring, 0x00000000);
497
 
   OUT_RING(ring, 0x00000000);
498
 
   OUT_RING(ring, 0x00000000);
499
 
   OUT_RING(ring, 0x00000000);
500
 
 
501
 
   OUT_REG(ring, A6XX_GRAS_2D_SRC_TL_X(0), A6XX_GRAS_2D_SRC_BR_X(0),
502
 
           A6XX_GRAS_2D_SRC_TL_Y(0), A6XX_GRAS_2D_SRC_BR_Y(0));
503
 
 
504
 
   OUT_PKT4(ring, REG_A6XX_GRAS_2D_DST_TL, 2);
505
 
   OUT_RING(ring, A6XX_GRAS_2D_DST_TL_X(0) | A6XX_GRAS_2D_DST_TL_Y(0));
506
 
   OUT_RING(ring, A6XX_GRAS_2D_DST_BR_X(zsbuf->lrz_width - 1) |
507
 
                     A6XX_GRAS_2D_DST_BR_Y(zsbuf->lrz_height - 1));
508
 
 
509
 
   fd6_event_write(batch, ring, (enum vgt_event_type)0x3f, false);
510
 
 
511
 
   if (screen->info->a6xx.magic.RB_DBG_ECO_CNTL_blit != screen->info->a6xx.magic.RB_DBG_ECO_CNTL) {
512
 
      /* This a non-context register, so we have to WFI before changing. */
513
 
      OUT_WFI5(ring);
514
 
      OUT_PKT4(ring, REG_A6XX_RB_DBG_ECO_CNTL, 1);
515
 
      OUT_RING(ring, screen->info->a6xx.magic.RB_DBG_ECO_CNTL_blit);
516
 
   }
517
 
 
518
 
   OUT_PKT7(ring, CP_BLIT, 1);
519
 
   OUT_RING(ring, CP_BLIT_0_OP(BLIT_OP_SCALE));
520
 
 
521
 
   if (screen->info->a6xx.magic.RB_DBG_ECO_CNTL_blit != screen->info->a6xx.magic.RB_DBG_ECO_CNTL) {
522
 
      OUT_WFI5(ring);
523
 
      OUT_PKT4(ring, REG_A6XX_RB_DBG_ECO_CNTL, 1);
524
 
      OUT_RING(ring, screen->info->a6xx.magic.RB_DBG_ECO_CNTL);
525
 
   }
526
 
 
527
 
   fd6_event_write(batch, ring, PC_CCU_FLUSH_COLOR_TS, true);
528
 
   fd6_event_write(batch, ring, PC_CCU_FLUSH_DEPTH_TS, true);
529
 
   fd6_event_write(batch, ring, CACHE_FLUSH_TS, true);
530
 
   fd_wfi(batch, ring);
531
 
 
532
 
   fd6_cache_inv(batch, ring);
 
536
template <chip CHIP, fd6_pipeline_type PIPELINE>
 
537
static void
 
538
fd6_draw_vbos(struct fd_context *ctx, const struct pipe_draw_info *info,
 
539
              unsigned drawid_offset,
 
540
              const struct pipe_draw_indirect_info *indirect,
 
541
              const struct pipe_draw_start_count_bias *draws,
 
542
              unsigned num_draws,
 
543
              unsigned index_offset)
 
544
   assert_dt
 
545
{
 
546
   /* Non-indirect case is where we are more likely to see a high draw rate: */
 
547
   if (likely(!indirect)) {
 
548
      if (info->index_size) {
 
549
         draw_vbos<CHIP, PIPELINE, DRAW_DIRECT_OP_INDEXED>(
 
550
               ctx, info, drawid_offset, NULL, draws, num_draws, index_offset);
 
551
      } else {
 
552
         draw_vbos<CHIP, PIPELINE, DRAW_DIRECT_OP_NORMAL>(
 
553
               ctx, info, drawid_offset, NULL, draws, num_draws, index_offset);
 
554
      }
 
555
   } else if (indirect->count_from_stream_output) {
 
556
      draw_vbos<CHIP, PIPELINE, DRAW_INDIRECT_OP_XFB>(
 
557
            ctx, info, drawid_offset, indirect, draws, num_draws, index_offset);
 
558
   } else if (indirect->indirect_draw_count && info->index_size) {
 
559
      draw_vbos<CHIP, PIPELINE, DRAW_INDIRECT_OP_INDIRECT_COUNT_INDEXED>(
 
560
            ctx, info, drawid_offset, indirect, draws, num_draws, index_offset);
 
561
   } else if (indirect->indirect_draw_count) {
 
562
      draw_vbos<CHIP, PIPELINE, DRAW_INDIRECT_OP_INDIRECT_COUNT>(
 
563
            ctx, info, drawid_offset, indirect, draws, num_draws, index_offset);
 
564
   } else if (info->index_size) {
 
565
      draw_vbos<CHIP, PIPELINE, DRAW_INDIRECT_OP_INDEXED>(
 
566
            ctx, info, drawid_offset, indirect, draws, num_draws, index_offset);
 
567
   } else {
 
568
      draw_vbos<CHIP, PIPELINE, DRAW_INDIRECT_OP_NORMAL>(
 
569
            ctx, info, drawid_offset, indirect, draws, num_draws, index_offset);
 
570
   }
 
571
}
 
572
 
 
573
template <chip CHIP>
 
574
static void
 
575
fd6_update_draw(struct fd_context *ctx)
 
576
{
 
577
   const uint32_t gs_tess_stages = BIT(MESA_SHADER_TESS_CTRL) |
 
578
         BIT(MESA_SHADER_TESS_EVAL) | BIT(MESA_SHADER_GEOMETRY);
 
579
 
 
580
   if (ctx->bound_shader_stages & gs_tess_stages) {
 
581
      ctx->draw_vbos = fd6_draw_vbos<CHIP, HAS_TESS_GS>;
 
582
   } else {
 
583
      ctx->draw_vbos = fd6_draw_vbos<CHIP, NO_TESS_GS>;
 
584
   }
533
585
}
534
586
 
535
587
static bool
536
 
is_z32(enum pipe_format format)
 
588
do_lrz_clear(struct fd_context *ctx, enum fd_buffer_mask buffers)
537
589
{
538
 
   switch (format) {
539
 
   case PIPE_FORMAT_Z32_FLOAT_S8X24_UINT:
540
 
   case PIPE_FORMAT_Z32_UNORM:
541
 
   case PIPE_FORMAT_Z32_FLOAT:
542
 
      return true;
543
 
   default:
 
590
   struct pipe_framebuffer_state *pfb = &ctx->batch->framebuffer;
 
591
 
 
592
   if (!pfb->zsbuf)
544
593
      return false;
545
 
   }
 
594
 
 
595
   struct fd_resource *zsbuf = fd_resource(pfb->zsbuf->texture);
 
596
 
 
597
   return (buffers & FD_BUFFER_DEPTH) && zsbuf->lrz;
546
598
}
547
599
 
548
600
static bool
551
603
          unsigned stencil) assert_dt
552
604
{
553
605
   struct pipe_framebuffer_state *pfb = &ctx->batch->framebuffer;
554
 
   const bool has_depth = pfb->zsbuf;
 
606
   struct fd_batch_subpass *subpass = ctx->batch->subpass;
555
607
   unsigned color_buffers = buffers >> 2;
556
608
 
557
 
   /* If we're clearing after draws, fallback to 3D pipe clears.  We could
558
 
    * use blitter clears in the draw batch but then we'd have to patch up the
559
 
    * gmem offsets. This doesn't seem like a useful thing to optimize for
560
 
    * however.*/
561
 
   if (ctx->batch->num_draws > 0)
562
 
      return false;
563
 
 
564
 
   if (has_depth && (buffers & FD_BUFFER_DEPTH)) {
 
609
   if (pfb->samples > 1) {
 
610
      /* we need to do multisample clear on 3d pipe, so fallback to u_blitter.
 
611
       * But we do this ourselves so that we can still benefit from LRZ, as
 
612
       * normally zfunc==ALWAYS would invalidate LRZ.  So we want to mark the
 
613
       * LRZ state as valid *after* the fallback clear.
 
614
       */
 
615
      fd_blitter_clear(&ctx->base, (unsigned)buffers, color, depth, stencil);
 
616
   }
 
617
 
 
618
   /* If we are clearing after draws, split out a new subpass:
 
619
    */
 
620
   if (subpass->num_draws > 0) {
 
621
      /* If we won't be able to do any fast-clears, avoid pointlessly
 
622
       * splitting out a new subpass:
 
623
       */
 
624
      if (pfb->samples > 1 && !do_lrz_clear(ctx, buffers))
 
625
         return true;
 
626
 
 
627
      subpass = fd_batch_create_subpass(ctx->batch);
 
628
 
 
629
      /* If doing an LRZ clear, replace the existing LRZ buffer with a
 
630
       * freshly allocated one so that we have valid LRZ state for the
 
631
       * new pass.  Otherwise unconditional writes to the depth buffer
 
632
       * would cause LRZ state to be invalid.
 
633
       */
 
634
      if (do_lrz_clear(ctx, buffers)) {
 
635
         struct fd_resource *zsbuf = fd_resource(pfb->zsbuf->texture);
 
636
 
 
637
         fd_bo_del(subpass->lrz);
 
638
         subpass->lrz = fd_bo_new(ctx->screen->dev, fd_bo_size(zsbuf->lrz),
 
639
                                  FD_BO_NOMAP, "lrz");
 
640
         fd_bo_del(zsbuf->lrz);
 
641
         zsbuf->lrz = fd_bo_ref(subpass->lrz);
 
642
      }
 
643
   }
 
644
 
 
645
   if (do_lrz_clear(ctx, buffers)) {
565
646
      struct fd_resource *zsbuf = fd_resource(pfb->zsbuf->texture);
566
 
      if (zsbuf->lrz && !is_z32(pfb->zsbuf->format)) {
567
 
         zsbuf->lrz_valid = true;
568
 
         zsbuf->lrz_direction = FD_LRZ_UNKNOWN;
569
 
         fd6_clear_lrz(ctx->batch, zsbuf, depth);
570
 
      }
 
647
 
 
648
      zsbuf->lrz_valid = true;
 
649
      zsbuf->lrz_direction = FD_LRZ_UNKNOWN;
 
650
      subpass->clear_depth = depth;
 
651
      subpass->fast_cleared |= FD_BUFFER_LRZ;
 
652
 
 
653
      STATIC_ASSERT((FD_BUFFER_LRZ & FD_BUFFER_ALL) == 0);
571
654
   }
572
655
 
573
 
   /* we need to do multisample clear on 3d pipe, so fallback to u_blitter: */
 
656
   /* We've already done the fallback 3d clear: */
574
657
   if (pfb->samples > 1)
575
 
      return false;
 
658
      return true;
576
659
 
577
660
   u_foreach_bit (i, color_buffers)
578
 
      ctx->batch->clear_color[i] = *color;
 
661
      subpass->clear_color[i] = *color;
579
662
   if (buffers & FD_BUFFER_DEPTH)
580
 
      ctx->batch->clear_depth = depth;
 
663
      subpass->clear_depth = depth;
581
664
   if (buffers & FD_BUFFER_STENCIL)
582
 
      ctx->batch->clear_stencil = stencil;
 
665
      subpass->clear_stencil = stencil;
583
666
 
584
 
   ctx->batch->fast_cleared |= buffers;
 
667
   subpass->fast_cleared |= buffers;
585
668
 
586
669
   return true;
587
670
}
588
671
 
 
672
template <chip CHIP>
589
673
void
590
 
fd6_draw_init(struct pipe_context *pctx) disable_thread_safety_analysis
 
674
fd6_draw_init(struct pipe_context *pctx)
 
675
   disable_thread_safety_analysis
591
676
{
592
677
   struct fd_context *ctx = fd_context(pctx);
593
 
   ctx->draw_vbos = fd6_draw_vbos;
594
678
   ctx->clear = fd6_clear;
 
679
   ctx->update_draw = fd6_update_draw<CHIP>;
 
680
   fd6_update_draw<CHIP>(ctx);
595
681
}
 
682
 
 
683
/* Teach the compiler about needed variants: */
 
684
template void fd6_draw_init<A6XX>(struct pipe_context *pctx);
 
685
template void fd6_draw_init<A7XX>(struct pipe_context *pctx);