2
* Copyright © 2019 Valve Corporation
4
* Permission is hereby granted, free of charge, to any person obtaining a
5
* copy of this software and associated documentation files (the "Software"),
6
* to deal in the Software without restriction, including without limitation
7
* the rights to use, copy, modify, merge, publish, distribute, sublicense,
8
* and/or sell copies of the Software, and to permit persons to whom the
9
* Software is furnished to do so, subject to the following conditions:
11
* The above copyright notice and this permission notice (including the next
12
* paragraph) shall be included in all copies or substantial portions of the
15
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
18
* THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19
* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
20
* FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
25
#include "aco_builder.h"
28
#include "util/u_math.h"
37
enum WQMState : uint8_t {
40
WQM = 1 << 1, /* with control flow applied */
43
enum mask_type : uint8_t {
44
mask_type_global = 1 << 0,
45
mask_type_exact = 1 << 1,
46
mask_type_wqm = 1 << 2,
47
mask_type_loop = 1 << 3, /* active lanes of a loop */
52
/* state for WQM propagation */
53
std::set<unsigned> worklist;
54
std::vector<bool> branch_wqm; /* true if the branch condition in this block should be in wqm */
55
wqm_ctx(Program* program_)
56
: program(program_), branch_wqm(program->blocks.size())
58
for (unsigned i = 0; i < program->blocks.size(); i++)
65
uint16_t num_exec_masks;
66
bool has_divergent_break;
67
bool has_divergent_continue;
68
bool has_discard; /* has a discard or demote */
69
loop_info(Block* b, uint16_t num, bool breaks, bool cont, bool discard)
70
: loop_header(b), num_exec_masks(num), has_divergent_break(breaks),
71
has_divergent_continue(cont), has_discard(discard)
76
std::vector<std::pair<Operand, uint8_t>>
77
exec; /* Vector of exec masks. Either a temporary or const -1. */
78
std::vector<WQMState> instr_needs;
84
std::vector<block_info> info;
85
std::vector<loop_info> loop;
86
bool handle_wqm = false;
87
exec_ctx(Program* program_) : program(program_), info(program->blocks.size()) {}
91
needs_exact(aco_ptr<Instruction>& instr)
93
if (instr->isMUBUF()) {
94
return instr->mubuf().disable_wqm;
95
} else if (instr->isMTBUF()) {
96
return instr->mtbuf().disable_wqm;
97
} else if (instr->isMIMG()) {
98
return instr->mimg().disable_wqm;
99
} else if (instr->isFlatLike()) {
100
return instr->flatlike().disable_wqm;
102
return instr->isEXP();
107
mark_block_wqm(wqm_ctx& ctx, unsigned block_idx)
109
if (ctx.branch_wqm[block_idx])
112
for (Block& block : ctx.program->blocks) {
113
if (block.index >= block_idx && block.kind & block_kind_top_level)
115
ctx.branch_wqm[block.index] = true;
116
ctx.worklist.insert(block.index);
121
get_block_needs(wqm_ctx& ctx, exec_ctx& exec_ctx, Block* block)
123
block_info& info = exec_ctx.info[block->index];
125
std::vector<WQMState> instr_needs(block->instructions.size());
127
bool propagate_wqm = ctx.branch_wqm[block->index];
128
for (int i = block->instructions.size() - 1; i >= 0; --i) {
129
aco_ptr<Instruction>& instr = block->instructions[i];
131
if (instr->opcode == aco_opcode::p_wqm)
132
propagate_wqm = true;
134
bool pred_by_exec = needs_exec_mask(instr.get()) ||
135
instr->opcode == aco_opcode::p_logical_end ||
138
if (needs_exact(instr))
139
instr_needs[i] = Exact;
140
else if (propagate_wqm && pred_by_exec)
141
instr_needs[i] = WQM;
143
instr_needs[i] = Unspecified;
145
info.block_needs |= instr_needs[i];
148
info.instr_needs = instr_needs;
150
/* for "if (<cond>) <wqm code>" or "while (<cond>) <wqm code>",
151
* <cond> should be computed in WQM */
152
if (info.block_needs & WQM) {
153
mark_block_wqm(ctx, block->index);
158
calculate_wqm_needs(exec_ctx& exec_ctx)
160
wqm_ctx ctx(exec_ctx.program);
162
while (!ctx.worklist.empty()) {
163
unsigned block_index = *std::prev(ctx.worklist.end());
164
ctx.worklist.erase(std::prev(ctx.worklist.end()));
166
Block& block = exec_ctx.program->blocks[block_index];
167
get_block_needs(ctx, exec_ctx, &block);
170
exec_ctx.handle_wqm = true;
174
get_exec_op(Operand t)
177
return Operand(exec, t.regClass());
183
transition_to_WQM(exec_ctx& ctx, Builder bld, unsigned idx)
185
if (ctx.info[idx].exec.back().second & mask_type_wqm)
187
if (ctx.info[idx].exec.back().second & mask_type_global) {
188
Operand exec_mask = ctx.info[idx].exec.back().first;
189
if (exec_mask.isUndefined()) {
190
exec_mask = bld.copy(bld.def(bld.lm), Operand(exec, bld.lm));
191
ctx.info[idx].exec.back().first = exec_mask;
194
exec_mask = bld.sop1(Builder::s_wqm, Definition(exec, bld.lm), bld.def(s1, scc),
195
get_exec_op(exec_mask));
196
ctx.info[idx].exec.emplace_back(exec_mask, mask_type_global | mask_type_wqm);
199
/* otherwise, the WQM mask should be one below the current mask */
200
ctx.info[idx].exec.pop_back();
201
assert(ctx.info[idx].exec.back().second & mask_type_wqm);
202
assert(ctx.info[idx].exec.back().first.size() == bld.lm.size());
203
assert(ctx.info[idx].exec.back().first.isTemp());
204
ctx.info[idx].exec.back().first =
205
bld.copy(Definition(exec, bld.lm), ctx.info[idx].exec.back().first);
209
transition_to_Exact(exec_ctx& ctx, Builder bld, unsigned idx)
211
if (ctx.info[idx].exec.back().second & mask_type_exact)
213
/* We can't remove the loop exec mask, because that can cause exec.size() to
214
* be less than num_exec_masks. The loop exec mask also needs to be kept
215
* around for various uses. */
216
if ((ctx.info[idx].exec.back().second & mask_type_global) &&
217
!(ctx.info[idx].exec.back().second & mask_type_loop)) {
218
ctx.info[idx].exec.pop_back();
219
assert(ctx.info[idx].exec.back().second & mask_type_exact);
220
assert(ctx.info[idx].exec.back().first.size() == bld.lm.size());
221
assert(ctx.info[idx].exec.back().first.isTemp());
222
ctx.info[idx].exec.back().first =
223
bld.copy(Definition(exec, bld.lm), ctx.info[idx].exec.back().first);
226
/* otherwise, we create an exact mask and push to the stack */
227
Operand wqm = ctx.info[idx].exec.back().first;
228
if (wqm.isUndefined()) {
229
wqm = bld.sop1(Builder::s_and_saveexec, bld.def(bld.lm), bld.def(s1, scc),
230
Definition(exec, bld.lm), ctx.info[idx].exec[0].first, Operand(exec, bld.lm));
232
bld.sop2(Builder::s_and, Definition(exec, bld.lm), bld.def(s1, scc),
233
ctx.info[idx].exec[0].first, wqm);
235
ctx.info[idx].exec.back().first = Operand(wqm);
236
ctx.info[idx].exec.emplace_back(Operand(bld.lm), mask_type_exact);
240
add_coupling_code(exec_ctx& ctx, Block* block, std::vector<aco_ptr<Instruction>>& instructions)
242
unsigned idx = block->index;
243
Builder bld(ctx.program, &instructions);
244
std::vector<unsigned>& preds = block->linear_preds;
248
aco_ptr<Instruction>& startpgm = block->instructions[0];
249
assert(startpgm->opcode == aco_opcode::p_startpgm);
250
bld.insert(std::move(startpgm));
252
Operand start_exec(bld.lm);
254
/* exec seems to need to be manually initialized with combined shaders */
255
if (ctx.program->stage.num_sw_stages() > 1 || ctx.program->stage.hw == HWStage::NGG) {
256
start_exec = Operand::c32_or_c64(-1u, bld.lm == s2);
257
bld.copy(Definition(exec, bld.lm), start_exec);
260
if (ctx.handle_wqm) {
261
ctx.info[0].exec.emplace_back(start_exec, mask_type_global | mask_type_exact);
262
/* if this block needs WQM, initialize already */
263
if (ctx.info[0].block_needs & WQM)
264
transition_to_WQM(ctx, bld, 0);
266
uint8_t mask = mask_type_global;
267
if (ctx.program->needs_wqm) {
268
bld.sop1(Builder::s_wqm, Definition(exec, bld.lm), bld.def(s1, scc),
269
Operand(exec, bld.lm));
270
mask |= mask_type_wqm;
272
mask |= mask_type_exact;
274
ctx.info[0].exec.emplace_back(start_exec, mask);
280
/* loop entry block */
281
if (block->kind & block_kind_loop_header) {
282
assert(preds[0] == idx - 1);
283
ctx.info[idx].exec = ctx.info[idx - 1].exec;
284
loop_info& info = ctx.loop.back();
285
while (ctx.info[idx].exec.size() > info.num_exec_masks)
286
ctx.info[idx].exec.pop_back();
288
/* create ssa names for outer exec masks */
289
if (info.has_discard) {
290
aco_ptr<Pseudo_instruction> phi;
291
for (int i = 0; i < info.num_exec_masks - 1; i++) {
292
phi.reset(create_instruction<Pseudo_instruction>(aco_opcode::p_linear_phi,
293
Format::PSEUDO, preds.size(), 1));
294
phi->definitions[0] = bld.def(bld.lm);
295
phi->operands[0] = get_exec_op(ctx.info[preds[0]].exec[i].first);
296
ctx.info[idx].exec[i].first = bld.insert(std::move(phi));
300
/* create ssa name for restore mask */
301
if (info.has_divergent_break) {
302
/* this phi might be trivial but ensures a parallelcopy on the loop header */
303
aco_ptr<Pseudo_instruction> phi{create_instruction<Pseudo_instruction>(
304
aco_opcode::p_linear_phi, Format::PSEUDO, preds.size(), 1)};
305
phi->definitions[0] = bld.def(bld.lm);
306
phi->operands[0] = get_exec_op(ctx.info[preds[0]].exec[info.num_exec_masks - 1].first);
307
ctx.info[idx].exec.back().first = bld.insert(std::move(phi));
310
/* create ssa name for loop active mask */
311
aco_ptr<Pseudo_instruction> phi{create_instruction<Pseudo_instruction>(
312
aco_opcode::p_linear_phi, Format::PSEUDO, preds.size(), 1)};
313
if (info.has_divergent_continue)
314
phi->definitions[0] = bld.def(bld.lm);
316
phi->definitions[0] = Definition(exec, bld.lm);
317
phi->operands[0] = get_exec_op(ctx.info[preds[0]].exec.back().first);
318
Temp loop_active = bld.insert(std::move(phi));
320
if (info.has_divergent_break) {
322
(ctx.info[idx].exec.back().second & (mask_type_wqm | mask_type_exact)) | mask_type_loop;
323
ctx.info[idx].exec.emplace_back(loop_active, mask_type);
325
ctx.info[idx].exec.back().first = Operand(loop_active);
326
ctx.info[idx].exec.back().second |= mask_type_loop;
329
/* create a parallelcopy to move the active mask to exec */
331
if (info.has_divergent_continue) {
332
while (block->instructions[i]->opcode != aco_opcode::p_logical_start) {
333
bld.insert(std::move(block->instructions[i]));
336
uint8_t mask_type = ctx.info[idx].exec.back().second & (mask_type_wqm | mask_type_exact);
337
assert(ctx.info[idx].exec.back().first.size() == bld.lm.size());
338
ctx.info[idx].exec.emplace_back(
339
bld.copy(Definition(exec, bld.lm), ctx.info[idx].exec.back().first), mask_type);
345
/* loop exit block */
346
if (block->kind & block_kind_loop_exit) {
347
Block* header = ctx.loop.back().loop_header;
348
loop_info& info = ctx.loop.back();
350
for (ASSERTED unsigned pred : preds)
351
assert(ctx.info[pred].exec.size() >= info.num_exec_masks);
353
/* fill the loop header phis */
354
std::vector<unsigned>& header_preds = header->linear_preds;
356
if (info.has_discard) {
357
while (instr_idx < info.num_exec_masks - 1) {
358
aco_ptr<Instruction>& phi = header->instructions[instr_idx];
359
assert(phi->opcode == aco_opcode::p_linear_phi);
360
for (unsigned i = 1; i < phi->operands.size(); i++)
361
phi->operands[i] = get_exec_op(ctx.info[header_preds[i]].exec[instr_idx].first);
367
aco_ptr<Instruction>& phi = header->instructions[instr_idx++];
368
assert(phi->opcode == aco_opcode::p_linear_phi);
369
for (unsigned i = 1; i < phi->operands.size(); i++)
371
get_exec_op(ctx.info[header_preds[i]].exec[info.num_exec_masks - 1].first);
374
if (info.has_divergent_break) {
375
aco_ptr<Instruction>& phi = header->instructions[instr_idx];
376
assert(phi->opcode == aco_opcode::p_linear_phi);
377
for (unsigned i = 1; i < phi->operands.size(); i++)
379
get_exec_op(ctx.info[header_preds[i]].exec[info.num_exec_masks].first);
382
assert(!(block->kind & block_kind_top_level) || info.num_exec_masks <= 2);
384
/* create the loop exit phis if not trivial */
385
for (unsigned exec_idx = 0; exec_idx < info.num_exec_masks; exec_idx++) {
386
Operand same = ctx.info[preds[0]].exec[exec_idx].first;
387
uint8_t type = ctx.info[header_preds[0]].exec[exec_idx].second;
390
for (unsigned i = 1; i < preds.size() && trivial; i++) {
391
if (ctx.info[preds[i]].exec[exec_idx].first != same)
396
ctx.info[idx].exec.emplace_back(same, type);
398
/* create phi for loop footer */
399
aco_ptr<Pseudo_instruction> phi{create_instruction<Pseudo_instruction>(
400
aco_opcode::p_linear_phi, Format::PSEUDO, preds.size(), 1)};
401
phi->definitions[0] = bld.def(bld.lm);
402
if (exec_idx == info.num_exec_masks - 1u) {
403
phi->definitions[0] = Definition(exec, bld.lm);
405
for (unsigned i = 0; i < phi->operands.size(); i++)
406
phi->operands[i] = get_exec_op(ctx.info[preds[i]].exec[exec_idx].first);
407
ctx.info[idx].exec.emplace_back(bld.insert(std::move(phi)), type);
411
assert(ctx.info[idx].exec.size() == info.num_exec_masks);
414
} else if (preds.size() == 1) {
415
ctx.info[idx].exec = ctx.info[preds[0]].exec;
417
assert(preds.size() == 2);
418
/* if one of the predecessors ends in exact mask, we pop it from stack */
419
unsigned num_exec_masks =
420
std::min(ctx.info[preds[0]].exec.size(), ctx.info[preds[1]].exec.size());
422
if (block->kind & block_kind_merge)
424
if (block->kind & block_kind_top_level)
425
num_exec_masks = std::min(num_exec_masks, 2u);
427
/* create phis for diverged exec masks */
428
for (unsigned i = 0; i < num_exec_masks; i++) {
429
/* skip trivial phis */
430
if (ctx.info[preds[0]].exec[i].first == ctx.info[preds[1]].exec[i].first) {
431
Operand t = ctx.info[preds[0]].exec[i].first;
432
/* discard/demote can change the state of the current exec mask */
433
assert(!t.isTemp() ||
434
ctx.info[preds[0]].exec[i].second == ctx.info[preds[1]].exec[i].second);
435
uint8_t mask = ctx.info[preds[0]].exec[i].second & ctx.info[preds[1]].exec[i].second;
436
ctx.info[idx].exec.emplace_back(t, mask);
440
Temp phi = bld.pseudo(aco_opcode::p_linear_phi, bld.def(bld.lm),
441
get_exec_op(ctx.info[preds[0]].exec[i].first),
442
get_exec_op(ctx.info[preds[1]].exec[i].first));
443
uint8_t mask_type = ctx.info[preds[0]].exec[i].second & ctx.info[preds[1]].exec[i].second;
444
ctx.info[idx].exec.emplace_back(phi, mask_type);
449
while (block->instructions[i]->opcode == aco_opcode::p_phi ||
450
block->instructions[i]->opcode == aco_opcode::p_linear_phi) {
451
bld.insert(std::move(block->instructions[i]));
455
/* try to satisfy the block's needs */
456
if (ctx.handle_wqm) {
457
if (block->kind & block_kind_top_level && ctx.info[idx].exec.size() == 2) {
458
if (ctx.info[idx].block_needs == 0 || ctx.info[idx].block_needs == Exact) {
459
ctx.info[idx].exec.back().second |= mask_type_global;
460
transition_to_Exact(ctx, bld, idx);
461
ctx.handle_wqm = false;
466
/* restore exec mask after divergent control flow */
467
if (block->kind & (block_kind_loop_exit | block_kind_merge) &&
468
!ctx.info[idx].exec.back().first.isUndefined()) {
469
Operand restore = ctx.info[idx].exec.back().first;
470
assert(restore.size() == bld.lm.size());
471
bld.copy(Definition(exec, bld.lm), restore);
472
if (!restore.isConstant())
473
ctx.info[idx].exec.back().first = Operand(bld.lm);
480
process_instructions(exec_ctx& ctx, Block* block, std::vector<aco_ptr<Instruction>>& instructions,
484
if (ctx.info[block->index].exec.back().second & mask_type_wqm) {
487
assert(!ctx.handle_wqm || ctx.info[block->index].exec.back().second & mask_type_exact);
491
/* if the block doesn't need both, WQM and Exact, we can skip processing the instructions */
492
bool process = (ctx.handle_wqm && (ctx.info[block->index].block_needs & state) !=
493
(ctx.info[block->index].block_needs & (WQM | Exact))) ||
494
block->kind & block_kind_uses_discard || block->kind & block_kind_needs_lowering;
496
std::vector<aco_ptr<Instruction>>::iterator it = std::next(block->instructions.begin(), idx);
497
instructions.insert(instructions.end(),
498
std::move_iterator<std::vector<aco_ptr<Instruction>>::iterator>(it),
499
std::move_iterator<std::vector<aco_ptr<Instruction>>::iterator>(
500
block->instructions.end()));
504
Builder bld(ctx.program, &instructions);
506
for (; idx < block->instructions.size(); idx++) {
507
aco_ptr<Instruction> instr = std::move(block->instructions[idx]);
509
WQMState needs = ctx.handle_wqm ? ctx.info[block->index].instr_needs[idx] : Unspecified;
511
if (needs == WQM && state != WQM) {
512
transition_to_WQM(ctx, bld, block->index);
514
} else if (needs == Exact && state != Exact) {
515
transition_to_Exact(ctx, bld, block->index);
519
if (instr->opcode == aco_opcode::p_discard_if) {
520
Operand current_exec = Operand(exec, bld.lm);
522
if (ctx.info[block->index].exec.size() >= 2) {
524
/* Preserve the WQM mask */
525
ctx.info[block->index].exec[1].second &= ~mask_type_global;
526
} else if (block->kind & block_kind_top_level) {
527
/* Transition to Exact without extra instruction. Since needs != WQM, we won't need
530
ctx.info[block->index].exec.resize(1);
531
assert(ctx.info[block->index].exec[0].second == (mask_type_exact | mask_type_global));
532
current_exec = get_exec_op(ctx.info[block->index].exec.back().first);
533
ctx.info[block->index].exec[0].first = Operand(bld.lm);
537
Temp cond, exit_cond;
538
if (instr->operands[0].isConstant()) {
539
assert(instr->operands[0].constantValue() == -1u);
540
/* save condition and set exec to zero */
541
exit_cond = bld.tmp(s1);
543
bld.sop1(Builder::s_and_saveexec, bld.def(bld.lm), bld.scc(Definition(exit_cond)),
544
Definition(exec, bld.lm), Operand::zero(), Operand(exec, bld.lm));
546
cond = instr->operands[0].getTemp();
547
/* discard from current exec */
548
exit_cond = bld.sop2(Builder::s_andn2, Definition(exec, bld.lm), bld.def(s1, scc),
554
/* discard from inner to outer exec mask on stack */
555
int num = ctx.info[block->index].exec.size() - 2;
556
for (int i = num; i >= 0; i--) {
557
Instruction* andn2 = bld.sop2(Builder::s_andn2, bld.def(bld.lm), bld.def(s1, scc),
558
ctx.info[block->index].exec[i].first, cond);
559
ctx.info[block->index].exec[i].first = Operand(andn2->definitions[0].getTemp());
560
exit_cond = andn2->definitions[1].getTemp();
563
instr->opcode = aco_opcode::p_exit_early_if;
564
instr->operands[0] = bld.scc(exit_cond);
565
assert(!ctx.handle_wqm || (ctx.info[block->index].exec[0].second & mask_type_wqm) == 0);
567
} else if (instr->opcode == aco_opcode::p_is_helper) {
568
Definition dst = instr->definitions[0];
569
assert(dst.size() == bld.lm.size());
570
if (state == Exact) {
571
instr.reset(create_instruction<SOP1_instruction>(bld.w64or32(Builder::s_mov),
572
Format::SOP1, 1, 1));
573
instr->operands[0] = Operand::zero();
574
instr->definitions[0] = dst;
576
std::pair<Operand, uint8_t>& exact_mask = ctx.info[block->index].exec[0];
577
assert(exact_mask.second & mask_type_exact);
579
instr.reset(create_instruction<SOP2_instruction>(bld.w64or32(Builder::s_andn2),
580
Format::SOP2, 2, 2));
581
instr->operands[0] = Operand(exec, bld.lm); /* current exec */
582
instr->operands[1] = Operand(exact_mask.first);
583
instr->definitions[0] = dst;
584
instr->definitions[1] = bld.def(s1, scc);
586
} else if (instr->opcode == aco_opcode::p_demote_to_helper) {
587
/* turn demote into discard_if with only exact masks */
588
assert(ctx.info[block->index].exec[0].second == (mask_type_exact | mask_type_global));
591
Temp cond, exit_cond;
592
if (instr->operands[0].isConstant()) {
593
assert(instr->operands[0].constantValue() == -1u);
594
/* transition to exact and set exec to zero */
595
exit_cond = bld.tmp(s1);
597
bld.sop1(Builder::s_and_saveexec, bld.def(bld.lm), bld.scc(Definition(exit_cond)),
598
Definition(exec, bld.lm), Operand::zero(), Operand(exec, bld.lm));
600
num = ctx.info[block->index].exec.size() - 2;
601
if (!(ctx.info[block->index].exec.back().second & mask_type_exact)) {
602
ctx.info[block->index].exec.back().first = Operand(cond);
603
ctx.info[block->index].exec.emplace_back(Operand(bld.lm), mask_type_exact);
606
/* demote_if: transition to exact */
607
if (block->kind & block_kind_top_level && ctx.info[block->index].exec.size() == 2 &&
608
ctx.info[block->index].exec.back().second & mask_type_global) {
609
/* We don't need to actually copy anything into exact, since the s_andn2
610
* instructions later will do that.
612
ctx.info[block->index].exec.pop_back();
614
transition_to_Exact(ctx, bld, block->index);
616
assert(instr->operands[0].isTemp());
617
cond = instr->operands[0].getTemp();
618
num = ctx.info[block->index].exec.size() - 1;
621
for (int i = num; i >= 0; i--) {
622
if (ctx.info[block->index].exec[i].second & mask_type_exact) {
624
bld.sop2(Builder::s_andn2, bld.def(bld.lm), bld.def(s1, scc),
625
get_exec_op(ctx.info[block->index].exec[i].first), cond);
626
if (i == (int)ctx.info[block->index].exec.size() - 1)
627
andn2->definitions[0] = Definition(exec, bld.lm);
629
ctx.info[block->index].exec[i].first = Operand(andn2->definitions[0].getTemp());
630
exit_cond = andn2->definitions[1].getTemp();
635
instr->opcode = aco_opcode::p_exit_early_if;
636
instr->operands[0] = bld.scc(exit_cond);
639
} else if (instr->opcode == aco_opcode::p_elect) {
640
bool all_lanes_enabled = ctx.info[block->index].exec.back().first.constantEquals(-1u);
641
Definition dst = instr->definitions[0];
643
if (all_lanes_enabled) {
644
bld.copy(Definition(dst), Operand::c32_or_c64(1u, dst.size() == 2));
646
Temp first_lane_idx = bld.sop1(Builder::s_ff1_i32, bld.def(s1), Operand(exec, bld.lm));
647
bld.sop2(Builder::s_lshl, Definition(dst), bld.def(s1, scc),
648
Operand::c32_or_c64(1u, dst.size() == 2), Operand(first_lane_idx));
654
bld.insert(std::move(instr));
659
add_branch_code(exec_ctx& ctx, Block* block)
661
unsigned idx = block->index;
662
Builder bld(ctx.program, block);
664
if (idx == ctx.program->blocks.size() - 1)
667
/* try to disable wqm handling */
668
if (ctx.handle_wqm && block->kind & block_kind_top_level) {
669
if (ctx.info[idx].exec.size() == 3) {
670
assert(ctx.info[idx].exec[1].second == mask_type_wqm);
671
ctx.info[idx].exec.pop_back();
673
assert(ctx.info[idx].exec.size() <= 2);
675
if (!(ctx.info[idx].instr_needs.back() & WQM)) {
676
/* transition to Exact if the branch doesn't need WQM */
677
aco_ptr<Instruction> branch = std::move(block->instructions.back());
678
block->instructions.pop_back();
679
ctx.info[idx].exec.back().second |= mask_type_global;
680
transition_to_Exact(ctx, bld, idx);
681
bld.insert(std::move(branch));
682
ctx.handle_wqm = false;
686
if (block->kind & block_kind_loop_preheader) {
687
/* collect information about the succeeding loop */
688
bool has_divergent_break = false;
689
bool has_divergent_continue = false;
690
bool has_discard = false;
691
unsigned loop_nest_depth = ctx.program->blocks[idx + 1].loop_nest_depth;
693
for (unsigned i = idx + 1; ctx.program->blocks[i].loop_nest_depth >= loop_nest_depth; i++) {
694
Block& loop_block = ctx.program->blocks[i];
696
if (loop_block.kind & block_kind_uses_discard)
698
if (loop_block.loop_nest_depth != loop_nest_depth)
701
if (loop_block.kind & block_kind_uniform)
703
else if (loop_block.kind & block_kind_break)
704
has_divergent_break = true;
705
else if (loop_block.kind & block_kind_continue)
706
has_divergent_continue = true;
709
unsigned num_exec_masks = ctx.info[idx].exec.size();
710
if (block->kind & block_kind_top_level)
711
num_exec_masks = std::min(num_exec_masks, 2u);
713
ctx.loop.emplace_back(&ctx.program->blocks[block->linear_succs[0]], num_exec_masks,
714
has_divergent_break, has_divergent_continue, has_discard);
717
/* For normal breaks, this is the exec mask. For discard+break, it's the
718
* old exec mask before it was zero'd.
720
Operand break_cond = Operand(exec, bld.lm);
722
if (block->kind & block_kind_continue_or_break) {
723
assert(ctx.program->blocks[ctx.program->blocks[block->linear_succs[1]].linear_succs[0]].kind &
724
block_kind_loop_header);
725
assert(ctx.program->blocks[ctx.program->blocks[block->linear_succs[0]].linear_succs[0]].kind &
726
block_kind_loop_exit);
727
assert(block->instructions.back()->opcode == aco_opcode::p_branch);
728
block->instructions.pop_back();
730
bool need_parallelcopy = false;
731
while (!(ctx.info[idx].exec.back().second & mask_type_loop)) {
732
ctx.info[idx].exec.pop_back();
733
need_parallelcopy = true;
736
if (need_parallelcopy)
737
ctx.info[idx].exec.back().first =
738
bld.copy(Definition(exec, bld.lm), ctx.info[idx].exec.back().first);
739
bld.branch(aco_opcode::p_cbranch_nz, bld.def(s2), Operand(exec, bld.lm),
740
block->linear_succs[1], block->linear_succs[0]);
744
if (block->kind & block_kind_uniform) {
745
Pseudo_branch_instruction& branch = block->instructions.back()->branch();
746
if (branch.opcode == aco_opcode::p_branch) {
747
branch.target[0] = block->linear_succs[0];
749
branch.target[0] = block->linear_succs[1];
750
branch.target[1] = block->linear_succs[0];
755
if (block->kind & block_kind_branch) {
756
// orig = s_and_saveexec_b64
757
assert(block->linear_succs.size() == 2);
758
assert(block->instructions.back()->opcode == aco_opcode::p_cbranch_z);
759
Temp cond = block->instructions.back()->operands[0].getTemp();
760
block->instructions.pop_back();
762
uint8_t mask_type = ctx.info[idx].exec.back().second & (mask_type_wqm | mask_type_exact);
763
if (ctx.info[idx].exec.back().first.constantEquals(-1u)) {
764
bld.copy(Definition(exec, bld.lm), cond);
766
Temp old_exec = bld.sop1(Builder::s_and_saveexec, bld.def(bld.lm), bld.def(s1, scc),
767
Definition(exec, bld.lm), cond, Operand(exec, bld.lm));
769
ctx.info[idx].exec.back().first = Operand(old_exec);
772
/* add next current exec to the stack */
773
ctx.info[idx].exec.emplace_back(Operand(bld.lm), mask_type);
775
bld.branch(aco_opcode::p_cbranch_z, bld.def(s2), Operand(exec, bld.lm),
776
block->linear_succs[1], block->linear_succs[0]);
780
if (block->kind & block_kind_invert) {
781
// exec = s_andn2_b64 (original_exec, exec)
782
assert(block->instructions.back()->opcode == aco_opcode::p_branch);
783
block->instructions.pop_back();
784
assert(ctx.info[idx].exec.size() >= 2);
785
Operand orig_exec = ctx.info[idx].exec[ctx.info[idx].exec.size() - 2].first;
786
bld.sop2(Builder::s_andn2, Definition(exec, bld.lm), bld.def(s1, scc), orig_exec,
787
Operand(exec, bld.lm));
789
bld.branch(aco_opcode::p_cbranch_z, bld.def(s2), Operand(exec, bld.lm),
790
block->linear_succs[1], block->linear_succs[0]);
794
if (block->kind & block_kind_break) {
795
// loop_mask = s_andn2_b64 (loop_mask, exec)
796
assert(block->instructions.back()->opcode == aco_opcode::p_branch);
797
block->instructions.pop_back();
800
for (int exec_idx = ctx.info[idx].exec.size() - 2; exec_idx >= 0; exec_idx--) {
802
Operand exec_mask = ctx.info[idx].exec[exec_idx].first;
803
exec_mask = bld.sop2(Builder::s_andn2, bld.def(bld.lm), bld.scc(Definition(cond)),
804
exec_mask, break_cond);
805
ctx.info[idx].exec[exec_idx].first = exec_mask;
806
if (ctx.info[idx].exec[exec_idx].second & mask_type_loop)
810
/* check if the successor is the merge block, otherwise set exec to 0 */
811
// TODO: this could be done better by directly branching to the merge block
812
unsigned succ_idx = ctx.program->blocks[block->linear_succs[1]].linear_succs[0];
813
Block& succ = ctx.program->blocks[succ_idx];
814
if (!(succ.kind & block_kind_invert || succ.kind & block_kind_merge)) {
815
bld.copy(Definition(exec, bld.lm), Operand::zero(bld.lm.bytes()));
818
bld.branch(aco_opcode::p_cbranch_nz, bld.def(s2), bld.scc(cond), block->linear_succs[1],
819
block->linear_succs[0]);
823
if (block->kind & block_kind_continue) {
824
assert(block->instructions.back()->opcode == aco_opcode::p_branch);
825
block->instructions.pop_back();
828
for (int exec_idx = ctx.info[idx].exec.size() - 2; exec_idx >= 0; exec_idx--) {
829
if (ctx.info[idx].exec[exec_idx].second & mask_type_loop)
832
Operand exec_mask = ctx.info[idx].exec[exec_idx].first;
833
exec_mask = bld.sop2(Builder::s_andn2, bld.def(bld.lm), bld.scc(Definition(cond)),
834
exec_mask, Operand(exec, bld.lm));
835
ctx.info[idx].exec[exec_idx].first = exec_mask;
837
assert(cond != Temp());
839
/* check if the successor is the merge block, otherwise set exec to 0 */
840
// TODO: this could be done better by directly branching to the merge block
841
unsigned succ_idx = ctx.program->blocks[block->linear_succs[1]].linear_succs[0];
842
Block& succ = ctx.program->blocks[succ_idx];
843
if (!(succ.kind & block_kind_invert || succ.kind & block_kind_merge)) {
844
bld.copy(Definition(exec, bld.lm), Operand::zero(bld.lm.bytes()));
847
bld.branch(aco_opcode::p_cbranch_nz, bld.def(s2), bld.scc(cond), block->linear_succs[1],
848
block->linear_succs[0]);
854
process_block(exec_ctx& ctx, Block* block)
856
std::vector<aco_ptr<Instruction>> instructions;
857
instructions.reserve(block->instructions.size());
859
unsigned idx = add_coupling_code(ctx, block, instructions);
861
assert(block->index != ctx.program->blocks.size() - 1 ||
862
ctx.info[block->index].exec.size() <= 2);
864
process_instructions(ctx, block, instructions, idx);
866
block->instructions = std::move(instructions);
868
add_branch_code(ctx, block);
871
} /* end namespace */
874
insert_exec_mask(Program* program)
876
exec_ctx ctx(program);
878
if (program->needs_wqm && program->needs_exact)
879
calculate_wqm_needs(ctx);
881
for (Block& block : program->blocks)
882
process_block(ctx, &block);
added
removed
src/amd/compiler/aco_insert_exec_mask.cpp
