2
* Copyright 2011 Christoph Bumiller
4
* Permission is hereby granted, free of charge, to any person obtaining a
5
* copy of this software and associated documentation files (the "Software"),
6
* to deal in the Software without restriction, including without limitation
7
* the rights to use, copy, modify, merge, publish, distribute, sublicense,
8
* and/or sell copies of the Software, and to permit persons to whom the
9
* Software is furnished to do so, subject to the following conditions:
11
* The above copyright notice and this permission notice shall be included in
12
* all copies or substantial portions of the Software.
14
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
15
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
16
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
17
* THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR
18
* OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
19
* ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
20
* OTHER DEALINGS IN THE SOFTWARE.
23
#include "codegen/nv50_ir.h"
24
#include "codegen/nv50_ir_target.h"
25
#include "codegen/nv50_ir_build_util.h"
28
#include "util/u_math.h"
34
Instruction::isNop() const
36
if (op == OP_PHI || op == OP_SPLIT || op == OP_MERGE || op == OP_CONSTRAINT)
38
if (terminator || join) // XXX: should terminator imply flow ?
42
if (!fixed && op == OP_NOP)
45
if (defExists(0) && def(0).rep()->reg.data.id < 0) {
46
for (int d = 1; defExists(d); ++d)
47
if (def(d).rep()->reg.data.id >= 0)
48
WARN("part of vector result is unused !\n");
52
if (op == OP_MOV || op == OP_UNION) {
53
if (!getDef(0)->equals(getSrc(0)))
56
if (!def(0).rep()->equals(getSrc(1)))
64
bool Instruction::isDead() const
69
op == OP_SUSTB || op == OP_SUSTP || op == OP_SUREDP || op == OP_SUREDB ||
73
for (int d = 0; defExists(d); ++d)
74
if (getDef(d)->refCount() || getDef(d)->reg.data.id >= 0)
77
if (terminator || asFlow())
85
// =============================================================================
87
class CopyPropagation : public Pass
90
virtual bool visit(BasicBlock *);
93
// Propagate all MOVs forward to make subsequent optimization easier, except if
94
// the sources stem from a phi, in which case we don't want to mess up potential
95
// swaps $rX <-> $rY, i.e. do not create live range overlaps of phi src and def.
97
CopyPropagation::visit(BasicBlock *bb)
99
Instruction *mov, *si, *next;
101
for (mov = bb->getEntry(); mov; mov = next) {
103
if (mov->op != OP_MOV || mov->fixed || !mov->getSrc(0)->asLValue())
105
if (mov->getPredicate())
107
if (mov->def(0).getFile() != mov->src(0).getFile())
109
si = mov->getSrc(0)->getInsn();
110
if (mov->getDef(0)->reg.data.id < 0 && si && si->op != OP_PHI) {
112
mov->def(0).replace(mov->getSrc(0), false);
113
delete_Instruction(prog, mov);
119
// =============================================================================
121
class MergeSplits : public Pass
124
virtual bool visit(BasicBlock *);
127
// For SPLIT / MERGE pairs that operate on the same registers, replace the
128
// post-merge def with the SPLIT's source.
130
MergeSplits::visit(BasicBlock *bb)
132
Instruction *i, *next, *si;
134
for (i = bb->getEntry(); i; i = next) {
136
if (i->op != OP_MERGE || typeSizeof(i->dType) != 8)
138
si = i->getSrc(0)->getInsn();
139
if (si->op != OP_SPLIT || si != i->getSrc(1)->getInsn())
141
i->def(0).replace(si->getSrc(0), false);
142
delete_Instruction(prog, i);
148
// =============================================================================
150
class LoadPropagation : public Pass
153
virtual bool visit(BasicBlock *);
155
void checkSwapSrc01(Instruction *);
157
bool isCSpaceLoad(Instruction *);
158
bool isImmdLoad(Instruction *);
159
bool isAttribOrSharedLoad(Instruction *);
163
LoadPropagation::isCSpaceLoad(Instruction *ld)
165
return ld && ld->op == OP_LOAD && ld->src(0).getFile() == FILE_MEMORY_CONST;
169
LoadPropagation::isImmdLoad(Instruction *ld)
171
if (!ld || (ld->op != OP_MOV) ||
172
((typeSizeof(ld->dType) != 4) && (typeSizeof(ld->dType) != 8)))
175
// A 0 can be replaced with a register, so it doesn't count as an immediate.
177
return ld->src(0).getImmediate(val) && !val.isInteger(0);
181
LoadPropagation::isAttribOrSharedLoad(Instruction *ld)
184
(ld->op == OP_VFETCH ||
185
(ld->op == OP_LOAD &&
186
(ld->src(0).getFile() == FILE_SHADER_INPUT ||
187
ld->src(0).getFile() == FILE_MEMORY_SHARED)));
191
LoadPropagation::checkSwapSrc01(Instruction *insn)
193
const Target *targ = prog->getTarget();
194
if (!targ->getOpInfo(insn).commutative) {
195
if (insn->op != OP_SET && insn->op != OP_SLCT &&
196
insn->op != OP_SUB && insn->op != OP_XMAD)
198
// XMAD is only commutative if both the CBCC and MRG flags are not set.
199
if (insn->op == OP_XMAD &&
200
(insn->subOp & NV50_IR_SUBOP_XMAD_CMODE_MASK) == NV50_IR_SUBOP_XMAD_CBCC)
202
if (insn->op == OP_XMAD && (insn->subOp & NV50_IR_SUBOP_XMAD_MRG))
205
if (insn->src(1).getFile() != FILE_GPR)
207
// This is the special OP_SET used for alphatesting, we can't reverse its
208
// arguments as that will confuse the fixup code.
209
if (insn->op == OP_SET && insn->subOp)
212
Instruction *i0 = insn->getSrc(0)->getInsn();
213
Instruction *i1 = insn->getSrc(1)->getInsn();
215
// Swap sources to inline the less frequently used source. That way,
216
// optimistically, it will eventually be able to remove the instruction.
217
int i0refs = insn->getSrc(0)->refCount();
218
int i1refs = insn->getSrc(1)->refCount();
220
if ((isCSpaceLoad(i0) || isImmdLoad(i0)) && targ->insnCanLoad(insn, 1, i0)) {
221
if ((!isImmdLoad(i1) && !isCSpaceLoad(i1)) ||
222
!targ->insnCanLoad(insn, 1, i1) ||
224
insn->swapSources(0, 1);
228
if (isAttribOrSharedLoad(i1)) {
229
if (!isAttribOrSharedLoad(i0))
230
insn->swapSources(0, 1);
237
if (insn->op == OP_SET || insn->op == OP_SET_AND ||
238
insn->op == OP_SET_OR || insn->op == OP_SET_XOR)
239
insn->asCmp()->setCond = reverseCondCode(insn->asCmp()->setCond);
241
if (insn->op == OP_SLCT)
242
insn->asCmp()->setCond = inverseCondCode(insn->asCmp()->setCond);
244
if (insn->op == OP_SUB) {
245
insn->src(0).mod = insn->src(0).mod ^ Modifier(NV50_IR_MOD_NEG);
246
insn->src(1).mod = insn->src(1).mod ^ Modifier(NV50_IR_MOD_NEG);
248
if (insn->op == OP_XMAD) {
250
uint16_t h1 = (insn->subOp >> 1 & NV50_IR_SUBOP_XMAD_H1(0)) |
251
(insn->subOp << 1 & NV50_IR_SUBOP_XMAD_H1(1));
252
insn->subOp = (insn->subOp & ~NV50_IR_SUBOP_XMAD_H1_MASK) | h1;
257
LoadPropagation::visit(BasicBlock *bb)
259
const Target *targ = prog->getTarget();
262
for (Instruction *i = bb->getEntry(); i; i = next) {
265
if (i->op == OP_CALL) // calls have args as sources, they must be in regs
268
if (i->op == OP_PFETCH) // pfetch expects arg1 to be a reg
274
for (int s = 0; i->srcExists(s); ++s) {
275
Instruction *ld = i->getSrc(s)->getInsn();
277
if (!ld || ld->fixed || (ld->op != OP_LOAD && ld->op != OP_MOV))
279
if (ld->op == OP_LOAD && ld->subOp == NV50_IR_SUBOP_LOAD_LOCKED)
281
if (!targ->insnCanLoad(i, s, ld))
285
i->setSrc(s, ld->getSrc(0));
286
if (ld->src(0).isIndirect(0))
287
i->setIndirect(s, 0, ld->getIndirect(0, 0));
289
if (ld->getDef(0)->refCount() == 0)
290
delete_Instruction(prog, ld);
296
// =============================================================================
298
class IndirectPropagation : public Pass
301
virtual bool visit(BasicBlock *);
307
IndirectPropagation::visit(BasicBlock *bb)
309
const Target *targ = prog->getTarget();
312
for (Instruction *i = bb->getEntry(); i; i = next) {
315
bld.setPosition(i, false);
317
for (int s = 0; i->srcExists(s); ++s) {
320
if (!i->src(s).isIndirect(0))
322
insn = i->getIndirect(s, 0)->getInsn();
325
if (insn->op == OP_ADD && !isFloatType(insn->dType)) {
326
if (insn->src(0).getFile() != targ->nativeFile(FILE_ADDRESS) ||
327
!insn->src(1).getImmediate(imm) ||
328
!targ->insnCanLoadOffset(i, s, imm.reg.data.s32))
330
i->setIndirect(s, 0, insn->getSrc(0));
331
i->setSrc(s, cloneShallow(func, i->getSrc(s)));
332
i->src(s).get()->reg.data.offset += imm.reg.data.u32;
333
} else if (insn->op == OP_SUB && !isFloatType(insn->dType)) {
334
if (insn->src(0).getFile() != targ->nativeFile(FILE_ADDRESS) ||
335
!insn->src(1).getImmediate(imm) ||
336
!targ->insnCanLoadOffset(i, s, -imm.reg.data.s32))
338
i->setIndirect(s, 0, insn->getSrc(0));
339
i->setSrc(s, cloneShallow(func, i->getSrc(s)));
340
i->src(s).get()->reg.data.offset -= imm.reg.data.u32;
341
} else if (insn->op == OP_MOV) {
342
if (!insn->src(0).getImmediate(imm) ||
343
!targ->insnCanLoadOffset(i, s, imm.reg.data.s32))
345
i->setIndirect(s, 0, NULL);
346
i->setSrc(s, cloneShallow(func, i->getSrc(s)));
347
i->src(s).get()->reg.data.offset += imm.reg.data.u32;
348
} else if (insn->op == OP_SHLADD) {
349
if (!insn->src(2).getImmediate(imm) ||
350
!targ->insnCanLoadOffset(i, s, imm.reg.data.s32))
352
i->setIndirect(s, 0, bld.mkOp2v(
353
OP_SHL, TYPE_U32, bld.getSSA(), insn->getSrc(0), insn->getSrc(1)));
354
i->setSrc(s, cloneShallow(func, i->getSrc(s)));
355
i->src(s).get()->reg.data.offset += imm.reg.data.u32;
362
// =============================================================================
364
// Evaluate constant expressions.
365
class ConstantFolding : public Pass
368
ConstantFolding() : foldCount(0) {}
369
bool foldAll(Program *);
372
virtual bool visit(BasicBlock *);
374
void expr(Instruction *, ImmediateValue&, ImmediateValue&);
375
void expr(Instruction *, ImmediateValue&, ImmediateValue&, ImmediateValue&);
376
/* true if i was deleted */
377
bool opnd(Instruction *i, ImmediateValue&, int s);
378
void opnd3(Instruction *, ImmediateValue&);
380
void unary(Instruction *, const ImmediateValue&);
382
void tryCollapseChainedMULs(Instruction *, const int s, ImmediateValue&);
384
CmpInstruction *findOriginForTestWithZero(Value *);
386
bool createMul(DataType ty, Value *def, Value *a, int64_t b, Value *c);
388
unsigned int foldCount;
393
// TODO: remember generated immediates and only revisit these
395
ConstantFolding::foldAll(Program *prog)
397
unsigned int iterCount = 0;
402
} while (foldCount && ++iterCount < 2);
407
ConstantFolding::visit(BasicBlock *bb)
409
Instruction *i, *next;
411
for (i = bb->getEntry(); i; i = next) {
413
if (i->op == OP_MOV || i->op == OP_CALL)
416
ImmediateValue src0, src1, src2;
418
if (i->srcExists(2) &&
419
i->src(0).getImmediate(src0) &&
420
i->src(1).getImmediate(src1) &&
421
i->src(2).getImmediate(src2)) {
422
expr(i, src0, src1, src2);
424
if (i->srcExists(1) &&
425
i->src(0).getImmediate(src0) && i->src(1).getImmediate(src1)) {
428
if (i->srcExists(0) && i->src(0).getImmediate(src0)) {
429
if (opnd(i, src0, 0))
432
if (i->srcExists(1) && i->src(1).getImmediate(src1)) {
433
if (opnd(i, src1, 1))
436
if (i->srcExists(2) && i->src(2).getImmediate(src2))
443
ConstantFolding::findOriginForTestWithZero(Value *value)
447
Instruction *insn = value->getInsn();
451
if (insn->asCmp() && insn->op != OP_SLCT)
452
return insn->asCmp();
454
/* Sometimes mov's will sneak in as a result of other folding. This gets
457
if (insn->op == OP_MOV)
458
return findOriginForTestWithZero(insn->getSrc(0));
460
/* Deal with AND 1.0 here since nv50 can't fold into boolean float */
461
if (insn->op == OP_AND) {
464
if (!insn->src(s).getImmediate(imm)) {
466
if (!insn->src(s).getImmediate(imm))
469
if (imm.reg.data.f32 != 1.0f)
471
/* TODO: Come up with a way to handle the condition being inverted */
472
if (insn->src(!s).mod != Modifier(0))
474
return findOriginForTestWithZero(insn->getSrc(!s));
481
Modifier::applyTo(ImmediateValue& imm) const
483
if (!bits) // avoid failure if imm.reg.type is unhandled (e.g. b128)
485
switch (imm.reg.type) {
487
if (bits & NV50_IR_MOD_ABS)
488
imm.reg.data.f32 = fabsf(imm.reg.data.f32);
489
if (bits & NV50_IR_MOD_NEG)
490
imm.reg.data.f32 = -imm.reg.data.f32;
491
if (bits & NV50_IR_MOD_SAT) {
492
if (imm.reg.data.f32 < 0.0f)
493
imm.reg.data.f32 = 0.0f;
495
if (imm.reg.data.f32 > 1.0f)
496
imm.reg.data.f32 = 1.0f;
498
assert(!(bits & NV50_IR_MOD_NOT));
501
case TYPE_S8: // NOTE: will be extended
504
case TYPE_U8: // NOTE: treated as signed
507
if (bits & NV50_IR_MOD_ABS)
508
imm.reg.data.s32 = (imm.reg.data.s32 >= 0) ?
509
imm.reg.data.s32 : -imm.reg.data.s32;
510
if (bits & NV50_IR_MOD_NEG)
511
imm.reg.data.s32 = -imm.reg.data.s32;
512
if (bits & NV50_IR_MOD_NOT)
513
imm.reg.data.s32 = ~imm.reg.data.s32;
517
if (bits & NV50_IR_MOD_ABS)
518
imm.reg.data.f64 = fabs(imm.reg.data.f64);
519
if (bits & NV50_IR_MOD_NEG)
520
imm.reg.data.f64 = -imm.reg.data.f64;
521
if (bits & NV50_IR_MOD_SAT) {
522
if (imm.reg.data.f64 < 0.0)
523
imm.reg.data.f64 = 0.0;
525
if (imm.reg.data.f64 > 1.0)
526
imm.reg.data.f64 = 1.0;
528
assert(!(bits & NV50_IR_MOD_NOT));
532
assert(!"invalid/unhandled type");
533
imm.reg.data.u64 = 0;
539
Modifier::getOp() const
542
case NV50_IR_MOD_ABS: return OP_ABS;
543
case NV50_IR_MOD_NEG: return OP_NEG;
544
case NV50_IR_MOD_SAT: return OP_SAT;
545
case NV50_IR_MOD_NOT: return OP_NOT;
554
ConstantFolding::expr(Instruction *i,
555
ImmediateValue &imm0, ImmediateValue &imm1)
557
struct Storage *const a = &imm0.reg, *const b = &imm1.reg;
559
DataType type = i->dType;
561
memset(&res.data, 0, sizeof(res.data));
565
int bits = b->data.u32;
567
uint32_t data = a->data.u32 & (0xffffffff >> (32 - bits));
568
if (bits < 32 && (data & (1 << (bits - 1))))
569
data = data - (1 << bits);
575
res.data.u32 = ((1 << b->data.u32) - 1) << a->data.u32;
580
if (i->dnz && i->dType == TYPE_F32) {
581
if (!isfinite(a->data.f32))
583
if (!isfinite(b->data.f32))
588
res.data.f32 = a->data.f32 * b->data.f32 * exp2f(i->postFactor);
590
case TYPE_F64: res.data.f64 = a->data.f64 * b->data.f64; break;
592
if (i->subOp == NV50_IR_SUBOP_MUL_HIGH) {
593
res.data.s32 = ((int64_t)a->data.s32 * b->data.s32) >> 32;
598
if (i->subOp == NV50_IR_SUBOP_MUL_HIGH) {
599
res.data.u32 = ((uint64_t)a->data.u32 * b->data.u32) >> 32;
602
res.data.u32 = a->data.u32 * b->data.u32; break;
608
if (b->data.u32 == 0)
611
case TYPE_F32: res.data.f32 = a->data.f32 / b->data.f32; break;
612
case TYPE_F64: res.data.f64 = a->data.f64 / b->data.f64; break;
613
case TYPE_S32: res.data.s32 = a->data.s32 / b->data.s32; break;
614
case TYPE_U32: res.data.u32 = a->data.u32 / b->data.u32; break;
621
case TYPE_F32: res.data.f32 = a->data.f32 + b->data.f32; break;
622
case TYPE_F64: res.data.f64 = a->data.f64 + b->data.f64; break;
624
case TYPE_U32: res.data.u32 = a->data.u32 + b->data.u32; break;
631
case TYPE_F32: res.data.f32 = a->data.f32 - b->data.f32; break;
632
case TYPE_F64: res.data.f64 = a->data.f64 - b->data.f64; break;
634
case TYPE_U32: res.data.u32 = a->data.u32 - b->data.u32; break;
641
case TYPE_F32: res.data.f32 = pow(a->data.f32, b->data.f32); break;
642
case TYPE_F64: res.data.f64 = pow(a->data.f64, b->data.f64); break;
649
case TYPE_F32: res.data.f32 = MAX2(a->data.f32, b->data.f32); break;
650
case TYPE_F64: res.data.f64 = MAX2(a->data.f64, b->data.f64); break;
651
case TYPE_S32: res.data.s32 = MAX2(a->data.s32, b->data.s32); break;
652
case TYPE_U32: res.data.u32 = MAX2(a->data.u32, b->data.u32); break;
659
case TYPE_F32: res.data.f32 = MIN2(a->data.f32, b->data.f32); break;
660
case TYPE_F64: res.data.f64 = MIN2(a->data.f64, b->data.f64); break;
661
case TYPE_S32: res.data.s32 = MIN2(a->data.s32, b->data.s32); break;
662
case TYPE_U32: res.data.u32 = MIN2(a->data.u32, b->data.u32); break;
668
res.data.u64 = a->data.u64 & b->data.u64;
671
res.data.u64 = a->data.u64 | b->data.u64;
674
res.data.u64 = a->data.u64 ^ b->data.u64;
677
res.data.u32 = a->data.u32 << b->data.u32;
681
case TYPE_S32: res.data.s32 = a->data.s32 >> b->data.u32; break;
682
case TYPE_U32: res.data.u32 = a->data.u32 >> b->data.u32; break;
688
if (a->data.u32 != b->data.u32)
690
res.data.u32 = a->data.u32;
693
int offset = b->data.u32 & 0xff;
694
int width = (b->data.u32 >> 8) & 0xff;
701
if (width + offset < 32) {
703
lshift = 32 - width - offset;
705
if (i->subOp == NV50_IR_SUBOP_EXTBF_REV)
706
res.data.u32 = util_bitreverse(a->data.u32);
708
res.data.u32 = a->data.u32;
710
case TYPE_S32: res.data.s32 = (res.data.s32 << lshift) >> rshift; break;
711
case TYPE_U32: res.data.u32 = (res.data.u32 << lshift) >> rshift; break;
718
res.data.u32 = util_bitcount(a->data.u32 & b->data.u32);
721
// The two arguments to pfetch are logically added together. Normally
722
// the second argument will not be constant, but that can happen.
723
res.data.u32 = a->data.u32 + b->data.u32;
731
res.data.u64 = (((uint64_t)b->data.u32) << 32) | a->data.u32;
742
i->src(0).mod = Modifier(0);
743
i->src(1).mod = Modifier(0);
746
i->setSrc(0, new_ImmediateValue(i->bb->getProgram(), res.data.u32));
749
i->getSrc(0)->reg.data = res.data;
750
i->getSrc(0)->reg.type = type;
751
i->getSrc(0)->reg.size = typeSizeof(type);
756
ImmediateValue src0, src1 = *i->getSrc(0)->asImm();
758
// Move the immediate into position 1, where we know it might be
759
// emittable. However it might not be anyways, as there may be other
760
// restrictions, so move it into a separate LValue.
761
bld.setPosition(i, false);
764
i->setSrc(1, bld.mkMov(bld.getSSA(type), i->getSrc(0), type)->getDef(0));
765
i->setSrc(0, i->getSrc(2));
766
i->src(0).mod = i->src(2).mod;
769
if (i->src(0).getImmediate(src0))
776
// Leave PFETCH alone... we just folded its 2 args into 1.
779
i->op = i->saturate ? OP_SAT : OP_MOV;
781
unary(i, *i->getSrc(0)->asImm());
788
ConstantFolding::expr(Instruction *i,
789
ImmediateValue &imm0,
790
ImmediateValue &imm1,
791
ImmediateValue &imm2)
793
struct Storage *const a = &imm0.reg, *const b = &imm1.reg, *const c = &imm2.reg;
796
memset(&res.data, 0, sizeof(res.data));
800
for (int n = 0; n < 32; n++) {
801
uint8_t lut = ((a->data.u32 >> n) & 1) << 2 |
802
((b->data.u32 >> n) & 1) << 1 |
803
((c->data.u32 >> n) & 1);
804
res.data.u32 |= !!(i->subOp & (1 << lut)) << n;
809
uint64_t input = (uint64_t)c->data.u32 << 32 | a->data.u32;
810
uint16_t permt = b->data.u32;
811
for (int n = 0 ; n < 4; n++, permt >>= 4)
812
res.data.u32 |= ((input >> ((permt & 0xf) * 8)) & 0xff) << n * 8;
817
int offset = b->data.u32 & 0xff;
818
int width = (b->data.u32 >> 8) & 0xff;
819
unsigned bitmask = ((1 << width) - 1) << offset;
820
res.data.u32 = ((a->data.u32 << offset) & bitmask) | (c->data.u32 & ~bitmask);
827
res.data.f32 = a->data.f32 * b->data.f32 * exp2f(i->postFactor) +
831
res.data.f64 = a->data.f64 * b->data.f64 + c->data.f64;
834
if (i->subOp == NV50_IR_SUBOP_MUL_HIGH) {
835
res.data.s32 = ((int64_t)a->data.s32 * b->data.s32 >> 32) + c->data.s32;
840
if (i->subOp == NV50_IR_SUBOP_MUL_HIGH) {
841
res.data.u32 = ((uint64_t)a->data.u32 * b->data.u32 >> 32) + c->data.u32;
844
res.data.u32 = a->data.u32 * b->data.u32 + c->data.u32;
852
res.data.u32 = (a->data.u32 << b->data.u32) + c->data.u32;
859
i->src(0).mod = Modifier(0);
860
i->src(1).mod = Modifier(0);
861
i->src(2).mod = Modifier(0);
863
i->setSrc(0, new_ImmediateValue(i->bb->getProgram(), res.data.u32));
867
i->getSrc(0)->reg.data = res.data;
868
i->getSrc(0)->reg.type = i->dType;
869
i->getSrc(0)->reg.size = typeSizeof(i->dType);
875
ConstantFolding::unary(Instruction *i, const ImmediateValue &imm)
879
if (i->dType != TYPE_F32)
882
case OP_NEG: res.data.f32 = -imm.reg.data.f32; break;
883
case OP_ABS: res.data.f32 = fabsf(imm.reg.data.f32); break;
884
case OP_SAT: res.data.f32 = SATURATE(imm.reg.data.f32); break;
885
case OP_RCP: res.data.f32 = 1.0f / imm.reg.data.f32; break;
886
case OP_RSQ: res.data.f32 = 1.0f / sqrtf(imm.reg.data.f32); break;
887
case OP_LG2: res.data.f32 = log2f(imm.reg.data.f32); break;
888
case OP_EX2: res.data.f32 = exp2f(imm.reg.data.f32); break;
889
case OP_SIN: res.data.f32 = sinf(imm.reg.data.f32); break;
890
case OP_COS: res.data.f32 = cosf(imm.reg.data.f32); break;
891
case OP_SQRT: res.data.f32 = sqrtf(imm.reg.data.f32); break;
894
// these should be handled in subsequent OP_SIN/COS/EX2
895
res.data.f32 = imm.reg.data.f32;
901
i->setSrc(0, new_ImmediateValue(i->bb->getProgram(), res.data.f32));
902
i->src(0).mod = Modifier(0);
906
ConstantFolding::tryCollapseChainedMULs(Instruction *mul2,
907
const int s, ImmediateValue& imm2)
909
const int t = s ? 0 : 1;
911
Instruction *mul1 = NULL; // mul1 before mul2
913
float f = imm2.reg.data.f32 * exp2f(mul2->postFactor);
916
assert(mul2->op == OP_MUL && mul2->dType == TYPE_F32);
918
if (mul2->getSrc(t)->refCount() == 1) {
919
insn = mul2->getSrc(t)->getInsn();
920
if (!mul2->src(t).mod && insn->op == OP_MUL && insn->dType == TYPE_F32)
922
if (mul1 && !mul1->saturate) {
925
if (mul1->src(s1 = 0).getImmediate(imm1) ||
926
mul1->src(s1 = 1).getImmediate(imm1)) {
927
bld.setPosition(mul1, false);
929
// d = mul a, imm2 -> d = mul r, (imm1 * imm2)
930
mul1->setSrc(s1, bld.loadImm(NULL, f * imm1.reg.data.f32));
931
mul1->src(s1).mod = Modifier(0);
932
mul2->def(0).replace(mul1->getDef(0), false);
933
mul1->saturate = mul2->saturate;
935
if (prog->getTarget()->isPostMultiplySupported(OP_MUL, f, e)) {
937
// d = mul c, imm -> d = mul_x_imm a, b
938
mul1->postFactor = e;
939
mul2->def(0).replace(mul1->getDef(0), false);
941
mul1->src(0).mod *= Modifier(NV50_IR_MOD_NEG);
942
mul1->saturate = mul2->saturate;
947
if (mul2->getDef(0)->refCount() == 1 && !mul2->saturate) {
949
// d = mul b, c -> d = mul_x_imm a, c
951
insn = (*mul2->getDef(0)->uses.begin())->getInsn();
956
s2 = insn->getSrc(0) == mul1->getDef(0) ? 0 : 1;
958
if (insn->op == OP_MUL && insn->dType == TYPE_F32)
959
if (!insn->src(s2).mod && !insn->src(t2).getImmediate(imm1))
961
if (mul2 && prog->getTarget()->isPostMultiplySupported(OP_MUL, f, e)) {
962
mul2->postFactor = e;
963
mul2->setSrc(s2, mul1->src(t));
965
mul2->src(s2).mod *= Modifier(NV50_IR_MOD_NEG);
971
ConstantFolding::opnd3(Instruction *i, ImmediateValue &imm2)
976
if (imm2.isInteger(0)) {
984
if (imm2.isInteger(0)) {
997
ConstantFolding::createMul(DataType ty, Value *def, Value *a, int64_t b, Value *c)
999
const Target *target = prog->getTarget();
1000
int64_t absB = llabs(b);
1002
//a * (2^shl) -> a << shl
1003
if (b >= 0 && util_is_power_of_two_or_zero64(b)) {
1004
int shl = util_logbase2_64(b);
1006
Value *res = c ? bld.getSSA(typeSizeof(ty)) : def;
1007
bld.mkOp2(OP_SHL, ty, res, a, bld.mkImm(shl));
1009
bld.mkOp2(OP_ADD, ty, def, res, c);
1014
//a * (2^shl + 1) -> a << shl + a
1015
//a * -(2^shl + 1) -> -a << shl + a
1016
//a * (2^shl - 1) -> a << shl - a
1017
//a * -(2^shl - 1) -> -a << shl - a
1018
if (typeSizeof(ty) == 4 &&
1019
(util_is_power_of_two_or_zero64(absB - 1) ||
1020
util_is_power_of_two_or_zero64(absB + 1)) &&
1021
target->isOpSupported(OP_SHLADD, TYPE_U32)) {
1022
bool subA = util_is_power_of_two_or_zero64(absB + 1);
1023
int shl = subA ? util_logbase2_64(absB + 1) : util_logbase2_64(absB - 1);
1025
Value *res = c ? bld.getSSA() : def;
1026
Instruction *insn = bld.mkOp3(OP_SHLADD, TYPE_U32, res, a, bld.mkImm(shl), a);
1028
insn->src(0).mod = Modifier(NV50_IR_MOD_NEG);
1030
insn->src(2).mod = Modifier(NV50_IR_MOD_NEG);
1033
bld.mkOp2(OP_ADD, TYPE_U32, def, res, c);
1038
if (typeSizeof(ty) == 4 && b >= 0 && b <= 0xffff &&
1039
target->isOpSupported(OP_XMAD, TYPE_U32)) {
1040
Value *tmp = bld.mkOp3v(OP_XMAD, TYPE_U32, bld.getSSA(),
1041
a, bld.mkImm((uint32_t)b), c ? c : bld.mkImm(0));
1042
bld.mkOp3(OP_XMAD, TYPE_U32, def, a, bld.mkImm((uint32_t)b), tmp)->subOp =
1043
NV50_IR_SUBOP_XMAD_PSL | NV50_IR_SUBOP_XMAD_H1(0);
1052
ConstantFolding::opnd(Instruction *i, ImmediateValue &imm0, int s)
1055
const operation op = i->op;
1056
Instruction *newi = i;
1057
bool deleted = false;
1061
bld.setPosition(i, false);
1063
uint8_t size = i->getDef(0)->reg.size;
1064
uint8_t bitsize = size * 8;
1065
uint32_t mask = (1ULL << bitsize) - 1;
1066
assert(bitsize <= 32);
1068
uint64_t val = imm0.reg.data.u64;
1069
for (int8_t d = 0; i->defExists(d); ++d) {
1070
Value *def = i->getDef(d);
1071
assert(def->reg.size == size);
1073
newi = bld.mkMov(def, bld.mkImm((uint32_t)(val & mask)), TYPE_U32);
1076
delete_Instruction(prog, i);
1081
if (i->dType == TYPE_F32 && !i->precise)
1082
tryCollapseChainedMULs(i, s, imm0);
1084
if (i->subOp == NV50_IR_SUBOP_MUL_HIGH) {
1085
assert(!isFloatType(i->sType));
1086
if (imm0.isInteger(1) && i->dType == TYPE_S32) {
1087
bld.setPosition(i, false);
1088
// Need to set to the sign value, which is a compare.
1089
newi = bld.mkCmp(OP_SET, CC_LT, TYPE_S32, i->getDef(0),
1090
TYPE_S32, i->getSrc(t), bld.mkImm(0));
1091
delete_Instruction(prog, i);
1093
} else if (imm0.isInteger(0) || imm0.isInteger(1)) {
1094
// The high bits can't be set in this case (either mul by 0 or
1098
i->setSrc(0, new_ImmediateValue(prog, 0u));
1099
i->src(0).mod = Modifier(0);
1101
} else if (!imm0.isNegative() && imm0.isPow2()) {
1102
// Translate into a shift
1106
imm0.reg.data.u32 = 32 - imm0.reg.data.u32;
1107
i->setSrc(0, i->getSrc(t));
1108
i->src(0).mod = i->src(t).mod;
1109
i->setSrc(1, new_ImmediateValue(prog, imm0.reg.data.u32));
1113
if (imm0.isInteger(0)) {
1116
i->setSrc(0, new_ImmediateValue(prog, 0u));
1117
i->src(0).mod = Modifier(0);
1121
if (!i->postFactor && (imm0.isInteger(1) || imm0.isInteger(-1))) {
1122
if (imm0.isNegative())
1123
i->src(t).mod = i->src(t).mod ^ Modifier(NV50_IR_MOD_NEG);
1125
i->op = i->src(t).mod.getOp();
1127
i->setSrc(0, i->getSrc(1));
1128
i->src(0).mod = i->src(1).mod;
1131
if (i->op != OP_CVT)
1135
if (!i->postFactor && (imm0.isInteger(2) || imm0.isInteger(-2))) {
1136
if (imm0.isNegative())
1137
i->src(t).mod = i->src(t).mod ^ Modifier(NV50_IR_MOD_NEG);
1140
i->setSrc(s, i->getSrc(t));
1141
i->src(s).mod = i->src(t).mod;
1143
if (!isFloatType(i->dType) && !i->src(t).mod) {
1144
bld.setPosition(i, false);
1145
int64_t b = typeSizeof(i->dType) == 8 ? imm0.reg.data.s64 : imm0.reg.data.s32;
1146
if (createMul(i->dType, i->getDef(0), i->getSrc(t), b, NULL)) {
1147
delete_Instruction(prog, i);
1151
if (i->postFactor && i->sType == TYPE_F32) {
1152
/* Can't emit a postfactor with an immediate, have to fold it in */
1153
i->setSrc(s, new_ImmediateValue(
1154
prog, imm0.reg.data.f32 * exp2f(i->postFactor)));
1160
if (imm0.isInteger(0)) {
1161
i->setSrc(0, i->getSrc(2));
1162
i->src(0).mod = i->src(2).mod;
1166
i->op = i->src(0).mod.getOp();
1167
if (i->op != OP_CVT)
1170
if (i->subOp != NV50_IR_SUBOP_MUL_HIGH &&
1171
(imm0.isInteger(1) || imm0.isInteger(-1))) {
1172
if (imm0.isNegative())
1173
i->src(t).mod = i->src(t).mod ^ Modifier(NV50_IR_MOD_NEG);
1175
i->setSrc(0, i->getSrc(1));
1176
i->src(0).mod = i->src(1).mod;
1178
i->setSrc(1, i->getSrc(2));
1179
i->src(1).mod = i->src(2).mod;
1184
if (!isFloatType(i->dType) && !i->subOp && !i->src(t).mod && !i->src(2).mod) {
1185
bld.setPosition(i, false);
1186
int64_t b = typeSizeof(i->dType) == 8 ? imm0.reg.data.s64 : imm0.reg.data.s32;
1187
if (createMul(i->dType, i->getDef(0), i->getSrc(t), b, i->getSrc(2))) {
1188
delete_Instruction(prog, i);
1194
if (imm0.isInteger(0) && s == 0 && typeSizeof(i->dType) == 8 &&
1195
!isFloatType(i->dType))
1201
if (imm0.isInteger(0)) {
1203
i->setSrc(0, i->getSrc(1));
1204
i->src(0).mod = i->src(1).mod;
1205
if (i->op == OP_SUB)
1206
i->src(0).mod = i->src(0).mod ^ Modifier(NV50_IR_MOD_NEG);
1209
i->op = i->src(0).mod.getOp();
1210
if (i->op != OP_CVT)
1211
i->src(0).mod = Modifier(0);
1216
if (s != 1 || (i->dType != TYPE_S32 && i->dType != TYPE_U32))
1218
bld.setPosition(i, false);
1219
if (imm0.reg.data.u32 == 0) {
1222
if (imm0.reg.data.u32 == 1) {
1226
if (i->dType == TYPE_U32 && imm0.isPow2()) {
1228
i->setSrc(1, bld.mkImm(util_logbase2(imm0.reg.data.u32)));
1230
if (i->dType == TYPE_U32) {
1233
const uint32_t d = imm0.reg.data.u32;
1236
uint32_t l = util_logbase2(d);
1237
if (((uint32_t)1 << l) < d)
1239
m = (((uint64_t)1 << 32) * (((uint64_t)1 << l) - d)) / d + 1;
1241
s = l ? (l - 1) : 0;
1245
mul = bld.mkOp2(OP_MUL, TYPE_U32, tA, i->getSrc(0),
1246
bld.loadImm(NULL, m));
1247
mul->subOp = NV50_IR_SUBOP_MUL_HIGH;
1248
bld.mkOp2(OP_SUB, TYPE_U32, tB, i->getSrc(0), tA);
1251
bld.mkOp2(OP_SHR, TYPE_U32, tA, tB, bld.mkImm(r));
1254
tB = s ? bld.getSSA() : i->getDef(0);
1255
newi = bld.mkOp2(OP_ADD, TYPE_U32, tB, mul->getDef(0), tA);
1257
bld.mkOp2(OP_SHR, TYPE_U32, i->getDef(0), tB, bld.mkImm(s));
1259
delete_Instruction(prog, i);
1262
if (imm0.reg.data.s32 == -1) {
1268
const int32_t d = imm0.reg.data.s32;
1270
int32_t l = util_logbase2(static_cast<unsigned>(abs(d)));
1271
if ((1 << l) < abs(d))
1275
m = ((uint64_t)1 << (32 + l - 1)) / abs(d) + 1 - ((uint64_t)1 << 32);
1279
bld.mkOp3(OP_MAD, TYPE_S32, tA, i->getSrc(0), bld.loadImm(NULL, m),
1280
i->getSrc(0))->subOp = NV50_IR_SUBOP_MUL_HIGH;
1282
bld.mkOp2(OP_SHR, TYPE_S32, tB, tA, bld.mkImm(l - 1));
1286
bld.mkCmp(OP_SET, CC_LT, TYPE_S32, tA, TYPE_S32, i->getSrc(0), bld.mkImm(0));
1287
tD = (d < 0) ? bld.getSSA() : i->getDef(0)->asLValue();
1288
newi = bld.mkOp2(OP_SUB, TYPE_U32, tD, tB, tA);
1290
bld.mkOp1(OP_NEG, TYPE_S32, i->getDef(0), tB);
1292
delete_Instruction(prog, i);
1298
if (s == 1 && imm0.isPow2()) {
1299
bld.setPosition(i, false);
1300
if (i->sType == TYPE_U32) {
1302
i->setSrc(1, bld.loadImm(NULL, imm0.reg.data.u32 - 1));
1303
} else if (i->sType == TYPE_S32) {
1304
// Do it on the absolute value of the input, and then restore the
1305
// sign. The only odd case is MIN_INT, but that should work out
1306
// as well, since MIN_INT mod any power of 2 is 0.
1308
// Technically we don't have to do any of this since MOD is
1309
// undefined with negative arguments in GLSL, but this seems like
1310
// the nice thing to do.
1311
Value *abs = bld.mkOp1v(OP_ABS, TYPE_S32, bld.getSSA(), i->getSrc(0));
1312
Value *neg, *v1, *v2;
1313
bld.mkCmp(OP_SET, CC_LT, TYPE_S32,
1314
(neg = bld.getSSA(1, prog->getTarget()->nativeFile(FILE_PREDICATE))),
1315
TYPE_S32, i->getSrc(0), bld.loadImm(NULL, 0));
1316
Value *mod = bld.mkOp2v(OP_AND, TYPE_U32, bld.getSSA(), abs,
1317
bld.loadImm(NULL, imm0.reg.data.u32 - 1));
1318
bld.mkOp1(OP_NEG, TYPE_S32, (v1 = bld.getSSA()), mod)
1319
->setPredicate(CC_P, neg);
1320
bld.mkOp1(OP_MOV, TYPE_S32, (v2 = bld.getSSA()), mod)
1321
->setPredicate(CC_NOT_P, neg);
1322
newi = bld.mkOp2(OP_UNION, TYPE_S32, i->getDef(0), v1, v2);
1324
delete_Instruction(prog, i);
1327
} else if (s == 1) {
1328
// In this case, we still want the optimized lowering that we get
1329
// from having division by an immediate.
1331
// a % b == a - (a/b) * b
1332
bld.setPosition(i, false);
1333
Value *div = bld.mkOp2v(OP_DIV, i->sType, bld.getSSA(),
1334
i->getSrc(0), i->getSrc(1));
1335
newi = bld.mkOp2(OP_ADD, i->sType, i->getDef(0), i->getSrc(0),
1336
bld.mkOp2v(OP_MUL, i->sType, bld.getSSA(), div, i->getSrc(1)));
1337
// TODO: Check that target supports this. In this case, we know that
1339
newi->src(1).mod = Modifier(NV50_IR_MOD_NEG);
1341
delete_Instruction(prog, i);
1346
case OP_SET: // TODO: SET_AND,OR,XOR
1348
/* This optimizes the case where the output of a set is being compared
1349
* to zero. Since the set can only produce 0/-1 (int) or 0/1 (float), we
1350
* can be a lot cleverer in our comparison.
1352
CmpInstruction *si = findOriginForTestWithZero(i->getSrc(t));
1354
if (imm0.reg.data.u32 != 0 || !si)
1357
ccZ = (CondCode)((unsigned int)i->asCmp()->setCond & ~CC_U);
1358
// We do everything assuming var (cmp) 0, reverse the condition if 0 is
1361
ccZ = reverseCondCode(ccZ);
1362
// If there is a negative modifier, we need to undo that, by flipping
1363
// the comparison to zero.
1364
if (i->src(t).mod.neg())
1365
ccZ = reverseCondCode(ccZ);
1366
// If this is a signed comparison, we expect the input to be a regular
1367
// boolean, i.e. 0/-1. However the rest of the logic assumes that true
1368
// is positive, so just flip the sign.
1369
if (i->sType == TYPE_S32) {
1370
assert(!isFloatType(si->dType));
1371
ccZ = reverseCondCode(ccZ);
1374
case CC_LT: cc = CC_FL; break; // bool < 0 -- this is never true
1375
case CC_GE: cc = CC_TR; break; // bool >= 0 -- this is always true
1376
case CC_EQ: cc = inverseCondCode(cc); break; // bool == 0 -- !bool
1377
case CC_LE: cc = inverseCondCode(cc); break; // bool <= 0 -- !bool
1378
case CC_GT: break; // bool > 0 -- bool
1379
case CC_NE: break; // bool != 0 -- bool
1384
// Update the condition of this SET to be identical to the origin set,
1385
// but with the updated condition code. The original SET should get
1388
i->asCmp()->setCond = cc;
1389
i->setSrc(0, si->src(0));
1390
i->setSrc(1, si->src(1));
1391
if (si->srcExists(2))
1392
i->setSrc(2, si->src(2));
1393
i->sType = si->sType;
1399
Instruction *src = i->getSrc(t)->getInsn();
1400
ImmediateValue imm1;
1401
if (imm0.reg.data.u32 == 0) {
1403
i->setSrc(0, new_ImmediateValue(prog, 0u));
1404
i->src(0).mod = Modifier(0);
1406
} else if (imm0.reg.data.u32 == ~0U) {
1407
i->op = i->src(t).mod.getOp();
1409
i->setSrc(0, i->getSrc(t));
1410
i->src(0).mod = i->src(t).mod;
1413
} else if (src->asCmp()) {
1414
CmpInstruction *cmp = src->asCmp();
1415
if (!cmp || cmp->op == OP_SLCT || cmp->getDef(0)->refCount() > 1)
1417
if (!prog->getTarget()->isOpSupported(cmp->op, TYPE_F32))
1419
if (imm0.reg.data.f32 != 1.0)
1421
if (cmp->dType != TYPE_U32)
1424
cmp->dType = TYPE_F32;
1425
if (i->src(t).mod != Modifier(0)) {
1426
assert(i->src(t).mod == Modifier(NV50_IR_MOD_NOT));
1427
i->src(t).mod = Modifier(0);
1428
cmp->setCond = inverseCondCode(cmp->setCond);
1433
i->setSrc(0, i->getSrc(t));
1436
} else if (prog->getTarget()->isOpSupported(OP_EXTBF, TYPE_U32) &&
1437
src->op == OP_SHR &&
1438
src->src(1).getImmediate(imm1) &&
1439
i->src(t).mod == Modifier(0) &&
1440
util_is_power_of_two_or_zero(imm0.reg.data.u32 + 1)) {
1441
// low byte = offset, high byte = width
1442
uint32_t ext = (util_last_bit(imm0.reg.data.u32) << 8) | imm1.reg.data.u32;
1444
i->setSrc(0, src->getSrc(0));
1445
i->setSrc(1, new_ImmediateValue(prog, ext));
1446
} else if (src->op == OP_SHL &&
1447
src->src(1).getImmediate(imm1) &&
1448
i->src(t).mod == Modifier(0) &&
1449
util_is_power_of_two_or_zero(~imm0.reg.data.u32 + 1) &&
1450
util_last_bit(~imm0.reg.data.u32) <= imm1.reg.data.u32) {
1454
i->setSrc(0, i->getSrc(t));
1463
if (s != 1 || i->src(0).mod != Modifier(0))
1466
if (imm0.reg.data.u32 == 0) {
1471
// try to concatenate shifts
1472
Instruction *si = i->getSrc(0)->getInsn();
1475
ImmediateValue imm1;
1478
if (si->src(1).getImmediate(imm1)) {
1479
bld.setPosition(i, false);
1480
i->setSrc(0, si->getSrc(0));
1481
i->setSrc(1, bld.loadImm(NULL, imm0.reg.data.u32 + imm1.reg.data.u32));
1485
if (si->src(1).getImmediate(imm1) && imm0.reg.data.u32 == imm1.reg.data.u32) {
1486
bld.setPosition(i, false);
1488
i->setSrc(0, si->getSrc(0));
1489
i->setSrc(1, bld.loadImm(NULL, ~((1 << imm0.reg.data.u32) - 1)));
1494
if (isFloatType(si->dType))
1498
if (si->src(1).getImmediate(imm1))
1500
else if (si->src(0).getImmediate(imm1))
1505
bld.setPosition(i, false);
1508
i->dType = si->dType;
1509
i->sType = si->sType;
1510
i->setSrc(0, si->getSrc(!muls));
1511
i->setSrc(1, bld.loadImm(NULL, imm1.reg.data.u32 << imm0.reg.data.u32));
1516
if (isFloatType(si->dType))
1518
if (si->op != OP_SUB && si->src(0).getImmediate(imm1))
1520
else if (si->src(1).getImmediate(imm1))
1524
if (si->src(!adds).mod != Modifier(0))
1526
// SHL(ADD(x, y), z) = ADD(SHL(x, z), SHL(y, z))
1528
// This is more operations, but if one of x, y is an immediate, then
1529
// we can get a situation where (a) we can use ISCADD, or (b)
1530
// propagate the add bit into an indirect load.
1531
bld.setPosition(i, false);
1533
i->setSrc(adds, bld.loadImm(NULL, imm1.reg.data.u32 << imm0.reg.data.u32));
1534
i->setSrc(!adds, bld.mkOp2v(OP_SHL, i->dType,
1535
bld.getSSA(i->def(0).getSize(), i->def(0).getFile()),
1537
bld.mkImm(imm0.reg.data.u32)));
1562
case TYPE_S32: res = util_last_bit_signed(imm0.reg.data.s32) - 1; break;
1563
case TYPE_U32: res = util_last_bit(imm0.reg.data.u32) - 1; break;
1567
if (i->subOp == NV50_IR_SUBOP_BFIND_SAMT && res >= 0)
1569
bld.setPosition(i, false); /* make sure bld is init'ed */
1570
i->setSrc(0, bld.mkImm(res));
1577
uint32_t res = util_bitreverse(imm0.reg.data.u32);
1578
i->setSrc(0, new_ImmediateValue(i->bb->getProgram(), res));
1583
// Only deal with 1-arg POPCNT here
1584
if (i->srcExists(1))
1586
uint32_t res = util_bitcount(imm0.reg.data.u32);
1587
i->setSrc(0, new_ImmediateValue(i->bb->getProgram(), res));
1595
// TODO: handle 64-bit values properly
1596
if (typeSizeof(i->dType) == 8 || typeSizeof(i->sType) == 8)
1599
// TODO: handle single byte/word extractions
1603
bld.setPosition(i, true); /* make sure bld is init'ed */
1605
#define CASE(type, dst, fmin, fmax, imin, imax, umin, umax) \
1607
switch (i->sType) { \
1609
res.data.dst = util_iround(i->saturate ? \
1610
CLAMP(imm0.reg.data.f64, fmin, fmax) : \
1611
imm0.reg.data.f64); \
1614
res.data.dst = util_iround(i->saturate ? \
1615
CLAMP(imm0.reg.data.f32, fmin, fmax) : \
1616
imm0.reg.data.f32); \
1619
res.data.dst = i->saturate ? \
1620
CLAMP(imm0.reg.data.s32, imin, imax) : \
1621
imm0.reg.data.s32; \
1624
res.data.dst = i->saturate ? \
1625
CLAMP(imm0.reg.data.u32, umin, umax) : \
1626
imm0.reg.data.u32; \
1629
res.data.dst = i->saturate ? \
1630
CLAMP(imm0.reg.data.s16, imin, imax) : \
1631
imm0.reg.data.s16; \
1634
res.data.dst = i->saturate ? \
1635
CLAMP(imm0.reg.data.u16, umin, umax) : \
1636
imm0.reg.data.u16; \
1638
default: return false; \
1640
i->setSrc(0, bld.mkImm(res.data.dst)); \
1644
CASE(TYPE_U16, u16, 0, UINT16_MAX, 0, UINT16_MAX, 0, UINT16_MAX);
1645
CASE(TYPE_S16, s16, INT16_MIN, INT16_MAX, INT16_MIN, INT16_MAX, 0, INT16_MAX);
1646
CASE(TYPE_U32, u32, 0, UINT32_MAX, 0, INT32_MAX, 0, UINT32_MAX);
1647
CASE(TYPE_S32, s32, INT32_MIN, INT32_MAX, INT32_MIN, INT32_MAX, 0, INT32_MAX);
1651
res.data.f32 = i->saturate ?
1652
SATURATE(imm0.reg.data.f64) :
1656
res.data.f32 = i->saturate ?
1657
SATURATE(imm0.reg.data.f32) :
1660
case TYPE_U16: res.data.f32 = (float) imm0.reg.data.u16; break;
1661
case TYPE_U32: res.data.f32 = (float) imm0.reg.data.u32; break;
1662
case TYPE_S16: res.data.f32 = (float) imm0.reg.data.s16; break;
1663
case TYPE_S32: res.data.f32 = (float) imm0.reg.data.s32; break;
1667
i->setSrc(0, bld.mkImm(res.data.f32));
1672
res.data.f64 = i->saturate ?
1673
SATURATE(imm0.reg.data.f64) :
1677
res.data.f64 = i->saturate ?
1678
SATURATE(imm0.reg.data.f32) :
1681
case TYPE_U16: res.data.f64 = (double) imm0.reg.data.u16; break;
1682
case TYPE_U32: res.data.f64 = (double) imm0.reg.data.u32; break;
1683
case TYPE_S16: res.data.f64 = (double) imm0.reg.data.s16; break;
1684
case TYPE_S32: res.data.f64 = (double) imm0.reg.data.s32; break;
1688
i->setSrc(0, bld.mkImm(res.data.f64));
1695
i->setType(i->dType); /* Remove i->sType, which we don't need anymore */
1698
i->src(0).mod = Modifier(0); /* Clear the already applied modifier */
1705
// This can get left behind some of the optimizations which simplify
1706
// saturatable values.
1707
if (newi->op == OP_MOV && newi->saturate) {
1711
if (newi->src(0).getImmediate(tmp))
1720
// =============================================================================
1722
// Merge modifier operations (ABS, NEG, NOT) into ValueRefs where allowed.
1723
class ModifierFolding : public Pass
1726
virtual bool visit(BasicBlock *);
1730
ModifierFolding::visit(BasicBlock *bb)
1732
const Target *target = prog->getTarget();
1734
Instruction *i, *next, *mi;
1737
for (i = bb->getEntry(); i; i = next) {
1740
if (false && i->op == OP_SUB) {
1741
// turn "sub" into "add neg" (do we really want this ?)
1743
i->src(0).mod = i->src(0).mod ^ Modifier(NV50_IR_MOD_NEG);
1746
for (int s = 0; s < 3 && i->srcExists(s); ++s) {
1747
mi = i->getSrc(s)->getInsn();
1749
mi->predSrc >= 0 || mi->getDef(0)->refCount() > 8)
1751
if (i->sType == TYPE_U32 && mi->dType == TYPE_S32) {
1752
if ((i->op != OP_ADD &&
1754
(mi->op != OP_ABS &&
1758
if (i->sType != mi->dType) {
1761
if ((mod = Modifier(mi->op)) == Modifier(0))
1763
mod *= mi->src(0).mod;
1765
if ((i->op == OP_ABS) || i->src(s).mod.abs()) {
1766
// abs neg [abs] = abs
1767
mod = mod & Modifier(~(NV50_IR_MOD_NEG | NV50_IR_MOD_ABS));
1769
if ((i->op == OP_NEG) && mod.neg()) {
1771
// neg as both opcode and modifier on same insn is prohibited
1772
// neg neg abs = abs, neg neg = identity
1773
mod = mod & Modifier(~NV50_IR_MOD_NEG);
1774
i->op = mod.getOp();
1775
mod = mod & Modifier(~NV50_IR_MOD_ABS);
1776
if (mod == Modifier(0))
1780
if (target->isModSupported(i, s, mod)) {
1781
i->setSrc(s, mi->getSrc(0));
1782
i->src(s).mod *= mod;
1786
if (i->op == OP_SAT) {
1787
mi = i->getSrc(0)->getInsn();
1789
mi->getDef(0)->refCount() <= 1 && target->isSatSupported(mi)) {
1791
mi->setDef(0, i->getDef(0));
1792
delete_Instruction(prog, i);
1800
// =============================================================================
1802
// MUL + ADD -> MAD/FMA
1803
// MIN/MAX(a, a) -> a, etc.
1804
// SLCT(a, b, const) -> cc(const) ? a : b
1806
// MUL(MUL(a, b), const) -> MUL_Xconst(a, b)
1807
// EXTBF(RDSV(COMBINED_TID)) -> RDSV(TID)
1808
class AlgebraicOpt : public Pass
1811
virtual bool visit(BasicBlock *);
1813
void handleABS(Instruction *);
1814
bool handleADD(Instruction *);
1815
bool tryADDToMADOrSAD(Instruction *, operation toOp);
1816
void handleMINMAX(Instruction *);
1817
void handleRCP(Instruction *);
1818
void handleSLCT(Instruction *);
1819
void handleLOGOP(Instruction *);
1820
void handleCVT_NEG(Instruction *);
1821
void handleCVT_CVT(Instruction *);
1822
void handleCVT_EXTBF(Instruction *);
1823
void handleSUCLAMP(Instruction *);
1824
void handleNEG(Instruction *);
1825
void handleEXTBF_RDSV(Instruction *);
1831
AlgebraicOpt::handleABS(Instruction *abs)
1833
Instruction *sub = abs->getSrc(0)->getInsn();
1836
!prog->getTarget()->isOpSupported(OP_SAD, abs->dType))
1838
// hidden conversion ?
1839
ty = intTypeToSigned(sub->dType);
1840
if (abs->dType != abs->sType || ty != abs->sType)
1843
if ((sub->op != OP_ADD && sub->op != OP_SUB) ||
1844
sub->src(0).getFile() != FILE_GPR || sub->src(0).mod ||
1845
sub->src(1).getFile() != FILE_GPR || sub->src(1).mod)
1848
Value *src0 = sub->getSrc(0);
1849
Value *src1 = sub->getSrc(1);
1851
if (sub->op == OP_ADD) {
1852
Instruction *neg = sub->getSrc(1)->getInsn();
1853
if (neg && neg->op != OP_NEG) {
1854
neg = sub->getSrc(0)->getInsn();
1855
src0 = sub->getSrc(1);
1857
if (!neg || neg->op != OP_NEG ||
1858
neg->dType != neg->sType || neg->sType != ty)
1860
src1 = neg->getSrc(0);
1864
abs->moveSources(1, 2); // move sources >=1 up by 2
1866
abs->setType(sub->dType);
1867
abs->setSrc(0, src0);
1868
abs->setSrc(1, src1);
1869
bld.setPosition(abs, false);
1870
abs->setSrc(2, bld.loadImm(bld.getSSA(typeSizeof(ty)), 0));
1874
AlgebraicOpt::handleADD(Instruction *add)
1876
Value *src0 = add->getSrc(0);
1877
Value *src1 = add->getSrc(1);
1879
if (src0->reg.file != FILE_GPR || src1->reg.file != FILE_GPR)
1882
bool changed = false;
1883
// we can't optimize to MAD if the add is precise
1884
if (!add->precise && prog->getTarget()->isOpSupported(OP_MAD, add->dType))
1885
changed = tryADDToMADOrSAD(add, OP_MAD);
1886
if (!changed && prog->getTarget()->isOpSupported(OP_SAD, add->dType))
1887
changed = tryADDToMADOrSAD(add, OP_SAD);
1891
// ADD(SAD(a,b,0), c) -> SAD(a,b,c)
1892
// ADD(MUL(a,b), c) -> MAD(a,b,c)
1894
AlgebraicOpt::tryADDToMADOrSAD(Instruction *add, operation toOp)
1896
Value *src0 = add->getSrc(0);
1897
Value *src1 = add->getSrc(1);
1900
const operation srcOp = toOp == OP_SAD ? OP_SAD : OP_MUL;
1901
const Modifier modBad = Modifier(~((toOp == OP_MAD) ? NV50_IR_MOD_NEG : 0));
1904
if (src0->refCount() == 1 &&
1905
src0->getUniqueInsn() && src0->getUniqueInsn()->op == srcOp)
1908
if (src1->refCount() == 1 &&
1909
src1->getUniqueInsn() && src1->getUniqueInsn()->op == srcOp)
1914
src = add->getSrc(s);
1916
if (src->getUniqueInsn() && src->getUniqueInsn()->bb != add->bb)
1919
if (src->getInsn()->saturate || src->getInsn()->postFactor ||
1920
src->getInsn()->dnz || src->getInsn()->precise)
1923
if (toOp == OP_SAD) {
1925
if (!src->getInsn()->src(2).getImmediate(imm))
1927
if (!imm.isInteger(0))
1931
if (typeSizeof(add->dType) != typeSizeof(src->getInsn()->dType) ||
1932
isFloatType(add->dType) != isFloatType(src->getInsn()->dType))
1935
mod[0] = add->src(0).mod;
1936
mod[1] = add->src(1).mod;
1937
mod[2] = src->getUniqueInsn()->src(0).mod;
1938
mod[3] = src->getUniqueInsn()->src(1).mod;
1940
if (((mod[0] | mod[1]) | (mod[2] | mod[3])) & modBad)
1944
add->subOp = src->getInsn()->subOp; // potentially mul-high
1945
add->dnz = src->getInsn()->dnz;
1946
add->dType = src->getInsn()->dType; // sign matters for imad hi
1947
add->sType = src->getInsn()->sType;
1949
add->setSrc(2, add->src(s ? 0 : 1));
1951
add->setSrc(0, src->getInsn()->getSrc(0));
1952
add->src(0).mod = mod[2] ^ mod[s];
1953
add->setSrc(1, src->getInsn()->getSrc(1));
1954
add->src(1).mod = mod[3];
1960
AlgebraicOpt::handleMINMAX(Instruction *minmax)
1962
Value *src0 = minmax->getSrc(0);
1963
Value *src1 = minmax->getSrc(1);
1965
if (src0 != src1 || src0->reg.file != FILE_GPR)
1967
if (minmax->src(0).mod == minmax->src(1).mod) {
1968
if (minmax->def(0).mayReplace(minmax->src(0))) {
1969
minmax->def(0).replace(minmax->src(0), false);
1970
delete_Instruction(prog, minmax);
1972
minmax->op = OP_CVT;
1973
minmax->setSrc(1, NULL);
1977
// min(x, -x) = -abs(x)
1978
// min(x, -abs(x)) = -abs(x)
1979
// min(x, abs(x)) = x
1980
// max(x, -abs(x)) = x
1981
// max(x, abs(x)) = abs(x)
1982
// max(x, -x) = abs(x)
1987
// rcp(sqrt(a)) = rsq(a)
1989
AlgebraicOpt::handleRCP(Instruction *rcp)
1991
Instruction *si = rcp->getSrc(0)->getUniqueInsn();
1996
if (si->op == OP_RCP) {
1997
Modifier mod = rcp->src(0).mod * si->src(0).mod;
1998
rcp->op = mod.getOp();
1999
rcp->setSrc(0, si->getSrc(0));
2000
} else if (si->op == OP_SQRT) {
2002
rcp->setSrc(0, si->getSrc(0));
2003
rcp->src(0).mod = rcp->src(0).mod * si->src(0).mod;
2008
AlgebraicOpt::handleSLCT(Instruction *slct)
2010
if (slct->getSrc(2)->reg.file == FILE_IMMEDIATE) {
2011
if (slct->getSrc(2)->asImm()->compare(slct->asCmp()->setCond, 0.0f))
2012
slct->setSrc(0, slct->getSrc(1));
2014
if (slct->getSrc(0) != slct->getSrc(1)) {
2018
slct->setSrc(1, NULL);
2019
slct->setSrc(2, NULL);
2023
AlgebraicOpt::handleLOGOP(Instruction *logop)
2025
Value *src0 = logop->getSrc(0);
2026
Value *src1 = logop->getSrc(1);
2028
if (src0->reg.file != FILE_GPR || src1->reg.file != FILE_GPR)
2032
if ((logop->op == OP_AND || logop->op == OP_OR) &&
2033
logop->def(0).mayReplace(logop->src(0))) {
2034
logop->def(0).replace(logop->src(0), false);
2035
delete_Instruction(prog, logop);
2038
// try AND(SET, SET) -> SET_AND(SET)
2039
Instruction *set0 = src0->getInsn();
2040
Instruction *set1 = src1->getInsn();
2042
if (!set0 || set0->fixed || !set1 || set1->fixed)
2044
if (set1->op != OP_SET) {
2045
Instruction *xchg = set0;
2048
if (set1->op != OP_SET)
2051
operation redOp = (logop->op == OP_AND ? OP_SET_AND :
2052
logop->op == OP_XOR ? OP_SET_XOR : OP_SET_OR);
2053
if (!prog->getTarget()->isOpSupported(redOp, set1->sType))
2055
if (set0->op != OP_SET &&
2056
set0->op != OP_SET_AND &&
2057
set0->op != OP_SET_OR &&
2058
set0->op != OP_SET_XOR)
2060
if (set0->getDef(0)->refCount() > 1 &&
2061
set1->getDef(0)->refCount() > 1)
2063
if (set0->getPredicate() || set1->getPredicate())
2065
// check that they don't source each other
2066
for (int s = 0; s < 2; ++s)
2067
if (set0->getSrc(s) == set1->getDef(0) ||
2068
set1->getSrc(s) == set0->getDef(0))
2071
set0 = cloneForward(func, set0);
2072
set1 = cloneShallow(func, set1);
2073
logop->bb->insertAfter(logop, set1);
2074
logop->bb->insertAfter(logop, set0);
2076
set0->dType = TYPE_U8;
2077
set0->getDef(0)->reg.file = FILE_PREDICATE;
2078
set0->getDef(0)->reg.size = 1;
2079
set1->setSrc(2, set0->getDef(0));
2081
set1->setDef(0, logop->getDef(0));
2082
delete_Instruction(prog, logop);
2086
// F2I(NEG(SET with result 1.0f/0.0f)) -> SET with result -1/0
2088
// F2I(NEG(I2F(ABS(SET))))
2090
AlgebraicOpt::handleCVT_NEG(Instruction *cvt)
2092
Instruction *insn = cvt->getSrc(0)->getInsn();
2093
if (cvt->sType != TYPE_F32 ||
2094
cvt->dType != TYPE_S32 || cvt->src(0).mod != Modifier(0))
2096
if (!insn || insn->op != OP_NEG || insn->dType != TYPE_F32)
2098
if (insn->src(0).mod != Modifier(0))
2100
insn = insn->getSrc(0)->getInsn();
2102
// check for nv50 SET(-1,0) -> SET(1.0f/0.0f) chain and nvc0's f32 SET
2103
if (insn && insn->op == OP_CVT &&
2104
insn->dType == TYPE_F32 &&
2105
insn->sType == TYPE_S32) {
2106
insn = insn->getSrc(0)->getInsn();
2107
if (!insn || insn->op != OP_ABS || insn->sType != TYPE_S32 ||
2110
insn = insn->getSrc(0)->getInsn();
2111
if (!insn || insn->op != OP_SET || insn->dType != TYPE_U32)
2114
if (!insn || insn->op != OP_SET || insn->dType != TYPE_F32) {
2118
Instruction *bset = cloneShallow(func, insn);
2119
bset->dType = TYPE_U32;
2120
bset->setDef(0, cvt->getDef(0));
2121
cvt->bb->insertAfter(cvt, bset);
2122
delete_Instruction(prog, cvt);
2125
// F2I(TRUNC()) and so on can be expressed as a single CVT. If the earlier CVT
2126
// does a type conversion, this becomes trickier as there might be range
2127
// changes/etc. We could handle those in theory as long as the range was being
2128
// reduced or kept the same.
2130
AlgebraicOpt::handleCVT_CVT(Instruction *cvt)
2132
Instruction *insn = cvt->getSrc(0)->getInsn();
2137
insn->dType != insn->sType ||
2138
insn->dType != cvt->sType)
2141
RoundMode rnd = insn->rnd;
2158
if (!isFloatType(cvt->dType) || !isFloatType(insn->sType))
2159
rnd = (RoundMode)(rnd & 3);
2162
cvt->setSrc(0, insn->getSrc(0));
2163
cvt->src(0).mod *= insn->src(0).mod;
2164
cvt->sType = insn->sType;
2167
// Some shaders extract packed bytes out of words and convert them to
2168
// e.g. float. The Fermi+ CVT instruction can extract those directly, as can
2169
// nv50 for word sizes.
2171
// CVT(EXTBF(x, byte/word))
2172
// CVT(AND(bytemask, x))
2173
// CVT(AND(bytemask, SHR(x, 8/16/24)))
2174
// CVT(SHR(x, 16/24))
2176
AlgebraicOpt::handleCVT_EXTBF(Instruction *cvt)
2178
Instruction *insn = cvt->getSrc(0)->getInsn();
2181
unsigned width, offset = 0;
2182
if ((cvt->sType != TYPE_U32 && cvt->sType != TYPE_S32) || !insn)
2184
if (insn->op == OP_EXTBF && insn->src(1).getImmediate(imm)) {
2185
width = (imm.reg.data.u32 >> 8) & 0xff;
2186
offset = imm.reg.data.u32 & 0xff;
2187
arg = insn->getSrc(0);
2189
if (width != 8 && width != 16)
2191
if (width == 8 && offset & 0x7)
2193
if (width == 16 && offset & 0xf)
2195
} else if (insn->op == OP_AND) {
2197
if (insn->src(0).getImmediate(imm))
2199
else if (insn->src(1).getImmediate(imm))
2204
if (imm.reg.data.u32 == 0xff)
2206
else if (imm.reg.data.u32 == 0xffff)
2211
arg = insn->getSrc(!s);
2212
Instruction *shift = arg->getInsn();
2214
if (shift && shift->op == OP_SHR &&
2215
shift->sType == cvt->sType &&
2216
shift->src(1).getImmediate(imm) &&
2217
((width == 8 && (imm.reg.data.u32 & 0x7) == 0) ||
2218
(width == 16 && (imm.reg.data.u32 & 0xf) == 0))) {
2219
arg = shift->getSrc(0);
2220
offset = imm.reg.data.u32;
2222
// We just AND'd the high bits away, which means this is effectively an
2224
cvt->sType = TYPE_U32;
2225
} else if (insn->op == OP_SHR &&
2226
insn->sType == cvt->sType &&
2227
insn->src(1).getImmediate(imm)) {
2228
arg = insn->getSrc(0);
2229
if (imm.reg.data.u32 == 24) {
2232
} else if (imm.reg.data.u32 == 16) {
2243
// Irrespective of what came earlier, we can undo a shift on the argument
2244
// by adjusting the offset.
2245
Instruction *shift = arg->getInsn();
2246
if (shift && shift->op == OP_SHL &&
2247
shift->src(1).getImmediate(imm) &&
2248
((width == 8 && (imm.reg.data.u32 & 0x7) == 0) ||
2249
(width == 16 && (imm.reg.data.u32 & 0xf) == 0)) &&
2250
imm.reg.data.u32 <= offset) {
2251
arg = shift->getSrc(0);
2252
offset -= imm.reg.data.u32;
2255
// The unpackSnorm lowering still leaves a few shifts behind, but it's too
2256
// annoying to detect them.
2259
cvt->sType = cvt->sType == TYPE_U32 ? TYPE_U8 : TYPE_S8;
2261
assert(width == 16);
2262
cvt->sType = cvt->sType == TYPE_U32 ? TYPE_U16 : TYPE_S16;
2264
cvt->setSrc(0, arg);
2265
cvt->subOp = offset >> 3;
2268
// SUCLAMP dst, (ADD b imm), k, 0 -> SUCLAMP dst, b, k, imm (if imm fits s6)
2270
AlgebraicOpt::handleSUCLAMP(Instruction *insn)
2273
int32_t val = insn->getSrc(2)->asImm()->reg.data.s32;
2277
assert(insn->srcExists(0) && insn->src(0).getFile() == FILE_GPR);
2279
// look for ADD (TODO: only count references by non-SUCLAMP)
2280
if (insn->getSrc(0)->refCount() > 1)
2282
add = insn->getSrc(0)->getInsn();
2283
if (!add || add->op != OP_ADD ||
2284
(add->dType != TYPE_U32 &&
2285
add->dType != TYPE_S32))
2288
// look for immediate
2289
for (s = 0; s < 2; ++s)
2290
if (add->src(s).getImmediate(imm))
2295
// determine if immediate fits
2296
val += imm.reg.data.s32;
2297
if (val > 31 || val < -32)
2299
// determine if other addend fits
2300
if (add->src(s).getFile() != FILE_GPR || add->src(s).mod != Modifier(0))
2303
bld.setPosition(insn, false); // make sure bld is init'ed
2305
insn->setSrc(2, bld.mkImm(val));
2306
insn->setSrc(0, add->getSrc(s));
2309
// NEG(AND(SET, 1)) -> SET
2311
AlgebraicOpt::handleNEG(Instruction *i) {
2312
Instruction *src = i->getSrc(0)->getInsn();
2316
if (isFloatType(i->sType) || !src || src->op != OP_AND)
2319
if (src->src(0).getImmediate(imm))
2321
else if (src->src(1).getImmediate(imm))
2326
if (!imm.isInteger(1))
2329
Instruction *set = src->getSrc(b)->getInsn();
2330
if ((set->op == OP_SET || set->op == OP_SET_AND ||
2331
set->op == OP_SET_OR || set->op == OP_SET_XOR) &&
2332
!isFloatType(set->dType)) {
2333
i->def(0).replace(set->getDef(0), false);
2337
// EXTBF(RDSV(COMBINED_TID)) -> RDSV(TID)
2339
AlgebraicOpt::handleEXTBF_RDSV(Instruction *i)
2341
Instruction *rdsv = i->getSrc(0)->getUniqueInsn();
2342
if (rdsv->op != OP_RDSV ||
2343
rdsv->getSrc(0)->asSym()->reg.data.sv.sv != SV_COMBINED_TID)
2345
// Avoid creating more RDSV instructions
2346
if (rdsv->getDef(0)->refCount() > 1)
2350
if (!i->src(1).getImmediate(imm))
2354
if (imm.isInteger(0x1000))
2357
if (imm.isInteger(0x0a10))
2360
if (imm.isInteger(0x061a))
2365
bld.setPosition(i, false);
2368
i->setSrc(0, bld.mkSysVal(SV_TID, index));
2373
AlgebraicOpt::visit(BasicBlock *bb)
2376
for (Instruction *i = bb->getEntry(); i; i = next) {
2403
if (prog->getTarget()->isOpSupported(OP_EXTBF, TYPE_U32))
2413
handleEXTBF_RDSV(i);
2423
// =============================================================================
2425
// ADD(SHL(a, b), c) -> SHLADD(a, b, c)
2426
// MUL(a, b) -> a few XMADs
2427
// MAD/FMA(a, b, c) -> a few XMADs
2428
class LateAlgebraicOpt : public Pass
2431
virtual bool visit(Instruction *);
2433
void handleADD(Instruction *);
2434
void handleMULMAD(Instruction *);
2435
bool tryADDToSHLADD(Instruction *);
2441
LateAlgebraicOpt::handleADD(Instruction *add)
2443
Value *src0 = add->getSrc(0);
2444
Value *src1 = add->getSrc(1);
2446
if (src0->reg.file != FILE_GPR || src1->reg.file != FILE_GPR)
2449
if (prog->getTarget()->isOpSupported(OP_SHLADD, add->dType))
2450
tryADDToSHLADD(add);
2453
// ADD(SHL(a, b), c) -> SHLADD(a, b, c)
2455
LateAlgebraicOpt::tryADDToSHLADD(Instruction *add)
2457
Value *src0 = add->getSrc(0);
2458
Value *src1 = add->getSrc(1);
2464
if (add->saturate || add->usesFlags() || typeSizeof(add->dType) == 8
2465
|| isFloatType(add->dType))
2468
if (src0->getUniqueInsn() && src0->getUniqueInsn()->op == OP_SHL)
2471
if (src1->getUniqueInsn() && src1->getUniqueInsn()->op == OP_SHL)
2476
src = add->getSrc(s);
2477
shl = src->getUniqueInsn();
2479
if (shl->bb != add->bb || shl->usesFlags() || shl->subOp || shl->src(0).mod)
2482
if (!shl->src(1).getImmediate(imm))
2485
add->op = OP_SHLADD;
2486
add->setSrc(2, add->src(!s));
2487
// SHL can't have any modifiers, but the ADD source may have had
2488
// one. Preserve it.
2489
add->setSrc(0, shl->getSrc(0));
2491
add->src(0).mod = add->src(1).mod;
2492
add->setSrc(1, new_ImmediateValue(shl->bb->getProgram(), imm.reg.data.u32));
2493
add->src(1).mod = Modifier(0);
2498
// MUL(a, b) -> a few XMADs
2499
// MAD/FMA(a, b, c) -> a few XMADs
2501
LateAlgebraicOpt::handleMULMAD(Instruction *i)
2503
// TODO: handle NV50_IR_SUBOP_MUL_HIGH
2504
if (!prog->getTarget()->isOpSupported(OP_XMAD, TYPE_U32))
2506
if (isFloatType(i->dType) || typeSizeof(i->dType) != 4)
2508
if (i->subOp || i->usesFlags() || i->flagsDef >= 0)
2511
assert(!i->src(0).mod);
2512
assert(!i->src(1).mod);
2513
assert(i->op == OP_MUL ? 1 : !i->src(2).mod);
2515
bld.setPosition(i, false);
2517
Value *a = i->getSrc(0);
2518
Value *b = i->getSrc(1);
2519
Value *c = i->op == OP_MUL ? bld.mkImm(0) : i->getSrc(2);
2521
Value *tmp0 = bld.getSSA();
2522
Value *tmp1 = bld.getSSA();
2524
Instruction *insn = bld.mkOp3(OP_XMAD, TYPE_U32, tmp0, b, a, c);
2525
insn->setPredicate(i->cc, i->getPredicate());
2527
insn = bld.mkOp3(OP_XMAD, TYPE_U32, tmp1, b, a, bld.mkImm(0));
2528
insn->setPredicate(i->cc, i->getPredicate());
2529
insn->subOp = NV50_IR_SUBOP_XMAD_MRG | NV50_IR_SUBOP_XMAD_H1(1);
2531
Value *pred = i->getPredicate();
2532
i->setPredicate(i->cc, NULL);
2538
i->subOp = NV50_IR_SUBOP_XMAD_PSL | NV50_IR_SUBOP_XMAD_CBCC;
2539
i->subOp |= NV50_IR_SUBOP_XMAD_H1(0) | NV50_IR_SUBOP_XMAD_H1(1);
2541
i->setPredicate(i->cc, pred);
2545
LateAlgebraicOpt::visit(Instruction *i)
2563
// =============================================================================
2565
// Split 64-bit MUL and MAD
2566
class Split64BitOpPreRA : public Pass
2569
virtual bool visit(BasicBlock *);
2570
void split64MulMad(Function *, Instruction *, DataType);
2576
Split64BitOpPreRA::visit(BasicBlock *bb)
2578
Instruction *i, *next;
2581
for (i = bb->getEntry(); i; i = next) {
2586
case TYPE_U64: hTy = TYPE_U32; break;
2587
case TYPE_S64: hTy = TYPE_S32; break;
2592
if (i->op == OP_MAD || i->op == OP_MUL)
2593
split64MulMad(func, i, hTy);
2600
Split64BitOpPreRA::split64MulMad(Function *fn, Instruction *i, DataType hTy)
2602
assert(i->op == OP_MAD || i->op == OP_MUL);
2603
assert(!isFloatType(i->dType) && !isFloatType(i->sType));
2604
assert(typeSizeof(hTy) == 4);
2606
bld.setPosition(i, true);
2608
Value *zero = bld.mkImm(0u);
2609
Value *carry = bld.getSSA(1, FILE_FLAGS);
2611
// We want to compute `d = a * b (+ c)?`, where a, b, c and d are 64-bit
2612
// values (a, b and c might be 32-bit values), using 32-bit operations. This
2613
// gives the following operations:
2614
// * `d.low = low(a.low * b.low) (+ c.low)?`
2615
// * `d.high = low(a.high * b.low) + low(a.low * b.high)
2616
// + high(a.low * b.low) (+ c.high)?`
2618
// To compute the high bits, we can split in the following operations:
2619
// * `tmp1 = low(a.high * b.low) (+ c.high)?`
2620
// * `tmp2 = low(a.low * b.high) + tmp1`
2621
// * `d.high = high(a.low * b.low) + tmp2`
2623
// mkSplit put lower bits at index 0 and higher bits at index 1
2626
if (i->getSrc(0)->reg.size == 8)
2627
bld.mkSplit(op1, 4, i->getSrc(0));
2629
op1[0] = i->getSrc(0);
2633
if (i->getSrc(1)->reg.size == 8)
2634
bld.mkSplit(op2, 4, i->getSrc(1));
2636
op2[0] = i->getSrc(1);
2640
Value *op3[2] = { NULL, NULL };
2641
if (i->op == OP_MAD) {
2642
if (i->getSrc(2)->reg.size == 8)
2643
bld.mkSplit(op3, 4, i->getSrc(2));
2645
op3[0] = i->getSrc(2);
2650
Value *tmpRes1Hi = bld.getSSA();
2651
if (i->op == OP_MAD)
2652
bld.mkOp3(OP_MAD, hTy, tmpRes1Hi, op1[1], op2[0], op3[1]);
2654
bld.mkOp2(OP_MUL, hTy, tmpRes1Hi, op1[1], op2[0]);
2656
Value *tmpRes2Hi = bld.mkOp3v(OP_MAD, hTy, bld.getSSA(), op1[0], op2[1], tmpRes1Hi);
2658
Value *def[2] = { bld.getSSA(), bld.getSSA() };
2660
// If it was a MAD, add the carry from the low bits
2661
// It is not needed if it was a MUL, since we added high(a.low * b.low) to
2663
if (i->op == OP_MAD)
2664
bld.mkOp3(OP_MAD, hTy, def[0], op1[0], op2[0], op3[0])->setFlagsDef(1, carry);
2666
bld.mkOp2(OP_MUL, hTy, def[0], op1[0], op2[0]);
2668
Instruction *hiPart3 = bld.mkOp3(OP_MAD, hTy, def[1], op1[0], op2[0], tmpRes2Hi);
2669
hiPart3->subOp = NV50_IR_SUBOP_MUL_HIGH;
2670
if (i->op == OP_MAD)
2671
hiPart3->setFlagsSrc(3, carry);
2673
bld.mkOp2(OP_MERGE, i->dType, i->getDef(0), def[0], def[1]);
2675
delete_Instruction(fn->getProgram(), i);
2678
// =============================================================================
2681
updateLdStOffset(Instruction *ldst, int32_t offset, Function *fn)
2683
if (offset != ldst->getSrc(0)->reg.data.offset) {
2684
if (ldst->getSrc(0)->refCount() > 1)
2685
ldst->setSrc(0, cloneShallow(fn, ldst->getSrc(0)));
2686
ldst->getSrc(0)->reg.data.offset = offset;
2690
// Combine loads and stores, forward stores to loads where possible.
2691
class MemoryOpt : public Pass
2699
const Value *rel[2];
2707
bool overlaps(const Instruction *ldst) const;
2709
inline void link(Record **);
2710
inline void unlink(Record **);
2711
inline void set(const Instruction *ldst);
2717
Record *loads[DATA_FILE_COUNT];
2718
Record *stores[DATA_FILE_COUNT];
2720
MemoryPool recordPool;
2723
virtual bool visit(BasicBlock *);
2724
bool runOpt(BasicBlock *);
2726
Record **getList(const Instruction *);
2728
Record *findRecord(const Instruction *, bool load, bool& isAdjacent) const;
2730
// merge @insn into load/store instruction from @rec
2731
bool combineLd(Record *rec, Instruction *ld);
2732
bool combineSt(Record *rec, Instruction *st);
2734
bool replaceLdFromLd(Instruction *ld, Record *ldRec);
2735
bool replaceLdFromSt(Instruction *ld, Record *stRec);
2736
bool replaceStFromSt(Instruction *restrict st, Record *stRec);
2738
void addRecord(Instruction *ldst);
2739
void purgeRecords(Instruction *const st, DataFile);
2740
void lockStores(Instruction *const ld);
2747
MemoryOpt::MemoryOpt() : recordPool(sizeof(MemoryOpt::Record), 6)
2749
for (int i = 0; i < DATA_FILE_COUNT; ++i) {
2759
for (unsigned int i = 0; i < DATA_FILE_COUNT; ++i) {
2761
for (it = loads[i]; it; it = next) {
2763
recordPool.release(it);
2766
for (it = stores[i]; it; it = next) {
2768
recordPool.release(it);
2775
MemoryOpt::combineLd(Record *rec, Instruction *ld)
2777
int32_t offRc = rec->offset;
2778
int32_t offLd = ld->getSrc(0)->reg.data.offset;
2779
int sizeRc = rec->size;
2780
int sizeLd = typeSizeof(ld->dType);
2781
int size = sizeRc + sizeLd;
2784
if (!prog->getTarget()->
2785
isAccessSupported(ld->getSrc(0)->reg.file, typeOfSize(size)))
2787
// no unaligned loads
2788
if (((size == 0x8) && (MIN2(offLd, offRc) & 0x7)) ||
2789
((size == 0xc) && (MIN2(offLd, offRc) & 0xf)))
2791
// for compute indirect loads are not guaranteed to be aligned
2792
if (prog->getType() == Program::TYPE_COMPUTE && rec->rel[0])
2795
assert(sizeRc + sizeLd <= 16 && offRc != offLd);
2797
// lock any stores that overlap with the load being merged into the
2801
for (j = 0; sizeRc; sizeRc -= rec->insn->getDef(j)->reg.size, ++j);
2803
if (offLd < offRc) {
2805
for (sz = 0, d = 0; sz < sizeLd; sz += ld->getDef(d)->reg.size, ++d);
2806
// d: nr of definitions in ld
2807
// j: nr of definitions in rec->insn, move:
2808
for (d = d + j - 1; j > 0; --j, --d)
2809
rec->insn->setDef(d, rec->insn->getDef(j - 1));
2811
if (rec->insn->getSrc(0)->refCount() > 1)
2812
rec->insn->setSrc(0, cloneShallow(func, rec->insn->getSrc(0)));
2813
rec->offset = rec->insn->getSrc(0)->reg.data.offset = offLd;
2819
// move definitions of @ld to @rec->insn
2820
for (j = 0; sizeLd; ++j, ++d) {
2821
sizeLd -= ld->getDef(j)->reg.size;
2822
rec->insn->setDef(d, ld->getDef(j));
2826
rec->insn->getSrc(0)->reg.size = size;
2827
rec->insn->setType(typeOfSize(size));
2829
delete_Instruction(prog, ld);
2835
MemoryOpt::combineSt(Record *rec, Instruction *st)
2837
int32_t offRc = rec->offset;
2838
int32_t offSt = st->getSrc(0)->reg.data.offset;
2839
int sizeRc = rec->size;
2840
int sizeSt = typeSizeof(st->dType);
2842
int size = sizeRc + sizeSt;
2844
Value *src[4]; // no modifiers in ValueRef allowed for st
2847
if (!prog->getTarget()->
2848
isAccessSupported(st->getSrc(0)->reg.file, typeOfSize(size)))
2850
// no unaligned stores
2851
if (size == 8 && MIN2(offRc, offSt) & 0x7)
2853
// for compute indirect stores are not guaranteed to be aligned
2854
if (prog->getType() == Program::TYPE_COMPUTE && rec->rel[0])
2857
// There's really no great place to put this in a generic manner. Seemingly
2858
// wide stores at 0x60 don't work in GS shaders on SM50+. Don't combine
2860
if (prog->getTarget()->getChipset() >= NVISA_GM107_CHIPSET &&
2861
prog->getType() == Program::TYPE_GEOMETRY &&
2862
st->getSrc(0)->reg.file == FILE_SHADER_OUTPUT &&
2863
rec->rel[0] == NULL &&
2864
MIN2(offRc, offSt) == 0x60)
2867
// remove any existing load/store records for the store being merged into
2868
// the existing record.
2869
purgeRecords(st, DATA_FILE_COUNT);
2871
st->takeExtraSources(0, extra); // save predicate and indirect address
2873
if (offRc < offSt) {
2874
// save values from @st
2875
for (s = 0; sizeSt; ++s) {
2876
sizeSt -= st->getSrc(s + 1)->reg.size;
2877
src[s] = st->getSrc(s + 1);
2879
// set record's values as low sources of @st
2880
for (j = 1; sizeRc; ++j) {
2881
sizeRc -= rec->insn->getSrc(j)->reg.size;
2882
st->setSrc(j, rec->insn->getSrc(j));
2884
// set saved values as high sources of @st
2885
for (k = j, j = 0; j < s; ++j)
2886
st->setSrc(k++, src[j]);
2888
updateLdStOffset(st, offRc, func);
2890
for (j = 1; sizeSt; ++j)
2891
sizeSt -= st->getSrc(j)->reg.size;
2892
for (s = 1; sizeRc; ++j, ++s) {
2893
sizeRc -= rec->insn->getSrc(s)->reg.size;
2894
st->setSrc(j, rec->insn->getSrc(s));
2896
rec->offset = offSt;
2898
st->putExtraSources(0, extra); // restore pointer and predicate
2900
delete_Instruction(prog, rec->insn);
2903
rec->insn->getSrc(0)->reg.size = size;
2904
rec->insn->setType(typeOfSize(size));
2909
MemoryOpt::Record::set(const Instruction *ldst)
2911
const Symbol *mem = ldst->getSrc(0)->asSym();
2912
fileIndex = mem->reg.fileIndex;
2913
rel[0] = ldst->getIndirect(0, 0);
2914
rel[1] = ldst->getIndirect(0, 1);
2915
offset = mem->reg.data.offset;
2916
base = mem->getBase();
2917
size = typeSizeof(ldst->sType);
2921
MemoryOpt::Record::link(Record **list)
2931
MemoryOpt::Record::unlink(Record **list)
2941
MemoryOpt::Record **
2942
MemoryOpt::getList(const Instruction *insn)
2944
if (insn->op == OP_LOAD || insn->op == OP_VFETCH)
2945
return &loads[insn->src(0).getFile()];
2946
return &stores[insn->src(0).getFile()];
2950
MemoryOpt::addRecord(Instruction *i)
2952
Record **list = getList(i);
2953
Record *it = reinterpret_cast<Record *>(recordPool.allocate());
2962
MemoryOpt::findRecord(const Instruction *insn, bool load, bool& isAdj) const
2964
const Symbol *sym = insn->getSrc(0)->asSym();
2965
const int size = typeSizeof(insn->sType);
2967
Record *it = load ? loads[sym->reg.file] : stores[sym->reg.file];
2969
for (; it; it = it->next) {
2970
if (it->locked && insn->op != OP_LOAD && insn->op != OP_VFETCH)
2972
if ((it->offset >> 4) != (sym->reg.data.offset >> 4) ||
2973
it->rel[0] != insn->getIndirect(0, 0) ||
2974
it->fileIndex != sym->reg.fileIndex ||
2975
it->rel[1] != insn->getIndirect(0, 1))
2978
if (it->offset < sym->reg.data.offset) {
2979
if (it->offset + it->size >= sym->reg.data.offset) {
2980
isAdj = (it->offset + it->size == sym->reg.data.offset);
2983
if (!(it->offset & 0x7))
2987
isAdj = it->offset != sym->reg.data.offset;
2988
if (size <= it->size && !isAdj)
2991
if (!(sym->reg.data.offset & 0x7))
2992
if (it->offset - size <= sym->reg.data.offset)
3000
MemoryOpt::replaceLdFromSt(Instruction *ld, Record *rec)
3002
Instruction *st = rec->insn;
3003
int32_t offSt = rec->offset;
3004
int32_t offLd = ld->getSrc(0)->reg.data.offset;
3007
for (s = 1; offSt != offLd && st->srcExists(s); ++s)
3008
offSt += st->getSrc(s)->reg.size;
3012
for (d = 0; ld->defExists(d) && st->srcExists(s); ++d, ++s) {
3013
if (ld->getDef(d)->reg.size != st->getSrc(s)->reg.size)
3015
if (st->getSrc(s)->reg.file != FILE_GPR)
3017
ld->def(d).replace(st->src(s), false);
3024
MemoryOpt::replaceLdFromLd(Instruction *ldE, Record *rec)
3026
Instruction *ldR = rec->insn;
3027
int32_t offR = rec->offset;
3028
int32_t offE = ldE->getSrc(0)->reg.data.offset;
3031
assert(offR <= offE);
3032
for (dR = 0; offR < offE && ldR->defExists(dR); ++dR)
3033
offR += ldR->getDef(dR)->reg.size;
3037
for (dE = 0; ldE->defExists(dE) && ldR->defExists(dR); ++dE, ++dR) {
3038
if (ldE->getDef(dE)->reg.size != ldR->getDef(dR)->reg.size)
3040
ldE->def(dE).replace(ldR->getDef(dR), false);
3043
delete_Instruction(prog, ldE);
3048
MemoryOpt::replaceStFromSt(Instruction *restrict st, Record *rec)
3050
const Instruction *const ri = rec->insn;
3053
int32_t offS = st->getSrc(0)->reg.data.offset;
3054
int32_t offR = rec->offset;
3055
int32_t endS = offS + typeSizeof(st->dType);
3056
int32_t endR = offR + typeSizeof(ri->dType);
3058
rec->size = MAX2(endS, endR) - MIN2(offS, offR);
3060
st->takeExtraSources(0, extra);
3066
// get non-replaced sources of ri
3067
for (s = 1; offR < offS; offR += ri->getSrc(s)->reg.size, ++s)
3068
vals[k++] = ri->getSrc(s);
3070
// get replaced sources of st
3071
for (s = 1; st->srcExists(s); offS += st->getSrc(s)->reg.size, ++s)
3072
vals[k++] = st->getSrc(s);
3073
// skip replaced sources of ri
3074
for (s = n; offR < endS; offR += ri->getSrc(s)->reg.size, ++s);
3075
// get non-replaced sources after values covered by st
3076
for (; offR < endR; offR += ri->getSrc(s)->reg.size, ++s)
3077
vals[k++] = ri->getSrc(s);
3078
assert((unsigned int)k <= ARRAY_SIZE(vals));
3079
for (s = 0; s < k; ++s)
3080
st->setSrc(s + 1, vals[s]);
3081
st->setSrc(0, ri->getSrc(0));
3085
for (j = 1; offR < endS; offR += ri->getSrc(j++)->reg.size);
3086
for (s = 1; offS < endS; offS += st->getSrc(s++)->reg.size);
3087
for (; offR < endR; offR += ri->getSrc(j++)->reg.size)
3088
st->setSrc(s++, ri->getSrc(j));
3090
st->putExtraSources(0, extra);
3092
delete_Instruction(prog, rec->insn);
3095
rec->offset = st->getSrc(0)->reg.data.offset;
3097
st->setType(typeOfSize(rec->size));
3103
MemoryOpt::Record::overlaps(const Instruction *ldst) const
3108
// This assumes that images/buffers can't overlap. They can.
3109
// TODO: Plumb the restrict logic through, and only skip when it's a
3110
// restrict situation, or there can implicitly be no writes.
3111
if (this->fileIndex != that.fileIndex && this->rel[1] == that.rel[1])
3114
if (this->rel[0] || that.rel[0])
3115
return this->base == that.base;
3118
(this->offset < that.offset + that.size) &&
3119
(this->offset + this->size > that.offset);
3122
// We must not eliminate stores that affect the result of @ld if
3123
// we find later stores to the same location, and we may no longer
3124
// merge them with later stores.
3125
// The stored value can, however, still be used to determine the value
3126
// returned by future loads.
3128
MemoryOpt::lockStores(Instruction *const ld)
3130
for (Record *r = stores[ld->src(0).getFile()]; r; r = r->next)
3131
if (!r->locked && r->overlaps(ld))
3135
// Prior loads from the location of @st are no longer valid.
3136
// Stores to the location of @st may no longer be used to derive
3137
// the value at it nor be coalesced into later stores.
3139
MemoryOpt::purgeRecords(Instruction *const st, DataFile f)
3142
f = st->src(0).getFile();
3144
for (Record *r = loads[f]; r; r = r->next)
3145
if (!st || r->overlaps(st))
3146
r->unlink(&loads[f]);
3148
for (Record *r = stores[f]; r; r = r->next)
3149
if (!st || r->overlaps(st))
3150
r->unlink(&stores[f]);
3154
MemoryOpt::visit(BasicBlock *bb)
3156
bool ret = runOpt(bb);
3157
// Run again, one pass won't combine 4 32 bit ld/st to a single 128 bit ld/st
3158
// where 96 bit memory operations are forbidden.
3165
MemoryOpt::runOpt(BasicBlock *bb)
3167
Instruction *ldst, *next;
3169
bool isAdjacent = true;
3171
for (ldst = bb->getEntry(); ldst; ldst = next) {
3176
if (ldst->op == OP_LOAD || ldst->op == OP_VFETCH) {
3177
if (ldst->subOp == NV50_IR_SUBOP_LOAD_LOCKED) {
3178
purgeRecords(ldst, ldst->src(0).getFile());
3181
if (ldst->isDead()) {
3182
// might have been produced by earlier optimization
3183
delete_Instruction(prog, ldst);
3187
if (ldst->op == OP_STORE || ldst->op == OP_EXPORT) {
3188
if (ldst->subOp == NV50_IR_SUBOP_STORE_UNLOCKED) {
3189
purgeRecords(ldst, ldst->src(0).getFile());
3192
if (typeSizeof(ldst->dType) == 4 &&
3193
ldst->src(1).getFile() == FILE_GPR &&
3194
ldst->getSrc(1)->getInsn()->op == OP_NOP) {
3195
delete_Instruction(prog, ldst);
3200
// TODO: maybe have all fixed ops act as barrier ?
3201
if (ldst->op == OP_CALL ||
3202
ldst->op == OP_BAR ||
3203
ldst->op == OP_MEMBAR) {
3204
purgeRecords(NULL, FILE_MEMORY_LOCAL);
3205
purgeRecords(NULL, FILE_MEMORY_GLOBAL);
3206
purgeRecords(NULL, FILE_MEMORY_SHARED);
3207
purgeRecords(NULL, FILE_SHADER_OUTPUT);
3209
if (ldst->op == OP_ATOM || ldst->op == OP_CCTL) {
3210
if (ldst->src(0).getFile() == FILE_MEMORY_GLOBAL) {
3211
purgeRecords(NULL, FILE_MEMORY_LOCAL);
3212
purgeRecords(NULL, FILE_MEMORY_GLOBAL);
3213
purgeRecords(NULL, FILE_MEMORY_SHARED);
3215
purgeRecords(NULL, ldst->src(0).getFile());
3218
if (ldst->op == OP_EMIT || ldst->op == OP_RESTART) {
3219
purgeRecords(NULL, FILE_SHADER_OUTPUT);
3223
if (ldst->getPredicate()) // TODO: handle predicated ld/st
3225
if (ldst->perPatch) // TODO: create separate per-patch lists
3229
DataFile file = ldst->src(0).getFile();
3231
// if ld l[]/g[] look for previous store to eliminate the reload
3232
if (file == FILE_MEMORY_GLOBAL || file == FILE_MEMORY_LOCAL) {
3233
// TODO: shared memory ?
3234
rec = findRecord(ldst, false, isAdjacent);
3235
if (rec && !isAdjacent)
3236
keep = !replaceLdFromSt(ldst, rec);
3239
// or look for ld from the same location and replace this one
3240
rec = keep ? findRecord(ldst, true, isAdjacent) : NULL;
3243
keep = !replaceLdFromLd(ldst, rec);
3245
// or combine a previous load with this one
3246
keep = !combineLd(rec, ldst);
3251
rec = findRecord(ldst, false, isAdjacent);
3254
keep = !replaceStFromSt(ldst, rec);
3256
keep = !combineSt(rec, ldst);
3259
purgeRecords(ldst, DATA_FILE_COUNT);
3269
// =============================================================================
3271
// Turn control flow into predicated instructions (after register allocation !).
3273
// Could move this to before register allocation on NVC0 and also handle nested
3275
class FlatteningPass : public Pass
3278
FlatteningPass() : gpr_unit(0) {}
3281
virtual bool visit(Function *);
3282
virtual bool visit(BasicBlock *);
3284
bool tryPredicateConditional(BasicBlock *);
3285
void predicateInstructions(BasicBlock *, Value *pred, CondCode cc);
3286
void tryPropagateBranch(BasicBlock *);
3287
inline bool isConstantCondition(Value *pred);
3288
inline bool mayPredicate(const Instruction *, const Value *pred) const;
3289
inline void removeFlow(Instruction *);
3295
FlatteningPass::isConstantCondition(Value *pred)
3297
Instruction *insn = pred->getUniqueInsn();
3299
if (insn->op != OP_SET || insn->srcExists(2))
3302
for (int s = 0; s < 2 && insn->srcExists(s); ++s) {
3303
Instruction *ld = insn->getSrc(s)->getUniqueInsn();
3306
if (ld->op != OP_MOV && ld->op != OP_LOAD)
3308
if (ld->src(0).isIndirect(0))
3310
file = ld->src(0).getFile();
3312
file = insn->src(s).getFile();
3313
// catch $r63 on NVC0 and $r63/$r127 on NV50. Unfortunately maxGPR is
3314
// in register "units", which can vary between targets.
3315
if (file == FILE_GPR) {
3316
Value *v = insn->getSrc(s);
3317
int bytes = v->reg.data.id * MIN2(v->reg.size, 4);
3318
int units = bytes >> gpr_unit;
3319
if (units > prog->maxGPR)
3320
file = FILE_IMMEDIATE;
3323
if (file != FILE_IMMEDIATE && file != FILE_MEMORY_CONST)
3330
FlatteningPass::removeFlow(Instruction *insn)
3332
FlowInstruction *term = insn ? insn->asFlow() : NULL;
3335
Graph::Edge::Type ty = term->bb->cfg.outgoing().getType();
3337
if (term->op == OP_BRA) {
3338
// TODO: this might get more difficult when we get arbitrary BRAs
3339
if (ty == Graph::Edge::CROSS || ty == Graph::Edge::BACK)
3342
if (term->op != OP_JOIN)
3345
Value *pred = term->getPredicate();
3347
delete_Instruction(prog, term);
3349
if (pred && pred->refCount() == 0) {
3350
Instruction *pSet = pred->getUniqueInsn();
3351
pred->join->reg.data.id = -1; // deallocate
3353
delete_Instruction(prog, pSet);
3358
FlatteningPass::predicateInstructions(BasicBlock *bb, Value *pred, CondCode cc)
3360
for (Instruction *i = bb->getEntry(); i; i = i->next) {
3363
assert(!i->getPredicate());
3364
i->setPredicate(cc, pred);
3366
removeFlow(bb->getExit());
3370
FlatteningPass::mayPredicate(const Instruction *insn, const Value *pred) const
3372
if (insn->isPseudo())
3374
// TODO: calls where we don't know which registers are modified
3376
if (!prog->getTarget()->mayPredicate(insn, pred))
3378
for (int d = 0; insn->defExists(d); ++d)
3379
if (insn->getDef(d)->equals(pred))
3384
// If we jump to BRA/RET/EXIT, replace the jump with it.
3385
// NOTE: We do not update the CFG anymore here !
3387
// TODO: Handle cases where we skip over a branch (maybe do that elsewhere ?):
3389
// @p0 bra BB:2 -> @!p0 bra BB:3 iff (!) BB:2 immediately adjoins BB:1
3397
FlatteningPass::tryPropagateBranch(BasicBlock *bb)
3399
for (Instruction *i = bb->getExit(); i && i->op == OP_BRA; i = i->prev) {
3400
BasicBlock *bf = i->asFlow()->target.bb;
3402
if (bf->getInsnCount() != 1)
3405
FlowInstruction *bra = i->asFlow();
3406
FlowInstruction *rep = bf->getExit()->asFlow();
3408
if (!rep || rep->getPredicate())
3410
if (rep->op != OP_BRA &&
3411
rep->op != OP_JOIN &&
3415
// TODO: If there are multiple branches to @rep, only the first would
3416
// be replaced, so only remove them after this pass is done ?
3417
// Also, need to check all incident blocks for fall-through exits and
3418
// add the branch there.
3420
bra->target.bb = rep->target.bb;
3421
if (bf->cfg.incidentCount() == 1)
3427
FlatteningPass::visit(Function *fn)
3429
gpr_unit = prog->getTarget()->getFileUnit(FILE_GPR);
3435
FlatteningPass::visit(BasicBlock *bb)
3437
if (tryPredicateConditional(bb))
3440
// try to attach join to previous instruction
3441
if (prog->getTarget()->hasJoin) {
3442
Instruction *insn = bb->getExit();
3443
if (insn && insn->op == OP_JOIN && !insn->getPredicate()) {
3445
if (insn && !insn->getPredicate() &&
3447
insn->op != OP_DISCARD &&
3448
insn->op != OP_TEXBAR &&
3449
!isTextureOp(insn->op) && // probably just nve4
3450
!isSurfaceOp(insn->op) && // not confirmed
3451
insn->op != OP_LINTERP && // probably just nve4
3452
insn->op != OP_PINTERP && // probably just nve4
3453
((insn->op != OP_LOAD && insn->op != OP_STORE && insn->op != OP_ATOM) ||
3454
(typeSizeof(insn->dType) <= 4 && !insn->src(0).isIndirect(0))) &&
3457
bb->remove(bb->getExit());
3463
tryPropagateBranch(bb);
3469
FlatteningPass::tryPredicateConditional(BasicBlock *bb)
3471
BasicBlock *bL = NULL, *bR = NULL;
3472
unsigned int nL = 0, nR = 0, limit = 12;
3476
mask = bb->initiatesSimpleConditional();
3480
assert(bb->getExit());
3481
Value *pred = bb->getExit()->getPredicate();
3484
if (isConstantCondition(pred))
3487
Graph::EdgeIterator ei = bb->cfg.outgoing();
3490
bL = BasicBlock::get(ei.getNode());
3491
for (insn = bL->getEntry(); insn; insn = insn->next, ++nL)
3492
if (!mayPredicate(insn, pred))
3495
return false; // too long, do a real branch
3500
bR = BasicBlock::get(ei.getNode());
3501
for (insn = bR->getEntry(); insn; insn = insn->next, ++nR)
3502
if (!mayPredicate(insn, pred))
3505
return false; // too long, do a real branch
3509
predicateInstructions(bL, pred, bb->getExit()->cc);
3511
predicateInstructions(bR, pred, inverseCondCode(bb->getExit()->cc));
3514
bb->remove(bb->joinAt);
3517
removeFlow(bb->getExit()); // delete the branch/join at the fork point
3519
// remove potential join operations at the end of the conditional
3520
if (prog->getTarget()->joinAnterior) {
3521
bb = BasicBlock::get((bL ? bL : bR)->cfg.outgoing().getNode());
3522
if (bb->getEntry() && bb->getEntry()->op == OP_JOIN)
3523
removeFlow(bb->getEntry());
3529
// =============================================================================
3531
// Fold Immediate into MAD; must be done after register allocation due to
3532
// constraint SDST == SSRC2
3534
// Does NVC0+ have other situations where this pass makes sense?
3535
class PostRaLoadPropagation : public Pass
3538
virtual bool visit(Instruction *);
3540
void handleMADforNV50(Instruction *);
3541
void handleMADforNVC0(Instruction *);
3545
post_ra_dead(Instruction *i)
3547
for (int d = 0; i->defExists(d); ++d)
3548
if (i->getDef(d)->refCount())
3553
// Fold Immediate into MAD; must be done after register allocation due to
3554
// constraint SDST == SSRC2
3556
PostRaLoadPropagation::handleMADforNV50(Instruction *i)
3558
if (i->def(0).getFile() != FILE_GPR ||
3559
i->src(0).getFile() != FILE_GPR ||
3560
i->src(1).getFile() != FILE_GPR ||
3561
i->src(2).getFile() != FILE_GPR ||
3562
i->getDef(0)->reg.data.id != i->getSrc(2)->reg.data.id)
3565
if (i->getDef(0)->reg.data.id >= 64 ||
3566
i->getSrc(0)->reg.data.id >= 64)
3569
if (i->flagsSrc >= 0 && i->getSrc(i->flagsSrc)->reg.data.id != 0)
3572
if (i->getPredicate())
3576
Instruction *def = i->getSrc(1)->getInsn();
3578
if (def && def->op == OP_SPLIT && typeSizeof(def->sType) == 4)
3579
def = def->getSrc(0)->getInsn();
3580
if (def && def->op == OP_MOV && def->src(0).getFile() == FILE_IMMEDIATE) {
3581
vtmp = i->getSrc(1);
3582
if (isFloatType(i->sType)) {
3583
i->setSrc(1, def->getSrc(0));
3586
// getImmediate() has side-effects on the argument so this *shouldn't*
3587
// be folded into the assert()
3588
ASSERTED bool ret = def->src(0).getImmediate(val);
3590
if (i->getSrc(1)->reg.data.id & 1)
3591
val.reg.data.u32 >>= 16;
3592
val.reg.data.u32 &= 0xffff;
3593
i->setSrc(1, new_ImmediateValue(prog, val.reg.data.u32));
3596
/* There's no post-RA dead code elimination, so do it here
3597
* XXX: if we add more code-removing post-RA passes, we might
3598
* want to create a post-RA dead-code elim pass */
3599
if (post_ra_dead(vtmp->getInsn())) {
3600
Value *src = vtmp->getInsn()->getSrc(0);
3601
// Careful -- splits will have already been removed from the
3602
// functions. Don't double-delete.
3603
if (vtmp->getInsn()->bb)
3604
delete_Instruction(prog, vtmp->getInsn());
3605
if (src->getInsn() && post_ra_dead(src->getInsn()))
3606
delete_Instruction(prog, src->getInsn());
3612
PostRaLoadPropagation::handleMADforNVC0(Instruction *i)
3614
if (i->def(0).getFile() != FILE_GPR ||
3615
i->src(0).getFile() != FILE_GPR ||
3616
i->src(1).getFile() != FILE_GPR ||
3617
i->src(2).getFile() != FILE_GPR ||
3618
i->getDef(0)->reg.data.id != i->getSrc(2)->reg.data.id)
3621
// TODO: gm107 can also do this for S32, maybe other chipsets as well
3622
if (i->dType != TYPE_F32)
3625
if ((i->src(2).mod | Modifier(NV50_IR_MOD_NEG)) != Modifier(NV50_IR_MOD_NEG))
3631
if (i->src(0).getImmediate(val))
3633
else if (i->src(1).getImmediate(val))
3638
if ((i->src(s).mod | Modifier(NV50_IR_MOD_NEG)) != Modifier(NV50_IR_MOD_NEG))
3642
i->swapSources(0, 1);
3644
Instruction *imm = i->getSrc(1)->getInsn();
3645
i->setSrc(1, imm->getSrc(0));
3646
if (post_ra_dead(imm))
3647
delete_Instruction(prog, imm);
3651
PostRaLoadPropagation::visit(Instruction *i)
3656
if (prog->getTarget()->getChipset() < 0xc0)
3657
handleMADforNV50(i);
3659
handleMADforNVC0(i);
3668
// =============================================================================
3670
// Common subexpression elimination. Stupid O^2 implementation.
3671
class LocalCSE : public Pass
3674
virtual bool visit(BasicBlock *);
3676
inline bool tryReplace(Instruction **, Instruction *);
3678
DLList ops[OP_LAST + 1];
3681
class GlobalCSE : public Pass
3684
virtual bool visit(BasicBlock *);
3688
Instruction::isActionEqual(const Instruction *that) const
3690
if (this->op != that->op ||
3691
this->dType != that->dType ||
3692
this->sType != that->sType)
3694
if (this->cc != that->cc)
3697
if (this->asTex()) {
3698
if (memcmp(&this->asTex()->tex,
3699
&that->asTex()->tex,
3700
sizeof(this->asTex()->tex)))
3703
if (this->asCmp()) {
3704
if (this->asCmp()->setCond != that->asCmp()->setCond)
3707
if (this->asFlow()) {
3710
if (this->op == OP_PHI && this->bb != that->bb) {
3711
/* TODO: we could probably be a bit smarter here by following the
3712
* control flow, but honestly, it is quite painful to check */
3715
if (this->ipa != that->ipa ||
3716
this->lanes != that->lanes ||
3717
this->perPatch != that->perPatch)
3719
if (this->postFactor != that->postFactor)
3723
if (this->subOp != that->subOp ||
3724
this->saturate != that->saturate ||
3725
this->rnd != that->rnd ||
3726
this->ftz != that->ftz ||
3727
this->dnz != that->dnz ||
3728
this->cache != that->cache ||
3729
this->mask != that->mask)
3736
Instruction::isResultEqual(const Instruction *that) const
3740
// NOTE: location of discard only affects tex with liveOnly and quadops
3741
if (!this->defExists(0) && this->op != OP_DISCARD)
3744
if (!isActionEqual(that))
3747
if (this->predSrc != that->predSrc)
3750
for (d = 0; this->defExists(d); ++d) {
3751
if (!that->defExists(d) ||
3752
!this->getDef(d)->equals(that->getDef(d), false))
3755
if (that->defExists(d))
3758
for (s = 0; this->srcExists(s); ++s) {
3759
if (!that->srcExists(s))
3761
if (this->src(s).mod != that->src(s).mod)
3763
if (!this->getSrc(s)->equals(that->getSrc(s), true))
3766
if (that->srcExists(s))
3769
if (op == OP_LOAD || op == OP_VFETCH || op == OP_ATOM) {
3770
switch (src(0).getFile()) {
3771
case FILE_MEMORY_CONST:
3772
case FILE_SHADER_INPUT:
3774
case FILE_SHADER_OUTPUT:
3775
return bb->getProgram()->getType() == Program::TYPE_TESSELLATION_EVAL;
3784
// pull through common expressions from different in-blocks
3786
GlobalCSE::visit(BasicBlock *bb)
3788
Instruction *phi, *next, *ik;
3791
// TODO: maybe do this with OP_UNION, too
3793
for (phi = bb->getPhi(); phi && phi->op == OP_PHI; phi = next) {
3795
if (phi->getSrc(0)->refCount() > 1)
3797
ik = phi->getSrc(0)->getInsn();
3799
continue; // probably a function input
3800
if (ik->defCount(0xff) > 1)
3801
continue; // too painful to check if we can really push this forward
3802
for (s = 1; phi->srcExists(s); ++s) {
3803
if (phi->getSrc(s)->refCount() > 1)
3805
if (!phi->getSrc(s)->getInsn() ||
3806
!phi->getSrc(s)->getInsn()->isResultEqual(ik))
3809
if (!phi->srcExists(s)) {
3810
assert(ik->op != OP_PHI);
3811
Instruction *entry = bb->getEntry();
3813
if (!entry || entry->op != OP_JOIN)
3816
bb->insertAfter(entry, ik);
3817
ik->setDef(0, phi->getDef(0));
3818
delete_Instruction(prog, phi);
3826
LocalCSE::tryReplace(Instruction **ptr, Instruction *i)
3828
Instruction *old = *ptr;
3830
// TODO: maybe relax this later (causes trouble with OP_UNION)
3831
if (i->isPredicated())
3834
if (!old->isResultEqual(i))
3837
for (int d = 0; old->defExists(d); ++d)
3838
old->def(d).replace(i->getDef(d), false);
3839
delete_Instruction(prog, old);
3845
LocalCSE::visit(BasicBlock *bb)
3847
unsigned int replaced;
3850
Instruction *ir, *next;
3854
// will need to know the order of instructions
3856
for (ir = bb->getFirst(); ir; ir = ir->next)
3857
ir->serial = serial++;
3859
for (ir = bb->getFirst(); ir; ir = next) {
3866
ops[ir->op].insert(ir);
3870
for (s = 0; ir->srcExists(s); ++s)
3871
if (ir->getSrc(s)->asLValue())
3872
if (!src || ir->getSrc(s)->refCount() < src->refCount())
3873
src = ir->getSrc(s);
3876
for (Value::UseIterator it = src->uses.begin();
3877
it != src->uses.end(); ++it) {
3878
Instruction *ik = (*it)->getInsn();
3879
if (ik && ik->bb == ir->bb && ik->serial < ir->serial)
3880
if (tryReplace(&ir, ik))
3884
DLLIST_FOR_EACH(&ops[ir->op], iter)
3886
Instruction *ik = reinterpret_cast<Instruction *>(iter.get());
3887
if (tryReplace(&ir, ik))
3893
ops[ir->op].insert(ir);
3897
for (unsigned int i = 0; i <= OP_LAST; ++i)
3905
// =============================================================================
3907
// Remove computations of unused values.
3908
class DeadCodeElim : public Pass
3911
DeadCodeElim() : deadCount(0) {}
3912
bool buryAll(Program *);
3915
virtual bool visit(BasicBlock *);
3917
void checkSplitLoad(Instruction *ld); // for partially dead loads
3919
unsigned int deadCount;
3923
DeadCodeElim::buryAll(Program *prog)
3927
if (!this->run(prog, false, false))
3929
} while (deadCount);
3935
DeadCodeElim::visit(BasicBlock *bb)
3939
for (Instruction *i = bb->getExit(); i; i = prev) {
3943
delete_Instruction(prog, i);
3945
if (i->defExists(1) &&
3947
(i->op == OP_VFETCH || i->op == OP_LOAD)) {
3950
if (i->defExists(0) && !i->getDef(0)->refCount()) {
3951
if (i->op == OP_ATOM ||
3952
i->op == OP_SUREDP ||
3953
i->op == OP_SUREDB) {
3954
const Target *targ = prog->getTarget();
3955
if (targ->getChipset() >= NVISA_GF100_CHIPSET ||
3956
i->subOp != NV50_IR_SUBOP_ATOM_CAS)
3958
if (i->op == OP_ATOM && i->subOp == NV50_IR_SUBOP_ATOM_EXCH) {
3959
i->cache = CACHE_CV;
3963
} else if (i->op == OP_LOAD && i->subOp == NV50_IR_SUBOP_LOAD_LOCKED) {
3964
i->setDef(0, i->getDef(1));
3972
// Each load can go into up to 4 destinations, any of which might potentially
3973
// be dead (i.e. a hole). These can always be split into 2 loads, independent
3974
// of where the holes are. We find the first contiguous region, put it into
3975
// the first load, and then put the second contiguous region into the second
3976
// load. There can be at most 2 contiguous regions.
3978
// Note that there are some restrictions, for example it's not possible to do
3979
// a 64-bit load that's not 64-bit aligned, so such a load has to be split
3980
// up. Also hardware doesn't support 96-bit loads, so those also have to be
3981
// split into a 64-bit and 32-bit load.
3983
DeadCodeElim::checkSplitLoad(Instruction *ld1)
3985
Instruction *ld2 = NULL; // can get at most 2 loads
3988
int32_t addr1, addr2;
3989
int32_t size1, size2;
3991
uint32_t mask = 0xffffffff;
3993
for (d = 0; ld1->defExists(d); ++d)
3994
if (!ld1->getDef(d)->refCount() && ld1->getDef(d)->reg.data.id < 0)
3996
if (mask == 0xffffffff)
3999
addr1 = ld1->getSrc(0)->reg.data.offset;
4003
// Compute address/width for first load
4004
for (d = 0; ld1->defExists(d); ++d) {
4005
if (mask & (1 << d)) {
4006
if (size1 && (addr1 & 0x7))
4008
def1[n1] = ld1->getDef(d);
4009
size1 += def1[n1++]->reg.size;
4012
addr1 += ld1->getDef(d)->reg.size;
4018
// Scale back the size of the first load until it can be loaded. This
4019
// typically happens for TYPE_B96 loads.
4021
!prog->getTarget()->isAccessSupported(ld1->getSrc(0)->reg.file,
4022
typeOfSize(size1))) {
4023
size1 -= def1[--n1]->reg.size;
4027
// Compute address/width for second load
4028
for (addr2 = addr1 + size1; ld1->defExists(d); ++d) {
4029
if (mask & (1 << d)) {
4030
assert(!size2 || !(addr2 & 0x7));
4031
def2[n2] = ld1->getDef(d);
4032
size2 += def2[n2++]->reg.size;
4035
addr2 += ld1->getDef(d)->reg.size;
4041
// Make sure that we've processed all the values
4042
for (; ld1->defExists(d); ++d)
4043
assert(!(mask & (1 << d)));
4045
updateLdStOffset(ld1, addr1, func);
4046
ld1->setType(typeOfSize(size1));
4047
for (d = 0; d < 4; ++d)
4048
ld1->setDef(d, (d < n1) ? def1[d] : NULL);
4053
ld2 = cloneShallow(func, ld1);
4054
updateLdStOffset(ld2, addr2, func);
4055
ld2->setType(typeOfSize(size2));
4056
for (d = 0; d < 4; ++d)
4057
ld2->setDef(d, (d < n2) ? def2[d] : NULL);
4059
ld1->bb->insertAfter(ld1, ld2);
4062
// =============================================================================
4064
#define RUN_PASS(l, n, f) \
4065
if (level >= (l)) { \
4066
if (dbgFlags & NV50_IR_DEBUG_VERBOSE) \
4067
INFO("PEEPHOLE: %s\n", #n); \
4069
if (!pass.f(this)) \
4074
Program::optimizeSSA(int level)
4076
RUN_PASS(1, DeadCodeElim, buryAll);
4077
RUN_PASS(1, CopyPropagation, run);
4078
RUN_PASS(1, MergeSplits, run);
4079
RUN_PASS(2, GlobalCSE, run);
4080
RUN_PASS(1, LocalCSE, run);
4081
RUN_PASS(2, AlgebraicOpt, run);
4082
RUN_PASS(2, ModifierFolding, run); // before load propagation -> less checks
4083
RUN_PASS(1, ConstantFolding, foldAll);
4084
RUN_PASS(0, Split64BitOpPreRA, run);
4085
RUN_PASS(2, LateAlgebraicOpt, run);
4086
RUN_PASS(1, LoadPropagation, run);
4087
RUN_PASS(1, IndirectPropagation, run);
4088
RUN_PASS(2, MemoryOpt, run);
4089
RUN_PASS(2, LocalCSE, run);
4090
RUN_PASS(0, DeadCodeElim, buryAll);
4096
Program::optimizePostRA(int level)
4098
RUN_PASS(2, FlatteningPass, run);
4099
RUN_PASS(2, PostRaLoadPropagation, run);
added
removed
src/gallium/drivers/nouveau/codegen/nv50_ir_peephole.cpp
